flowyml 1.7.1__py3-none-any.whl → 1.8.0__py3-none-any.whl

flowyml/core/image_builder.py

@@ -0,0 +1,129 @@
+import subprocess
+from pathlib import Path
+from flowyml.stacks.components import DockerConfig
+
+
+class DockerImageBuilder:
+    """Handles building and pushing Docker images for remote execution."""
+
+    def build_image(self, docker_config: DockerConfig, tag: str) -> str:
+        """Build a Docker image from the configuration.
+
+        Args:
+            docker_config: The Docker configuration.
+            tag: The tag to apply to the built image.
+
+        Returns:
+            The full image tag that was built.
+        """
+        build_context = Path(docker_config.build_context)
+        if not build_context.exists():
+            raise FileNotFoundError(f"Build context not found: {build_context}")
+
+        # Auto-generate Dockerfile if needed
+        dockerfile_path = self._ensure_dockerfile(docker_config, build_context)
+
+        cmd = [
+            "docker",
+            "build",
+            "-t",
+            tag,
+            "-f",
+            str(dockerfile_path),
+            str(build_context),
+        ]
+
+        # Add build args
+        for k, v in docker_config.build_args.items():
+            cmd.extend(["--build-arg", f"{k}={v}"])
+
+        print(f"🐳 Building image: {tag}")
+        try:
+            subprocess.run(cmd, check=True)
+            print("✅ Build successful!")
+            return tag
+        except subprocess.CalledProcessError as e:
+            raise RuntimeError(f"Docker build failed: {e}")
+
+    def _ensure_dockerfile(self, config: DockerConfig, context: Path) -> Path:
+        """Get path to Dockerfile or generate one."""
+        if config.dockerfile:
+            path = context / config.dockerfile
+            if not path.exists():
+                # Try absolute path
+                path = Path(config.dockerfile)
+                if not path.exists():
+                    raise FileNotFoundError(f"Dockerfile not found: {config.dockerfile}")
+            return path
+
+        # Generate temporary Dockerfile
+        generated_path = context / ".flowyml.Dockerfile"
+        content = self._generate_dockerfile_content(config)
+        generated_path.write_text(content)
+        return generated_path
+
+    def _generate_dockerfile_content(self, config: DockerConfig) -> str:
+        """Generate Dockerfile content based on requirements.
+
+        Prioritizes:
+        1. uv.lock -> uv sync
+        2. poetry.lock -> poetry install
+        3. requirements.txt -> uv pip install
+        4. list -> uv pip install
+        """
+        lines = [f"FROM {config.base_image}", "WORKDIR /app"]
+
+        # Install system dependencies if any
+        # lines.append("RUN apt-get update && apt-get install -y ...")
+
+        context_path = Path(config.build_context)
+
+        # 0. Always install uv as it's our preferred installer for pip/reqs too
+        # We install it via the official installer script for speed and isolation
+        lines.append("RUN pip install uv")
+        lines.append("ENV VIRTUAL_ENV=/app/.venv")
+        lines.append('ENV PATH="$VIRTUAL_ENV/bin:$PATH"')
+
+        # 1. Check for uv.lock
+        if (context_path / "uv.lock").exists():
+            print("📦 Detected uv based project")
+            lines.append("COPY pyproject.toml uv.lock ./")
+            # Create venv and sync
+            lines.append("RUN uv venv && uv sync --frozen --no-install-project")
+
+        # 2. Check for poetry.lock
+        elif (context_path / "poetry.lock").exists() or (context_path / "pyproject.toml").exists():
+            print("📦 Detected Poetry based project")
+            lines.append("RUN pip install poetry")
+            lines.append("COPY pyproject.toml poetry.lock* ./")
+            lines.append("RUN poetry config virtualenvs.in-project true")
+            lines.append("RUN poetry install --no-interaction --no-ansi --no-root")
+            # Add local venv to path if poetry created one
+            lines.append('ENV PATH="/app/.venv/bin:$PATH"')
+
+        # 3. Check for requirements.txt (Use uv for speed)
+        elif (context_path / "requirements.txt").exists():
+            print("📦 Detected requirements.txt")
+            lines.append("COPY requirements.txt .")
+            lines.append("RUN uv venv && uv pip install -r requirements.txt")
+
+        # 4. Check for dynamic requirements list (Use uv for speed)
+        elif config.requirements:
+            print("📦 Detected dynamic requirements list")
+            reqs_str = " ".join([f'"{r}"' for r in config.requirements])
+            lines.append(f"RUN uv venv && uv pip install {reqs_str}")
+
+        # Copy source code
+        lines.append("COPY . .")
+
+        # Install project itself if needed (for uv/poetry)
+        if (context_path / "uv.lock").exists():
+            lines.append("RUN uv sync --frozen")
+        elif (context_path / "poetry.lock").exists():
+            lines.append("RUN poetry install --no-interaction --no-ansi")
+
+        # Env vars
+        for k, v in config.env_vars.items():
+            lines.append(f"ENV {k}={v}")
+
+        return "\n".join(lines)

flowyml/core/log_streamer.py

@@ -0,0 +1,227 @@
+"""Real-time log streaming utilities.
+
+This module provides utilities for capturing and streaming logs from
+pipeline executions to connected WebSocket clients.
+"""
+
+import asyncio
+import contextlib
+import logging
+import sys
+import threading
+from collections import deque
+from datetime import datetime
+from io import StringIO
+from collections.abc import Callable
+
+from flowyml.ui.backend.routers.websocket import manager as ws_manager
+
+
+class LogBuffer:
+    """Thread-safe log buffer with configurable max size."""
+
+    def __init__(self, max_size: int = 1000):
+        self._buffer: deque[dict] = deque(maxlen=max_size)
+        self._lock = threading.Lock()
+
+    def append(self, entry: dict) -> None:
+        """Append a log entry to the buffer."""
+        with self._lock:
+            self._buffer.append(entry)
+
+    def get_recent(self, count: int = 100) -> list[dict]:
+        """Get recent log entries."""
+        with self._lock:
+            return list(self._buffer)[-count:]
+
+    def clear(self) -> None:
+        """Clear the buffer."""
+        with self._lock:
+            self._buffer.clear()
+
+
+class LogStreamer:
+    """Captures and streams logs to WebSocket clients.
+
+    This class hooks into Python's logging system and stdout/stderr
+    to capture all output and stream it to connected WebSocket clients.
+
+    Example:
+        ```python
+        streamer = LogStreamer(run_id="abc123")
+
+        # Start capturing
+        streamer.start()
+
+        # Your pipeline code runs here...
+        print("Processing step 1...")
+        logger.info("Step 1 complete")
+
+        # Stop capturing
+        streamer.stop()
+        ```
+    """
+
+    def __init__(self, run_id: str, step_name: str = "__all__", buffer_size: int = 1000):
+        self.run_id = run_id
+        self.step_name = step_name
+        self.buffer = LogBuffer(max_size=buffer_size)
+        self._active = False
+        self._original_stdout = None
+        self._original_stderr = None
+        self._log_handler = None
+        self._loop: asyncio.AbstractEventLoop | None = None
+
+    def start(self) -> None:
+        """Start capturing logs."""
+        if self._active:
+            return
+
+        self._active = True
+
+        # Try to get the event loop
+        try:
+            self._loop = asyncio.get_running_loop()
+        except RuntimeError:
+            self._loop = None
+
+        # Install stdout/stderr hooks
+        self._original_stdout = sys.stdout
+        self._original_stderr = sys.stderr
+        sys.stdout = _StreamWrapper(self._original_stdout, self._on_output, "stdout")
+        sys.stderr = _StreamWrapper(self._original_stderr, self._on_output, "stderr")
+
+        # Install logging handler
+        self._log_handler = _StreamLoggingHandler(self._on_log)
+        logging.root.addHandler(self._log_handler)
+
+    def stop(self) -> None:
+        """Stop capturing logs."""
+        if not self._active:
+            return
+
+        self._active = False
+
+        # Restore stdout/stderr
+        if self._original_stdout:
+            sys.stdout = self._original_stdout
+        if self._original_stderr:
+            sys.stderr = self._original_stderr
+
+        # Remove logging handler
+        if self._log_handler:
+            logging.root.removeHandler(self._log_handler)
+            self._log_handler = None
+
+    def _on_output(self, text: str, stream: str) -> None:
+        """Handle stdout/stderr output."""
+        if not text.strip():
+            return
+
+        entry = {
+            "timestamp": datetime.now().isoformat(),
+            "level": "ERROR" if stream == "stderr" else "INFO",
+            "message": text.strip(),
+            "source": stream,
+        }
+
+        self.buffer.append(entry)
+        self._broadcast(entry)
+
+    def _on_log(self, record: logging.LogRecord) -> None:
+        """Handle log record."""
+        entry = {
+            "timestamp": datetime.now().isoformat(),
+            "level": record.levelname,
+            "message": record.getMessage(),
+            "source": "logging",
+            "logger": record.name,
+        }
+
+        self.buffer.append(entry)
+        self._broadcast(entry)
+
+    def _broadcast(self, entry: dict) -> None:
+        """Broadcast log entry to WebSocket clients."""
+        content = f"[{entry['timestamp'][:19]}] [{entry['level']}] {entry['message']}"
+
+        if self._loop and self._loop.is_running():
+            # Schedule the coroutine on the event loop
+            asyncio.run_coroutine_threadsafe(
+                ws_manager.broadcast_log(self.run_id, self.step_name, content),
+                self._loop,
+            )
+
+    def get_history(self, count: int = 100) -> list[dict]:
+        """Get recent log history."""
+        return self.buffer.get_recent(count)
+
+    def __enter__(self) -> "LogStreamer":
+        """Context manager entry."""
+        self.start()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+        """Context manager exit."""
+        self.stop()
+
+
+class _StreamWrapper:
+    """Wrapper for stdout/stderr to capture output."""
+
+    def __init__(self, original: StringIO, callback: Callable, stream_name: str):
+        self._original = original
+        self._callback = callback
+        self._stream_name = stream_name
+
+    def write(self, text: str) -> int:
+        """Write to the stream and callback."""
+        # Write to original
+        result = self._original.write(text)
+        # Callback for streaming
+        self._callback(text, self._stream_name)
+        return result
+
+    def flush(self) -> None:
+        """Flush the stream."""
+        self._original.flush()
+
+    def __getattr__(self, name: str):
+        """Proxy other attributes to original stream."""
+        return getattr(self._original, name)
+
+
+class _StreamLoggingHandler(logging.Handler):
+    """Logging handler that calls a callback for each record."""
+
+    def __init__(self, callback: Callable):
+        super().__init__()
+        self._callback = callback
+
+    def emit(self, record: logging.LogRecord) -> None:
+        """Emit a log record."""
+        with contextlib.suppress(Exception):
+            self._callback(record)
+
+
+# Global registry of active streamers
+_active_streamers: dict[str, LogStreamer] = {}
+
+
+def get_streamer(run_id: str) -> LogStreamer | None:
+    """Get an active streamer by run ID."""
+    return _active_streamers.get(run_id)
+
+
+def create_streamer(run_id: str, step_name: str = "__all__") -> LogStreamer:
+    """Create and register a new log streamer."""
+    streamer = LogStreamer(run_id, step_name)
+    _active_streamers[run_id] = streamer
+    return streamer
+
+
+def remove_streamer(run_id: str) -> None:
+    """Remove a streamer from the registry."""
+    if run_id in _active_streamers:
+        _active_streamers[run_id].stop()
+        del _active_streamers[run_id]

flowyml/core/orchestrator.py

@@ -68,6 +68,22 @@ class LocalOrchestrator(Orchestrator):
         result = PipelineResult(run_id, pipeline.name)
         result.attach_configs(resources, docker_config)
 
+        # Save run as "running" immediately so artifacts can reference run_id
+        # This ensures FK constraints are satisfied when artifacts are created during step execution
+        if pipeline.metadata_store:
+            try:
+                initial_metadata = {
+                    "run_id": run_id,
+                    "pipeline_name": pipeline.name,
+                    "status": "running",
+                    "start_time": result.start_time.isoformat() if result.start_time else None,
+                    "project": getattr(pipeline, "_project_name", None),
+                    "context": dict(pipeline.context) if pipeline.context else {},
+                }
+                pipeline.metadata_store.save_run(run_id, initial_metadata)
+            except Exception:
+                pass  # Silently continue if initial save fails
+
         # Run pipeline start hooks
         hooks = get_global_hooks()
         hooks.run_pipeline_start_hooks(pipeline)
@@ -266,8 +282,8 @@ class LocalOrchestrator(Orchestrator):
                     pipeline._save_pipeline_definition()
                     return result
 
-                # Get context parameters for this step
-                context_params = pipeline.context.inject_params(step.func)
+                # Get all context parameters for this step (to allow conditions to access any of them)
+                context_params = pipeline.context.to_dict()
 
                 # Update display - step starting
                 if hasattr(pipeline, "_display") and pipeline._display:
@@ -285,6 +301,7 @@ class LocalOrchestrator(Orchestrator):
                     artifact_store=pipeline.stack.artifact_store if pipeline.stack else None,
                     run_id=run_id,
                     project_name=pipeline.name,
+                    all_outputs=step_outputs,
                 )
 
                 # Run step end hooks
@@ -390,6 +407,7 @@ class LocalOrchestrator(Orchestrator):
             """Context object for conditional evaluation.
 
             Provides access to step outputs via ctx.steps['step_name'].outputs['output_name']
+            and context parameters via ctx.params
             """
 
             def __init__(self, result: "PipelineResult", pipeline: "Pipeline"):
@@ -397,6 +415,13 @@ class LocalOrchestrator(Orchestrator):
                 self.pipeline = pipeline
                 self._steps_cache = None
 
+            @property
+            def params(self):
+                """Get pipeline context parameters as a dictionary."""
+                if self.pipeline.context:
+                    return self.pipeline.context._params
+                return {}
+
             @property
             def steps(self):
                 """Lazy-load steps dictionary with outputs."""
@@ -627,6 +652,12 @@ class LocalOrchestrator(Orchestrator):
                     if step_obj.name not in result.step_results:
                         # Execute the selected step
                         # The check above prevents re-execution of the same step
+                        # If step has inputs defined, copy them to the step object for proper input mapping
+                        if hasattr(selected_step, "_step_inputs") and selected_step._step_inputs:
+                            step_obj.inputs = selected_step._step_inputs
+                        elif hasattr(selected_step, "inputs"):
+                            step_obj.inputs = selected_step.inputs or []
+
                         self._execute_conditional_step(
                             pipeline,
                             step_obj,
@@ -659,10 +690,31 @@ class LocalOrchestrator(Orchestrator):
         step_inputs = {}
         sig = inspect.signature(step.func)
         params = [p for p in sig.parameters.values() if p.name not in ("self", "cls")]
-
+        assigned_params = set()
+
+        # First, try to map from declared inputs (like "model/trained" -> function param)
+        if step.inputs:
+            for i, input_name in enumerate(step.inputs):
+                if input_name not in step_outputs:
+                    continue
+                val = step_outputs[input_name]
+                # Try to match input name directly to a parameter
+                param_match = next((p for p in params if p.name == input_name), None)
+                if param_match:
+                    step_inputs[param_match.name] = val
+                    assigned_params.add(param_match.name)
+                elif i < len(params):
+                    # Positional fallback - use parameter at same position
+                    target_param = params[i]
+                    if target_param.name not in assigned_params:
+                        step_inputs[target_param.name] = val
+                        assigned_params.add(target_param.name)
+
+        # Then, try direct parameter name matching from step_outputs
         for param in params:
-            if param.name in step_outputs:
+            if param.name not in assigned_params and param.name in step_outputs:
                 step_inputs[param.name] = step_outputs[param.name]
+                assigned_params.add(param.name)
 
         # Get context parameters
         context_params = pipeline.context.inject_params(step.func)
@@ -746,6 +798,9 @@ class LocalOrchestrator(Orchestrator):
         else:
             if step_def.outputs:
                 outputs_to_process[step_def.outputs[0]] = step_result.output
+            else:
+                # Fallback: use step name as output key
+                outputs_to_process[step_result.step_name] = step_result.output
 
         # Save and update state
         for name, value in outputs_to_process.items():

flowyml/core/pipeline.py

@@ -70,13 +70,15 @@ class PipelineResult:
             "start_time": self.start_time.isoformat(),
             "end_time": self.end_time.isoformat() if self.end_time else None,
             "duration_seconds": self.duration_seconds,
-            "resource_config": self.resource_config.to_dict()
-            if hasattr(self.resource_config, "to_dict")
-            else self.resource_config,
-            "docker_config": self.docker_config.to_dict()
-            if hasattr(self.docker_config, "to_dict")
-            else self.docker_config,
-            "remote_job_id": self.remote_job_id,
+            "metadata": {
+                "resources": self.resource_config.to_dict()
+                if hasattr(self.resource_config, "to_dict")
+                else self.resource_config,
+                "docker": self.docker_config.to_dict()
+                if hasattr(self.docker_config, "to_dict")
+                else self.docker_config,
+                "remote_job_id": self.remote_job_id,
+            },
             "steps": {
                 name: {
                     "success": result.success,
@@ -243,10 +245,17 @@ class Pipeline:
         # Metadata store for UI integration - use same store as UI
         from flowyml.storage.metadata import SQLiteMetadataStore
         from flowyml.utils.config import get_config
+        import os
 
         config = get_config()
-        # Use the same metadata database path as the UI to ensure visibility
-        self.metadata_store = SQLiteMetadataStore(db_path=str(config.metadata_db))
+        # Use simple environment variable check to allow connecting to shared DB
+        db_url = os.environ.get("FLOWYML_DATABASE_URL")
+
+        if db_url:
+            self.metadata_store = SQLiteMetadataStore(db_url=db_url)
+        else:
+            # Use the same metadata database path as the UI to ensure visibility
+            self.metadata_store = SQLiteMetadataStore(db_path=str(config.metadata_db))
 
         if stack:
             self._apply_stack(stack, locked=True)
@@ -488,6 +497,21 @@ class Pipeline:
         resource_config = self._coerce_resource_config(resources)
         docker_cfg = self._coerce_docker_config(docker_config)
 
+        # Prepare Docker Image if running on a stack
+        if self.stack and docker_cfg:
+            try:
+                # This handles building/pushing or validating the URI
+                project_name = getattr(self, "project_name", None)
+                docker_cfg.image = self.stack.prepare_docker_image(
+                    docker_cfg,
+                    pipeline_name=self.name,
+                    project_name=project_name,
+                )
+            except Exception as e:
+                # If preparation fails (e.g. build error), we should probably fail the run
+                # or at least warn. For now, we'll fail to prevent running with bad config
+                raise RuntimeError(f"Failed to prepare docker image: {e}") from e
+
         # Initialize display system for beautiful CLI output
         display = None
         try:
@@ -972,6 +996,26 @@ class Pipeline:
                     if is_asset:
                         # Handle flowyml Asset
                         asset_type = value.__class__.__name__
+                        # Get properties
+                        props = (
+                            self._sanitize_for_json(value.metadata.properties)
+                            if hasattr(value.metadata, "properties")
+                            else {}
+                        )
+
+                        # For Dataset assets, include the full data for visualization
+                        # This enables histograms and statistics in the UI
+                        data_value = None
+                        if asset_type == "Dataset" and value.data is not None:
+                            try:
+                                # Store full data as JSON-serializable dict
+                                data_value = self._sanitize_for_json(value.data)
+                                props["_full_data"] = data_value
+                            except Exception:
+                                data_value = str(value.data)[:1000]
+                        else:
+                            data_value = str(value.data)[:1000] if value.data else None
+
                         artifact_metadata = {
                             "artifact_id": artifact_id,
                             "name": value.name,
@@ -979,12 +1023,20 @@ class Pipeline:
                             "run_id": result.run_id,
                             "step": step_name,
                             "path": None,
-                            "value": str(value.data)[:1000] if value.data else None,
+                            "value": data_value if isinstance(data_value, str) else None,
                             "created_at": datetime.now().isoformat(),
-                            "properties": self._sanitize_for_json(value.metadata.properties)
-                            if hasattr(value.metadata, "properties")
-                            else {},
+                            "properties": props,
                         }
+
+                        # For Dataset, also include the data directly in the artifact
+                        if asset_type == "Dataset" and isinstance(data_value, dict):
+                            artifact_metadata["data"] = data_value
+
+                        # Include training_history if present (for Model assets with Keras training)
+                        # This enables interactive training charts in the UI
+                        if hasattr(value, "training_history") and value.training_history:
+                            artifact_metadata["training_history"] = value.training_history
+
                         self.metadata_store.save_artifact(artifact_id, artifact_metadata)
 
                         # Special handling for Metrics asset