flowyml 1.7.0__py3-none-any.whl → 1.7.2__py3-none-any.whl

flowyml/core/executor.py

@@ -87,10 +87,16 @@ class MonitorThread(threading.Thread):
             # Fallback to environment variable or default
             self.api_url = os.getenv("FLOWYML_SERVER_URL", "http://localhost:8080")
 
-    def stop(self):
+    def stop(self, error: str | None = None):
+        """Stop the monitor thread.
+
+        Args:
+            error: Optional error message to send as final log entry
+        """
+        self._final_error = error
         self._stop_event.set()
 
-    def _flush_logs(self):
+    def _flush_logs(self, level: str = "INFO"):
         """Send captured logs to the server."""
         if not self.log_capture:
             return
@@ -105,7 +111,20 @@ class MonitorThread(threading.Thread):
                 f"{self.api_url}/api/runs/{self.run_id}/steps/{self.step_name}/logs",
                 json={
                     "content": content,
-                    "level": "INFO",
+                    "level": level,
+                    "timestamp": datetime.now().isoformat(),
+                },
+                timeout=2,
+            )
+
+    def _send_error(self, error: str):
+        """Send error message to the server."""
+        with contextlib.suppress(Exception):
+            requests.post(
+                f"{self.api_url}/api/runs/{self.run_id}/steps/{self.step_name}/logs",
+                json={
+                    "content": f"ERROR: {error}",
+                    "level": "ERROR",
                     "timestamp": datetime.now().isoformat(),
                 },
                 timeout=2,
@@ -137,6 +156,10 @@ class MonitorThread(threading.Thread):
         # Final log flush
         self._flush_logs()
 
+        # Send error if there was one
+        if hasattr(self, "_final_error") and self._final_error:
+            self._send_error(self._final_error)
+
 
 # Keep HeartbeatThread as an alias for backwards compatibility
 HeartbeatThread = MonitorThread
@@ -335,8 +358,8 @@ class LocalExecutor(Executor):
 
                         sys.stderr = original_stderr
 
-                    # Stop monitor thread
-                    if monitor_thread:
+                    # Stop monitor thread (only if not already stopped in exception handler)
+                    if monitor_thread and not monitor_thread._stop_event.is_set():
                         monitor_thread.stop()
                         monitor_thread.join()
 
@@ -374,6 +397,7 @@ class LocalExecutor(Executor):
 
             except Exception as e:
                 last_error = str(e)
+                error_traceback = traceback.format_exc()
                 retries += 1
 
                 if attempt < max_retries:
@@ -382,12 +406,17 @@ class LocalExecutor(Executor):
                     time.sleep(wait_time)
                     continue
 
-                # All retries exhausted
+                # All retries exhausted - send error to logs
+                if monitor_thread:
+                    monitor_thread.stop(error=f"{last_error}\n{error_traceback}")
+                    monitor_thread.join()
+                    monitor_thread = None  # Prevent double-stop in finally
+
                 duration = time.time() - start_time
                 return ExecutionResult(
                     step_name=step.name,
                     success=False,
-                    error=f"{last_error}\n{traceback.format_exc()}",
+                    error=f"{last_error}\n{error_traceback}",
                     duration_seconds=duration,
                     retries=retries,
                 )
@@ -438,11 +467,41 @@ class LocalExecutor(Executor):
             # Find the step object
             step = next(s for s in step_group.steps if s.name == step_name)
 
-            # Prepare inputs for this step
+            # Prepare inputs for this step - map input names to function parameters
             step_inputs = {}
-            for input_name in step.inputs:
-                if input_name in step_outputs:
-                    step_inputs[input_name] = step_outputs[input_name]
+
+            # Get function signature to properly map inputs to parameters
+            sig = inspect.signature(step.func)
+            params = list(sig.parameters.values())
+            # Filter out self/cls
+            params = [p for p in params if p.name not in ("self", "cls")]
+            assigned_params = set()
+
+            if step.inputs:
+                for i, input_name in enumerate(step.inputs):
+                    if input_name not in step_outputs:
+                        continue
+
+                    val = step_outputs[input_name]
+
+                    # Check if input name matches a parameter directly
+                    param_match = next((p for p in params if p.name == input_name), None)
+
+                    if param_match:
+                        step_inputs[param_match.name] = val
+                        assigned_params.add(param_match.name)
+                    elif i < len(params):
+                        # Positional fallback - use the parameter at the same position
+                        target_param = params[i]
+                        if target_param.name not in assigned_params:
+                            step_inputs[target_param.name] = val
+                            assigned_params.add(target_param.name)
+
+            # Auto-map parameters from available outputs by name
+            for param in params:
+                if param.name in step_outputs and param.name not in step_inputs:
+                    step_inputs[param.name] = step_outputs[param.name]
+                    assigned_params.add(param.name)
 
             # Inject context parameters for this specific step
             if context is not None:

flowyml/core/orchestrator.py

@@ -390,6 +390,7 @@ class LocalOrchestrator(Orchestrator):
             """Context object for conditional evaluation.
 
             Provides access to step outputs via ctx.steps['step_name'].outputs['output_name']
+            and context parameters via ctx.params
             """
 
             def __init__(self, result: "PipelineResult", pipeline: "Pipeline"):
@@ -397,6 +398,13 @@ class LocalOrchestrator(Orchestrator):
                 self.pipeline = pipeline
                 self._steps_cache = None
 
+            @property
+            def params(self):
+                """Get pipeline context parameters as a dictionary."""
+                if self.pipeline.context:
+                    return self.pipeline.context._params
+                return {}
+
             @property
             def steps(self):
                 """Lazy-load steps dictionary with outputs."""
@@ -627,6 +635,12 @@ class LocalOrchestrator(Orchestrator):
                     if step_obj.name not in result.step_results:
                         # Execute the selected step
                         # The check above prevents re-execution of the same step
+                        # If step has inputs defined, copy them to the step object for proper input mapping
+                        if hasattr(selected_step, "_step_inputs") and selected_step._step_inputs:
+                            step_obj.inputs = selected_step._step_inputs
+                        elif hasattr(selected_step, "inputs"):
+                            step_obj.inputs = selected_step.inputs or []
+
                         self._execute_conditional_step(
                             pipeline,
                             step_obj,
@@ -659,10 +673,31 @@ class LocalOrchestrator(Orchestrator):
         step_inputs = {}
         sig = inspect.signature(step.func)
         params = [p for p in sig.parameters.values() if p.name not in ("self", "cls")]
-
+        assigned_params = set()
+
+        # First, try to map from declared inputs (like "model/trained" -> function param)
+        if step.inputs:
+            for i, input_name in enumerate(step.inputs):
+                if input_name not in step_outputs:
+                    continue
+                val = step_outputs[input_name]
+                # Try to match input name directly to a parameter
+                param_match = next((p for p in params if p.name == input_name), None)
+                if param_match:
+                    step_inputs[param_match.name] = val
+                    assigned_params.add(param_match.name)
+                elif i < len(params):
+                    # Positional fallback - use parameter at same position
+                    target_param = params[i]
+                    if target_param.name not in assigned_params:
+                        step_inputs[target_param.name] = val
+                        assigned_params.add(target_param.name)
+
+        # Then, try direct parameter name matching from step_outputs
         for param in params:
-            if param.name in step_outputs:
+            if param.name not in assigned_params and param.name in step_outputs:
                 step_inputs[param.name] = step_outputs[param.name]
+                assigned_params.add(param.name)
 
         # Get context parameters
         context_params = pipeline.context.inject_params(step.func)

flowyml/core/pipeline.py

@@ -972,6 +972,26 @@ class Pipeline:
                     if is_asset:
                         # Handle flowyml Asset
                         asset_type = value.__class__.__name__
+                        # Get properties
+                        props = (
+                            self._sanitize_for_json(value.metadata.properties)
+                            if hasattr(value.metadata, "properties")
+                            else {}
+                        )
+
+                        # For Dataset assets, include the full data for visualization
+                        # This enables histograms and statistics in the UI
+                        data_value = None
+                        if asset_type == "Dataset" and value.data:
+                            try:
+                                # Store full data as JSON-serializable dict
+                                data_value = self._sanitize_for_json(value.data)
+                                props["_full_data"] = data_value
+                            except Exception:
+                                data_value = str(value.data)[:1000]
+                        else:
+                            data_value = str(value.data)[:1000] if value.data else None
+
                         artifact_metadata = {
                             "artifact_id": artifact_id,
                             "name": value.name,
@@ -979,12 +999,20 @@ class Pipeline:
                             "run_id": result.run_id,
                             "step": step_name,
                             "path": None,
-                            "value": str(value.data)[:1000] if value.data else None,
+                            "value": data_value if isinstance(data_value, str) else None,
                             "created_at": datetime.now().isoformat(),
-                            "properties": self._sanitize_for_json(value.metadata.properties)
-                            if hasattr(value.metadata, "properties")
-                            else {},
+                            "properties": props,
                         }
+
+                        # For Dataset, also include the data directly in the artifact
+                        if asset_type == "Dataset" and isinstance(data_value, dict):
+                            artifact_metadata["data"] = data_value
+
+                        # Include training_history if present (for Model assets with Keras training)
+                        # This enables interactive training charts in the UI
+                        if hasattr(value, "training_history") and value.training_history:
+                            artifact_metadata["training_history"] = value.training_history
+
                         self.metadata_store.save_artifact(artifact_id, artifact_metadata)
 
                         # Special handling for Metrics asset

flowyml/core/scheduler.py

@@ -224,6 +224,36 @@ class SchedulerPersistence:
                     logger.error(f"Failed to load schedule {name}: {e}")
         return schedules
 
+    def load_schedule(self, name: str) -> Schedule | None:
+        """Load a single schedule from database by name.
+
+        Returns None if not found. Creates a Schedule without a pipeline_func
+        (the schedule will be enabled/disabled but won't actually run until
+        a pipeline function is registered).
+        """
+        with self.engine.connect() as conn:
+            stmt = select(self.schedules.c.name, self.schedules.c.data).where(
+                self.schedules.c.name == name,
+            )
+            result = conn.execute(stmt)
+            row = result.fetchone()
+            if row:
+                try:
+                    data = json.loads(row.data)
+                    # Create a minimal Schedule for enable/disable operations
+                    # without requiring the pipeline function
+                    return Schedule(
+                        pipeline_name=data.get("pipeline_name", name),
+                        pipeline_func=lambda: None,  # Placeholder - not for execution
+                        schedule_type=data.get("schedule_type", ""),
+                        schedule_value=data.get("schedule_value", ""),
+                        timezone=data.get("timezone", "UTC"),
+                        enabled=data.get("enabled", True),
+                    )
+                except Exception as e:
+                    logger.error(f"Failed to load schedule {name}: {e}")
+        return None
+
     def delete_schedule(self, name: str) -> None:
         """Delete schedule from database using SQLAlchemy."""
         with self.engine.connect() as conn:
@@ -251,6 +281,29 @@ class SchedulerPersistence:
             conn.execute(stmt)
             conn.commit()
 
+    def list_all_schedules(self) -> list[dict[str, Any]]:
+        """List all schedules from database without requiring pipeline functions.
+
+        This is useful for displaying schedules in the UI regardless of whether
+        the pipeline code is loaded.
+        """
+        schedules = []
+        with self.engine.connect() as conn:
+            stmt = select(self.schedules.c.name, self.schedules.c.data, self.schedules.c.updated_at)
+            result = conn.execute(stmt)
+            for row in result:
+                try:
+                    data = json.loads(row.data)
+                    data["name"] = row.name
+                    if row.updated_at:
+                        data["updated_at"] = (
+                            row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else str(row.updated_at)
+                        )
+                    schedules.append(data)
+                except Exception as e:
+                    logger.error(f"Failed to parse schedule {row.name}: {e}")
+        return schedules
+
     def get_history(self, schedule_name: str, limit: int = 50) -> list[dict[str, Any]]:
         """Get execution history for a schedule using SQLAlchemy."""
         history = []
@@ -533,11 +586,17 @@ class PipelineScheduler:
         return schedule
 
     def unschedule(self, name: str) -> None:
-        """Remove a scheduled pipeline."""
+        """Remove a scheduled pipeline.
+
+        Handles both in-memory schedules and persisted schedules.
+        """
+        # Remove from in-memory schedules if present
         if name in self.schedules:
             del self.schedules[name]
-            if self._persistence:
-                self._persistence.delete_schedule(name)
+
+        # Always try to remove from persistence (handles schedules created by other processes)
+        if self._persistence:
+            self._persistence.delete_schedule(name)
 
     def clear(self) -> None:
         """Remove all schedules."""
@@ -550,18 +609,42 @@ class PipelineScheduler:
                 conn.commit()
 
     def enable(self, name: str) -> None:
-        """Enable a schedule."""
+        """Enable a schedule.
+
+        Handles both in-memory schedules and persisted schedules.
+        """
         if name in self.schedules:
             self.schedules[name].enabled = True
             if self._persistence:
                 self._persistence.save_schedule(self.schedules[name])
+        elif self._persistence:
+            # Schedule might be in persistence but not loaded in memory
+            # Load it, update, and save back
+            schedule = self._persistence.load_schedule(name)
+            if schedule:
+                schedule.enabled = True
+                self._persistence.save_schedule(schedule)
+                # Also add to in-memory schedules
+                self.schedules[name] = schedule
 
     def disable(self, name: str) -> None:
-        """Disable a schedule."""
+        """Disable a schedule.
+
+        Handles both in-memory schedules and persisted schedules.
+        """
         if name in self.schedules:
             self.schedules[name].enabled = False
             if self._persistence:
                 self._persistence.save_schedule(self.schedules[name])
+        elif self._persistence:
+            # Schedule might be in persistence but not loaded in memory
+            # Load it, update, and save back
+            schedule = self._persistence.load_schedule(name)
+            if schedule:
+                schedule.enabled = False
+                self._persistence.save_schedule(schedule)
+                # Also add to in-memory schedules
+                self.schedules[name] = schedule
 
     def _run_pipeline(self, schedule: Schedule) -> None:
         """Run a scheduled pipeline."""