flowyml 1.7.0__py3-none-any.whl → 1.7.2__py3-none-any.whl

flowyml/integrations/keras.py

@@ -1,4 +1,9 @@
-"""Keras integration for flowyml."""
+"""Keras integration for flowyml.
+
+This module provides seamless integration between Keras and FlowyML,
+enabling automatic tracking of training metrics, model artifacts, and
+interactive visualization in the FlowyML dashboard.
+"""
 
 from pathlib import Path
 from datetime import datetime
@@ -19,17 +24,42 @@ from flowyml.storage.metadata import SQLiteMetadataStore
 class FlowymlKerasCallback(keras.callbacks.Callback if keras else object):
     """Keras callback for flowyml tracking with automatic training history collection.
 
-    Automatically logs:
-    - Training metrics (loss, accuracy, etc.) per epoch
-    - Complete training history for visualization
-    - Model checkpoints with training history attached
-    - Model architecture
-    - Training parameters
+    This callback integrates Keras training with FlowyML's tracking and visualization
+    system. It **automatically and dynamically** captures ALL metrics that Keras logs
+    during training - no configuration needed!
+
+    Features:
+    - **Automatic metric capture**: Whatever metrics you compile your model with
+      (loss, accuracy, mae, f1_score, custom metrics) are automatically tracked
+    - **Dynamic chart generation**: The UI generates charts for all captured metrics
+    - **Real-time updates**: Training progress is visible in the dashboard as it happens
+    - **Zero configuration**: Just add the callback and everything works automatically
 
     Example:
         >>> from flowyml.integrations.keras import FlowymlKerasCallback
-        >>> callback = FlowymlKerasCallback(experiment_name="my-experiment", project="my-project", auto_log_history=True)
-        >>> model.fit(x_train, y_train, epochs=50, callbacks=[callback])
+        >>>
+        >>> # Create callback - that's all you need!
+        >>> callback = FlowymlKerasCallback(
+        ...     experiment_name="my-experiment",
+        ...     project="my-project",
+        ... )
+        >>>
+        >>> # Compile with any metrics you want - they'll all be tracked
+        >>> model.compile(
+        ...     optimizer="adam",
+        ...     loss="mse",
+        ...     metrics=["mae", "mape"],  # All automatically captured!
+        ... )
+        >>>
+        >>> # Train with validation data - both train & val metrics captured
+        >>> history = model.fit(x_train, y_train, validation_data=(x_val, y_val), epochs=50, callbacks=[callback])
+        >>>
+        >>> # Get training history for use in your step output
+        >>> training_history = callback.get_training_history()
+
+    The training metrics will be visible in the FlowyML dashboard at:
+    - Run Details page → Training Metrics section
+    - Model artifacts → Training History charts
     """
 
     def __init__(
@@ -40,16 +70,22 @@ class FlowymlKerasCallback(keras.callbacks.Callback if keras else object):
         log_model: bool = True,
         log_every_epoch: bool = True,
         auto_log_history: bool = True,
+        live_update_interval: int = 1,
         metadata_store: SQLiteMetadataStore | None = None,
     ):
-        """Args:
-        experiment_name: Name of the experiment
-        run_name: Optional run name (defaults to timestamp)
-        project: Project name for organizing runs
-        log_model: Whether to save the model as an artifact
-        log_every_epoch: Whether to log metrics every epoch
-        auto_log_history: Whether to automatically collect training history
-        metadata_store: Optional metadata store override.
+        """Initialize the FlowyML Keras callback.
+
+        Args:
+            experiment_name: Name of the experiment for grouping runs.
+            run_name: Optional run name (defaults to timestamp-based name).
+            project: Project name for organizing runs in the dashboard.
+            log_model: Whether to save the model as an artifact after training.
+            log_every_epoch: Whether to log metrics to the database every epoch.
+            auto_log_history: Whether to automatically collect training history
+                for visualization. Highly recommended for dashboard charts.
+            live_update_interval: How often (in epochs) to update the live
+                training history artifact. Set to 1 for real-time updates.
+            metadata_store: Optional metadata store override for custom storage.
         """
         if keras is None:
             raise ImportError("Keras is not installed. Please install tensorflow or keras.")
@@ -61,6 +97,7 @@ class FlowymlKerasCallback(keras.callbacks.Callback if keras else object):
         self.log_model = log_model
         self.log_every_epoch = log_every_epoch
         self.auto_log_history = auto_log_history
+        self.live_update_interval = live_update_interval
 
         self.metadata_store = metadata_store or SQLiteMetadataStore()
 
@@ -70,15 +107,12 @@ class FlowymlKerasCallback(keras.callbacks.Callback if keras else object):
         # Track params
         self.params_logged = False
 
-        # Training history accumulator
-        self.training_history = {
-            "epochs": [],
-            "train_loss": [],
-            "train_accuracy": [],
-            "val_loss": [],
-            "val_accuracy": [],
-        }
-        self.custom_metrics = set()
+        # Training history artifact ID (for live updates)
+        self._history_artifact_id = f"{self.run_name}_training_history"
+
+        # DYNAMIC training history - only epochs is pre-initialized
+        # All other metrics are added dynamically as Keras logs them
+        self._training_history = {"epochs": []}
 
     def on_train_begin(self, logs=None) -> None:
         """Log initial parameters."""
@@ -117,51 +151,190 @@ class FlowymlKerasCallback(keras.callbacks.Callback if keras else object):
             self.params_logged = True
 
     def on_epoch_end(self, epoch, logs=None) -> None:
-        """Log metrics at the end of each epoch and accumulate training history."""
-        if logs:
-            # Log metrics to DB (existing behavior)
-            if self.log_every_epoch:
-                for k, v in logs.items():
-                    self.metadata_store.save_metric(
-                        run_id=self.run_name,
-                        name=k,
-                        value=float(v),
-                        step=epoch,
-                    )
-
-                # Update experiment run
-                self.metadata_store.log_experiment_run(
-                    experiment_id=self.experiment_name,
+        """Dynamically capture ALL metrics at the end of each epoch.
+
+        This method automatically captures whatever metrics Keras logs,
+        without requiring any configuration or hardcoded metric names.
+        """
+        if not logs:
+            return
+
+        # Log metrics to DB
+        if self.log_every_epoch:
+            for k, v in logs.items():
+                self.metadata_store.save_metric(
                     run_id=self.run_name,
-                    metrics=logs,
+                    name=k,
+                    value=float(v),
+                    step=epoch,
                 )
 
-            # Accumulate training history (NEW)
-            if self.auto_log_history:
-                self.training_history["epochs"].append(epoch + 1)  # 1-indexed
-
-                # Standard metrics
-                if "loss" in logs:
-                    self.training_history["train_loss"].append(float(logs["loss"]))
-                if "accuracy" in logs or "acc" in logs:
-                    acc_key = "accuracy" if "accuracy" in logs else "acc"
-                    self.training_history["train_accuracy"].append(float(logs[acc_key]))
-                if "val_loss" in logs:
-                    self.training_history["val_loss"].append(float(logs["val_loss"]))
-                if "val_accuracy" in logs or "val_acc" in logs:
-                    val_acc_key = "val_accuracy" if "val_accuracy" in logs else "val_acc"
-                    self.training_history["val_accuracy"].append(float(logs[val_acc_key]))
-
-                # Custom metrics
-                for metric_name, value in logs.items():
-                    if metric_name not in ["loss", "accuracy", "acc", "val_loss", "val_accuracy", "val_acc"]:
-                        if metric_name not in self.custom_metrics:
-                            self.custom_metrics.add(metric_name)
-                            self.training_history[metric_name] = []
-                        self.training_history[metric_name].append(float(value))
+            # Update experiment run
+            self.metadata_store.log_experiment_run(
+                experiment_id=self.experiment_name,
+                run_id=self.run_name,
+                metrics=logs,
+            )
+
+        # Accumulate training history for visualization
+        if self.auto_log_history:
+            # Record epoch number (1-indexed for display)
+            self._training_history["epochs"].append(epoch + 1)
+
+            # DYNAMICALLY capture ALL metrics from Keras logs
+            for metric_name, value in logs.items():
+                # Normalize metric name for consistent display
+                display_name = self._normalize_metric_name(metric_name)
+
+                # Initialize list if this is a new metric
+                if display_name not in self._training_history:
+                    self._training_history[display_name] = []
+
+                # Append the value
+                self._training_history[display_name].append(float(value))
+
+            # Save live training history artifact for real-time UI updates
+            if (epoch + 1) % self.live_update_interval == 0:
+                self._save_live_training_history()
+
+    def _normalize_metric_name(self, name: str) -> str:
+        """Normalize metric names for consistent display.
+
+        Converts Keras metric names to user-friendly display names:
+        - 'loss' -> 'train_loss'
+        - 'val_loss' -> 'val_loss' (unchanged)
+        - 'mae' -> 'train_mae'
+        - 'val_mae' -> 'val_mae' (unchanged)
+        - 'accuracy' -> 'train_accuracy'
+        - 'acc' -> 'train_accuracy'
+        """
+        # Validation metrics (val_*) stay as-is
+        if name.startswith("val_"):
+            return name
+
+        # Special case: 'acc' -> 'train_accuracy'
+        if name == "acc":
+            return "train_accuracy"
+
+        # Training metrics: add 'train_' prefix for clarity
+        if name == "loss":
+            return "train_loss"
+
+        # For other metrics (mae, accuracy, custom), add 'train_' prefix
+        return f"train_{name}"
+
+    def get_training_history(self) -> dict:
+        """Get the accumulated training history for use in step outputs.
+
+        This is the recommended way to include training history in your
+        Model asset, ensuring it's linked to the pipeline run.
+
+        Returns:
+            dict: Training history with epochs and all captured metrics.
+                  Only includes metrics that have data (non-empty lists).
+
+        Example:
+            >>> callback = FlowymlKerasCallback(...)
+            >>> model.fit(..., callbacks=[callback])
+            >>> history = callback.get_training_history()
+            >>> return Model.create(
+            ...     data=model,
+            ...     name="my_model",
+            ...     training_history=history,  # Automatically displayed in UI!
+            ... )
+        """
+        # Return cleaned history (only non-empty metrics)
+        return {k: v for k, v in self._training_history.items() if v and len(v) > 0}
+
+    def _save_live_training_history(self) -> None:
+        """Save current training history as an artifact for live UI updates."""
+        cleaned_history = self.get_training_history()
+
+        if not cleaned_history.get("epochs"):
+            return  # Nothing to save yet
+
+        # Calculate summary metrics dynamically
+        summary_metrics = {}
+        for key, values in cleaned_history.items():
+            if key == "epochs" or not values:
+                continue
+
+            # For loss-like metrics (lower is better)
+            if "loss" in key or "mae" in key or "mse" in key or "error" in key.lower():
+                summary_metrics[f"final_{key}"] = values[-1]
+                summary_metrics[f"best_{key}"] = min(values)
+            # For accuracy-like metrics (higher is better)
+            elif "accuracy" in key or "acc" in key or "f1" in key or "precision" in key or "recall" in key:
+                summary_metrics[f"final_{key}"] = values[-1]
+                summary_metrics[f"best_{key}"] = max(values)
+            # For other metrics, just store final value
+            else:
+                summary_metrics[f"final_{key}"] = values[-1]
+
+        # Save/update the training history artifact
+        self.metadata_store.save_artifact(
+            artifact_id=self._history_artifact_id,
+            metadata={
+                "artifact_id": self._history_artifact_id,
+                "name": f"training-history-{self.experiment_name}",
+                "type": "training_history",
+                "run_id": self.run_name,
+                "project": self.project,
+                "properties": {
+                    "experiment": self.experiment_name,
+                    "epochs_completed": len(cleaned_history.get("epochs", [])),
+                    "status": "training",
+                    **summary_metrics,
+                },
+                "training_history": cleaned_history,
+                "created_at": datetime.now().isoformat(),
+                "updated_at": datetime.now().isoformat(),
+            },
+        )
 
     def on_train_end(self, logs=None) -> None:
-        """Save model at the end of training with complete training history."""
+        """Save model and finalize training history at the end of training."""
+        cleaned_history = self.get_training_history()
+
+        # Calculate final metrics dynamically
+        final_metrics = {}
+        for key, values in cleaned_history.items():
+            if key == "epochs" or not values:
+                continue
+
+            # For loss-like metrics
+            if "loss" in key or "mae" in key or "mse" in key or "error" in key.lower():
+                final_metrics[f"final_{key}"] = values[-1]
+                final_metrics[f"best_{key}"] = min(values)
+            # For accuracy-like metrics
+            elif "accuracy" in key or "acc" in key or "f1" in key:
+                final_metrics[f"final_{key}"] = values[-1]
+                final_metrics[f"best_{key}"] = max(values)
+            else:
+                final_metrics[f"final_{key}"] = values[-1]
+
+        # Update training history artifact with final status
+        self.metadata_store.save_artifact(
+            artifact_id=self._history_artifact_id,
+            metadata={
+                "artifact_id": self._history_artifact_id,
+                "name": f"training-history-{self.experiment_name}",
+                "type": "training_history",
+                "run_id": self.run_name,
+                "project": self.project,
+                "properties": {
+                    "experiment": self.experiment_name,
+                    "epochs_completed": len(cleaned_history.get("epochs", [])),
+                    "status": "completed",
+                    **final_metrics,
+                },
+                "training_history": cleaned_history,
+                "created_at": datetime.now().isoformat(),
+                "updated_at": datetime.now().isoformat(),
+            },
+        )
+
+        # Save model artifact if enabled
         if self.log_model:
             # Create artifacts directory
             artifact_dir = Path(f".flowyml/artifacts/{self.run_name}")
@@ -170,21 +343,7 @@ class FlowymlKerasCallback(keras.callbacks.Callback if keras else object):
             model_path = artifact_dir / "model.keras"
             self.model.save(model_path)
 
-            # Clean up empty history lists
-            cleaned_history = {
-                k: v
-                for k, v in self.training_history.items()
-                if v  # Only include non-empty lists
-            }
-
-            # Calculate final metrics
-            final_metrics = {}
-            if "train_loss" in cleaned_history and cleaned_history["train_loss"]:
-                final_metrics["loss"] = cleaned_history["train_loss"][-1]
-            if "train_accuracy" in cleaned_history and cleaned_history["train_accuracy"]:
-                final_metrics["accuracy"] = cleaned_history["train_accuracy"][-1]
-
-            # Save model artifact with training history
+            # Save model artifact with training history attached
             artifact_id = str(uuid.uuid4())
             self.metadata_store.save_artifact(
                 artifact_id=artifact_id,
@@ -201,7 +360,13 @@ class FlowymlKerasCallback(keras.callbacks.Callback if keras else object):
                         "optimizer": str(self.model.optimizer.__class__.__name__),
                         **final_metrics,
                     },
-                    "training_history": cleaned_history,  # NEW: UI will display this!
+                    "training_history": cleaned_history,
                     "created_at": datetime.now().isoformat(),
                 },
             )
+
+    # Expose training_history as property for backwards compatibility
+    @property
+    def training_history(self) -> dict:
+        """Return the accumulated training history as a dictionary."""
+        return self.get_training_history()

flowyml/storage/sql.py

@@ -249,6 +249,9 @@ class SQLMetadataStore(MetadataStore):
             stmt = select(self.runs).where(self.runs.c.run_id == run_id)
             existing = conn.execute(stmt).fetchone()
 
+            # Ensure run_id is included in metadata for consistency
+            metadata_with_id = {**metadata, "run_id": run_id}
+
             values = {
                 "run_id": run_id,
                 "pipeline_name": metadata.get("pipeline_name"),
@@ -256,7 +259,7 @@ class SQLMetadataStore(MetadataStore):
                 "start_time": metadata.get("start_time"),
                 "end_time": metadata.get("end_time"),
                 "duration": metadata.get("duration"),
-                "metadata": json.dumps(metadata),
+                "metadata": json.dumps(metadata_with_id),
                 "project": metadata.get("project"),
             }
 
@@ -299,7 +302,10 @@ class SQLMetadataStore(MetadataStore):
             stmt = select(self.runs.c.metadata).where(self.runs.c.run_id == run_id)
             row = conn.execute(stmt).fetchone()
             if row:
-                return json.loads(row[0])
+                data = json.loads(row[0])
+                # Ensure run_id is always included in the returned dict
+                data["run_id"] = run_id
+                return data
             return None
 
     def update_run_project(self, run_id: str, project_name: str) -> None:
@@ -325,12 +331,18 @@ class SQLMetadataStore(MetadataStore):
     def list_runs(self, limit: int | None = None) -> list[dict]:
         """List all runs."""
         with self.engine.connect() as conn:
-            stmt = select(self.runs.c.metadata).order_by(self.runs.c.created_at.desc())
+            stmt = select(self.runs.c.run_id, self.runs.c.metadata).order_by(self.runs.c.created_at.desc())
             if limit:
                 stmt = stmt.limit(limit)
 
             rows = conn.execute(stmt).fetchall()
-            return [json.loads(row[0]) for row in rows]
+            runs = []
+            for row in rows:
+                data = json.loads(row[1])
+                # Ensure run_id is always included in the returned dict
+                data["run_id"] = row[0]
+                runs.append(data)
+            return runs
 
     def list_pipelines(self, project: str = None) -> list[str]:
         """List all unique pipeline names."""
@@ -402,7 +414,7 @@ class SQLMetadataStore(MetadataStore):
     def query(self, **filters) -> list[dict]:
         """Query runs with filters."""
         with self.engine.connect() as conn:
-            stmt = select(self.runs.c.metadata)
+            stmt = select(self.runs.c.run_id, self.runs.c.metadata)
 
             for key, value in filters.items():
                 if hasattr(self.runs.c, key):
@@ -410,7 +422,13 @@ class SQLMetadataStore(MetadataStore):
 
             stmt = stmt.order_by(self.runs.c.created_at.desc())
             rows = conn.execute(stmt).fetchall()
-            return [json.loads(row[0]) for row in rows]
+            runs = []
+            for row in rows:
+                data = json.loads(row[1])
+                # Ensure run_id is always included in the returned dict
+                data["run_id"] = row[0]
+                runs.append(data)
+            return runs
 
     def save_metric(self, run_id: str, name: str, value: float, step: int = 0) -> None:
         """Save a single metric value."""

flowyml/ui/backend/routers/runs.py

@@ -484,3 +484,115 @@ async def get_run_logs(run_id: str):
     logs = await anyio.to_thread.run_sync(read_all_logs)
 
     return {"logs": logs}
+
+
+@router.get("/{run_id}/training-history")
+async def get_training_history(run_id: str):
+    """Get training history (per-epoch metrics) for a run.
+
+    This combines:
+    1. Training history from model artifacts (saved by FlowymlKerasCallback)
+    2. Per-epoch metrics saved in the metrics table
+
+    Returns a consolidated training history suitable for visualization.
+    """
+    store = _find_store_for_run(run_id)
+
+    # Get per-epoch metrics from the metrics table
+    metrics = store.get_metrics(run_id)
+
+    # Build training history from metrics table
+    # Group metrics by step (epoch) and name
+    epoch_metrics = {}
+    for m in metrics:
+        step = m.get("step", 0)
+        name = m.get("name", "unknown")
+        value = m.get("value", 0)
+
+        if step not in epoch_metrics:
+            epoch_metrics[step] = {}
+        epoch_metrics[step][name] = value
+
+    # Convert to chart-friendly format
+    training_history_from_metrics = {
+        "epochs": [],
+        "train_loss": [],
+        "val_loss": [],
+        "train_accuracy": [],
+        "val_accuracy": [],
+        "mae": [],
+        "val_mae": [],
+    }
+
+    # Standard metric name mappings
+    metric_mappings = {
+        "loss": "train_loss",
+        "val_loss": "val_loss",
+        "accuracy": "train_accuracy",
+        "acc": "train_accuracy",
+        "val_accuracy": "val_accuracy",
+        "val_acc": "val_accuracy",
+        "mae": "mae",
+        "val_mae": "val_mae",
+    }
+
+    # Track custom metrics
+    custom_metrics = set()
+
+    if epoch_metrics:
+        sorted_epochs = sorted(epoch_metrics.keys())
+        for epoch in sorted_epochs:
+            training_history_from_metrics["epochs"].append(epoch + 1)  # 1-indexed for display
+
+            epoch_data = epoch_metrics[epoch]
+            for metric_name, value in epoch_data.items():
+                # Map to standard name or track as custom
+                standard_name = metric_mappings.get(metric_name)
+                if standard_name:
+                    training_history_from_metrics[standard_name].append(value)
+                else:
+                    # Custom metric
+                    if metric_name not in custom_metrics:
+                        custom_metrics.add(metric_name)
+                        training_history_from_metrics[metric_name] = []
+                    training_history_from_metrics[metric_name].append(value)
+
+    # Also try to get training history from model artifacts
+    artifacts = store.list_assets(run_id=run_id)
+    artifact_history = None
+
+    for artifact in artifacts:
+        # Check if artifact has training_history
+        if artifact.get("training_history"):
+            artifact_history = artifact.get("training_history")
+            break
+        # Also check in metadata/properties
+        metadata = artifact.get("metadata", {})
+        if isinstance(metadata, str):
+            try:
+                metadata = json.loads(metadata)
+            except Exception:
+                metadata = {}
+        if metadata.get("training_history"):
+            artifact_history = metadata.get("training_history")
+            break
+
+    # Prefer artifact history if it has more data, otherwise use metrics
+    if artifact_history and len(artifact_history.get("epochs", [])) > len(
+        training_history_from_metrics.get("epochs", []),
+    ):
+        final_history = artifact_history
+    elif training_history_from_metrics.get("epochs"):
+        final_history = training_history_from_metrics
+    else:
+        final_history = artifact_history or {}
+
+    # Clean up empty arrays
+    cleaned_history = {k: v for k, v in final_history.items() if v and (not isinstance(v, list) or len(v) > 0)}
+
+    return {
+        "training_history": cleaned_history,
+        "has_history": len(cleaned_history.get("epochs", [])) > 0,
+        "total_epochs": len(cleaned_history.get("epochs", [])),
+        "source": "artifact" if artifact_history else "metrics",
+    }

flowyml/ui/backend/routers/schedules.py

@@ -25,21 +25,41 @@ class ScheduleRequest(BaseModel):
 
 @router.get("/")
 async def list_schedules():
-    """List all active schedules."""
-    # Convert Schedule objects to dicts for JSON serialization
-    schedules = scheduler.list_schedules()
-    return [
-        {
-            "pipeline_name": s.pipeline_name,
-            "schedule_type": s.schedule_type,
-            "schedule_value": s.schedule_value,
-            "enabled": s.enabled,
-            "last_run": s.last_run,
-            "next_run": s.next_run,
-            "timezone": s.timezone,
-        }
-        for s in schedules
-    ]
+    """List all active schedules.
+
+    This reads schedules from the shared database, so schedules created
+    by user code (e.g., in scripts) are visible in the UI.
+    """
+    # First, get schedules from the in-memory scheduler
+    memory_schedules = []
+    for s in scheduler.list_schedules():
+        memory_schedules.append(
+            {
+                "pipeline_name": s.pipeline_name,
+                "schedule_type": s.schedule_type,
+                "schedule_value": s.schedule_value,
+                "enabled": s.enabled,
+                "last_run": s.last_run.isoformat() if s.last_run else None,
+                "next_run": s.next_run.isoformat() if s.next_run else None,
+                "timezone": s.timezone,
+            },
+        )
+
+    # Also read directly from the persistence database to get schedules
+    # created by other processes (e.g., user scripts)
+    db_schedules = []
+    if scheduler._persistence:
+        db_schedules = scheduler._persistence.list_all_schedules()
+
+    # Merge: prefer memory schedules (more up-to-date), but include db-only ones
+    memory_names = {s["pipeline_name"] for s in memory_schedules}
+    result = list(memory_schedules)
+
+    for db_sched in db_schedules:
+        if db_sched.get("pipeline_name") not in memory_names:
+            result.append(db_sched)
+
+    return result
 
 
 @router.get("/health")