flowyml 1.7.1__py3-none-any.whl → 1.8.0__py3-none-any.whl

flowyml/integrations/keras.py

@@ -1,4 +1,9 @@
-"""Keras integration for flowyml."""
+"""Keras integration for flowyml.
+
+This module provides seamless integration between Keras and FlowyML,
+enabling automatic tracking of training metrics, model artifacts, and
+interactive visualization in the FlowyML dashboard.
+"""
 
 from pathlib import Path
 from datetime import datetime
@@ -19,17 +24,42 @@ from flowyml.storage.metadata import SQLiteMetadataStore
 class FlowymlKerasCallback(keras.callbacks.Callback if keras else object):
     """Keras callback for flowyml tracking with automatic training history collection.
 
-    Automatically logs:
-    - Training metrics (loss, accuracy, etc.) per epoch
-    - Complete training history for visualization
-    - Model checkpoints with training history attached
-    - Model architecture
-    - Training parameters
+    This callback integrates Keras training with FlowyML's tracking and visualization
+    system. It **automatically and dynamically** captures ALL metrics that Keras logs
+    during training - no configuration needed!
+
+    Features:
+    - **Automatic metric capture**: Whatever metrics you compile your model with
+      (loss, accuracy, mae, f1_score, custom metrics) are automatically tracked
+    - **Dynamic chart generation**: The UI generates charts for all captured metrics
+    - **Real-time updates**: Training progress is visible in the dashboard as it happens
+    - **Zero configuration**: Just add the callback and everything works automatically
 
     Example:
         >>> from flowyml.integrations.keras import FlowymlKerasCallback
-        >>> callback = FlowymlKerasCallback(experiment_name="my-experiment", project="my-project", auto_log_history=True)
-        >>> model.fit(x_train, y_train, epochs=50, callbacks=[callback])
+        >>>
+        >>> # Create callback - that's all you need!
+        >>> callback = FlowymlKerasCallback(
+        ...     experiment_name="my-experiment",
+        ...     project="my-project",
+        ... )
+        >>>
+        >>> # Compile with any metrics you want - they'll all be tracked
+        >>> model.compile(
+        ...     optimizer="adam",
+        ...     loss="mse",
+        ...     metrics=["mae", "mape"],  # All automatically captured!
+        ... )
+        >>>
+        >>> # Train with validation data - both train & val metrics captured
+        >>> history = model.fit(x_train, y_train, validation_data=(x_val, y_val), epochs=50, callbacks=[callback])
+        >>>
+        >>> # Get training history for use in your step output
+        >>> training_history = callback.get_training_history()
+
+    The training metrics will be visible in the FlowyML dashboard at:
+    - Run Details page → Training Metrics section
+    - Model artifacts → Training History charts
     """
 
     def __init__(
@@ -40,16 +70,22 @@ class FlowymlKerasCallback(keras.callbacks.Callback if keras else object):
         log_model: bool = True,
         log_every_epoch: bool = True,
         auto_log_history: bool = True,
+        live_update_interval: int = 1,
         metadata_store: SQLiteMetadataStore | None = None,
     ):
-        """Args:
-        experiment_name: Name of the experiment
-        run_name: Optional run name (defaults to timestamp)
-        project: Project name for organizing runs
-        log_model: Whether to save the model as an artifact
-        log_every_epoch: Whether to log metrics every epoch
-        auto_log_history: Whether to automatically collect training history
-        metadata_store: Optional metadata store override.
+        """Initialize the FlowyML Keras callback.
+
+        Args:
+            experiment_name: Name of the experiment for grouping runs.
+            run_name: Optional run name (defaults to timestamp-based name).
+            project: Project name for organizing runs in the dashboard.
+            log_model: Whether to save the model as an artifact after training.
+            log_every_epoch: Whether to log metrics to the database every epoch.
+            auto_log_history: Whether to automatically collect training history
+                for visualization. Highly recommended for dashboard charts.
+            live_update_interval: How often (in epochs) to update the live
+                training history artifact. Set to 1 for real-time updates.
+            metadata_store: Optional metadata store override for custom storage.
         """
         if keras is None:
             raise ImportError("Keras is not installed. Please install tensorflow or keras.")
@@ -61,6 +97,7 @@ class FlowymlKerasCallback(keras.callbacks.Callback if keras else object):
         self.log_model = log_model
         self.log_every_epoch = log_every_epoch
         self.auto_log_history = auto_log_history
+        self.live_update_interval = live_update_interval
 
         self.metadata_store = metadata_store or SQLiteMetadataStore()
 
@@ -70,15 +107,12 @@ class FlowymlKerasCallback(keras.callbacks.Callback if keras else object):
         # Track params
         self.params_logged = False
 
-        # Training history accumulator
-        self.training_history = {
-            "epochs": [],
-            "train_loss": [],
-            "train_accuracy": [],
-            "val_loss": [],
-            "val_accuracy": [],
-        }
-        self.custom_metrics = set()
+        # Training history artifact ID (for live updates)
+        self._history_artifact_id = f"{self.run_name}_training_history"
+
+        # DYNAMIC training history - only epochs is pre-initialized
+        # All other metrics are added dynamically as Keras logs them
+        self._training_history = {"epochs": []}
 
     def on_train_begin(self, logs=None) -> None:
         """Log initial parameters."""
@@ -117,51 +151,190 @@ class FlowymlKerasCallback(keras.callbacks.Callback if keras else object):
             self.params_logged = True
 
     def on_epoch_end(self, epoch, logs=None) -> None:
-        """Log metrics at the end of each epoch and accumulate training history."""
-        if logs:
-            # Log metrics to DB (existing behavior)
-            if self.log_every_epoch:
-                for k, v in logs.items():
-                    self.metadata_store.save_metric(
-                        run_id=self.run_name,
-                        name=k,
-                        value=float(v),
-                        step=epoch,
-                    )
-
-                # Update experiment run
-                self.metadata_store.log_experiment_run(
-                    experiment_id=self.experiment_name,
+        """Dynamically capture ALL metrics at the end of each epoch.
+
+        This method automatically captures whatever metrics Keras logs,
+        without requiring any configuration or hardcoded metric names.
+        """
+        if not logs:
+            return
+
+        # Log metrics to DB
+        if self.log_every_epoch:
+            for k, v in logs.items():
+                self.metadata_store.save_metric(
                     run_id=self.run_name,
-                    metrics=logs,
+                    name=k,
+                    value=float(v),
+                    step=epoch,
                 )
 
-            # Accumulate training history (NEW)
-            if self.auto_log_history:
-                self.training_history["epochs"].append(epoch + 1)  # 1-indexed
-
-                # Standard metrics
-                if "loss" in logs:
-                    self.training_history["train_loss"].append(float(logs["loss"]))
-                if "accuracy" in logs or "acc" in logs:
-                    acc_key = "accuracy" if "accuracy" in logs else "acc"
-                    self.training_history["train_accuracy"].append(float(logs[acc_key]))
-                if "val_loss" in logs:
-                    self.training_history["val_loss"].append(float(logs["val_loss"]))
-                if "val_accuracy" in logs or "val_acc" in logs:
-                    val_acc_key = "val_accuracy" if "val_accuracy" in logs else "val_acc"
-                    self.training_history["val_accuracy"].append(float(logs[val_acc_key]))
-
-                # Custom metrics
-                for metric_name, value in logs.items():
-                    if metric_name not in ["loss", "accuracy", "acc", "val_loss", "val_accuracy", "val_acc"]:
-                        if metric_name not in self.custom_metrics:
-                            self.custom_metrics.add(metric_name)
-                            self.training_history[metric_name] = []
-                        self.training_history[metric_name].append(float(value))
+            # Update experiment run
+            self.metadata_store.log_experiment_run(
+                experiment_id=self.experiment_name,
+                run_id=self.run_name,
+                metrics=logs,
+            )
+
+        # Accumulate training history for visualization
+        if self.auto_log_history:
+            # Record epoch number (1-indexed for display)
+            self._training_history["epochs"].append(epoch + 1)
+
+            # DYNAMICALLY capture ALL metrics from Keras logs
+            for metric_name, value in logs.items():
+                # Normalize metric name for consistent display
+                display_name = self._normalize_metric_name(metric_name)
+
+                # Initialize list if this is a new metric
+                if display_name not in self._training_history:
+                    self._training_history[display_name] = []
+
+                # Append the value
+                self._training_history[display_name].append(float(value))
+
+            # Save live training history artifact for real-time UI updates
+            if (epoch + 1) % self.live_update_interval == 0:
+                self._save_live_training_history()
+
+    def _normalize_metric_name(self, name: str) -> str:
+        """Normalize metric names for consistent display.
+
+        Converts Keras metric names to user-friendly display names:
+        - 'loss' -> 'train_loss'
+        - 'val_loss' -> 'val_loss' (unchanged)
+        - 'mae' -> 'train_mae'
+        - 'val_mae' -> 'val_mae' (unchanged)
+        - 'accuracy' -> 'train_accuracy'
+        - 'acc' -> 'train_accuracy'
+        """
+        # Validation metrics (val_*) stay as-is
+        if name.startswith("val_"):
+            return name
+
+        # Special case: 'acc' -> 'train_accuracy'
+        if name == "acc":
+            return "train_accuracy"
+
+        # Training metrics: add 'train_' prefix for clarity
+        if name == "loss":
+            return "train_loss"
+
+        # For other metrics (mae, accuracy, custom), add 'train_' prefix
+        return f"train_{name}"
+
+    def get_training_history(self) -> dict:
+        """Get the accumulated training history for use in step outputs.
+
+        This is the recommended way to include training history in your
+        Model asset, ensuring it's linked to the pipeline run.
+
+        Returns:
+            dict: Training history with epochs and all captured metrics.
+                  Only includes metrics that have data (non-empty lists).
+
+        Example:
+            >>> callback = FlowymlKerasCallback(...)
+            >>> model.fit(..., callbacks=[callback])
+            >>> history = callback.get_training_history()
+            >>> return Model.create(
+            ...     data=model,
+            ...     name="my_model",
+            ...     training_history=history,  # Automatically displayed in UI!
+            ... )
+        """
+        # Return cleaned history (only non-empty metrics)
+        return {k: v for k, v in self._training_history.items() if v and len(v) > 0}
+
+    def _save_live_training_history(self) -> None:
+        """Save current training history as an artifact for live UI updates."""
+        cleaned_history = self.get_training_history()
+
+        if not cleaned_history.get("epochs"):
+            return  # Nothing to save yet
+
+        # Calculate summary metrics dynamically
+        summary_metrics = {}
+        for key, values in cleaned_history.items():
+            if key == "epochs" or not values:
+                continue
+
+            # For loss-like metrics (lower is better)
+            if "loss" in key or "mae" in key or "mse" in key or "error" in key.lower():
+                summary_metrics[f"final_{key}"] = values[-1]
+                summary_metrics[f"best_{key}"] = min(values)
+            # For accuracy-like metrics (higher is better)
+            elif "accuracy" in key or "acc" in key or "f1" in key or "precision" in key or "recall" in key:
+                summary_metrics[f"final_{key}"] = values[-1]
+                summary_metrics[f"best_{key}"] = max(values)
+            # For other metrics, just store final value
+            else:
+                summary_metrics[f"final_{key}"] = values[-1]
+
+        # Save/update the training history artifact
+        self.metadata_store.save_artifact(
+            artifact_id=self._history_artifact_id,
+            metadata={
+                "artifact_id": self._history_artifact_id,
+                "name": f"training-history-{self.experiment_name}",
+                "type": "training_history",
+                "run_id": self.run_name,
+                "project": self.project,
+                "properties": {
+                    "experiment": self.experiment_name,
+                    "epochs_completed": len(cleaned_history.get("epochs", [])),
+                    "status": "training",
+                    **summary_metrics,
+                },
+                "training_history": cleaned_history,
+                "created_at": datetime.now().isoformat(),
+                "updated_at": datetime.now().isoformat(),
+            },
+        )
 
     def on_train_end(self, logs=None) -> None:
-        """Save model at the end of training with complete training history."""
+        """Save model and finalize training history at the end of training."""
+        cleaned_history = self.get_training_history()
+
+        # Calculate final metrics dynamically
+        final_metrics = {}
+        for key, values in cleaned_history.items():
+            if key == "epochs" or not values:
+                continue
+
+            # For loss-like metrics
+            if "loss" in key or "mae" in key or "mse" in key or "error" in key.lower():
+                final_metrics[f"final_{key}"] = values[-1]
+                final_metrics[f"best_{key}"] = min(values)
+            # For accuracy-like metrics
+            elif "accuracy" in key or "acc" in key or "f1" in key:
+                final_metrics[f"final_{key}"] = values[-1]
+                final_metrics[f"best_{key}"] = max(values)
+            else:
+                final_metrics[f"final_{key}"] = values[-1]
+
+        # Update training history artifact with final status
+        self.metadata_store.save_artifact(
+            artifact_id=self._history_artifact_id,
+            metadata={
+                "artifact_id": self._history_artifact_id,
+                "name": f"training-history-{self.experiment_name}",
+                "type": "training_history",
+                "run_id": self.run_name,
+                "project": self.project,
+                "properties": {
+                    "experiment": self.experiment_name,
+                    "epochs_completed": len(cleaned_history.get("epochs", [])),
+                    "status": "completed",
+                    **final_metrics,
+                },
+                "training_history": cleaned_history,
+                "created_at": datetime.now().isoformat(),
+                "updated_at": datetime.now().isoformat(),
+            },
+        )
+
+        # Save model artifact if enabled
         if self.log_model:
             # Create artifacts directory
             artifact_dir = Path(f".flowyml/artifacts/{self.run_name}")
@@ -170,21 +343,7 @@ class FlowymlKerasCallback(keras.callbacks.Callback if keras else object):
             model_path = artifact_dir / "model.keras"
             self.model.save(model_path)
 
-            # Clean up empty history lists
-            cleaned_history = {
-                k: v
-                for k, v in self.training_history.items()
-                if v  # Only include non-empty lists
-            }
-
-            # Calculate final metrics
-            final_metrics = {}
-            if "train_loss" in cleaned_history and cleaned_history["train_loss"]:
-                final_metrics["loss"] = cleaned_history["train_loss"][-1]
-            if "train_accuracy" in cleaned_history and cleaned_history["train_accuracy"]:
-                final_metrics["accuracy"] = cleaned_history["train_accuracy"][-1]
-
-            # Save model artifact with training history
+            # Save model artifact with training history attached
             artifact_id = str(uuid.uuid4())
             self.metadata_store.save_artifact(
                 artifact_id=artifact_id,
@@ -201,7 +360,13 @@ class FlowymlKerasCallback(keras.callbacks.Callback if keras else object):
                         "optimizer": str(self.model.optimizer.__class__.__name__),
                         **final_metrics,
                     },
-                    "training_history": cleaned_history,  # NEW: UI will display this!
+                    "training_history": cleaned_history,
                     "created_at": datetime.now().isoformat(),
                 },
             )
+
+    # Expose training_history as property for backwards compatibility
+    @property
+    def training_history(self) -> dict:
+        """Return the accumulated training history as a dictionary."""
+        return self.get_training_history()

flowyml/monitoring/alerts.py

@@ -52,6 +52,16 @@ class AlertManager:
             except Exception as e:
                 logger.error(f"Failed to handle alert: {e}")
 
+    def alert(
+        self,
+        message: str,
+        title: str = "Pipeline Alert",
+        level: AlertLevel = AlertLevel.INFO,
+        metadata: dict = None,
+    ) -> None:
+        """Convenience method for sending alerts."""
+        self.send_alert(title=title, message=message, level=level, metadata=metadata)
+
 
 # Global instance
 alert_manager = AlertManager()

flowyml/monitoring/notifications.py

@@ -1,11 +1,11 @@
 """Notification system for pipeline events."""
 
+import contextlib
 import os
-from typing import Any
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from datetime import datetime
-import contextlib
+from typing import Any
 
 
 @dataclass
@@ -43,10 +43,11 @@ class ConsoleNotifier(NotificationChannel):
 
 
 class SlackNotifier(NotificationChannel):
-    """Send notifications to Slack."""
+    """Send notifications to Slack with rich Block Kit formatting."""
 
-    def __init__(self, webhook_url: str | None = None):
+    def __init__(self, webhook_url: str | None = None, ui_url: str | None = None):
         self.webhook_url = webhook_url or os.getenv("SLACK_WEBHOOK_URL")
+        self.ui_url = ui_url or os.getenv("FLOWYML_UI_URL", "http://localhost:5173")
 
     def send(self, notification: Notification) -> bool:
         if not self.webhook_url:
@@ -55,26 +56,104 @@ class SlackNotifier(NotificationChannel):
         try:
             import requests
 
-            color = {
-                "info": "#36a64f",
-                "warning": "#ff9900",
-                "error": "#ff0000",
-                "success": "#00ff00",
-            }.get(notification.level, "#cccccc")
-
-            payload = {
-                "attachments": [
-                    {
-                        "color": color,
-                        "title": notification.title,
-                        "text": notification.message,
-                        "footer": "flowyml",
-                        "ts": int(notification.timestamp.timestamp()),
-                    },
-                ],
-            }
-
-            response = requests.post(self.webhook_url, json=payload)
+            from flowyml.monitoring.slack_blocks import build_simple_message
+
+            # Use rich Block Kit message
+            payload = build_simple_message(
+                title=notification.title,
+                message=notification.message,
+                level=notification.level,
+                metadata=notification.metadata if notification.metadata else None,
+            )
+
+            response = requests.post(self.webhook_url, json=payload, timeout=10)
+            return response.status_code == 200
+        except Exception:
+            return False
+
+    def send_pipeline_success(
+        self,
+        pipeline_name: str,
+        run_id: str,
+        duration: float,
+        metrics: dict[str, float] | None = None,
+    ) -> bool:
+        """Send a rich pipeline success notification."""
+        if not self.webhook_url:
+            return False
+
+        try:
+            import requests
+
+            from flowyml.monitoring.slack_blocks import build_pipeline_success_message
+
+            payload = build_pipeline_success_message(
+                pipeline_name=pipeline_name,
+                run_id=run_id,
+                duration=duration,
+                metrics=metrics,
+                ui_url=self.ui_url,
+            )
+
+            response = requests.post(self.webhook_url, json=payload, timeout=10)
+            return response.status_code == 200
+        except Exception:
+            return False
+
+    def send_pipeline_failure(
+        self,
+        pipeline_name: str,
+        run_id: str,
+        error: str,
+        step_name: str | None = None,
+    ) -> bool:
+        """Send a rich pipeline failure notification."""
+        if not self.webhook_url:
+            return False
+
+        try:
+            import requests
+
+            from flowyml.monitoring.slack_blocks import build_pipeline_failure_message
+
+            payload = build_pipeline_failure_message(
+                pipeline_name=pipeline_name,
+                run_id=run_id,
+                error=error,
+                step_name=step_name,
+                ui_url=self.ui_url,
+            )
+
+            response = requests.post(self.webhook_url, json=payload, timeout=10)
+            return response.status_code == 200
+        except Exception:
+            return False
+
+    def send_drift_warning(
+        self,
+        feature: str,
+        psi: float,
+        threshold: float = 0.2,
+        model_name: str | None = None,
+    ) -> bool:
+        """Send a rich drift warning notification."""
+        if not self.webhook_url:
+            return False
+
+        try:
+            import requests
+
+            from flowyml.monitoring.slack_blocks import build_drift_warning_message
+
+            payload = build_drift_warning_message(
+                feature=feature,
+                psi=psi,
+                threshold=threshold,
+                model_name=model_name,
+                ui_url=self.ui_url,
+            )
+
+            response = requests.post(self.webhook_url, json=payload, timeout=10)
             return response.status_code == 200
         except Exception:
             return False
@@ -105,8 +184,8 @@ class EmailNotifier(NotificationChannel):
 
         try:
             import smtplib
-            from email.mime.text import MIMEText
             from email.mime.multipart import MIMEMultipart
+            from email.mime.text import MIMEText
 
             msg = MIMEMultipart()
             msg["From"] = self.from_addr