flowyml 1.7.1__py3-none-any.whl → 1.8.0__py3-none-any.whl

flowyml/ui/backend/routers/plugins.py

@@ -34,7 +34,7 @@ class ImportStackRequest(BaseModel):
 
 @router.get("/available", response_model=list[PluginInfo])
 async def get_available_plugins():
-    """Get list of available plugins."""
+    """Get list of available FlowyML plugins."""
     import importlib.metadata
 
     # Helper to check if package is installed
@@ -45,51 +45,95 @@ async def get_available_plugins():
         except importlib.metadata.PackageNotFoundError:
             return False
 
-    # Mock data for now - in production this would query a plugin registry
+    # FlowyML Native Plugins
     plugins = [
         PluginInfo(
-            plugin_id="zenml-kubernetes",
-            name="zenml-kubernetes",
-            version="0.45.0",
-            author="ZenML",
-            description="Kubernetes orchestrator integration from ZenML ecosystem.",
-            downloads="12k",
-            stars="450",
-            tags=["orchestrator", "kubernetes", "zenml"],
-            installed=is_installed("zenml-kubernetes"),
+            plugin_id="flowyml-gcp",
+            name="FlowyML GCP",
+            version="1.8.0",
+            author="FlowyML",
+            description="Google Cloud Platform integration: Vertex AI orchestrator, GCS artifact store, and Cloud Run deployer.",
+            downloads="5.2k",
+            stars="180",
+            tags=["orchestrator", "artifact-store", "gcp", "vertex-ai"],
+            installed=is_installed("google-cloud-aiplatform"),
         ),
         PluginInfo(
-            plugin_id="zenml-mlflow",
-            name="zenml-mlflow",
-            version="0.45.0",
-            author="ZenML",
-            description="MLflow integration for experiment tracking and model deployment.",
+            plugin_id="flowyml-aws",
+            name="FlowyML AWS",
+            version="1.8.0",
+            author="FlowyML",
+            description="AWS integration: SageMaker orchestrator, S3 artifact store, and ECR container registry.",
+            downloads="4.8k",
+            stars="165",
+            tags=["orchestrator", "artifact-store", "aws", "sagemaker"],
+            installed=is_installed("boto3"),
+        ),
+        PluginInfo(
+            plugin_id="flowyml-kubernetes",
+            name="FlowyML Kubernetes",
+            version="1.8.0",
+            author="FlowyML",
+            description="Kubernetes orchestrator for running pipelines on K8s clusters with auto-scaling.",
+            downloads="3.5k",
+            stars="145",
+            tags=["orchestrator", "kubernetes", "container"],
+            installed=is_installed("kubernetes"),
+        ),
+        PluginInfo(
+            plugin_id="flowyml-mlflow",
+            name="FlowyML MLflow",
+            version="1.8.0",
+            author="FlowyML",
+            description="MLflow integration for experiment tracking, model registry, and deployment.",
+            downloads="6.1k",
+            stars="220",
+            tags=["tracking", "model-registry", "mlflow"],
+            installed=is_installed("mlflow"),
+        ),
+        PluginInfo(
+            plugin_id="flowyml-wandb",
+            name="FlowyML Weights & Biases",
+            version="1.8.0",
+            author="FlowyML",
+            description="W&B integration for experiment tracking, artifact versioning, and collaboration.",
+            downloads="4.2k",
+            stars="195",
+            tags=["tracking", "wandb", "experiment"],
+            installed=is_installed("wandb"),
+        ),
+        PluginInfo(
+            plugin_id="flowyml-pytorch",
+            name="FlowyML PyTorch",
+            version="1.8.0",
+            author="FlowyML",
+            description="PyTorch integration with automatic model serialization and distributed training support.",
             downloads="8.5k",
-            stars="320",
-            tags=["tracking", "mlflow", "zenml"],
-            installed=is_installed("zenml-mlflow"),
+            stars="310",
+            tags=["framework", "pytorch", "deep-learning"],
+            installed=is_installed("torch"),
         ),
         PluginInfo(
-            plugin_id="airflow-providers-google",
-            name="airflow-providers-google",
-            version="10.1.0",
-            author="Apache Airflow",
-            description="Google Cloud Platform providers for Airflow.",
-            downloads="50k",
-            stars="1.2k",
-            tags=["orchestrator", "gcp", "airflow"],
-            installed=is_installed("airflow-providers-google"),
+            plugin_id="flowyml-tensorflow",
+            name="FlowyML TensorFlow",
+            version="1.8.0",
+            author="FlowyML",
+            description="TensorFlow/Keras integration with automatic callbacks and model tracking.",
+            downloads="7.8k",
+            stars="290",
+            tags=["framework", "tensorflow", "keras"],
+            installed=is_installed("tensorflow"),
         ),
         PluginInfo(
-            plugin_id="aws-s3",
-            name="aws-s3",
-            version="1.0.0",
-            author="AWS",
-            description="S3 artifact store integration.",
-            downloads="15k",
-            stars="200",
-            tags=["artifact-store", "aws"],
-            installed=is_installed("aws-s3"),
+            plugin_id="flowyml-sklearn",
+            name="FlowyML Scikit-Learn",
+            version="1.8.0",
+            author="FlowyML",
+            description="Scikit-learn integration with automatic model serialization and metrics extraction.",
+            downloads="9.2k",
+            stars="340",
+            tags=["framework", "sklearn", "ml"],
+            installed=is_installed("scikit-learn"),
         ),
     ]
 
@@ -98,33 +142,41 @@ async def get_available_plugins():
 
 @router.get("/installed", response_model=list[dict[str, Any]])
 async def get_installed_plugins():
-    """Get list of installed plugins."""
+    """Get list of installed FlowyML plugins and integrations."""
     import importlib.metadata
 
     # Get all installed packages that could be plugins
     installed = []
 
-    # List of known plugin packages (you can expand this)
+    # FlowyML-related plugin packages
     potential_plugins = [
-        "zenml",
-        "zenml-kubernetes",
-        "zenml-mlflow",
-        "zenml-s3",
-        "airflow",
-        "airflow-providers-google",
-        "airflow-providers-aws",
-        "aws-s3",
-        "boto3",
-        "kubernetes",
+        # Cloud providers
+        ("google-cloud-aiplatform", "FlowyML GCP"),
+        ("google-cloud-storage", "GCS Storage"),
+        ("boto3", "FlowyML AWS"),
+        ("sagemaker", "AWS SageMaker"),
+        # Orchestrators
+        ("kubernetes", "FlowyML Kubernetes"),
+        ("kfp", "Kubeflow Pipelines"),
+        # Tracking & Registry
+        ("mlflow", "FlowyML MLflow"),
+        ("wandb", "FlowyML W&B"),
+        # ML Frameworks
+        ("torch", "FlowyML PyTorch"),
+        ("tensorflow", "FlowyML TensorFlow"),
+        ("keras", "FlowyML Keras"),
+        ("scikit-learn", "FlowyML Scikit-Learn"),
+        # Core
+        ("flowyml", "FlowyML Core"),
     ]
 
-    for package_name in potential_plugins:
+    for package_name, display_name in potential_plugins:
         try:
             dist = importlib.metadata.distribution(package_name)
             installed.append(
                 {
                     "id": package_name,
-                    "name": package_name,
+                    "name": display_name,
                     "version": dist.version,
                     "description": dist.metadata.get("Summary", ""),
                     "status": "active",

flowyml/ui/backend/routers/projects.py

@@ -14,10 +14,42 @@ def get_projects_manager() -> ProjectManager:
 
 @router.get("/")
 async def list_projects(manager: ProjectManager = Depends(get_projects_manager)):
-    """List all projects."""
+    """List all projects, including those discovered from run metadata."""
     try:
-        projects = manager.list_projects()
-        return {"projects": projects}
+        # Get explicitly created projects
+        explicit_projects = manager.list_projects()
+        project_names = {p.get("name") for p in explicit_projects if p.get("name")}
+
+        # Also discover projects from run metadata in global store
+        from flowyml.ui.backend.dependencies import get_store
+
+        store = get_store()
+
+        discovered_projects = []
+        try:
+            # Get all runs and extract unique project names
+            runs = store.list_runs(limit=1000)
+            for run in runs:
+                project_name = run.get("project")
+                if project_name and project_name not in project_names:
+                    project_names.add(project_name)
+                    # Create a synthetic project entry for discovered projects
+                    discovered_projects.append(
+                        {
+                            "name": project_name,
+                            "description": "Auto-discovered from pipeline runs",
+                            "created_at": run.get("start_time"),
+                            "pipelines": [],
+                            "tags": {},
+                            "discovered": True,  # Flag to indicate this wasn't explicitly created
+                        },
+                    )
+        except Exception:
+            pass  # Store might not be initialized
+
+        # Combine explicit and discovered projects
+        all_projects = explicit_projects + discovered_projects
+        return {"projects": all_projects}
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))
 
@@ -93,14 +125,65 @@ async def get_project_metrics(
     limit: int = 100,
     manager: ProjectManager = Depends(get_projects_manager),
 ):
-    """Get logged production metrics for a project."""
-    project = manager.get_project(project_name)
-    if not project:
-        raise HTTPException(status_code=404, detail="Project not found")
+    """Get logged metrics for a project (from model_metrics table and Metrics artifacts)."""
+    metrics = []
+
+    from flowyml.ui.backend.dependencies import get_store
+
+    store = get_store()
+
+    try:
+        # Get all runs for this project
+        all_runs = store.list_runs(limit=1000)
+        project_run_ids = {r.get("run_id") for r in all_runs if r.get("project") == project_name}
+
+        # 1. Try to get metrics from model_metrics table
+        all_model_metrics = store.list_model_metrics(limit=limit * 2)
+        for m in all_model_metrics:
+            if m.get("run_id") in project_run_ids or m.get("project") == project_name:
+                metrics.append(m)
+
+        # 2. Also extract metrics from Metrics artifacts
+        all_assets = store.list_assets(limit=500)
+        for asset in all_assets:
+            # Check if it's a metrics artifact for this project (case-insensitive type check)
+            asset_type = str(asset.get("type", "")).lower()
+            if asset_type == "metrics" and asset.get("run_id") in project_run_ids:
+                # Get properties which contain the metric values
+                props = asset.get("properties", {})
+                created_at = asset.get("created_at", "")
+                run_id = asset.get("run_id", "")
+                asset_name = asset.get("name", "evaluation")
+
+                # Convert artifact properties to metric entries
+                for key, value in props.items():
+                    if isinstance(value, (int, float)) and key not in ["samples"]:
+                        metrics.append(
+                            {
+                                "project": project_name,
+                                "model_name": asset_name,
+                                "run_id": run_id,
+                                "metric_name": key,
+                                "metric_value": value,
+                                "environment": "evaluation",
+                                "tags": {"source": "artifact"},
+                                "created_at": created_at,
+                            },
+                        )
+    except Exception as e:
+        import logging
+
+        logging.getLogger(__name__).warning(f"Error fetching metrics: {e}")
+
+    # Try explicit project as fallback
+    if not metrics:
+        project = manager.get_project(project_name)
+        if project:
+            metrics = project.list_model_metrics(model_name=model_name, limit=limit)
 
     return {
         "project": project_name,
-        "metrics": project.list_model_metrics(model_name=model_name, limit=limit),
+        "metrics": metrics[:limit],
     }
 
 

flowyml/ui/backend/routers/runs.py

@@ -163,6 +163,9 @@ async def get_run(run_id: str):
             for step_name, ts in _heartbeat_timestamps[run_id].items():
                 if step_name in run.get("steps", {}):
                     run["steps"][step_name]["last_heartbeat"] = ts
+            for step_name, metrics in _step_metrics.get(run_id, {}).items():
+                if step_name in run.get("steps", {}):
+                    run["steps"][step_name]["metrics"] = metrics
 
     return run
 
@@ -296,11 +299,14 @@ async def get_cloud_status(run_id: str):
 class HeartbeatRequest(BaseModel):
     step_name: str
     status: str = "running"
+    metrics: dict | None = None
 
 
-# In-memory storage for heartbeat timestamps
+# In-memory storage for heartbeat timestamps and metrics
 # Format: {run_id: {step_name: last_heartbeat_timestamp}}
 _heartbeat_timestamps: dict[str, dict[str, float]] = {}
+# Format: {run_id: {step_name: metrics_dict}}
+_step_metrics: dict[str, dict[str, dict]] = {}
 _heartbeat_lock = __import__("threading").Lock()
 
 # Heartbeat interval in seconds (should match executor's interval)
@@ -319,6 +325,14 @@ def _record_heartbeat(run_id: str, step_name: str) -> None:
         _heartbeat_timestamps[run_id][step_name] = time.time()
 
 
+def _record_step_metrics(run_id: str, step_name: str, metrics: dict) -> None:
+    """Record metrics for a step."""
+    with _heartbeat_lock:
+        if run_id not in _step_metrics:
+            _step_metrics[run_id] = {}
+        _step_metrics[run_id][step_name] = metrics
+
+
 def _get_dead_steps(run_id: str) -> list[str]:
     """Get list of steps that have missed too many heartbeats."""
     import time
@@ -342,6 +356,7 @@ def _cleanup_heartbeats(run_id: str) -> None:
     """Remove heartbeat tracking for a completed run."""
     with _heartbeat_lock:
         _heartbeat_timestamps.pop(run_id, None)
+        _step_metrics.pop(run_id, None)
 
 
 @router.post("/{run_id}/steps/{step_name}/heartbeat")
@@ -356,6 +371,10 @@ async def step_heartbeat(run_id: str, step_name: str, heartbeat: HeartbeatReques
     # Record heartbeat timestamp
     _record_heartbeat(run_id, step_name)
 
+    # Record metrics if present
+    if heartbeat.metrics:
+        _record_step_metrics(run_id, step_name, heartbeat.metrics)
+
     # Check if run is marked for stopping
     run = store.load_run(run_id)
     if not run:
@@ -484,3 +503,115 @@ async def get_run_logs(run_id: str):
     logs = await anyio.to_thread.run_sync(read_all_logs)
 
     return {"logs": logs}
+
+
+@router.get("/{run_id}/training-history")
+async def get_training_history(run_id: str):
+    """Get training history (per-epoch metrics) for a run.
+
+    This combines:
+    1. Training history from model artifacts (saved by FlowymlKerasCallback)
+    2. Per-epoch metrics saved in the metrics table
+
+    Returns a consolidated training history suitable for visualization.
+    """
+    store = _find_store_for_run(run_id)
+
+    # Get per-epoch metrics from the metrics table
+    metrics = store.get_metrics(run_id)
+
+    # Build training history from metrics table
+    # Group metrics by step (epoch) and name
+    epoch_metrics = {}
+    for m in metrics:
+        step = m.get("step", 0)
+        name = m.get("name", "unknown")
+        value = m.get("value", 0)
+
+        if step not in epoch_metrics:
+            epoch_metrics[step] = {}
+        epoch_metrics[step][name] = value
+
+    # Convert to chart-friendly format
+    training_history_from_metrics = {
+        "epochs": [],
+        "train_loss": [],
+        "val_loss": [],
+        "train_accuracy": [],
+        "val_accuracy": [],
+        "mae": [],
+        "val_mae": [],
+    }
+
+    # Standard metric name mappings
+    metric_mappings = {
+        "loss": "train_loss",
+        "val_loss": "val_loss",
+        "accuracy": "train_accuracy",
+        "acc": "train_accuracy",
+        "val_accuracy": "val_accuracy",
+        "val_acc": "val_accuracy",
+        "mae": "mae",
+        "val_mae": "val_mae",
+    }
+
+    # Track custom metrics
+    custom_metrics = set()
+
+    if epoch_metrics:
+        sorted_epochs = sorted(epoch_metrics.keys())
+        for epoch in sorted_epochs:
+            training_history_from_metrics["epochs"].append(epoch + 1)  # 1-indexed for display
+
+            epoch_data = epoch_metrics[epoch]
+            for metric_name, value in epoch_data.items():
+                # Map to standard name or track as custom
+                standard_name = metric_mappings.get(metric_name)
+                if standard_name:
+                    training_history_from_metrics[standard_name].append(value)
+                else:
+                    # Custom metric
+                    if metric_name not in custom_metrics:
+                        custom_metrics.add(metric_name)
+                        training_history_from_metrics[metric_name] = []
+                    training_history_from_metrics[metric_name].append(value)
+
+    # Also try to get training history from model artifacts
+    artifacts = store.list_assets(run_id=run_id)
+    artifact_history = None
+
+    for artifact in artifacts:
+        # Check if artifact has training_history
+        if artifact.get("training_history"):
+            artifact_history = artifact.get("training_history")
+            break
+        # Also check in metadata/properties
+        metadata = artifact.get("metadata", {})
+        if isinstance(metadata, str):
+            try:
+                metadata = json.loads(metadata)
+            except Exception:
+                metadata = {}
+        if metadata.get("training_history"):
+            artifact_history = metadata.get("training_history")
+            break
+
+    # Prefer artifact history if it has more data, otherwise use metrics
+    if artifact_history and len(artifact_history.get("epochs", [])) > len(
+        training_history_from_metrics.get("epochs", []),
+    ):
+        final_history = artifact_history
+    elif training_history_from_metrics.get("epochs"):
+        final_history = training_history_from_metrics
+    else:
+        final_history = artifact_history or {}
+
+    # Clean up empty arrays
+    cleaned_history = {k: v for k, v in final_history.items() if v and (not isinstance(v, list) or len(v) > 0)}
+
+    return {
+        "training_history": cleaned_history,
+        "has_history": len(cleaned_history.get("epochs", [])) > 0,
+        "total_epochs": len(cleaned_history.get("epochs", [])),
+        "source": "artifact" if artifact_history else "metrics",
+    }

flowyml/ui/backend/routers/schedules.py

@@ -4,11 +4,16 @@ from flowyml.core.scheduler import PipelineScheduler
 from flowyml.registry.pipeline_registry import pipeline_registry
 
 router = APIRouter()
-# Note: In a real app, the scheduler instance should be a singleton managed by the app state
-# For now, we instantiate it here, but it might not persist state across reloads if not handled carefully.
-# Ideally, the scheduler is started when the backend starts.
-scheduler = PipelineScheduler()
-scheduler.start()  # Start the scheduler thread
+_scheduler = None
+
+
+def get_scheduler():
+    """Get or initialize the scheduler singleton."""
+    global _scheduler
+    if _scheduler is None:
+        _scheduler = PipelineScheduler()
+        _scheduler.start()
+    return _scheduler
 
 
 class ScheduleRequest(BaseModel):
@@ -25,27 +30,47 @@ class ScheduleRequest(BaseModel):
 
 @router.get("/")
 async def list_schedules():
-    """List all active schedules."""
-    # Convert Schedule objects to dicts for JSON serialization
-    schedules = scheduler.list_schedules()
-    return [
-        {
-            "pipeline_name": s.pipeline_name,
-            "schedule_type": s.schedule_type,
-            "schedule_value": s.schedule_value,
-            "enabled": s.enabled,
-            "last_run": s.last_run,
-            "next_run": s.next_run,
-            "timezone": s.timezone,
-        }
-        for s in schedules
-    ]
+    """List all active schedules.
+
+    This reads schedules from the shared database, so schedules created
+    by user code (e.g., in scripts) are visible in the UI.
+    """
+    # First, get schedules from the in-memory scheduler
+    memory_schedules = []
+    for s in get_scheduler().list_schedules():
+        memory_schedules.append(
+            {
+                "pipeline_name": s.pipeline_name,
+                "schedule_type": s.schedule_type,
+                "schedule_value": s.schedule_value,
+                "enabled": s.enabled,
+                "last_run": s.last_run.isoformat() if s.last_run else None,
+                "next_run": s.next_run.isoformat() if s.next_run else None,
+                "timezone": s.timezone,
+            },
+        )
+
+    # Also read directly from the persistence database to get schedules
+    # created by other processes (e.g., user scripts)
+    db_schedules = []
+    if get_scheduler()._persistence:
+        db_schedules = get_scheduler()._persistence.list_all_schedules()
+
+    # Merge: prefer memory schedules (more up-to-date), but include db-only ones
+    memory_names = {s["pipeline_name"] for s in memory_schedules}
+    result = list(memory_schedules)
+
+    for db_sched in db_schedules:
+        if db_sched.get("pipeline_name") not in memory_names:
+            result.append(db_sched)
+
+    return result
 
 
 @router.get("/health")
 async def get_scheduler_health():
     """Get scheduler health metrics."""
-    return scheduler.health_check()
+    return get_scheduler().health_check()
 
 
 @router.post("/")
@@ -113,7 +138,7 @@ async def create_schedule(schedule: ScheduleRequest):
     # 2. Schedule it
     try:
         if schedule.schedule_type == "daily":
-            scheduler.schedule_daily(
+            get_scheduler().schedule_daily(
                 name=schedule.name,
                 pipeline_func=pipeline_func,
                 hour=schedule.hour,
@@ -121,14 +146,14 @@ async def create_schedule(schedule: ScheduleRequest):
                 timezone=schedule.timezone,
             )
         elif schedule.schedule_type == "hourly":
-            scheduler.schedule_hourly(
+            get_scheduler().schedule_hourly(
                 name=schedule.name,
                 pipeline_func=pipeline_func,
                 minute=schedule.minute,
                 timezone=schedule.timezone,
             )
         elif schedule.schedule_type == "interval":
-            scheduler.schedule_interval(
+            get_scheduler().schedule_interval(
                 name=schedule.name,
                 pipeline_func=pipeline_func,
                 seconds=schedule.interval_seconds,
@@ -137,7 +162,7 @@ async def create_schedule(schedule: ScheduleRequest):
         elif schedule.schedule_type == "cron":
             if not schedule.cron_expression:
                 raise HTTPException(status_code=400, detail="Cron expression required for cron schedule")
-            scheduler.schedule_cron(
+            get_scheduler().schedule_cron(
                 name=schedule.name,
                 pipeline_func=pipeline_func,
                 cron_expression=schedule.cron_expression,
@@ -157,28 +182,28 @@ async def create_schedule(schedule: ScheduleRequest):
 @router.delete("/{schedule_name}")
 async def delete_schedule(schedule_name: str):
     """Remove a schedule."""
-    scheduler.unschedule(schedule_name)
+    get_scheduler().unschedule(schedule_name)
     return {"status": "success", "message": f"Schedule {schedule_name} removed"}
 
 
 @router.post("/{schedule_name}/enable")
 async def enable_schedule(schedule_name: str):
     """Enable a schedule."""
-    scheduler.enable(schedule_name)
+    get_scheduler().enable(schedule_name)
     return {"status": "success"}
 
 
 @router.post("/{schedule_name}/disable")
 async def disable_schedule(schedule_name: str):
     """Disable a schedule."""
-    scheduler.disable(schedule_name)
+    get_scheduler().disable(schedule_name)
     return {"status": "success"}
 
 
 @router.get("/{schedule_name}/history")
 async def get_schedule_history(schedule_name: str, limit: int = 50):
     """Get execution history for a schedule."""
-    return scheduler.get_history(schedule_name, limit)
+    return get_scheduler().get_history(schedule_name, limit)
 
 
 @router.get("/registered-pipelines")