flowyml 1.7.2__py3-none-any.whl → 1.8.0__py3-none-any.whl

flowyml/ui/backend/routers/plugins.py

@@ -34,7 +34,7 @@ class ImportStackRequest(BaseModel):
 
 @router.get("/available", response_model=list[PluginInfo])
 async def get_available_plugins():
-    """Get list of available plugins."""
+    """Get list of available FlowyML plugins."""
     import importlib.metadata
 
     # Helper to check if package is installed
@@ -45,51 +45,95 @@ async def get_available_plugins():
         except importlib.metadata.PackageNotFoundError:
             return False
 
-    # Mock data for now - in production this would query a plugin registry
+    # FlowyML Native Plugins
     plugins = [
         PluginInfo(
-            plugin_id="zenml-kubernetes",
-            name="zenml-kubernetes",
-            version="0.45.0",
-            author="ZenML",
-            description="Kubernetes orchestrator integration from ZenML ecosystem.",
-            downloads="12k",
-            stars="450",
-            tags=["orchestrator", "kubernetes", "zenml"],
-            installed=is_installed("zenml-kubernetes"),
+            plugin_id="flowyml-gcp",
+            name="FlowyML GCP",
+            version="1.8.0",
+            author="FlowyML",
+            description="Google Cloud Platform integration: Vertex AI orchestrator, GCS artifact store, and Cloud Run deployer.",
+            downloads="5.2k",
+            stars="180",
+            tags=["orchestrator", "artifact-store", "gcp", "vertex-ai"],
+            installed=is_installed("google-cloud-aiplatform"),
         ),
         PluginInfo(
-            plugin_id="zenml-mlflow",
-            name="zenml-mlflow",
-            version="0.45.0",
-            author="ZenML",
-            description="MLflow integration for experiment tracking and model deployment.",
+            plugin_id="flowyml-aws",
+            name="FlowyML AWS",
+            version="1.8.0",
+            author="FlowyML",
+            description="AWS integration: SageMaker orchestrator, S3 artifact store, and ECR container registry.",
+            downloads="4.8k",
+            stars="165",
+            tags=["orchestrator", "artifact-store", "aws", "sagemaker"],
+            installed=is_installed("boto3"),
+        ),
+        PluginInfo(
+            plugin_id="flowyml-kubernetes",
+            name="FlowyML Kubernetes",
+            version="1.8.0",
+            author="FlowyML",
+            description="Kubernetes orchestrator for running pipelines on K8s clusters with auto-scaling.",
+            downloads="3.5k",
+            stars="145",
+            tags=["orchestrator", "kubernetes", "container"],
+            installed=is_installed("kubernetes"),
+        ),
+        PluginInfo(
+            plugin_id="flowyml-mlflow",
+            name="FlowyML MLflow",
+            version="1.8.0",
+            author="FlowyML",
+            description="MLflow integration for experiment tracking, model registry, and deployment.",
+            downloads="6.1k",
+            stars="220",
+            tags=["tracking", "model-registry", "mlflow"],
+            installed=is_installed("mlflow"),
+        ),
+        PluginInfo(
+            plugin_id="flowyml-wandb",
+            name="FlowyML Weights & Biases",
+            version="1.8.0",
+            author="FlowyML",
+            description="W&B integration for experiment tracking, artifact versioning, and collaboration.",
+            downloads="4.2k",
+            stars="195",
+            tags=["tracking", "wandb", "experiment"],
+            installed=is_installed("wandb"),
+        ),
+        PluginInfo(
+            plugin_id="flowyml-pytorch",
+            name="FlowyML PyTorch",
+            version="1.8.0",
+            author="FlowyML",
+            description="PyTorch integration with automatic model serialization and distributed training support.",
             downloads="8.5k",
-            stars="320",
-            tags=["tracking", "mlflow", "zenml"],
-            installed=is_installed("zenml-mlflow"),
+            stars="310",
+            tags=["framework", "pytorch", "deep-learning"],
+            installed=is_installed("torch"),
         ),
         PluginInfo(
-            plugin_id="airflow-providers-google",
-            name="airflow-providers-google",
-            version="10.1.0",
-            author="Apache Airflow",
-            description="Google Cloud Platform providers for Airflow.",
-            downloads="50k",
-            stars="1.2k",
-            tags=["orchestrator", "gcp", "airflow"],
-            installed=is_installed("airflow-providers-google"),
+            plugin_id="flowyml-tensorflow",
+            name="FlowyML TensorFlow",
+            version="1.8.0",
+            author="FlowyML",
+            description="TensorFlow/Keras integration with automatic callbacks and model tracking.",
+            downloads="7.8k",
+            stars="290",
+            tags=["framework", "tensorflow", "keras"],
+            installed=is_installed("tensorflow"),
         ),
         PluginInfo(
-            plugin_id="aws-s3",
-            name="aws-s3",
-            version="1.0.0",
-            author="AWS",
-            description="S3 artifact store integration.",
-            downloads="15k",
-            stars="200",
-            tags=["artifact-store", "aws"],
-            installed=is_installed("aws-s3"),
+            plugin_id="flowyml-sklearn",
+            name="FlowyML Scikit-Learn",
+            version="1.8.0",
+            author="FlowyML",
+            description="Scikit-learn integration with automatic model serialization and metrics extraction.",
+            downloads="9.2k",
+            stars="340",
+            tags=["framework", "sklearn", "ml"],
+            installed=is_installed("scikit-learn"),
         ),
     ]
 
@@ -98,33 +142,41 @@ async def get_available_plugins():
 
 @router.get("/installed", response_model=list[dict[str, Any]])
 async def get_installed_plugins():
-    """Get list of installed plugins."""
+    """Get list of installed FlowyML plugins and integrations."""
     import importlib.metadata
 
     # Get all installed packages that could be plugins
     installed = []
 
-    # List of known plugin packages (you can expand this)
+    # FlowyML-related plugin packages
     potential_plugins = [
-        "zenml",
-        "zenml-kubernetes",
-        "zenml-mlflow",
-        "zenml-s3",
-        "airflow",
-        "airflow-providers-google",
-        "airflow-providers-aws",
-        "aws-s3",
-        "boto3",
-        "kubernetes",
+        # Cloud providers
+        ("google-cloud-aiplatform", "FlowyML GCP"),
+        ("google-cloud-storage", "GCS Storage"),
+        ("boto3", "FlowyML AWS"),
+        ("sagemaker", "AWS SageMaker"),
+        # Orchestrators
+        ("kubernetes", "FlowyML Kubernetes"),
+        ("kfp", "Kubeflow Pipelines"),
+        # Tracking & Registry
+        ("mlflow", "FlowyML MLflow"),
+        ("wandb", "FlowyML W&B"),
+        # ML Frameworks
+        ("torch", "FlowyML PyTorch"),
+        ("tensorflow", "FlowyML TensorFlow"),
+        ("keras", "FlowyML Keras"),
+        ("scikit-learn", "FlowyML Scikit-Learn"),
+        # Core
+        ("flowyml", "FlowyML Core"),
     ]
 
-    for package_name in potential_plugins:
+    for package_name, display_name in potential_plugins:
         try:
             dist = importlib.metadata.distribution(package_name)
             installed.append(
                 {
                     "id": package_name,
-                    "name": package_name,
+                    "name": display_name,
                     "version": dist.version,
                     "description": dist.metadata.get("Summary", ""),
                     "status": "active",

flowyml/ui/backend/routers/projects.py

@@ -14,10 +14,42 @@ def get_projects_manager() -> ProjectManager:
 
 @router.get("/")
 async def list_projects(manager: ProjectManager = Depends(get_projects_manager)):
-    """List all projects."""
+    """List all projects, including those discovered from run metadata."""
     try:
-        projects = manager.list_projects()
-        return {"projects": projects}
+        # Get explicitly created projects
+        explicit_projects = manager.list_projects()
+        project_names = {p.get("name") for p in explicit_projects if p.get("name")}
+
+        # Also discover projects from run metadata in global store
+        from flowyml.ui.backend.dependencies import get_store
+
+        store = get_store()
+
+        discovered_projects = []
+        try:
+            # Get all runs and extract unique project names
+            runs = store.list_runs(limit=1000)
+            for run in runs:
+                project_name = run.get("project")
+                if project_name and project_name not in project_names:
+                    project_names.add(project_name)
+                    # Create a synthetic project entry for discovered projects
+                    discovered_projects.append(
+                        {
+                            "name": project_name,
+                            "description": "Auto-discovered from pipeline runs",
+                            "created_at": run.get("start_time"),
+                            "pipelines": [],
+                            "tags": {},
+                            "discovered": True,  # Flag to indicate this wasn't explicitly created
+                        },
+                    )
+        except Exception:
+            pass  # Store might not be initialized
+
+        # Combine explicit and discovered projects
+        all_projects = explicit_projects + discovered_projects
+        return {"projects": all_projects}
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))
 
@@ -93,14 +125,65 @@ async def get_project_metrics(
     limit: int = 100,
     manager: ProjectManager = Depends(get_projects_manager),
 ):
-    """Get logged production metrics for a project."""
-    project = manager.get_project(project_name)
-    if not project:
-        raise HTTPException(status_code=404, detail="Project not found")
+    """Get logged metrics for a project (from model_metrics table and Metrics artifacts)."""
+    metrics = []
+
+    from flowyml.ui.backend.dependencies import get_store
+
+    store = get_store()
+
+    try:
+        # Get all runs for this project
+        all_runs = store.list_runs(limit=1000)
+        project_run_ids = {r.get("run_id") for r in all_runs if r.get("project") == project_name}
+
+        # 1. Try to get metrics from model_metrics table
+        all_model_metrics = store.list_model_metrics(limit=limit * 2)
+        for m in all_model_metrics:
+            if m.get("run_id") in project_run_ids or m.get("project") == project_name:
+                metrics.append(m)
+
+        # 2. Also extract metrics from Metrics artifacts
+        all_assets = store.list_assets(limit=500)
+        for asset in all_assets:
+            # Check if it's a metrics artifact for this project (case-insensitive type check)
+            asset_type = str(asset.get("type", "")).lower()
+            if asset_type == "metrics" and asset.get("run_id") in project_run_ids:
+                # Get properties which contain the metric values
+                props = asset.get("properties", {})
+                created_at = asset.get("created_at", "")
+                run_id = asset.get("run_id", "")
+                asset_name = asset.get("name", "evaluation")
+
+                # Convert artifact properties to metric entries
+                for key, value in props.items():
+                    if isinstance(value, (int, float)) and key not in ["samples"]:
+                        metrics.append(
+                            {
+                                "project": project_name,
+                                "model_name": asset_name,
+                                "run_id": run_id,
+                                "metric_name": key,
+                                "metric_value": value,
+                                "environment": "evaluation",
+                                "tags": {"source": "artifact"},
+                                "created_at": created_at,
+                            },
+                        )
+    except Exception as e:
+        import logging
+
+        logging.getLogger(__name__).warning(f"Error fetching metrics: {e}")
+
+    # Try explicit project as fallback
+    if not metrics:
+        project = manager.get_project(project_name)
+        if project:
+            metrics = project.list_model_metrics(model_name=model_name, limit=limit)
 
     return {
         "project": project_name,
-        "metrics": project.list_model_metrics(model_name=model_name, limit=limit),
+        "metrics": metrics[:limit],
     }
 
 

flowyml/ui/backend/routers/runs.py

@@ -163,6 +163,9 @@ async def get_run(run_id: str):
             for step_name, ts in _heartbeat_timestamps[run_id].items():
                 if step_name in run.get("steps", {}):
                     run["steps"][step_name]["last_heartbeat"] = ts
+            for step_name, metrics in _step_metrics.get(run_id, {}).items():
+                if step_name in run.get("steps", {}):
+                    run["steps"][step_name]["metrics"] = metrics
 
     return run
 
@@ -296,11 +299,14 @@ async def get_cloud_status(run_id: str):
 class HeartbeatRequest(BaseModel):
     step_name: str
     status: str = "running"
+    metrics: dict | None = None
 
 
-# In-memory storage for heartbeat timestamps
+# In-memory storage for heartbeat timestamps and metrics
 # Format: {run_id: {step_name: last_heartbeat_timestamp}}
 _heartbeat_timestamps: dict[str, dict[str, float]] = {}
+# Format: {run_id: {step_name: metrics_dict}}
+_step_metrics: dict[str, dict[str, dict]] = {}
 _heartbeat_lock = __import__("threading").Lock()
 
 # Heartbeat interval in seconds (should match executor's interval)
@@ -319,6 +325,14 @@ def _record_heartbeat(run_id: str, step_name: str) -> None:
         _heartbeat_timestamps[run_id][step_name] = time.time()
 
 
+def _record_step_metrics(run_id: str, step_name: str, metrics: dict) -> None:
+    """Record metrics for a step."""
+    with _heartbeat_lock:
+        if run_id not in _step_metrics:
+            _step_metrics[run_id] = {}
+        _step_metrics[run_id][step_name] = metrics
+
+
 def _get_dead_steps(run_id: str) -> list[str]:
     """Get list of steps that have missed too many heartbeats."""
     import time
@@ -342,6 +356,7 @@ def _cleanup_heartbeats(run_id: str) -> None:
     """Remove heartbeat tracking for a completed run."""
     with _heartbeat_lock:
         _heartbeat_timestamps.pop(run_id, None)
+        _step_metrics.pop(run_id, None)
 
 
 @router.post("/{run_id}/steps/{step_name}/heartbeat")
@@ -356,6 +371,10 @@ async def step_heartbeat(run_id: str, step_name: str, heartbeat: HeartbeatReques
     # Record heartbeat timestamp
     _record_heartbeat(run_id, step_name)
 
+    # Record metrics if present
+    if heartbeat.metrics:
+        _record_step_metrics(run_id, step_name, heartbeat.metrics)
+
     # Check if run is marked for stopping
     run = store.load_run(run_id)
     if not run:

flowyml/ui/backend/routers/schedules.py

@@ -4,11 +4,16 @@ from flowyml.core.scheduler import PipelineScheduler
 from flowyml.registry.pipeline_registry import pipeline_registry
 
 router = APIRouter()
-# Note: In a real app, the scheduler instance should be a singleton managed by the app state
-# For now, we instantiate it here, but it might not persist state across reloads if not handled carefully.
-# Ideally, the scheduler is started when the backend starts.
-scheduler = PipelineScheduler()
-scheduler.start()  # Start the scheduler thread
+_scheduler = None
+
+
+def get_scheduler():
+    """Get or initialize the scheduler singleton."""
+    global _scheduler
+    if _scheduler is None:
+        _scheduler = PipelineScheduler()
+        _scheduler.start()
+    return _scheduler
 
 
 class ScheduleRequest(BaseModel):
@@ -32,7 +37,7 @@ async def list_schedules():
     """
     # First, get schedules from the in-memory scheduler
     memory_schedules = []
-    for s in scheduler.list_schedules():
+    for s in get_scheduler().list_schedules():
         memory_schedules.append(
             {
                 "pipeline_name": s.pipeline_name,
@@ -48,8 +53,8 @@ async def list_schedules():
     # Also read directly from the persistence database to get schedules
     # created by other processes (e.g., user scripts)
     db_schedules = []
-    if scheduler._persistence:
-        db_schedules = scheduler._persistence.list_all_schedules()
+    if get_scheduler()._persistence:
+        db_schedules = get_scheduler()._persistence.list_all_schedules()
 
     # Merge: prefer memory schedules (more up-to-date), but include db-only ones
     memory_names = {s["pipeline_name"] for s in memory_schedules}
@@ -65,7 +70,7 @@ async def list_schedules():
 @router.get("/health")
 async def get_scheduler_health():
     """Get scheduler health metrics."""
-    return scheduler.health_check()
+    return get_scheduler().health_check()
 
 
 @router.post("/")
@@ -133,7 +138,7 @@ async def create_schedule(schedule: ScheduleRequest):
     # 2. Schedule it
     try:
         if schedule.schedule_type == "daily":
-            scheduler.schedule_daily(
+            get_scheduler().schedule_daily(
                 name=schedule.name,
                 pipeline_func=pipeline_func,
                 hour=schedule.hour,
@@ -141,14 +146,14 @@ async def create_schedule(schedule: ScheduleRequest):
                 timezone=schedule.timezone,
             )
         elif schedule.schedule_type == "hourly":
-            scheduler.schedule_hourly(
+            get_scheduler().schedule_hourly(
                 name=schedule.name,
                 pipeline_func=pipeline_func,
                 minute=schedule.minute,
                 timezone=schedule.timezone,
             )
         elif schedule.schedule_type == "interval":
-            scheduler.schedule_interval(
+            get_scheduler().schedule_interval(
                 name=schedule.name,
                 pipeline_func=pipeline_func,
                 seconds=schedule.interval_seconds,
@@ -157,7 +162,7 @@ async def create_schedule(schedule: ScheduleRequest):
         elif schedule.schedule_type == "cron":
             if not schedule.cron_expression:
                 raise HTTPException(status_code=400, detail="Cron expression required for cron schedule")
-            scheduler.schedule_cron(
+            get_scheduler().schedule_cron(
                 name=schedule.name,
                 pipeline_func=pipeline_func,
                 cron_expression=schedule.cron_expression,
@@ -177,28 +182,28 @@ async def create_schedule(schedule: ScheduleRequest):
 @router.delete("/{schedule_name}")
 async def delete_schedule(schedule_name: str):
     """Remove a schedule."""
-    scheduler.unschedule(schedule_name)
+    get_scheduler().unschedule(schedule_name)
     return {"status": "success", "message": f"Schedule {schedule_name} removed"}
 
 
 @router.post("/{schedule_name}/enable")
 async def enable_schedule(schedule_name: str):
     """Enable a schedule."""
-    scheduler.enable(schedule_name)
+    get_scheduler().enable(schedule_name)
     return {"status": "success"}
 
 
 @router.post("/{schedule_name}/disable")
 async def disable_schedule(schedule_name: str):
     """Disable a schedule."""
-    scheduler.disable(schedule_name)
+    get_scheduler().disable(schedule_name)
     return {"status": "success"}
 
 
 @router.get("/{schedule_name}/history")
 async def get_schedule_history(schedule_name: str, limit: int = 50):
     """Get execution history for a schedule."""
-    return scheduler.get_history(schedule_name, limit)
+    return get_scheduler().get_history(schedule_name, limit)
 
 
 @router.get("/registered-pipelines")