flowyml 1.2.0__py3-none-any.whl → 1.3.0__py3-none-any.whl

flowyml/ui/backend/routers/metrics.py

@@ -0,0 +1,213 @@
+"""Model metrics logging API endpoints."""
+
+from fastapi import APIRouter, HTTPException, Depends, Security, Query
+from fastapi.security import HTTPAuthorizationCredentials
+from pydantic import BaseModel, Field
+from typing import Any
+
+from flowyml.ui.backend.auth import verify_api_token, security
+from flowyml.storage.metadata import SQLiteMetadataStore
+from flowyml.core.project import ProjectManager
+from flowyml.utils.config import get_config
+
+router = APIRouter()
+
+
+def require_permission(permission: str):
+    """Create dependency enforcing a given permission."""
+
+    async def _verify(credentials: HTTPAuthorizationCredentials = Security(security)):
+        return await verify_api_token(credentials, required_permission=permission)
+
+    return _verify
+
+
+def get_project_manager() -> ProjectManager:
+    """Get a project manager rooted at configured projects dir."""
+    config = get_config()
+    return ProjectManager(str(config.projects_dir))
+
+
+def get_global_store() -> SQLiteMetadataStore:
+    """Metadata store for shared metrics."""
+    config = get_config()
+    return SQLiteMetadataStore(str(config.metadata_db))
+
+
+class MetricsLogRequest(BaseModel):
+    """Payload for logging production model metrics."""
+
+    project: str = Field(..., description="Project identifier")
+    model_name: str = Field(..., description="Name of the model emitting metrics")
+    metrics: dict[str, float] = Field(..., description="Dictionary of metric_name -> value")
+    run_id: str | None = Field(None, description="Related run identifier (optional)")
+    environment: str | None = Field(None, description="Environment label (e.g., prod, staging)")
+    tags: dict[str, Any] | None = Field(default_factory=dict, description="Optional metadata tags")
+
+
+@router.post("/log")
+async def log_model_metrics(
+    payload: MetricsLogRequest,
+    token_data: dict = Depends(require_permission("write")),
+):
+    """Log production metrics for a model.
+
+    Requires tokens with the `write` permission. Project-scoped tokens
+    may only submit metrics for their project.
+    """
+    if token_data.get("project") and token_data["project"] != payload.project:
+        raise HTTPException(
+            status_code=403,
+            detail=f"Token is scoped to project '{token_data['project']}'",
+        )
+
+    if not payload.metrics:
+        raise HTTPException(status_code=400, detail="metrics dictionary cannot be empty")
+
+    numeric_metrics = {}
+    for name, value in payload.metrics.items():
+        try:
+            numeric_metrics[name] = float(value)
+        except (TypeError, ValueError):
+            raise HTTPException(
+                status_code=400,
+                detail=f"Metric '{name}' must be numeric.",
+            )
+
+    project_manager = get_project_manager()
+    project = project_manager.get_project(payload.project)
+    if not project:
+        project = project_manager.create_project(payload.project)
+
+    shared_store = get_global_store()
+    shared_store.log_model_metrics(
+        project=payload.project,
+        model_name=payload.model_name,
+        metrics=numeric_metrics,
+        run_id=payload.run_id,
+        environment=payload.environment,
+        tags=payload.tags,
+    )
+
+    project.log_model_metrics(
+        model_name=payload.model_name,
+        metrics=numeric_metrics,
+        run_id=payload.run_id,
+        environment=payload.environment,
+        tags=payload.tags,
+    )
+
+    return {
+        "project": payload.project,
+        "model_name": payload.model_name,
+        "logged_metrics": list(numeric_metrics.keys()),
+        "message": "Metrics logged successfully",
+    }
+
+
+@router.get("")
+async def list_model_metrics(
+    project: str | None = Query(default=None, description="Filter by project"),
+    model_name: str | None = Query(default=None, description="Filter by model"),
+    limit: int = Query(default=100, ge=1, le=500),
+    token_data: dict = Depends(require_permission("read")),
+):
+    """Retrieve the latest logged model metrics."""
+    if token_data.get("project"):
+        if project and token_data["project"] != project:
+            raise HTTPException(
+                status_code=403,
+                detail=f"Token is scoped to project '{token_data['project']}'",
+            )
+        project = token_data["project"]
+
+    store: SQLiteMetadataStore
+    if project:
+        project_manager = get_project_manager()
+        project_obj = project_manager.get_project(project)
+        if not project_obj:
+            raise HTTPException(status_code=404, detail=f"Project '{project}' not found")
+        store = project_obj.metadata_store
+    else:
+        store = get_global_store()
+
+    records = store.list_model_metrics(project=project, model_name=model_name, limit=limit)
+    return {"metrics": records}
+
+
+@router.get("/observability/orchestrator")
+async def get_orchestrator_metrics():
+    """Get orchestrator-level performance metrics."""
+    from datetime import datetime, timedelta
+    import sqlite3
+
+    store = get_global_store()
+    conn = sqlite3.connect(store.db_path)
+    cursor = conn.cursor()
+
+    thirty_days_ago = (datetime.now() - timedelta(days=30)).isoformat()
+    cursor.execute("SELECT COUNT(*) FROM runs WHERE created_at >= ?", (thirty_days_ago,))
+    total_runs = cursor.fetchone()[0]
+
+    cursor.execute(
+        "SELECT status, COUNT(*) FROM runs WHERE created_at >= ? GROUP BY status",
+        (thirty_days_ago,),
+    )
+    status_counts = dict(cursor.fetchall())
+
+    cursor.execute(
+        "SELECT AVG(duration) FROM runs WHERE created_at >= ? AND duration IS NOT NULL",
+        (thirty_days_ago,),
+    )
+    avg_duration = cursor.fetchone()[0] or 0
+
+    conn.close()
+
+    completed = status_counts.get("completed", 0)
+    success_rate = completed / total_runs if total_runs > 0 else 0
+
+    return {
+        "total_runs": total_runs,
+        "success_rate": success_rate,
+        "avg_duration_seconds": avg_duration,
+        "status_distribution": status_counts,
+        "period_days": 30,
+    }
+
+
+@router.get("/observability/cache")
+async def get_cache_metrics():
+    """Get cache performance metrics."""
+    from datetime import datetime, timedelta
+    import sqlite3
+    import json as json_lib
+
+    store = get_global_store()
+    conn = sqlite3.connect(store.db_path)
+    cursor = conn.cursor()
+
+    thirty_days_ago = (datetime.now() - timedelta(days=30)).isoformat()
+    cursor.execute("SELECT metadata FROM runs WHERE created_at >= ?", (thirty_days_ago,))
+
+    total_steps, cached_steps = 0, 0
+    for row in cursor.fetchall():
+        if not row[0]:
+            continue
+        try:
+            metadata = json_lib.loads(row[0])
+            for step_data in metadata.get("steps", {}).values():
+                total_steps += 1
+                if step_data.get("cached"):
+                    cached_steps += 1
+        except Exception:
+            continue
+
+    conn.close()
+    cache_hit_rate = cached_steps / total_steps if total_steps > 0 else 0
+
+    return {
+        "total_steps": total_steps,
+        "cached_steps": cached_steps,
+        "cache_hit_rate": cache_hit_rate,
+        "period_days": 30,
+    }

flowyml/ui/backend/routers/pipelines.py

@@ -1,25 +1,81 @@
 from fastapi import APIRouter, HTTPException
 from pydantic import BaseModel
 from flowyml.storage.metadata import SQLiteMetadataStore
+from flowyml.core.project import ProjectManager
 from flowyml.utils.config import get_config
+from typing import Optional
 
 router = APIRouter()
 
 
 def get_store():
     get_config()
-    # Assuming default path or from config
-    # The SQLiteMetadataStore defaults to .flowyml/metadata.db which is what we want for now
     return SQLiteMetadataStore()
 
 
+def _iter_metadata_stores():
+    """Yield tuples of (project_name, store) including global and project stores."""
+    stores = [(None, SQLiteMetadataStore())]
+    try:
+        manager = ProjectManager()
+        for project_meta in manager.list_projects():
+            name = project_meta.get("name")
+            if not name:
+                continue
+            project = manager.get_project(name)
+            if project:
+                stores.append((name, project.metadata_store))
+    except Exception:
+        pass
+    return stores
+
+
 @router.get("/")
-async def list_pipelines(project: str = None):
-    """List all unique pipelines, optionally filtered by project."""
+async def list_pipelines(project: Optional[str] = None, limit: int = 100):
+    """List all unique pipelines with details, optionally filtered by project."""
     try:
-        store = get_store()
-        pipelines = store.list_pipelines(project=project)
-        return {"pipelines": pipelines}
+        pipeline_map = {}  # pipeline_name -> data
+
+        for project_name, store in _iter_metadata_stores():
+            # Skip other projects if filtering
+            if project and project_name and project != project_name:
+                continue
+
+            # Get pipeline names from this store
+            store_pipeline_names = store.list_pipelines()
+
+            for name in store_pipeline_names:
+                # Get runs for this pipeline
+                filters = {"pipeline_name": name}
+                runs = store.query(**filters)
+
+                if not runs:
+                    continue
+
+                last_run = runs[0]
+                run_project = last_run.get("project") or project_name
+
+                # Skip if filtering by project and doesn't match
+                if project and run_project != project:
+                    continue
+
+                # Use composite key if we already have this pipeline from another project
+                key = f"{name}:{run_project}" if run_project else name
+
+                if key not in pipeline_map:
+                    pipeline_map[key] = {
+                        "name": name,
+                        "created": last_run.get("start_time"),
+                        "version": last_run.get("git_sha", "latest")[:7] if last_run.get("git_sha") else "1.0",
+                        "status": last_run.get("status", "unknown"),
+                        "run_count": len(runs),
+                        "last_run_id": last_run.get("run_id"),
+                        "project": run_project,
+                    }
+
+        # Return list of pipelines
+        enriched_pipelines = list(pipeline_map.values())[:limit]
+        return {"pipelines": enriched_pipelines}
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))
 

flowyml/ui/backend/routers/projects.py

@@ -1,17 +1,23 @@
-from fastapi import APIRouter, HTTPException
+from fastapi import APIRouter, HTTPException, Depends
 from flowyml.core.project import ProjectManager
+from flowyml.utils.config import get_config
 from pydantic import BaseModel
 
 router = APIRouter()
-manager = ProjectManager()
+
+
+def get_projects_manager() -> ProjectManager:
+    """Instantiate a ProjectManager bound to the current config."""
+    config = get_config()
+    return ProjectManager(str(config.projects_dir))
 
 
 @router.get("/")
-async def list_projects():
+async def list_projects(manager: ProjectManager = Depends(get_projects_manager)):
     """List all projects."""
     try:
         projects = manager.list_projects()
-        return projects
+        return {"projects": projects}
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))
 
@@ -22,7 +28,7 @@ class ProjectCreate(BaseModel):
 
 
 @router.post("/")
-async def create_project(project: ProjectCreate):
+async def create_project(project: ProjectCreate, manager: ProjectManager = Depends(get_projects_manager)):
     """Create a new project."""
     created_project = manager.create_project(project.name, project.description)
     return {
@@ -33,7 +39,7 @@ async def create_project(project: ProjectCreate):
 
 
 @router.get("/{project_name}")
-async def get_project(project_name: str):
+async def get_project(project_name: str, manager: ProjectManager = Depends(get_projects_manager)):
     """Get project details."""
     project = manager.get_project(project_name)
     if not project:
@@ -53,6 +59,7 @@ async def get_project_runs(
     project_name: str,
     pipeline_name: str | None = None,
     limit: int = 100,
+    manager: ProjectManager = Depends(get_projects_manager),
 ):
     """Get runs for a project."""
     project = manager.get_project(project_name)
@@ -68,6 +75,7 @@ async def get_project_artifacts(
     project_name: str,
     artifact_type: str | None = None,
     limit: int = 100,
+    manager: ProjectManager = Depends(get_projects_manager),
 ):
     """Get artifacts for a project."""
     project = manager.get_project(project_name)
@@ -78,8 +86,26 @@ async def get_project_artifacts(
     return artifacts
 
 
+@router.get("/{project_name}/metrics")
+async def get_project_metrics(
+    project_name: str,
+    model_name: str | None = None,
+    limit: int = 100,
+    manager: ProjectManager = Depends(get_projects_manager),
+):
+    """Get logged production metrics for a project."""
+    project = manager.get_project(project_name)
+    if not project:
+        raise HTTPException(status_code=404, detail="Project not found")
+
+    return {
+        "project": project_name,
+        "metrics": project.list_model_metrics(model_name=model_name, limit=limit),
+    }
+
+
 @router.delete("/{project_name}")
-async def delete_project(project_name: str):
+async def delete_project(project_name: str, manager: ProjectManager = Depends(get_projects_manager)):
     """Delete a project."""
     manager.delete_project(project_name, confirm=True)
     return {"deleted": True}

flowyml/ui/backend/routers/runs.py

@@ -1,6 +1,9 @@
 from fastapi import APIRouter, HTTPException
 from pydantic import BaseModel
 from flowyml.storage.metadata import SQLiteMetadataStore
+from flowyml.core.project import ProjectManager
+from typing import Optional
+import json
 
 router = APIRouter()
 
@@ -9,17 +12,62 @@ def get_store():
     return SQLiteMetadataStore()
 
 
+def _iter_metadata_stores():
+    """Yield tuples of (project_name, store) including global and project stores."""
+    stores: list[tuple[Optional[str], SQLiteMetadataStore]] = [(None, SQLiteMetadataStore())]
+    try:
+        manager = ProjectManager()
+        for project_meta in manager.list_projects():
+            name = project_meta.get("name")
+            if not name:
+                continue
+            project = manager.get_project(name)
+            if project:
+                stores.append((name, project.metadata_store))
+    except Exception:
+        pass
+    return stores
+
+
+def _deduplicate_runs(runs):
+    seen = {}
+    for run, project_name in runs:
+        run_id = run.get("run_id") or f"{project_name}:{len(seen)}"
+        if run_id in seen:
+            continue
+        entry = dict(run)
+        if project_name and not entry.get("project"):
+            entry["project"] = project_name
+        seen[run_id] = entry
+    return list(seen.values())
+
+
+def _sort_runs(runs):
+    def sort_key(run):
+        return run.get("start_time") or run.get("created_at") or ""
+
+    return sorted(runs, key=sort_key, reverse=True)
+
+
 @router.get("/")
 async def list_runs(limit: int = 20, project: str = None):
     """List all runs, optionally filtered by project."""
     try:
-        store = get_store()
-        runs = store.list_runs(limit=limit)
+        combined = []
+        for project_name, store in _iter_metadata_stores():
+            # Skip other projects if filtering by project name
+            if project and project_name and project != project_name:
+                continue
+            store_runs = store.list_runs(limit=limit)
+            for run in store_runs:
+                combined.append((run, project_name))
+
+        runs = _deduplicate_runs(combined)
 
-        # Filter by project if specified
         if project:
             runs = [r for r in runs if r.get("project") == project]
 
+        runs = _sort_runs(runs)[:limit]
         return {"runs": runs}
     except Exception as e:
         return {"runs": [], "error": str(e)}
@@ -28,8 +76,7 @@ async def list_runs(limit: int = 20, project: str = None):
 @router.get("/{run_id}")
 async def get_run(run_id: str):
     """Get details for a specific run."""
-    store = get_store()
-    run = store.load_run(run_id)
+    run, _ = _find_run(run_id)
     if not run:
         raise HTTPException(status_code=404, detail="Run not found")
     return run
@@ -38,7 +85,7 @@ async def get_run(run_id: str):
 @router.get("/{run_id}/metrics")
 async def get_run_metrics(run_id: str):
     """Get metrics for a specific run."""
-    store = get_store()
+    store = _find_store_for_run(run_id)
     metrics = store.get_metrics(run_id)
     return {"metrics": metrics}
 
@@ -46,7 +93,7 @@ async def get_run_metrics(run_id: str):
 @router.get("/{run_id}/artifacts")
 async def get_run_artifacts(run_id: str):
     """Get artifacts for a specific run."""
-    store = get_store()
+    store = _find_store_for_run(run_id)
     artifacts = store.list_assets(run_id=run_id)
     return {"artifacts": artifacts}
 
@@ -58,9 +105,104 @@ class ProjectUpdate(BaseModel):
 @router.put("/{run_id}/project")
 async def update_run_project(run_id: str, update: ProjectUpdate):
     """Update the project for a run."""
-    store = get_store()
+    store = _find_store_for_run(run_id)
     try:
         store.update_run_project(run_id, update.project_name)
         return {"status": "success", "project": update.project_name}
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))
+
+
+def _find_run(run_id: str):
+    for project_name, store in _iter_metadata_stores():
+        run = store.load_run(run_id)
+        if run:
+            if project_name and not run.get("project"):
+                run["project"] = project_name
+            return run, store
+    return None, None
+
+
+def _find_store_for_run(run_id: str) -> SQLiteMetadataStore:
+    _, store = _find_run(run_id)
+    if store:
+        return store
+    raise HTTPException(status_code=404, detail="Run not found")
+
+
+@router.get("/{run_id}/cloud-status")
+async def get_cloud_status(run_id: str):
+    """Get real-time status from cloud orchestrator for remote runs.
+
+    Returns cloud provider status if the run is remote, otherwise returns
+    status from metadata store.
+    """
+    run, store = _find_run(run_id)
+    if not run:
+        raise HTTPException(status_code=404, detail="Run not found")
+
+    # Get orchestrator info from run metadata
+    metadata = run.get("metadata", {})
+    if isinstance(metadata, str):
+        try:
+            metadata = json.loads(metadata)
+        except Exception:
+            metadata = {}
+
+    orchestrator_type = metadata.get("orchestrator_type", "local")
+
+    # If local run, just return metadata store status
+    if orchestrator_type == "local":
+        return {
+            "run_id": run_id,
+            "status": run.get("status", "unknown"),
+            "orchestrator_type": "local",
+            "is_remote": False,
+            "cloud_status": None,
+        }
+
+    # For remote runs, try to query cloud orchestrator
+    cloud_status = None
+    cloud_error = None
+
+    try:
+        # Import orchestrators dynamically to avoid errors if cloud SDKs aren't installed
+        from flowyml.utils.stack_config import load_active_stack
+
+        stack = load_active_stack()
+        if not stack or not stack.orchestrator:
+            cloud_error = "No active stack or orchestrator configured"
+        else:
+            orchestrator = stack.orchestrator
+
+            # Check if orchestrator has get_run_status method
+            if hasattr(orchestrator, "get_run_status"):
+                from flowyml.core.execution_status import ExecutionStatus
+
+                status = orchestrator.get_run_status(run_id)
+
+                # Convert ExecutionStatus to dict
+                if isinstance(status, ExecutionStatus):
+                    cloud_status = {
+                        "status": status.value,
+                        "is_finished": status.is_finished,
+                        "is_successful": status.is_successful,
+                    }
+                else:
+                    cloud_status = {"status": str(status)}
+            else:
+                cloud_error = f"Orchestrator {orchestrator_type} does not support status queries"
+
+    except ImportError as e:
+        cloud_error = f"Cloud SDK not available: {str(e)}"
+    except Exception as e:
+        cloud_error = f"Error querying cloud status: {str(e)}"
+
+    return {
+        "run_id": run_id,
+        "status": run.get("status", "unknown"),
+        "orchestrator_type": orchestrator_type,
+        "is_remote": True,
+        "cloud_status": cloud_status,
+        "cloud_error": cloud_error,
+    }