flowyml 1.1.0__py3-none-any.whl → 1.3.0__py3-none-any.whl

flowyml/ui/backend/routers/assets.py

@@ -1,5 +1,10 @@
 from fastapi import APIRouter, HTTPException
+from fastapi.responses import FileResponse
+from pydantic import BaseModel
 from flowyml.storage.metadata import SQLiteMetadataStore
+from flowyml.core.project import ProjectManager
+from typing import Optional
+from pathlib import Path
 
 router = APIRouter()
 
@@ -8,38 +13,374 @@ def get_store():
     return SQLiteMetadataStore()
 
 
+def _iter_metadata_stores():
+    stores = [(None, SQLiteMetadataStore())]
+    try:
+        manager = ProjectManager()
+        for project_meta in manager.list_projects():
+            name = project_meta.get("name")
+            if not name:
+                continue
+            project = manager.get_project(name)
+            if project:
+                stores.append((name, project.metadata_store))
+    except Exception:
+        pass
+    return stores
+
+
+def _dedupe_assets(assets):
+    dedup = {}
+    for asset, project_name in assets:
+        artifact_id = asset.get("artifact_id") or f"{project_name}:{len(dedup)}"
+        if artifact_id in dedup:
+            continue
+        entry = dict(asset)
+        if project_name and not entry.get("project"):
+            entry["project"] = project_name
+        dedup[artifact_id] = entry
+    return list(dedup.values())
+
+
 @router.get("/")
 async def list_assets(limit: int = 50, asset_type: str = None, run_id: str = None, project: str = None):
     """List all assets, optionally filtered by project."""
     try:
-        store = get_store()
+        combined = []
+        for project_name, store in _iter_metadata_stores():
+            if project and project_name and project != project_name:
+                continue
+
+            filters = {}
+            if asset_type:
+                filters["type"] = asset_type
+            if run_id:
+                filters["run_id"] = run_id
 
-        # Build filters
-        filters = {}
-        if asset_type:
-            filters["type"] = asset_type
-        if run_id:
-            filters["run_id"] = run_id
+            assets = store.list_assets(limit=limit, **filters)
+            for asset in assets:
+                combined.append((asset, project_name))
 
-        assets = store.list_assets(limit=limit, **filters)
+        assets = _dedupe_assets(combined)
 
-        # Filter by project if specified
         if project:
-            # Get all runs for this project
-            all_runs = store.list_runs(limit=1000)
-            project_run_ids = {r.get("run_id") for r in all_runs if r.get("project") == project}
-            assets = [a for a in assets if a.get("run_id") in project_run_ids]
+            assets = [a for a in assets if a.get("project") == project]
+
+        assets = assets[:limit]
 
         return {"assets": assets}
     except Exception as e:
         return {"assets": [], "error": str(e)}
 
 
+@router.get("/stats")
+async def get_asset_stats(project: Optional[str] = None):
+    """Get statistics about assets for the dashboard."""
+    try:
+        combined_assets = []
+        for project_name, store in _iter_metadata_stores():
+            if project and project_name and project != project_name:
+                continue
+
+            assets = store.list_assets(limit=1000)
+            for asset in assets:
+                combined_assets.append((asset, project_name))
+
+        assets = _dedupe_assets(combined_assets)
+
+        if project:
+            assets = [a for a in assets if a.get("project") == project]
+
+        # Calculate statistics
+        total_assets = len(assets)
+
+        # Count by type
+        type_counts = {}
+        for asset in assets:
+            asset_type = asset.get("type", "unknown")
+            type_counts[asset_type] = type_counts.get(asset_type, 0) + 1
+
+        # Calculate total storage (if size info available)
+        total_storage = sum(asset.get("size_bytes", 0) for asset in assets)
+
+        # Get recent assets (last 10)
+        sorted_assets = sorted(
+            assets,
+            key=lambda x: x.get("created_at", ""),
+            reverse=True,
+        )
+        recent_assets = sorted_assets[:10]
+
+        # Count by project
+        project_counts = {}
+        for asset in assets:
+            proj = asset.get("project", "default")
+            project_counts[proj] = project_counts.get(proj, 0) + 1
+
+        return {
+            "total_assets": total_assets,
+            "by_type": type_counts,
+            "total_storage_bytes": total_storage,
+            "recent_assets": recent_assets,
+            "by_project": project_counts,
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to get stats: {str(e)}")
+
+
+@router.get("/search")
+async def search_assets(q: str, limit: int = 50, project: Optional[str] = None):
+    """Search assets by name or properties."""
+    try:
+        combined_assets = []
+        for project_name, store in _iter_metadata_stores():
+            if project and project_name and project != project_name:
+                continue
+
+            assets = store.list_assets(limit=1000)
+            for asset in assets:
+                combined_assets.append((asset, project_name))
+
+        assets = _dedupe_assets(combined_assets)
+
+        if project:
+            assets = [a for a in assets if a.get("project") == project]
+
+        # Simple fuzzy search in name, type, and step
+        query_lower = q.lower()
+        matching_assets = []
+
+        for asset in assets:
+            name = asset.get("name", "").lower()
+            asset_type = asset.get("type", "").lower()
+            step = asset.get("step", "").lower()
+
+            # Check if query matches any field
+            if query_lower in name or query_lower in asset_type or query_lower in step:
+                matching_assets.append(asset)
+
+        return {
+            "assets": matching_assets[:limit],
+            "total": len(matching_assets),
+            "query": q,
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to search: {str(e)}")
+
+
+@router.get("/lineage")
+async def get_asset_lineage(
+    asset_id: Optional[str] = None,
+    project: Optional[str] = None,
+    depth: int = 3,
+):
+    """
+    Get lineage graph for artifacts.
+
+    Returns nodes (artifacts, runs, pipelines) and edges (relationships).
+    Can be scoped to a specific asset or all assets in a project.
+    """
+    try:
+        store = get_store()
+        nodes = []
+        edges = []
+        visited_artifacts = set()
+        visited_runs = set()
+
+        # Get starting artifacts
+        if asset_id:
+            artifact = store.load_artifact(asset_id)
+            if not artifact:
+                raise HTTPException(status_code=404, detail="Asset not found")
+            starting_artifacts = [artifact]
+        elif project:
+            # Get all artifacts for project
+            all_runs = store.list_runs(limit=1000)
+            project_run_ids = {r.get("run_id") for r in all_runs if r.get("project") == project}
+            all_artifacts = store.list_assets(limit=1000)
+            starting_artifacts = [a for a in all_artifacts if a.get("run_id") in project_run_ids]
+        else:
+            # Get recent artifacts
+            starting_artifacts = store.list_assets(limit=50)
+
+        # Build graph
+        for artifact in starting_artifacts:
+            artifact_id = artifact.get("artifact_id")
+            if not artifact_id or artifact_id in visited_artifacts:
+                continue
+
+            visited_artifacts.add(artifact_id)
+
+            # Add artifact node
+            nodes.append(
+                {
+                    "id": artifact_id,
+                    "type": "artifact",
+                    "label": artifact.get("name", artifact_id),
+                    "artifact_type": artifact.get("type", "unknown"),
+                    "metadata": {
+                        "created_at": artifact.get("created_at"),
+                        "properties": artifact.get("properties", {}),
+                        "size": artifact.get("size"),
+                        "run_id": artifact.get("run_id"),
+                    },
+                },
+            )
+
+            # Add relationship to run
+            run_id = artifact.get("run_id")
+            if run_id and run_id not in visited_runs:
+                visited_runs.add(run_id)
+                run = store.load_run(run_id)
+
+                if run:
+                    # Add run node
+                    nodes.append(
+                        {
+                            "id": run_id,
+                            "type": "run",
+                            "label": run.get("run_name", run_id[:8]),
+                            "metadata": {
+                                "pipeline_name": run.get("pipeline_name"),
+                                "status": run.get("status"),
+                                "project": run.get("project"),
+                            },
+                        },
+                    )
+
+                    # Add edge: run produces artifact
+                    edges.append(
+                        {
+                            "id": f"{run_id}->{artifact_id}",
+                            "source": run_id,
+                            "target": artifact_id,
+                            "type": "produces",
+                            "label": artifact.get("step", ""),
+                        },
+                    )
+
+                    # Find input artifacts (from DAG/steps metadata)
+                    metadata = run.get("metadata", {})
+                    # dag = metadata.get("dag", {})
+                    steps = metadata.get("steps", {})
+
+                    # Get step that produced this artifact
+                    artifact_step = artifact.get("step")
+                    if artifact_step and artifact_step in steps:
+                        step_info = steps.get(artifact_step, {})
+                        inputs = step_info.get("inputs", [])
+
+                        # Find artifacts that were inputs to this step
+                        for input_name in inputs:
+                            # Search for artifact with this name from the same run
+                            input_artifacts = [
+                                a for a in all_artifacts if a.get("name") == input_name and a.get("run_id") == run_id
+                            ]
+                            for input_artifact in input_artifacts:
+                                input_id = input_artifact.get("artifact_id")
+                                if input_id and input_id not in visited_artifacts:
+                                    visited_artifacts.add(input_id)
+                                    nodes.append(
+                                        {
+                                            "id": input_id,
+                                            "type": "artifact",
+                                            "label": input_artifact.get("name", input_id),
+                                            "artifact_type": input_artifact.get("type", "unknown"),
+                                            "metadata": {
+                                                "created_at": input_artifact.get("created_at"),
+                                                "properties": input_artifact.get("properties", {}),
+                                            },
+                                        },
+                                    )
+
+                                # Add edge: input artifact consumed by run
+                                edges.append(
+                                    {
+                                        "id": f"{input_id}->{run_id}",
+                                        "source": input_id,
+                                        "target": run_id,
+                                        "type": "consumes",
+                                        "label": artifact_step,
+                                    },
+                                )
+
+        return {
+            "nodes": nodes,
+            "edges": edges,
+            "metadata": {
+                "total_artifacts": len([n for n in nodes if n["type"] == "artifact"]),
+                "total_runs": len([n for n in nodes if n["type"] == "run"]),
+                "depth_used": depth,
+            },
+        }
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to build lineage: {str(e)}")
+
+
 @router.get("/{artifact_id}")
 async def get_asset(artifact_id: str):
     """Get details for a specific asset."""
-    store = get_store()
-    asset = store.load_artifact(artifact_id)
+    asset = _find_asset(artifact_id)
     if not asset:
         raise HTTPException(status_code=404, detail="Asset not found")
     return asset
+
+
+@router.get("/{artifact_id}/download")
+async def download_asset(artifact_id: str):
+    """Download the artifact file referenced by metadata."""
+    asset, _ = _find_asset_with_store(artifact_id)
+    if not asset:
+        raise HTTPException(status_code=404, detail="Asset not found")
+
+    artifact_path = asset.get("path")
+    if not artifact_path:
+        raise HTTPException(status_code=404, detail="Artifact path not available")
+
+    file_path = Path(artifact_path)
+    if not file_path.exists():
+        raise HTTPException(status_code=404, detail="Artifact file not found on disk")
+
+    return FileResponse(
+        path=file_path,
+        filename=file_path.name,
+        media_type="application/octet-stream",
+    )
+
+
+class ProjectUpdate(BaseModel):
+    project_name: str
+
+
+@router.put("/{artifact_id}/project")
+async def update_asset_project(artifact_id: str, update: ProjectUpdate):
+    """Update the project for an asset."""
+    try:
+        _, store = _find_asset_with_store(artifact_id)
+        if not store:
+            raise HTTPException(status_code=404, detail="Asset not found")
+
+        store.update_artifact_project(artifact_id, update.project_name)
+        return {"status": "success", "project": update.project_name}
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+def _find_asset(artifact_id: str):
+    asset, _ = _find_asset_with_store(artifact_id)
+    return asset
+
+
+def _find_asset_with_store(artifact_id: str):
+    for project_name, store in _iter_metadata_stores():
+        asset = store.load_artifact(artifact_id)
+        if asset:
+            if project_name and not asset.get("project"):
+                asset["project"] = project_name
+            return asset, store
+    return None, None

flowyml/ui/backend/routers/client.py

@@ -0,0 +1,46 @@
+from fastapi import APIRouter, Request
+from pydantic import BaseModel
+from typing import Any, Optional
+from flowyml.monitoring.alerts import alert_manager, AlertLevel
+
+router = APIRouter()
+
+
+class ClientError(BaseModel):
+    message: str
+    stack: Optional[str] = None
+    component_stack: Optional[str] = None
+    url: Optional[str] = None
+    user_agent: Optional[str] = None
+    additional_info: Optional[dict[str, Any]] = None
+
+
+@router.post("/errors")
+async def report_client_error(error: ClientError, request: Request):
+    """Report an error from the frontend client."""
+
+    # Construct a detailed message
+    details = f"Client Error: {error.message}\n"
+    if error.url:
+        details += f"URL: {error.url}\n"
+    if error.user_agent:
+        details += f"User Agent: {error.user_agent}\n"
+    if error.component_stack:
+        details += f"Component Stack:\n{error.component_stack}\n"
+    if error.stack:
+        details += f"Stack Trace:\n{error.stack}\n"
+
+    # Send alert
+    alert_manager.send_alert(
+        title=f"Frontend Error: {error.message[:50]}...",
+        message=details,
+        level=AlertLevel.ERROR,
+        metadata={
+            "source": "frontend",
+            "url": error.url,
+            "user_agent": error.user_agent,
+            "additional_info": error.additional_info,
+        },
+    )
+
+    return {"status": "recorded"}

flowyml/ui/backend/routers/execution.py

@@ -27,6 +27,7 @@ class PipelineExecutionRequest(BaseModel):
     parameters: dict[str, Any] = {}
     project: str | None = None
     dry_run: bool = False  # If True, validate but don't execute
+    retry_count: int = 0  # Number of retries on failure (0-5)
 
 
 class TokenRequest(BaseModel):
@@ -92,13 +93,23 @@ async def execute_pipeline(
 
         pipeline = getattr(module, request.pipeline_name)
 
-        # Execute the pipeline
-        result = pipeline.run(**request.parameters)
+        # Execute the pipeline with retry policy if specified
+        run_kwargs = request.parameters.copy()
+
+        if request.retry_count > 0:
+            from flowyml.core.retry import OrchestratorRetryPolicy
+
+            run_kwargs["retry_policy"] = OrchestratorRetryPolicy(
+                max_retries=min(request.retry_count, 5),  # Cap at 5
+            )
+
+        result = pipeline.run(**run_kwargs)
 
         return {
             "status": "completed",
             "run_id": result.run_id if hasattr(result, "run_id") else None,
             "pipeline": request.pipeline_name,
+            "retry_count": request.retry_count,
             "message": "Pipeline executed successfully",
         }
 

flowyml/ui/backend/routers/experiments.py

@@ -1,5 +1,7 @@
 from fastapi import APIRouter, HTTPException
 from flowyml.storage.metadata import SQLiteMetadataStore
+from flowyml.core.project import ProjectManager
+from typing import Optional
 
 router = APIRouter()
 
@@ -8,18 +10,48 @@ def get_store():
     return SQLiteMetadataStore()
 
 
+def _iter_metadata_stores():
+    """Yield tuples of (project_name, store) including global and project stores."""
+    stores = [(None, SQLiteMetadataStore())]
+    try:
+        manager = ProjectManager()
+        for project_meta in manager.list_projects():
+            name = project_meta.get("name")
+            if not name:
+                continue
+            project = manager.get_project(name)
+            if project:
+                stores.append((name, project.metadata_store))
+    except Exception:
+        pass
+    return stores
+
+
 @router.get("/")
-async def list_experiments(project: str = None):
+async def list_experiments(project: Optional[str] = None):
     """List all experiments, optionally filtered by project."""
     try:
-        store = get_store()
-        experiments = store.list_experiments()
+        combined_experiments = []
+
+        for project_name, store in _iter_metadata_stores():
+            # Skip other projects if filtering
+            if project and project_name and project != project_name:
+                continue
 
-        # Filter by project if specified
-        if project:
-            experiments = [e for e in experiments if e.get("project") == project]
+            store_experiments = store.list_experiments()
 
-        return {"experiments": experiments}
+            for exp in store_experiments:
+                # Add project info if not already present
+                if project_name and not exp.get("project"):
+                    exp["project"] = project_name
+
+                # Skip if filtering by project and doesn't match
+                if project and exp.get("project") != project:
+                    continue
+
+                combined_experiments.append(exp)
+
+        return {"experiments": combined_experiments}
     except Exception as e:
         return {"experiments": [], "error": str(e)}
 
@@ -27,11 +59,15 @@ async def list_experiments(project: str = None):
 @router.get("/{experiment_id}")
 async def get_experiment(experiment_id: str):
     """Get experiment details."""
-    store = get_store()
-    experiment = store.get_experiment(experiment_id)
-    if not experiment:
-        raise HTTPException(status_code=404, detail="Experiment not found")
-    return experiment
+    # Try to find experiment in any store
+    for project_name, store in _iter_metadata_stores():
+        experiment = store.get_experiment(experiment_id)
+        if experiment:
+            if project_name and not experiment.get("project"):
+                experiment["project"] = project_name
+            return experiment
+
+    raise HTTPException(status_code=404, detail="Experiment not found")
 
 
 @router.put("/{experiment_name}/project")