flowyml 1.3.0__py3-none-any.whl → 1.4.0__py3-none-any.whl

flowyml/ui/backend/dependencies.py

@@ -0,0 +1,28 @@
+"""Backend dependencies."""
+
+import os
+from flowyml.storage.sql import SQLMetadataStore
+from flowyml.utils.config import get_config
+
+_store = None
+
+
+def get_store() -> SQLMetadataStore:
+    """Get the metadata store instance.
+
+    Uses FLOWYML_DATABASE_URL if set, otherwise defaults to local SQLite.
+    """
+    global _store
+    if _store is None:
+        config = get_config()
+        db_url = os.environ.get("FLOWYML_DATABASE_URL")
+
+        # If no explicit URL, use the config's metadata_db path
+        if not db_url:
+            db_path = config.metadata_db
+            # Ensure it's a string path for SQLMetadataStore
+            _store = SQLMetadataStore(db_path=str(db_path))
+        else:
+            _store = SQLMetadataStore(db_url=db_url)
+
+    return _store

flowyml/ui/backend/main.py

@@ -23,6 +23,7 @@ from flowyml.ui.backend.routers import (
     plugins,
     metrics,
     client,
+    stats,
 )
 
 app = FastAPI(
@@ -75,85 +76,7 @@ app.include_router(execution.router, prefix="/api/execution", tags=["execution"]
 app.include_router(metrics.router, prefix="/api/metrics", tags=["metrics"])
 app.include_router(plugins.router, prefix="/api", tags=["plugins"])
 app.include_router(client.router, prefix="/api/client", tags=["client"])
-
-
-# Stats endpoint for dashboard
-@app.get("/api/stats")
-async def get_stats(project: str = None):
-    """Get overall statistics for the dashboard, optionally filtered by project."""
-    try:
-        from flowyml.storage.metadata import SQLiteMetadataStore
-
-        store = SQLiteMetadataStore()
-
-        # Get base stats
-        stats = store.get_statistics()
-
-        # Get run status counts (not in get_statistics yet)
-        # We can add this to get_statistics later, but for now let's query efficiently
-        import sqlite3
-
-        conn = sqlite3.connect(store.db_path)
-        cursor = conn.cursor()
-
-        if project:
-            cursor.execute(
-                "SELECT COUNT(*) FROM runs WHERE project = ? AND status = 'completed'",
-                [project],
-            )
-            completed_runs = cursor.fetchone()[0]
-
-            cursor.execute(
-                "SELECT COUNT(*) FROM runs WHERE project = ? AND status = 'failed'",
-                [project],
-            )
-            failed_runs = cursor.fetchone()[0]
-
-            cursor.execute(
-                "SELECT AVG(duration) FROM runs WHERE project = ? AND duration IS NOT NULL",
-                [project],
-            )
-            avg_duration = cursor.fetchone()[0] or 0
-
-            cursor.execute(
-                "SELECT COUNT(*) FROM runs WHERE project = ?",
-                [project],
-            )
-            total_runs = cursor.fetchone()[0]
-        else:
-            cursor.execute("SELECT COUNT(*) FROM runs WHERE status = 'completed'")
-            completed_runs = cursor.fetchone()[0]
-
-            cursor.execute("SELECT COUNT(*) FROM runs WHERE status = 'failed'")
-            failed_runs = cursor.fetchone()[0]
-
-            cursor.execute("SELECT AVG(duration) FROM runs WHERE duration IS NOT NULL")
-            avg_duration = cursor.fetchone()[0] or 0
-
-            cursor.execute("SELECT COUNT(*) FROM runs")
-            total_runs = cursor.fetchone()[0]
-
-        conn.close()
-
-        return {
-            "runs": total_runs if project else stats.get("total_runs", 0),
-            "completed_runs": completed_runs,
-            "failed_runs": failed_runs,
-            "pipelines": stats.get("total_pipelines", 0),  # TODO: filter by project
-            "artifacts": stats.get("total_artifacts", 0),  # TODO: filter by project
-            "avg_duration": avg_duration,
-        }
-    except Exception as e:
-        # Return default stats if there's an error
-        return {
-            "runs": 0,
-            "completed_runs": 0,
-            "failed_runs": 0,
-            "pipelines": 0,
-            "artifacts": 0,
-            "avg_duration": 0,
-            "error": str(e),
-        }
+app.include_router(stats.router, prefix="/api/stats", tags=["stats"])
 
 
 # Static file serving for frontend

flowyml/ui/backend/routers/assets.py

@@ -1,16 +1,20 @@
-from fastapi import APIRouter, HTTPException
+from fastapi import APIRouter, HTTPException, UploadFile, File
 from fastapi.responses import FileResponse
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 from flowyml.storage.metadata import SQLiteMetadataStore
 from flowyml.core.project import ProjectManager
-from typing import Optional
 from pathlib import Path
+from flowyml.ui.backend.dependencies import get_store
+import shutil
+import asyncio
+import contextlib
 
 router = APIRouter()
 
 
-def get_store():
-    return SQLiteMetadataStore()
+def _save_file_sync(src, dst):
+    with open(dst, "wb") as buffer:
+        shutil.copyfileobj(src, buffer)
 
 
 def _iter_metadata_stores():
@@ -73,8 +77,129 @@ async def list_assets(limit: int = 50, asset_type: str = None, run_id: str = Non
         return {"assets": [], "error": str(e)}
 
 
+class AssetCreate(BaseModel):
+    artifact_id: str
+    name: str
+    asset_type: str = Field(..., alias="type")
+    run_id: str
+    step: str
+    project: str | None = None
+    metadata: dict = {}
+    value: str | None = None
+
+
+@router.post("/")
+async def create_asset(asset: AssetCreate):
+    """Create or update an asset metadata."""
+    try:
+        store = get_store()
+
+        # Prepare metadata
+        metadata = asset.metadata.copy()
+        metadata.update(
+            {
+                "name": asset.name,
+                "type": asset.asset_type,
+                "run_id": asset.run_id,
+                "step": asset.step,
+                "project": asset.project,
+                "value": asset.value,
+            },
+        )
+
+        store.save_artifact(asset.artifact_id, metadata)
+        return {"status": "success", "artifact_id": asset.artifact_id}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.post("/{artifact_id}/upload")
+async def upload_asset_content(artifact_id: str, file: UploadFile = File(...)):
+    """Upload content for an artifact."""
+    try:
+        store = get_store()
+
+        # Get existing metadata to find path or create a new one
+        existing = store.load_artifact(artifact_id)
+
+        if not existing:
+            raise HTTPException(status_code=404, detail="Artifact metadata not found. Create metadata first.")
+
+        # Determine storage path
+        # We use the LocalArtifactStore logic here since the backend is running locally relative to itself
+        from flowyml.storage.artifacts import LocalArtifactStore
+        from flowyml.utils.config import get_config
+
+        config = get_config()
+        artifact_store = LocalArtifactStore(base_path=config.artifacts_dir)
+
+        # Construct a path if not present
+        if not existing.get("path"):
+            # Create a path structure: project/run_id/artifact_id/filename
+            project = existing.get("project", "default")
+            run_id = existing.get("run_id", "unknown")
+            filename = file.filename or "content"
+            rel_path = f"{project}/{run_id}/{artifact_id}/{filename}"
+        else:
+            rel_path = existing.get("path")
+            # If path is absolute, make it relative to artifacts dir if possible, or just use it
+            # But LocalArtifactStore expects relative paths usually, or handles absolute ones
+
+        # Save the file
+        full_path = artifact_store.base_path / rel_path
+        full_path.parent.mkdir(parents=True, exist_ok=True)
+
+        loop = asyncio.get_running_loop()
+        await loop.run_in_executor(None, _save_file_sync, file.file, full_path)
+
+        # Update metadata with path
+        existing["path"] = str(rel_path)
+        store.save_artifact(artifact_id, existing)
+
+        return {"status": "success", "path": str(rel_path)}
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.delete("/{artifact_id}")
+async def delete_asset(artifact_id: str):
+    """Delete an asset and its file."""
+    try:
+        store = get_store()
+
+        # Get metadata to find path
+        asset = store.load_artifact(artifact_id)
+        if not asset:
+            raise HTTPException(status_code=404, detail="Asset not found")
+
+        # Delete file if it exists locally (since backend is local to itself)
+        path = asset.get("path")
+        if path:
+            from flowyml.storage.artifacts import LocalArtifactStore
+            from flowyml.utils.config import get_config
+
+            config = get_config()
+            artifact_store = LocalArtifactStore(base_path=config.artifacts_dir)
+
+            with contextlib.suppress(Exception):
+                artifact_store.delete(path)
+
+        # Delete metadata
+        store.delete_artifact(artifact_id)
+
+        return {"status": "success", "artifact_id": artifact_id}
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
 @router.get("/stats")
-async def get_asset_stats(project: Optional[str] = None):
+async def get_asset_stats(project: str | None = None):
     """Get statistics about assets for the dashboard."""
     try:
         combined_assets = []
@@ -129,7 +254,7 @@ async def get_asset_stats(project: Optional[str] = None):
 
 
 @router.get("/search")
-async def search_assets(q: str, limit: int = 50, project: Optional[str] = None):
+async def search_assets(q: str, limit: int = 50, project: str | None = None):
     """Search assets by name or properties."""
     try:
         combined_assets = []
@@ -170,8 +295,8 @@ async def search_assets(q: str, limit: int = 50, project: Optional[str] = None):
 
 @router.get("/lineage")
 async def get_asset_lineage(
-    asset_id: Optional[str] = None,
-    project: Optional[str] = None,
+    asset_id: str | None = None,
+    project: str | None = None,
     depth: int = 3,
 ):
     """

flowyml/ui/backend/routers/client.py

@@ -1,6 +1,6 @@
 from fastapi import APIRouter, Request
 from pydantic import BaseModel
-from typing import Any, Optional
+from typing import Any
 from flowyml.monitoring.alerts import alert_manager, AlertLevel
 
 router = APIRouter()
@@ -8,11 +8,11 @@ router = APIRouter()
 
 class ClientError(BaseModel):
     message: str
-    stack: Optional[str] = None
-    component_stack: Optional[str] = None
-    url: Optional[str] = None
-    user_agent: Optional[str] = None
-    additional_info: Optional[dict[str, Any]] = None
+    stack: str | None = None
+    component_stack: str | None = None
+    url: str | None = None
+    user_agent: str | None = None
+    additional_info: dict[str, Any] | None = None
 
 
 @router.post("/errors")

flowyml/ui/backend/routers/experiments.py

@@ -1,15 +1,12 @@
 from fastapi import APIRouter, HTTPException
 from flowyml.storage.metadata import SQLiteMetadataStore
 from flowyml.core.project import ProjectManager
-from typing import Optional
+from flowyml.ui.backend.dependencies import get_store
+from pydantic import BaseModel
 
 router = APIRouter()
 
 
-def get_store():
-    return SQLiteMetadataStore()
-
-
 def _iter_metadata_stores():
     """Yield tuples of (project_name, store) including global and project stores."""
     stores = [(None, SQLiteMetadataStore())]
@@ -28,7 +25,7 @@ def _iter_metadata_stores():
 
 
 @router.get("/")
-async def list_experiments(project: Optional[str] = None):
+async def list_experiments(project: str | None = None):
     """List all experiments, optionally filtered by project."""
     try:
         combined_experiments = []
@@ -83,3 +80,53 @@ async def update_experiment_project(experiment_name: str, project_update: dict):
         return {"message": f"Updated experiment {experiment_name} to project {project_name}"}
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))
+
+
+class ExperimentCreate(BaseModel):
+    experiment_id: str
+    name: str
+    description: str = ""
+    tags: dict = {}
+    project: str | None = None
+
+
+@router.post("/")
+async def create_experiment(experiment: ExperimentCreate):
+    """Create or update an experiment."""
+    try:
+        store = get_store()
+        store.save_experiment(
+            experiment_id=experiment.experiment_id,
+            name=experiment.name,
+            description=experiment.description,
+            tags=experiment.tags,
+        )
+
+        if experiment.project:
+            store.update_experiment_project(experiment.name, experiment.project)
+
+        return {"status": "success", "experiment_id": experiment.experiment_id}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+class ExperimentRunLog(BaseModel):
+    run_id: str
+    metrics: dict | None = None
+    parameters: dict | None = None
+
+
+@router.post("/{experiment_id}/runs")
+async def log_experiment_run(experiment_id: str, log: ExperimentRunLog):
+    """Log a run to an experiment."""
+    try:
+        store = get_store()
+        store.log_experiment_run(
+            experiment_id=experiment_id,
+            run_id=log.run_id,
+            metrics=log.metrics,
+            parameters=log.parameters,
+        )
+        return {"status": "success"}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))

flowyml/ui/backend/routers/metrics.py

@@ -6,6 +6,7 @@ from pydantic import BaseModel, Field
 from typing import Any
 
 from flowyml.ui.backend.auth import verify_api_token, security
+from flowyml.ui.backend.dependencies import get_store
 from flowyml.storage.metadata import SQLiteMetadataStore
 from flowyml.core.project import ProjectManager
 from flowyml.utils.config import get_config
@@ -45,6 +46,24 @@ class MetricsLogRequest(BaseModel):
     tags: dict[str, Any] | None = Field(default_factory=dict, description="Optional metadata tags")
 
 
+class MetricLog(BaseModel):
+    run_id: str
+    name: str
+    value: float
+    step: int = 0
+
+
+@router.post("/")
+async def log_metric(metric: MetricLog):
+    """Log a single metric."""
+    try:
+        store = get_global_store()
+        store.save_metric(metric.run_id, metric.name, metric.value, metric.step)
+        return {"status": "success"}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
 @router.post("/log")
 async def log_model_metrics(
     payload: MetricsLogRequest,
@@ -138,76 +157,12 @@ async def list_model_metrics(
 @router.get("/observability/orchestrator")
 async def get_orchestrator_metrics():
     """Get orchestrator-level performance metrics."""
-    from datetime import datetime, timedelta
-    import sqlite3
-
-    store = get_global_store()
-    conn = sqlite3.connect(store.db_path)
-    cursor = conn.cursor()
-
-    thirty_days_ago = (datetime.now() - timedelta(days=30)).isoformat()
-    cursor.execute("SELECT COUNT(*) FROM runs WHERE created_at >= ?", (thirty_days_ago,))
-    total_runs = cursor.fetchone()[0]
-
-    cursor.execute(
-        "SELECT status, COUNT(*) FROM runs WHERE created_at >= ? GROUP BY status",
-        (thirty_days_ago,),
-    )
-    status_counts = dict(cursor.fetchall())
-
-    cursor.execute(
-        "SELECT AVG(duration) FROM runs WHERE created_at >= ? AND duration IS NOT NULL",
-        (thirty_days_ago,),
-    )
-    avg_duration = cursor.fetchone()[0] or 0
-
-    conn.close()
-
-    completed = status_counts.get("completed", 0)
-    success_rate = completed / total_runs if total_runs > 0 else 0
-
-    return {
-        "total_runs": total_runs,
-        "success_rate": success_rate,
-        "avg_duration_seconds": avg_duration,
-        "status_distribution": status_counts,
-        "period_days": 30,
-    }
+    store = get_store()
+    return store.get_orchestrator_metrics(days=30)
 
 
 @router.get("/observability/cache")
 async def get_cache_metrics():
     """Get cache performance metrics."""
-    from datetime import datetime, timedelta
-    import sqlite3
-    import json as json_lib
-
-    store = get_global_store()
-    conn = sqlite3.connect(store.db_path)
-    cursor = conn.cursor()
-
-    thirty_days_ago = (datetime.now() - timedelta(days=30)).isoformat()
-    cursor.execute("SELECT metadata FROM runs WHERE created_at >= ?", (thirty_days_ago,))
-
-    total_steps, cached_steps = 0, 0
-    for row in cursor.fetchall():
-        if not row[0]:
-            continue
-        try:
-            metadata = json_lib.loads(row[0])
-            for step_data in metadata.get("steps", {}).values():
-                total_steps += 1
-                if step_data.get("cached"):
-                    cached_steps += 1
-        except Exception:
-            continue
-
-    conn.close()
-    cache_hit_rate = cached_steps / total_steps if total_steps > 0 else 0
-
-    return {
-        "total_steps": total_steps,
-        "cached_steps": cached_steps,
-        "cache_hit_rate": cache_hit_rate,
-        "period_days": 30,
-    }
+    store = get_store()
+    return store.get_cache_metrics(days=30)

flowyml/ui/backend/routers/pipelines.py

@@ -1,21 +1,14 @@
 from fastapi import APIRouter, HTTPException
 from pydantic import BaseModel
-from flowyml.storage.metadata import SQLiteMetadataStore
+from flowyml.ui.backend.dependencies import get_store
 from flowyml.core.project import ProjectManager
-from flowyml.utils.config import get_config
-from typing import Optional
 
 router = APIRouter()
 
 
-def get_store():
-    get_config()
-    return SQLiteMetadataStore()
-
-
 def _iter_metadata_stores():
     """Yield tuples of (project_name, store) including global and project stores."""
-    stores = [(None, SQLiteMetadataStore())]
+    stores = [(None, get_store())]
     try:
         manager = ProjectManager()
         for project_meta in manager.list_projects():
@@ -31,7 +24,7 @@ def _iter_metadata_stores():
 
 
 @router.get("/")
-async def list_pipelines(project: Optional[str] = None, limit: int = 100):
+async def list_pipelines(project: str | None = None, limit: int = 100):
     """List all unique pipelines with details, optionally filtered by project."""
     try:
         pipeline_map = {}  # pipeline_name -> data
@@ -164,3 +157,19 @@ async def update_pipeline_project(pipeline_name: str, update: ProjectUpdate):
         return {"status": "success", "project": update.project_name}
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))
+
+
+class PipelineDefinitionCreate(BaseModel):
+    pipeline_name: str
+    definition: dict
+
+
+@router.post("/")
+async def save_pipeline_definition(data: PipelineDefinitionCreate):
+    """Save a pipeline definition."""
+    try:
+        store = get_store()
+        store.save_pipeline_definition(data.pipeline_name, data.definition)
+        return {"status": "success", "pipeline_name": data.pipeline_name}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))

flowyml/ui/backend/routers/runs.py

@@ -2,19 +2,15 @@ from fastapi import APIRouter, HTTPException
 from pydantic import BaseModel
 from flowyml.storage.metadata import SQLiteMetadataStore
 from flowyml.core.project import ProjectManager
-from typing import Optional
 import json
+from flowyml.ui.backend.dependencies import get_store
 
 router = APIRouter()
 
 
-def get_store():
-    return SQLiteMetadataStore()
-
-
 def _iter_metadata_stores():
     """Yield tuples of (project_name, store) including global and project stores."""
-    stores: list[tuple[Optional[str], SQLiteMetadataStore]] = [(None, SQLiteMetadataStore())]
+    stores: list[tuple[str | None, SQLiteMetadataStore]] = [(None, get_store())]
     try:
         manager = ProjectManager()
         for project_meta in manager.list_projects():
@@ -50,16 +46,43 @@ def _sort_runs(runs):
 
 
 @router.get("/")
-async def list_runs(limit: int = 20, project: str = None):
-    """List all runs, optionally filtered by project."""
+async def list_runs(
+    limit: int = 20,
+    project: str = None,
+    pipeline_name: str = None,
+    status: str = None,
+):
+    """List all runs, optionally filtered by project, pipeline_name, and status."""
     try:
         combined = []
         for project_name, store in _iter_metadata_stores():
             # Skip other projects if filtering by project name
             if project and project_name and project != project_name:
                 continue
-            store_runs = store.list_runs(limit=limit)
+
+            # Use store's query method if available for better performance, or list_runs
+            # SQLMetadataStore has query method.
+            if hasattr(store, "query"):
+                filters = {}
+                if pipeline_name:
+                    filters["pipeline_name"] = pipeline_name
+                if status:
+                    filters["status"] = status
+
+                # We can't pass limit to query easily if it doesn't support it,
+                # but SQLMetadataStore.query usually returns all matching.
+                # We'll slice later.
+                store_runs = store.query(**filters)
+            else:
+                store_runs = store.list_runs(limit=limit)
+
             for run in store_runs:
+                # Apply filters if store didn't (e.g. if we used list_runs or store doesn't support query)
+                if pipeline_name and run.get("pipeline_name") != pipeline_name:
+                    continue
+                if status and run.get("status") != status:
+                    continue
+
                 combined.append((run, project_name))
 
         runs = _deduplicate_runs(combined)
@@ -73,6 +96,50 @@ async def list_runs(limit: int = 20, project: str = None):
         return {"runs": [], "error": str(e)}
 
 
+class RunCreate(BaseModel):
+    run_id: str
+    pipeline_name: str
+    status: str = "pending"
+    start_time: str
+    end_time: str | None = None
+    duration: float | None = None
+    metadata: dict = {}
+    project: str | None = None
+    metrics: dict | None = None
+    parameters: dict | None = None
+
+
+@router.post("/")
+async def create_run(run: RunCreate):
+    """Create or update a run."""
+    try:
+        store = get_store()
+
+        # Prepare metadata dict
+        metadata = run.metadata.copy()
+        metadata.update(
+            {
+                "pipeline_name": run.pipeline_name,
+                "status": run.status,
+                "start_time": run.start_time,
+                "end_time": run.end_time,
+                "duration": run.duration,
+                "project": run.project,
+            },
+        )
+
+        if run.metrics:
+            metadata["metrics"] = run.metrics
+
+        if run.parameters:
+            metadata["parameters"] = run.parameters
+
+        store.save_run(run.run_id, metadata)
+        return {"status": "success", "run_id": run.run_id}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
 @router.get("/{run_id}")
 async def get_run(run_id: str):
     """Get details for a specific run."""