flowyml 1.3.0__py3-none-any.whl → 1.5.0__py3-none-any.whl

flowyml/ui/backend/dependencies.py

@@ -0,0 +1,28 @@
+"""Backend dependencies."""
+
+import os
+from flowyml.storage.sql import SQLMetadataStore
+from flowyml.utils.config import get_config
+
+_store = None
+
+
+def get_store() -> SQLMetadataStore:
+    """Get the metadata store instance.
+
+    Uses FLOWYML_DATABASE_URL if set, otherwise defaults to local SQLite.
+    """
+    global _store
+    if _store is None:
+        config = get_config()
+        db_url = os.environ.get("FLOWYML_DATABASE_URL")
+
+        # If no explicit URL, use the config's metadata_db path
+        if not db_url:
+            db_path = config.metadata_db
+            # Ensure it's a string path for SQLMetadataStore
+            _store = SQLMetadataStore(db_path=str(db_path))
+        else:
+            _store = SQLMetadataStore(db_url=db_url)
+
+    return _store

flowyml/ui/backend/main.py

@@ -23,6 +23,8 @@ from flowyml.ui.backend.routers import (
     plugins,
     metrics,
     client,
+    stats,
+    websocket,
 )
 
 app = FastAPI(
@@ -75,85 +77,8 @@ app.include_router(execution.router, prefix="/api/execution", tags=["execution"]
 app.include_router(metrics.router, prefix="/api/metrics", tags=["metrics"])
 app.include_router(plugins.router, prefix="/api", tags=["plugins"])
 app.include_router(client.router, prefix="/api/client", tags=["client"])
-
-
-# Stats endpoint for dashboard
-@app.get("/api/stats")
-async def get_stats(project: str = None):
-    """Get overall statistics for the dashboard, optionally filtered by project."""
-    try:
-        from flowyml.storage.metadata import SQLiteMetadataStore
-
-        store = SQLiteMetadataStore()
-
-        # Get base stats
-        stats = store.get_statistics()
-
-        # Get run status counts (not in get_statistics yet)
-        # We can add this to get_statistics later, but for now let's query efficiently
-        import sqlite3
-
-        conn = sqlite3.connect(store.db_path)
-        cursor = conn.cursor()
-
-        if project:
-            cursor.execute(
-                "SELECT COUNT(*) FROM runs WHERE project = ? AND status = 'completed'",
-                [project],
-            )
-            completed_runs = cursor.fetchone()[0]
-
-            cursor.execute(
-                "SELECT COUNT(*) FROM runs WHERE project = ? AND status = 'failed'",
-                [project],
-            )
-            failed_runs = cursor.fetchone()[0]
-
-            cursor.execute(
-                "SELECT AVG(duration) FROM runs WHERE project = ? AND duration IS NOT NULL",
-                [project],
-            )
-            avg_duration = cursor.fetchone()[0] or 0
-
-            cursor.execute(
-                "SELECT COUNT(*) FROM runs WHERE project = ?",
-                [project],
-            )
-            total_runs = cursor.fetchone()[0]
-        else:
-            cursor.execute("SELECT COUNT(*) FROM runs WHERE status = 'completed'")
-            completed_runs = cursor.fetchone()[0]
-
-            cursor.execute("SELECT COUNT(*) FROM runs WHERE status = 'failed'")
-            failed_runs = cursor.fetchone()[0]
-
-            cursor.execute("SELECT AVG(duration) FROM runs WHERE duration IS NOT NULL")
-            avg_duration = cursor.fetchone()[0] or 0
-
-            cursor.execute("SELECT COUNT(*) FROM runs")
-            total_runs = cursor.fetchone()[0]
-
-        conn.close()
-
-        return {
-            "runs": total_runs if project else stats.get("total_runs", 0),
-            "completed_runs": completed_runs,
-            "failed_runs": failed_runs,
-            "pipelines": stats.get("total_pipelines", 0),  # TODO: filter by project
-            "artifacts": stats.get("total_artifacts", 0),  # TODO: filter by project
-            "avg_duration": avg_duration,
-        }
-    except Exception as e:
-        # Return default stats if there's an error
-        return {
-            "runs": 0,
-            "completed_runs": 0,
-            "failed_runs": 0,
-            "pipelines": 0,
-            "artifacts": 0,
-            "avg_duration": 0,
-            "error": str(e),
-        }
+app.include_router(stats.router, prefix="/api/stats", tags=["stats"])
+app.include_router(websocket.router, tags=["websocket"])
 
 
 # Static file serving for frontend

flowyml/ui/backend/routers/assets.py

@@ -1,16 +1,20 @@
-from fastapi import APIRouter, HTTPException
+from fastapi import APIRouter, HTTPException, UploadFile, File
 from fastapi.responses import FileResponse
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 from flowyml.storage.metadata import SQLiteMetadataStore
 from flowyml.core.project import ProjectManager
-from typing import Optional
 from pathlib import Path
+from flowyml.ui.backend.dependencies import get_store
+import shutil
+import asyncio
+import contextlib
 
 router = APIRouter()
 
 
-def get_store():
-    return SQLiteMetadataStore()
+def _save_file_sync(src, dst):
+    with open(dst, "wb") as buffer:
+        shutil.copyfileobj(src, buffer)
 
 
 def _iter_metadata_stores():
@@ -73,8 +77,129 @@ async def list_assets(limit: int = 50, asset_type: str = None, run_id: str = Non
         return {"assets": [], "error": str(e)}
 
 
+class AssetCreate(BaseModel):
+    artifact_id: str
+    name: str
+    asset_type: str = Field(..., alias="type")
+    run_id: str
+    step: str
+    project: str | None = None
+    metadata: dict = {}
+    value: str | None = None
+
+
+@router.post("/")
+async def create_asset(asset: AssetCreate):
+    """Create or update an asset metadata."""
+    try:
+        store = get_store()
+
+        # Prepare metadata
+        metadata = asset.metadata.copy()
+        metadata.update(
+            {
+                "name": asset.name,
+                "type": asset.asset_type,
+                "run_id": asset.run_id,
+                "step": asset.step,
+                "project": asset.project,
+                "value": asset.value,
+            },
+        )
+
+        store.save_artifact(asset.artifact_id, metadata)
+        return {"status": "success", "artifact_id": asset.artifact_id}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.post("/{artifact_id}/upload")
+async def upload_asset_content(artifact_id: str, file: UploadFile = File(...)):
+    """Upload content for an artifact."""
+    try:
+        store = get_store()
+
+        # Get existing metadata to find path or create a new one
+        existing = store.load_artifact(artifact_id)
+
+        if not existing:
+            raise HTTPException(status_code=404, detail="Artifact metadata not found. Create metadata first.")
+
+        # Determine storage path
+        # We use the LocalArtifactStore logic here since the backend is running locally relative to itself
+        from flowyml.storage.artifacts import LocalArtifactStore
+        from flowyml.utils.config import get_config
+
+        config = get_config()
+        artifact_store = LocalArtifactStore(base_path=config.artifacts_dir)
+
+        # Construct a path if not present
+        if not existing.get("path"):
+            # Create a path structure: project/run_id/artifact_id/filename
+            project = existing.get("project", "default")
+            run_id = existing.get("run_id", "unknown")
+            filename = file.filename or "content"
+            rel_path = f"{project}/{run_id}/{artifact_id}/{filename}"
+        else:
+            rel_path = existing.get("path")
+            # If path is absolute, make it relative to artifacts dir if possible, or just use it
+            # But LocalArtifactStore expects relative paths usually, or handles absolute ones
+
+        # Save the file
+        full_path = artifact_store.base_path / rel_path
+        full_path.parent.mkdir(parents=True, exist_ok=True)
+
+        loop = asyncio.get_running_loop()
+        await loop.run_in_executor(None, _save_file_sync, file.file, full_path)
+
+        # Update metadata with path
+        existing["path"] = str(rel_path)
+        store.save_artifact(artifact_id, existing)
+
+        return {"status": "success", "path": str(rel_path)}
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.delete("/{artifact_id}")
+async def delete_asset(artifact_id: str):
+    """Delete an asset and its file."""
+    try:
+        store = get_store()
+
+        # Get metadata to find path
+        asset = store.load_artifact(artifact_id)
+        if not asset:
+            raise HTTPException(status_code=404, detail="Asset not found")
+
+        # Delete file if it exists locally (since backend is local to itself)
+        path = asset.get("path")
+        if path:
+            from flowyml.storage.artifacts import LocalArtifactStore
+            from flowyml.utils.config import get_config
+
+            config = get_config()
+            artifact_store = LocalArtifactStore(base_path=config.artifacts_dir)
+
+            with contextlib.suppress(Exception):
+                artifact_store.delete(path)
+
+        # Delete metadata
+        store.delete_artifact(artifact_id)
+
+        return {"status": "success", "artifact_id": artifact_id}
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
 @router.get("/stats")
-async def get_asset_stats(project: Optional[str] = None):
+async def get_asset_stats(project: str | None = None):
     """Get statistics about assets for the dashboard."""
     try:
         combined_assets = []
@@ -129,7 +254,7 @@ async def get_asset_stats(project: Optional[str] = None):
 
 
 @router.get("/search")
-async def search_assets(q: str, limit: int = 50, project: Optional[str] = None):
+async def search_assets(q: str, limit: int = 50, project: str | None = None):
     """Search assets by name or properties."""
     try:
         combined_assets = []
@@ -170,8 +295,8 @@ async def search_assets(q: str, limit: int = 50, project: Optional[str] = None):
 
 @router.get("/lineage")
 async def get_asset_lineage(
-    asset_id: Optional[str] = None,
-    project: Optional[str] = None,
+    asset_id: str | None = None,
+    project: str | None = None,
     depth: int = 3,
 ):
     """
@@ -351,6 +476,42 @@ async def download_asset(artifact_id: str):
     )
 
 
+@router.get("/{artifact_id}/content")
+async def get_asset_content(artifact_id: str):
+    """Get the artifact content for inline viewing."""
+    import mimetypes
+
+    asset, _ = _find_asset_with_store(artifact_id)
+    if not asset:
+        raise HTTPException(status_code=404, detail="Asset not found")
+
+    artifact_path = asset.get("path")
+    if not artifact_path:
+        raise HTTPException(status_code=404, detail="Artifact path not available")
+
+    # Handle relative paths for local store
+    from flowyml.utils.config import get_config
+
+    config = get_config()
+
+    file_path = Path(artifact_path)
+    if not file_path.is_absolute():
+        file_path = config.artifacts_dir / file_path
+
+    if not file_path.exists():
+        raise HTTPException(status_code=404, detail="Artifact file not found on disk")
+
+    # Guess mime type
+    mime_type, _ = mimetypes.guess_type(file_path.name)
+    if not mime_type:
+        mime_type = "text/plain"  # Default fallback
+
+    return FileResponse(
+        path=file_path,
+        media_type=mime_type,
+    )
+
+
 class ProjectUpdate(BaseModel):
     project_name: str
 

flowyml/ui/backend/routers/client.py

@@ -1,6 +1,6 @@
 from fastapi import APIRouter, Request
 from pydantic import BaseModel
-from typing import Any, Optional
+from typing import Any
 from flowyml.monitoring.alerts import alert_manager, AlertLevel
 
 router = APIRouter()
@@ -8,11 +8,11 @@ router = APIRouter()
 
 class ClientError(BaseModel):
     message: str
-    stack: Optional[str] = None
-    component_stack: Optional[str] = None
-    url: Optional[str] = None
-    user_agent: Optional[str] = None
-    additional_info: Optional[dict[str, Any]] = None
+    stack: str | None = None
+    component_stack: str | None = None
+    url: str | None = None
+    user_agent: str | None = None
+    additional_info: dict[str, Any] | None = None
 
 
 @router.post("/errors")

flowyml/ui/backend/routers/execution.py

@@ -97,10 +97,10 @@ async def execute_pipeline(
         run_kwargs = request.parameters.copy()
 
         if request.retry_count > 0:
-            from flowyml.core.retry import OrchestratorRetryPolicy
+            from flowyml.core.retry_policy import OrchestratorRetryPolicy
 
             run_kwargs["retry_policy"] = OrchestratorRetryPolicy(
-                max_retries=min(request.retry_count, 5),  # Cap at 5
+                max_attempts=min(request.retry_count, 5),  # Cap at 5
             )
 
         result = pipeline.run(**run_kwargs)

flowyml/ui/backend/routers/experiments.py

@@ -1,15 +1,12 @@
 from fastapi import APIRouter, HTTPException
 from flowyml.storage.metadata import SQLiteMetadataStore
 from flowyml.core.project import ProjectManager
-from typing import Optional
+from flowyml.ui.backend.dependencies import get_store
+from pydantic import BaseModel
 
 router = APIRouter()
 
 
-def get_store():
-    return SQLiteMetadataStore()
-
-
 def _iter_metadata_stores():
     """Yield tuples of (project_name, store) including global and project stores."""
     stores = [(None, SQLiteMetadataStore())]
@@ -28,7 +25,7 @@ def _iter_metadata_stores():
 
 
 @router.get("/")
-async def list_experiments(project: Optional[str] = None):
+async def list_experiments(project: str | None = None):
     """List all experiments, optionally filtered by project."""
     try:
         combined_experiments = []
@@ -83,3 +80,53 @@ async def update_experiment_project(experiment_name: str, project_update: dict):
         return {"message": f"Updated experiment {experiment_name} to project {project_name}"}
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))
+
+
+class ExperimentCreate(BaseModel):
+    experiment_id: str
+    name: str
+    description: str = ""
+    tags: dict = {}
+    project: str | None = None
+
+
+@router.post("/")
+async def create_experiment(experiment: ExperimentCreate):
+    """Create or update an experiment."""
+    try:
+        store = get_store()
+        store.save_experiment(
+            experiment_id=experiment.experiment_id,
+            name=experiment.name,
+            description=experiment.description,
+            tags=experiment.tags,
+        )
+
+        if experiment.project:
+            store.update_experiment_project(experiment.name, experiment.project)
+
+        return {"status": "success", "experiment_id": experiment.experiment_id}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+class ExperimentRunLog(BaseModel):
+    run_id: str
+    metrics: dict | None = None
+    parameters: dict | None = None
+
+
+@router.post("/{experiment_id}/runs")
+async def log_experiment_run(experiment_id: str, log: ExperimentRunLog):
+    """Log a run to an experiment."""
+    try:
+        store = get_store()
+        store.log_experiment_run(
+            experiment_id=experiment_id,
+            run_id=log.run_id,
+            metrics=log.metrics,
+            parameters=log.parameters,
+        )
+        return {"status": "success"}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))

flowyml/ui/backend/routers/metrics.py

@@ -6,6 +6,7 @@ from pydantic import BaseModel, Field
 from typing import Any
 
 from flowyml.ui.backend.auth import verify_api_token, security
+from flowyml.ui.backend.dependencies import get_store
 from flowyml.storage.metadata import SQLiteMetadataStore
 from flowyml.core.project import ProjectManager
 from flowyml.utils.config import get_config
@@ -45,6 +46,24 @@ class MetricsLogRequest(BaseModel):
     tags: dict[str, Any] | None = Field(default_factory=dict, description="Optional metadata tags")
 
 
+class MetricLog(BaseModel):
+    run_id: str
+    name: str
+    value: float
+    step: int = 0
+
+
+@router.post("/")
+async def log_metric(metric: MetricLog):
+    """Log a single metric."""
+    try:
+        store = get_global_store()
+        store.save_metric(metric.run_id, metric.name, metric.value, metric.step)
+        return {"status": "success"}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
 @router.post("/log")
 async def log_model_metrics(
     payload: MetricsLogRequest,
@@ -138,76 +157,12 @@ async def list_model_metrics(
 @router.get("/observability/orchestrator")
 async def get_orchestrator_metrics():
     """Get orchestrator-level performance metrics."""
-    from datetime import datetime, timedelta
-    import sqlite3
-
-    store = get_global_store()
-    conn = sqlite3.connect(store.db_path)
-    cursor = conn.cursor()
-
-    thirty_days_ago = (datetime.now() - timedelta(days=30)).isoformat()
-    cursor.execute("SELECT COUNT(*) FROM runs WHERE created_at >= ?", (thirty_days_ago,))
-    total_runs = cursor.fetchone()[0]
-
-    cursor.execute(
-        "SELECT status, COUNT(*) FROM runs WHERE created_at >= ? GROUP BY status",
-        (thirty_days_ago,),
-    )
-    status_counts = dict(cursor.fetchall())
-
-    cursor.execute(
-        "SELECT AVG(duration) FROM runs WHERE created_at >= ? AND duration IS NOT NULL",
-        (thirty_days_ago,),
-    )
-    avg_duration = cursor.fetchone()[0] or 0
-
-    conn.close()
-
-    completed = status_counts.get("completed", 0)
-    success_rate = completed / total_runs if total_runs > 0 else 0
-
-    return {
-        "total_runs": total_runs,
-        "success_rate": success_rate,
-        "avg_duration_seconds": avg_duration,
-        "status_distribution": status_counts,
-        "period_days": 30,
-    }
+    store = get_store()
+    return store.get_orchestrator_metrics(days=30)
 
 
 @router.get("/observability/cache")
 async def get_cache_metrics():
     """Get cache performance metrics."""
-    from datetime import datetime, timedelta
-    import sqlite3
-    import json as json_lib
-
-    store = get_global_store()
-    conn = sqlite3.connect(store.db_path)
-    cursor = conn.cursor()
-
-    thirty_days_ago = (datetime.now() - timedelta(days=30)).isoformat()
-    cursor.execute("SELECT metadata FROM runs WHERE created_at >= ?", (thirty_days_ago,))
-
-    total_steps, cached_steps = 0, 0
-    for row in cursor.fetchall():
-        if not row[0]:
-            continue
-        try:
-            metadata = json_lib.loads(row[0])
-            for step_data in metadata.get("steps", {}).values():
-                total_steps += 1
-                if step_data.get("cached"):
-                    cached_steps += 1
-        except Exception:
-            continue
-
-    conn.close()
-    cache_hit_rate = cached_steps / total_steps if total_steps > 0 else 0
-
-    return {
-        "total_steps": total_steps,
-        "cached_steps": cached_steps,
-        "cache_hit_rate": cache_hit_rate,
-        "period_days": 30,
-    }
+    store = get_store()
+    return store.get_cache_metrics(days=30)

flowyml/ui/backend/routers/pipelines.py

@@ -1,21 +1,14 @@
 from fastapi import APIRouter, HTTPException
 from pydantic import BaseModel
-from flowyml.storage.metadata import SQLiteMetadataStore
+from flowyml.ui.backend.dependencies import get_store
 from flowyml.core.project import ProjectManager
-from flowyml.utils.config import get_config
-from typing import Optional
 
 router = APIRouter()
 
 
-def get_store():
-    get_config()
-    return SQLiteMetadataStore()
-
-
 def _iter_metadata_stores():
     """Yield tuples of (project_name, store) including global and project stores."""
-    stores = [(None, SQLiteMetadataStore())]
+    stores = [(None, get_store())]
     try:
         manager = ProjectManager()
         for project_meta in manager.list_projects():
@@ -31,7 +24,7 @@ def _iter_metadata_stores():
 
 
 @router.get("/")
-async def list_pipelines(project: Optional[str] = None, limit: int = 100):
+async def list_pipelines(project: str | None = None, limit: int = 100):
     """List all unique pipelines with details, optionally filtered by project."""
     try:
         pipeline_map = {}  # pipeline_name -> data
@@ -164,3 +157,19 @@ async def update_pipeline_project(pipeline_name: str, update: ProjectUpdate):
         return {"status": "success", "project": update.project_name}
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))
+
+
+class PipelineDefinitionCreate(BaseModel):
+    pipeline_name: str
+    definition: dict
+
+
+@router.post("/")
+async def save_pipeline_definition(data: PipelineDefinitionCreate):
+    """Save a pipeline definition."""
+    try:
+        store = get_store()
+        store.save_pipeline_definition(data.pipeline_name, data.definition)
+        return {"status": "success", "pipeline_name": data.pipeline_name}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))