flowyml 1.1.0__py3-none-any.whl

flowyml/ui/backend/auth.py

@@ -0,0 +1,163 @@
+"""Authentication and authorization for flowyml API."""
+
+import secrets
+import hashlib
+import json
+from pathlib import Path
+from typing import Any
+from datetime import datetime
+from fastapi import HTTPException, Security
+from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+
+security = HTTPBearer(auto_error=False)
+
+
+class TokenManager:
+    """Manage API tokens for authentication."""
+
+    def __init__(self, tokens_file: str = ".flowyml/api_tokens.json"):
+        self.tokens_file = Path(tokens_file)
+        self.tokens_file.parent.mkdir(parents=True, exist_ok=True)
+        self._load_tokens()
+
+    def _load_tokens(self) -> None:
+        """Load tokens from file."""
+        if self.tokens_file.exists():
+            with open(self.tokens_file) as f:
+                self.tokens = json.load(f)
+        else:
+            self.tokens = {}
+            self._save_tokens()
+
+    def _save_tokens(self) -> None:
+        """Save tokens to file."""
+        with open(self.tokens_file, "w") as f:
+            json.dump(self.tokens, f, indent=2)
+
+    def _hash_token(self, token: str) -> str:
+        """Hash a token for secure storage."""
+        return hashlib.sha256(token.encode()).hexdigest()
+
+    def create_token(
+        self,
+        name: str,
+        project: str | None = None,
+        permissions: list = None,
+    ) -> str:
+        """Create a new API token.
+
+        Args:
+            name: Token name/description
+            project: Optional project scope
+            permissions: List of permissions
+
+        Returns:
+            The generated token
+        """
+        token = f"uf_{secrets.token_urlsafe(32)}"
+        token_hash = self._hash_token(token)
+
+        self.tokens[token_hash] = {
+            "name": name,
+            "project": project,
+            "permissions": permissions or ["read", "write", "execute"],
+            "created_at": datetime.now().isoformat(),
+            "last_used": None,
+        }
+
+        self._save_tokens()
+        return token
+
+    def verify_token(self, token: str) -> dict[str, Any] | None:
+        """Verify a token and return its metadata.
+
+        Args:
+            token: The token to verify
+
+        Returns:
+            Token metadata if valid, None otherwise
+        """
+        token_hash = self._hash_token(token)
+        token_data = self.tokens.get(token_hash)
+
+        if token_data:
+            # Update last used timestamp
+            token_data["last_used"] = datetime.now().isoformat()
+            self.tokens[token_hash] = token_data
+            self._save_tokens()
+
+        return token_data
+
+    def revoke_token(self, token: str) -> bool:
+        """Revoke a token.
+
+        Args:
+            token: The token to revoke
+
+        Returns:
+            True if revoked, False if not found
+        """
+        token_hash = self._hash_token(token)
+        if token_hash in self.tokens:
+            del self.tokens[token_hash]
+            self._save_tokens()
+            return True
+        return False
+
+    def list_tokens(self) -> list:
+        """List all tokens (without revealing the actual token values)."""
+        return [
+            {
+                "name": data["name"],
+                "project": data["project"],
+                "permissions": data["permissions"],
+                "created_at": data["created_at"],
+                "last_used": data["last_used"],
+            }
+            for data in self.tokens.values()
+        ]
+
+
+# Global token manager instance
+token_manager = TokenManager()
+
+
+async def verify_api_token(
+    credentials: HTTPAuthorizationCredentials = Security(security),
+    required_permission: str = "read",
+) -> dict[str, Any]:
+    """Verify API token from Authorization header.
+
+    Args:
+        credentials: HTTP authorization credentials
+        required_permission: Required permission level
+
+    Returns:
+        Token metadata
+
+    Raises:
+        HTTPException: If token is invalid or insufficient permissions
+    """
+    if not credentials:
+        raise HTTPException(
+            status_code=401,
+            detail="Not authenticated. Provide an API token in the Authorization header.",
+        )
+
+    token = credentials.credentials
+    token_data = token_manager.verify_token(token)
+
+    if not token_data:
+        raise HTTPException(
+            status_code=403,
+            detail="Invalid API token",
+        )
+
+    # Check permissions
+    if required_permission not in token_data["permissions"]:
+        raise HTTPException(
+            status_code=403,
+            detail=f"Insufficient permissions. Required: {required_permission}",
+        )
+
+    return token_data

flowyml/ui/backend/main.py

@@ -0,0 +1,187 @@
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.staticfiles import StaticFiles
+from fastapi.responses import FileResponse
+import os
+
+# Include API routers
+from flowyml.ui.backend.routers import (
+    pipelines,
+    runs,
+    assets,
+    experiments,
+    traces,
+    projects,
+    schedules,
+    notifications,
+    leaderboard,
+    execution,
+    plugins,
+)
+
+app = FastAPI(
+    title="flowyml UI",
+    description="Real-time UI for flowyml pipelines",
+    version="0.1.0",
+)
+
+# Configure CORS
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],  # For development
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+# Health check endpoint
+@app.get("/api/health")
+async def health_check():
+    return {"status": "ok", "version": "0.1.0"}
+
+
+@app.get("/api/config")
+async def get_public_config():
+    """Get public configuration."""
+    from flowyml.utils.config import get_config
+
+    config = get_config()
+    return {
+        "execution_mode": config.execution_mode,
+        "remote_server_url": config.remote_server_url,
+        "remote_ui_url": config.remote_ui_url,
+        "enable_ui": config.enable_ui,
+    }
+
+
+app.include_router(pipelines.router, prefix="/api/pipelines", tags=["pipelines"])
+app.include_router(runs.router, prefix="/api/runs", tags=["runs"])
+app.include_router(assets.router, prefix="/api/assets", tags=["assets"])
+app.include_router(experiments.router, prefix="/api/experiments", tags=["experiments"])
+app.include_router(traces.router, prefix="/api/traces", tags=["traces"])
+app.include_router(projects.router, prefix="/api/projects", tags=["projects"])
+app.include_router(schedules.router, prefix="/api/schedules", tags=["schedules"])
+app.include_router(notifications.router, prefix="/api/notifications", tags=["notifications"])
+app.include_router(leaderboard.router, prefix="/api/leaderboard", tags=["leaderboard"])
+app.include_router(execution.router, prefix="/api/execution", tags=["execution"])
+app.include_router(plugins.router, prefix="/api", tags=["plugins"])
+
+
+# Stats endpoint for dashboard
+@app.get("/api/stats")
+async def get_stats(project: str = None):
+    """Get overall statistics for the dashboard, optionally filtered by project."""
+    try:
+        from flowyml.storage.metadata import SQLiteMetadataStore
+
+        store = SQLiteMetadataStore()
+
+        # Get base stats
+        stats = store.get_statistics()
+
+        # Get run status counts (not in get_statistics yet)
+        # We can add this to get_statistics later, but for now let's query efficiently
+        import sqlite3
+
+        conn = sqlite3.connect(store.db_path)
+        cursor = conn.cursor()
+
+        if project:
+            cursor.execute(
+                "SELECT COUNT(*) FROM runs WHERE project = ? AND status = 'completed'",
+                [project],
+            )
+            completed_runs = cursor.fetchone()[0]
+
+            cursor.execute(
+                "SELECT COUNT(*) FROM runs WHERE project = ? AND status = 'failed'",
+                [project],
+            )
+            failed_runs = cursor.fetchone()[0]
+
+            cursor.execute(
+                "SELECT AVG(duration) FROM runs WHERE project = ? AND duration IS NOT NULL",
+                [project],
+            )
+            avg_duration = cursor.fetchone()[0] or 0
+
+            cursor.execute(
+                "SELECT COUNT(*) FROM runs WHERE project = ?",
+                [project],
+            )
+            total_runs = cursor.fetchone()[0]
+        else:
+            cursor.execute("SELECT COUNT(*) FROM runs WHERE status = 'completed'")
+            completed_runs = cursor.fetchone()[0]
+
+            cursor.execute("SELECT COUNT(*) FROM runs WHERE status = 'failed'")
+            failed_runs = cursor.fetchone()[0]
+
+            cursor.execute("SELECT AVG(duration) FROM runs WHERE duration IS NOT NULL")
+            avg_duration = cursor.fetchone()[0] or 0
+
+            cursor.execute("SELECT COUNT(*) FROM runs")
+            total_runs = cursor.fetchone()[0]
+
+        conn.close()
+
+        return {
+            "runs": total_runs if project else stats.get("total_runs", 0),
+            "completed_runs": completed_runs,
+            "failed_runs": failed_runs,
+            "pipelines": stats.get("total_pipelines", 0),  # TODO: filter by project
+            "artifacts": stats.get("total_artifacts", 0),  # TODO: filter by project
+            "avg_duration": avg_duration,
+        }
+    except Exception as e:
+        # Return default stats if there's an error
+        return {
+            "runs": 0,
+            "completed_runs": 0,
+            "failed_runs": 0,
+            "pipelines": 0,
+            "artifacts": 0,
+            "avg_duration": 0,
+            "error": str(e),
+        }
+
+
+# Static file serving for frontend
+# Path to frontend build
+frontend_dist = os.path.join(os.path.dirname(os.path.dirname(__file__)), "frontend", "dist")
+
+if os.path.exists(frontend_dist):
+    # Mount static assets
+    app.mount("/assets", StaticFiles(directory=os.path.join(frontend_dist, "assets")), name="assets")
+
+    # Serve index.html for root and other non-API routes
+    # Use a specific route for root
+    @app.get("/", include_in_schema=False)
+    async def serve_root():
+        return FileResponse(os.path.join(frontend_dist, "index.html"))
+
+    # For SPA routing, we need to serve index.html for common frontend routes
+    # But we can't use a catch-all because it interferes with API routes
+    # Instead, mount a StaticFiles handler for the root, but do it AFTER API routes
+    # Actually, let's try a different approach - use a custom middleware or exceptions
+
+    # The trick is to let FastAPI handle routes first, then catch 404s
+    from starlette.exceptions import HTTPException as StarletteHTTPException
+    from fastapi.exception_handlers import http_exception_handler
+
+    @app.exception_handler(StarletteHTTPException)
+    async def custom_http_exception_handler(request, exc):
+        # If it's a 404 and not an API route, serve the SPA
+        if exc.status_code == 404 and not request.url.path.startswith("/api"):
+            return FileResponse(os.path.join(frontend_dist, "index.html"))
+        # Otherwise, use the default handler
+        return await http_exception_handler(request, exc)
+else:
+
+    @app.get("/")
+    async def root():
+        return {
+            "message": "flowyml API is running.",
+            "detail": "Frontend not built. Run 'npm run build' in flowyml/ui/frontend to enable the UI.",
+        }

flowyml/ui/backend/routers/assets.py

@@ -0,0 +1,45 @@
+from fastapi import APIRouter, HTTPException
+from flowyml.storage.metadata import SQLiteMetadataStore
+
+router = APIRouter()
+
+
+def get_store():
+    return SQLiteMetadataStore()
+
+
+@router.get("/")
+async def list_assets(limit: int = 50, asset_type: str = None, run_id: str = None, project: str = None):
+    """List all assets, optionally filtered by project."""
+    try:
+        store = get_store()
+
+        # Build filters
+        filters = {}
+        if asset_type:
+            filters["type"] = asset_type
+        if run_id:
+            filters["run_id"] = run_id
+
+        assets = store.list_assets(limit=limit, **filters)
+
+        # Filter by project if specified
+        if project:
+            # Get all runs for this project
+            all_runs = store.list_runs(limit=1000)
+            project_run_ids = {r.get("run_id") for r in all_runs if r.get("project") == project}
+            assets = [a for a in assets if a.get("run_id") in project_run_ids]
+
+        return {"assets": assets}
+    except Exception as e:
+        return {"assets": [], "error": str(e)}
+
+
+@router.get("/{artifact_id}")
+async def get_asset(artifact_id: str):
+    """Get details for a specific asset."""
+    store = get_store()
+    asset = store.load_artifact(artifact_id)
+    if not asset:
+        raise HTTPException(status_code=404, detail="Asset not found")
+    return asset

flowyml/ui/backend/routers/execution.py

@@ -0,0 +1,179 @@
+"""Pipeline execution API endpoints."""
+
+from fastapi import APIRouter, HTTPException, Depends, Security
+from fastapi.security import HTTPAuthorizationCredentials
+from pydantic import BaseModel
+from typing import Any
+from flowyml.ui.backend.auth import verify_api_token, security
+import importlib
+
+router = APIRouter()
+
+
+def require_permission(permission: str):
+    """Create a dependency for checking permissions."""
+
+    async def _verify(credentials: HTTPAuthorizationCredentials = Security(security)):
+        return await verify_api_token(credentials, required_permission=permission)
+
+    return _verify
+
+
+class PipelineExecutionRequest(BaseModel):
+    """Pipeline execution request."""
+
+    pipeline_module: str  # e.g., "my_pipelines.training"
+    pipeline_name: str  # e.g., "training_pipeline"
+    parameters: dict[str, Any] = {}
+    project: str | None = None
+    dry_run: bool = False  # If True, validate but don't execute
+
+
+class TokenRequest(BaseModel):
+    """API token creation request."""
+
+    name: str
+    project: str | None = None
+    permissions: list = ["read", "write", "execute"]
+
+
+@router.post("/execute")
+async def execute_pipeline(
+    request: PipelineExecutionRequest,
+    token_data: dict = Depends(require_permission("execute")),
+):
+    """Execute a pipeline.
+
+    Requires 'execute' permission.
+
+    Example request:
+    ```json
+    {
+        "pipeline_module": "my_pipelines.training",
+        "pipeline_name": "training_pipeline",
+        "parameters": {"epochs": 10},
+        "project": "my_project",
+        "dry_run": false
+    }
+    ```
+    """
+    try:
+        # Check project scope if token is project-specific
+        if token_data.get("project") and token_data["project"] != request.project:
+            raise HTTPException(
+                status_code=403,
+                detail=f"Token is scoped to project '{token_data['project']}'",
+            )
+
+        if request.dry_run:
+            return {
+                "status": "validated",
+                "pipeline": request.pipeline_name,
+                "module": request.pipeline_module,
+                "parameters": request.parameters,
+                "message": "Pipeline configuration is valid (dry run)",
+            }
+
+        # Import the pipeline module
+        try:
+            module = importlib.import_module(request.pipeline_module)
+        except ImportError as e:
+            raise HTTPException(
+                status_code=404,
+                detail=f"Pipeline module not found: {request.pipeline_module}. Error: {str(e)}",
+            )
+
+        # Get the pipeline object
+        if not hasattr(module, request.pipeline_name):
+            raise HTTPException(
+                status_code=404,
+                detail=f"Pipeline '{request.pipeline_name}' not found in module '{request.pipeline_module}'",
+            )
+
+        pipeline = getattr(module, request.pipeline_name)
+
+        # Execute the pipeline
+        result = pipeline.run(**request.parameters)
+
+        return {
+            "status": "completed",
+            "run_id": result.run_id if hasattr(result, "run_id") else None,
+            "pipeline": request.pipeline_name,
+            "message": "Pipeline executed successfully",
+        }
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Pipeline execution failed: {str(e)}",
+        )
+
+
+@router.post("/tokens")
+async def create_token(
+    request: TokenRequest,
+    token_data: dict = Depends(require_permission("admin")),
+):
+    """Create a new API token.
+
+    Requires 'admin' permission or can be called without auth for initial setup.
+    """
+    from flowyml.ui.backend.auth import token_manager
+
+    try:
+        token = token_manager.create_token(
+            name=request.name,
+            project=request.project,
+            permissions=request.permissions,
+        )
+
+        return {
+            "token": token,
+            "name": request.name,
+            "project": request.project,
+            "permissions": request.permissions,
+            "message": "Token created successfully. Save this token - it won't be shown again!",
+        }
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to create token: {str(e)}",
+        )
+
+
+@router.get("/tokens")
+async def list_tokens():
+    """List all API tokens (without revealing token values)."""
+    from flowyml.ui.backend.auth import token_manager
+
+    # Allow listing tokens without auth (for UI to check if any exist)
+    return {"tokens": token_manager.list_tokens()}
+
+
+@router.post("/tokens/init")
+async def initialize_first_token():
+    """Create the first admin token (no auth required).
+
+    This endpoint can only be used if no tokens exist yet.
+    """
+    from flowyml.ui.backend.auth import token_manager
+
+    if token_manager.list_tokens():
+        raise HTTPException(
+            status_code=403,
+            detail="Tokens already exist. Use /api/execution/tokens with admin token to create more.",
+        )
+
+    token = token_manager.create_token(
+        name="Initial Admin Token",
+        project=None,
+        permissions=["read", "write", "execute", "admin"],
+    )
+
+    return {
+        "token": token,
+        "message": "Initial admin token created. Save this token - it won't be shown again!",
+        "next_steps": "Use this token to create additional tokens via /api/execution/tokens",
+    }

flowyml/ui/backend/routers/experiments.py

@@ -0,0 +1,49 @@
+from fastapi import APIRouter, HTTPException
+from flowyml.storage.metadata import SQLiteMetadataStore
+
+router = APIRouter()
+
+
+def get_store():
+    return SQLiteMetadataStore()
+
+
+@router.get("/")
+async def list_experiments(project: str = None):
+    """List all experiments, optionally filtered by project."""
+    try:
+        store = get_store()
+        experiments = store.list_experiments()
+
+        # Filter by project if specified
+        if project:
+            experiments = [e for e in experiments if e.get("project") == project]
+
+        return {"experiments": experiments}
+    except Exception as e:
+        return {"experiments": [], "error": str(e)}
+
+
+@router.get("/{experiment_id}")
+async def get_experiment(experiment_id: str):
+    """Get experiment details."""
+    store = get_store()
+    experiment = store.get_experiment(experiment_id)
+    if not experiment:
+        raise HTTPException(status_code=404, detail="Experiment not found")
+    return experiment
+
+
+@router.put("/{experiment_name}/project")
+async def update_experiment_project(experiment_name: str, project_update: dict):
+    """Update the project for an experiment."""
+    try:
+        store = get_store()
+        project_name = project_update.get("project_name")
+
+        # Update experiment project
+        store.update_experiment_project(experiment_name, project_name)
+
+        return {"message": f"Updated experiment {experiment_name} to project {project_name}"}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))