flowyml 1.7.1__py3-none-any.whl → 1.8.0__py3-none-any.whl

flowyml/ui/backend/main.py

@@ -8,7 +8,18 @@ import traceback
 
 from flowyml.monitoring.alerts import alert_manager, AlertLevel
 
-# Include API routers
+# OpenTelemetry Imports
+from opentelemetry import trace, metrics
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from opentelemetry.exporter.prometheus import PrometheusMetricReader
+from opentelemetry.sdk.resources import SERVICE_NAME, Resource
+from prometheus_client import make_asgi_app
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+
 from flowyml.ui.backend.routers import (
     pipelines,
     runs,
@@ -21,28 +32,149 @@ from flowyml.ui.backend.routers import (
     leaderboard,
     execution,
     plugins,
-    metrics,
+    metrics as metrics_router,
     client,
     stats,
     websocket,
+    deployments,
+    model_explorer,
+    auth,  # New Auth Router
+    templates,  # Pipeline Templates
+    ai_context,  # AI Assistant Context
+)
+
+# Initialize OpenTelemetry
+resource = Resource(
+    attributes={
+        SERVICE_NAME: "flowyml-backend",
+    },
 )
 
+# Tracing - Use OTLP exporter in production, console in development
+trace_provider = TracerProvider(resource=resource)
+_otlp_endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
+if _otlp_endpoint:
+    # Production: Use OTLP exporter (supports Jaeger, Honeycomb, Google Cloud Trace, etc.)
+    try:
+        from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+
+        trace_provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=_otlp_endpoint)))
+    except ImportError:
+        # Fallback to console if OTLP exporter not installed
+        trace_provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter()))
+else:
+    # Development: Console exporter for debugging
+    trace_provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter()))
+trace.set_tracer_provider(trace_provider)
+
+# Metrics (Prometheus)
+reader = PrometheusMetricReader()
+meter_provider = MeterProvider(resource=resource, metric_readers=[reader])
+metrics.set_meter_provider(meter_provider)
+
+# Routers included above
+
 app = FastAPI(
     title="flowyml UI",
     description="Real-time UI for flowyml pipelines",
     version="0.1.0",
 )
 
-# Configure CORS
+# Instrument FastAPI
+FastAPIInstrumentor.instrument_app(app)
+
+# Expose Prometheus metrics
+metrics_app = make_asgi_app()
+app.mount("/metrics", metrics_app)
+
+# Configure CORS - Production settings with localhost for development
+_is_production = os.getenv("FLOWYML_ENV") == "production"
+_cors_origins = (
+    [
+        # Production: Only allow specific origins
+        "https://flowyml.unicolab.ai",
+        "https://app.flowyml.io",
+    ]
+    if _is_production
+    else [
+        # Development: Allow localhost and common dev ports
+        "http://localhost:3000",
+        "http://localhost:5173",
+        "http://localhost:8080",
+        "http://127.0.0.1:3000",
+        "http://127.0.0.1:5173",
+        "http://127.0.0.1:8080",
+        "*",  # Fallback for dev flexibility
+    ]
+)
+
 app.add_middleware(
     CORSMiddleware,
-    allow_origins=["*"],  # For development
+    allow_origins=_cors_origins,
     allow_credentials=True,
     allow_methods=["*"],
     allow_headers=["*"],
 )
 
 
+class AuthMiddleware(BaseHTTPMiddleware):
+    async def dispatch(self, request: Request, call_next):
+        # 0. Skip if not in production to allow easy local development
+        if os.getenv("FLOWYML_ENV") != "production":
+            return await call_next(request)
+
+        # 1. Check if token auth is enabled via env var
+        api_token = os.getenv("FLOWYML_API_TOKEN")
+        if not api_token:
+            return await call_next(request)
+
+        # 2. Define public paths
+        path = request.url.path
+        if (
+            path in ["/api/health", "/metrics", "/docs", "/redoc", "/openapi.json"]
+            or path.startswith(("/assets", "/api/auth/login", "/api/auth/logout"))  # Allow login endpoints
+            or path == "/"
+            or request.method == "OPTIONS"
+        ):
+            return await call_next(request)
+
+        # 3. Check Auth Header OR Cookie
+        auth_header = request.headers.get("Authorization")
+        if not auth_header:
+            # Check for cookie
+            cookie_token = request.cookies.get("access_token")
+            if cookie_token:
+                auth_header = cookie_token  # Reuse validation logic below
+            else:
+                token_param = request.query_params.get("token")
+                if token_param == api_token:
+                    return await call_next(request)
+
+                return JSONResponse(
+                    status_code=401,
+                    content={"error": "Unauthorized", "message": "Missing authentication token"},
+                )
+
+        # 4. Validate Token
+        try:
+            scheme, token = auth_header.split()
+            if scheme.lower() != "bearer" or token != api_token:
+                return JSONResponse(
+                    status_code=401,
+                    content={"error": "Unauthorized", "message": "Invalid authentication token"},
+                )
+        except ValueError:
+            return JSONResponse(
+                status_code=401,
+                content={"error": "Unauthorized", "message": "Invalid Authorization header format"},
+            )
+
+        return await call_next(request)
+
+
+app.add_middleware(AuthMiddleware)
+
+
 # Health check endpoint
 @app.get("/api/health")
 async def health_check():
@@ -74,11 +206,16 @@ app.include_router(schedules.router, prefix="/api/schedules", tags=["schedules"]
 app.include_router(notifications.router, prefix="/api/notifications", tags=["notifications"])
 app.include_router(leaderboard.router, prefix="/api/leaderboard", tags=["leaderboard"])
 app.include_router(execution.router, prefix="/api/execution", tags=["execution"])
-app.include_router(metrics.router, prefix="/api/metrics", tags=["metrics"])
+app.include_router(metrics_router.router, prefix="/api/metrics", tags=["metrics"])
 app.include_router(plugins.router, prefix="/api", tags=["plugins"])
 app.include_router(client.router, prefix="/api/client", tags=["client"])
 app.include_router(stats.router, prefix="/api/stats", tags=["stats"])
 app.include_router(websocket.router, tags=["websocket"])
+app.include_router(deployments.router, prefix="/api", tags=["deployments"])
+app.include_router(model_explorer.router, prefix="/api", tags=["model-explorer"])
+app.include_router(auth.router, prefix="/api", tags=["auth"])
+app.include_router(templates.router, tags=["templates"])
+app.include_router(ai_context.router, tags=["ai"])  # AI Context for assistant
 
 
 # Static file serving for frontend
@@ -150,3 +287,10 @@ async def validation_exception_handler(request, exc):
         status_code=422,
         content={"error": "Validation Error", "detail": exc.errors()},
     )
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    port = int(os.environ.get("PORT", "8080"))
+    uvicorn.run("flowyml.ui.backend.main:app", host="0.0.0.0", port=port, reload=False)  # noqa: S104

flowyml/ui/backend/routers/ai_context.py

@@ -0,0 +1,226 @@
+"""
+AI Context Router - Provides comprehensive context for the AI assistant.
+
+This endpoint aggregates run details, logs, metrics, and step code to provide
+rich context for the in-browser AI assistant.
+"""
+
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+from ..dependencies import get_store
+import json
+from flowyml.utils.config import get_config
+
+router = APIRouter(prefix="/api/ai", tags=["ai"])
+
+
+class AIContextRequest(BaseModel):
+    """Request model for AI context."""
+
+    page_type: str  # 'run', 'pipeline', 'experiment', 'asset'
+    resource_id: str  # run_id, pipeline_name, etc.
+    include_logs: bool = True
+    include_code: bool = True
+    include_metrics: bool = True
+    max_log_lines: int = 100
+
+
+class AIContextResponse(BaseModel):
+    """Response model with comprehensive AI context."""
+
+    page_type: str
+    resource_id: str
+    summary: dict
+    details: dict
+    suggestions: list[str] = []
+
+
+def _summarize_run(run: dict, include_logs: bool = True, include_code: bool = True, max_log_lines: int = 100) -> dict:
+    """Generate a comprehensive summary of a run for AI context."""
+    steps_info = []
+    failed_steps = []
+
+    steps = run.get("steps", {})
+    if isinstance(steps, str):
+        try:
+            steps = json.loads(steps)
+        except Exception:
+            steps = {}
+
+    if steps:
+        for name, step_data in steps.items():
+            step_summary = {
+                "name": name,
+                "status": "success"
+                if step_data.get("success")
+                else ("failed" if step_data.get("error") else "pending"),
+                "duration": f"{step_data.get('duration', 0):.2f}s",
+                "cached": step_data.get("cached", False),
+            }
+
+            # Include error details for failed steps
+            if step_data.get("error"):
+                step_summary["error"] = str(step_data["error"])[:500]  # Truncate long errors
+                failed_steps.append(name)
+
+            # Include source code if requested
+            if include_code and step_data.get("source_code"):
+                step_summary["source_code"] = step_data["source_code"][:2000]  # Limit code size
+
+            # Include inputs/outputs
+            step_summary["inputs"] = step_data.get("inputs", [])[:5]
+            step_summary["outputs"] = step_data.get("outputs", [])[:5]
+
+            steps_info.append(step_summary)
+
+    summary = {
+        "run_id": str(run.get("run_id")),
+        "pipeline_name": run.get("pipeline_name"),
+        "status": run.get("status"),
+        "duration": f"{run.get('duration', 0):.2f}s" if run.get("duration") else None,
+        "start_time": run.get("start_time"),
+        "end_time": run.get("end_time"),
+        "total_steps": len(steps_info),
+        "successful_steps": len([s for s in steps_info if s["status"] == "success"]),
+        "failed_steps": len(failed_steps),
+        "cached_steps": len([s for s in steps_info if s["cached"]]),
+        "steps": steps_info,
+        "failed_step_names": failed_steps,
+        "context_params": run.get("context") or {},
+        "environment": run.get("environment") or {},
+    }
+
+    return summary
+
+
+def _get_run_logs(run_id: str, max_lines: int = 100) -> dict:
+    """Fetch recent logs for a run from filesystem."""
+    logs_by_step = {}
+    runs_dir = get_config().runs_dir
+    log_dir = runs_dir / run_id / "logs"
+
+    if not log_dir.exists():
+        return {}
+
+    for log_file in log_dir.glob("*.log"):
+        step_name = log_file.stem
+        try:
+            with open(log_file) as f:
+                lines = f.readlines()
+                # Get last N lines
+                recent_lines = lines[-max_lines:]
+                logs_by_step[step_name] = [{"message": line.strip(), "level": "INFO"} for line in recent_lines]
+        except Exception:
+            continue
+
+    return logs_by_step
+
+
+def _get_run_metrics(run_id: str) -> list:
+    """Fetch metrics for a run."""
+    metrics = []
+    store = get_store()
+
+    db_metrics = store.get_metrics(run_id)
+    if db_metrics:
+        for m in db_metrics[:50]:
+            metrics.append(
+                {
+                    "name": m.get("name"),
+                    "value": m.get("value"),
+                    "step": m.get("step"),
+                },
+            )
+
+    return metrics
+
+
+def _generate_suggestions(summary: dict) -> list[str]:
+    """Generate AI-friendly suggestions based on run data."""
+    suggestions = []
+
+    if summary.get("failed_steps", 0) > 0:
+        suggestions.append(f"Analyze the {summary['failed_steps']} failed step(s) and suggest fixes")
+
+    if summary.get("cached_steps", 0) == 0 and summary.get("total_steps", 0) > 3:
+        suggestions.append("Consider enabling caching for frequently-run steps")
+
+    duration_str = summary.get("duration")
+    if duration_str:
+        try:
+            duration = float(duration_str.replace("s", ""))
+            if duration > 300:
+                suggestions.append("The run took over 5 minutes - consider optimization opportunities")
+        except (ValueError, AttributeError):
+            pass
+
+    return suggestions
+
+
+@router.post("/context", response_model=AIContextResponse)
+async def get_ai_context(request: AIContextRequest):
+    """
+    Get comprehensive AI context for a specific page/resource.
+
+    This endpoint aggregates all relevant information that the AI assistant
+    might need to provide helpful, context-aware responses.
+    """
+    if request.page_type == "run":
+        # Fetch run data
+        store = get_store()
+        run = store.load_run(request.resource_id)
+
+        if not run:
+            raise HTTPException(status_code=404, detail="Run not found")
+
+        # Generate comprehensive summary
+        summary = _summarize_run(
+            run,
+            include_logs=request.include_logs,
+            include_code=request.include_code,
+            max_log_lines=request.max_log_lines,
+        )
+
+        # Fetch additional data
+        details = {}
+
+        if request.include_logs:
+            details["logs"] = _get_run_logs(request.resource_id, request.max_log_lines)
+
+        if request.include_metrics:
+            details["metrics"] = _get_run_metrics(request.resource_id)
+
+        # Generate suggestions
+        suggestions = _generate_suggestions(summary)
+
+        return AIContextResponse(
+            page_type=request.page_type,
+            resource_id=request.resource_id,
+            summary=summary,
+            details=details,
+            suggestions=suggestions,
+        )
+
+    # Add more page types as needed
+    raise HTTPException(status_code=400, detail=f"Unsupported page type: {request.page_type}")
+
+
+@router.get("/context/run/{run_id}")
+async def get_run_ai_context(
+    run_id: str,
+    include_logs: bool = True,
+    include_code: bool = True,
+    include_metrics: bool = True,
+):
+    """
+    Convenience endpoint for getting AI context for a specific run.
+    """
+    return await get_ai_context(
+        AIContextRequest(
+            page_type="run",
+            resource_id=run_id,
+            include_logs=include_logs,
+            include_code=include_code,
+            include_metrics=include_metrics,
+        ),
+    )

flowyml/ui/backend/routers/assets.py

@@ -1,7 +1,6 @@
 from fastapi import APIRouter, HTTPException, UploadFile, File
 from fastapi.responses import FileResponse
 from pydantic import BaseModel, Field
-from flowyml.storage.metadata import SQLiteMetadataStore
 from flowyml.core.project import ProjectManager
 from pathlib import Path
 from flowyml.ui.backend.dependencies import get_store
@@ -18,7 +17,8 @@ def _save_file_sync(src, dst):
 
 
 def _iter_metadata_stores():
-    stores = [(None, SQLiteMetadataStore())]
+    # Use the main configured store (PostgreSQL in Docker deployment)
+    stores = [(None, get_store())]
     try:
         manager = ProjectManager()
         for project_meta in manager.list_projects():
@@ -51,6 +51,14 @@ async def list_assets(limit: int = 50, asset_type: str = None, run_id: str = Non
     """List all assets, optionally filtered by project."""
     try:
         combined = []
+        project_run_ids = set()
+
+        # If filtering by project, first get all run_ids for that project
+        if project:
+            main_store = get_store()
+            all_runs = main_store.list_runs(limit=1000)
+            project_run_ids = {r.get("run_id") for r in all_runs if r.get("project") == project}
+
         for project_name, store in _iter_metadata_stores():
             if project and project_name and project != project_name:
                 continue
@@ -61,14 +69,25 @@ async def list_assets(limit: int = 50, asset_type: str = None, run_id: str = Non
             if run_id:
                 filters["run_id"] = run_id
 
-            assets = store.list_assets(limit=limit, **filters)
+            assets = store.list_assets(limit=limit * 2, **filters)  # Get more to filter
             for asset in assets:
                 combined.append((asset, project_name))
 
         assets = _dedupe_assets(combined)
 
+        # Enhanced filtering: include artifacts that either:
+        # 1. Have project field matching
+        # 2. Have run_id that belongs to the project
         if project:
-            assets = [a for a in assets if a.get("project") == project]
+            filtered_assets = []
+            for a in assets:
+                if a.get("project") == project:
+                    filtered_assets.append(a)
+                elif a.get("run_id") in project_run_ids:
+                    # Also include and tag with project
+                    a["project"] = project
+                    filtered_assets.append(a)
+            assets = filtered_assets
 
         assets = assets[:limit]
 

flowyml/ui/backend/routers/auth.py

@@ -0,0 +1,96 @@
+from fastapi import APIRouter, HTTPException, status, Response, Request
+from pydantic import BaseModel
+import os
+
+router = APIRouter(prefix="/auth", tags=["auth"])
+
+
+# --- Models ---
+class User(BaseModel):
+    username: str
+    role: str = "admin"
+
+
+class Token(BaseModel):
+    access_token: str
+    token_type: str
+
+
+class LoginRequest(BaseModel):
+    username: str
+    password: str
+
+
+# --- Config ---
+# For simplicity in this iteration, we use single-user admin auth
+ADMIN_USER = os.getenv("FLOWYML_ADMIN_USER", "admin")
+ADMIN_PASSWORD = os.getenv("FLOWYML_ADMIN_PASSWORD", "flowyml")
+# In production, this token is also a long-lived secret or signed JWT.
+# For simplicity, we reuse the API Token logic or generate a session token.
+# Here, we'll implement a simple session token mechanism or reuse API token.
+API_TOKEN = os.getenv("FLOWYML_API_TOKEN")
+
+# --- Routes ---
+
+
+@router.post("/login", response_model=Token)
+async def login(response: Response, login_data: LoginRequest):
+    """
+    Login with username and password.
+    Sets HttpOnly cookie for browser sessions.
+    Returns token for CLI/API use.
+    """
+    if login_data.username != ADMIN_USER or login_data.password != ADMIN_PASSWORD:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Incorrect username or password",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    # Use the API token as the session token for simplicity in this unified auth model
+    # Ensure API_TOKEN is set in production!
+    if not API_TOKEN:
+        # Fallback for dev if not set (though middleware skips auth in dev usually)
+        token = "dev-token-placeholder"  # noqa: S105
+    else:
+        token = API_TOKEN
+
+    # Set HttpOnly cookie
+    # accessible only by server, secure in prod
+    response.set_cookie(
+        key="access_token",
+        value=f"Bearer {token}",
+        httponly=True,
+        max_age=86400 * 7,  # 7 days
+        secure=os.getenv("FLOWYML_ENV") == "production",
+        samesite="lax",
+    )
+
+    return {"access_token": token, "token_type": "bearer"}
+
+
+@router.post("/logout")
+async def logout(response: Response):
+    """Clear session cookie."""
+    response.delete_cookie("access_token")
+    return {"message": "Logged out successfully"}
+
+
+@router.get("/me", response_model=User)
+async def get_current_user(request: Request):
+    """
+    Get current logged in user.
+    Used by frontend to verify session.
+    """
+    # If middleware let us through, we are authenticated.
+    # In 'production' mode, unauth requests are blocked by middleware.
+    # In 'development' mode, middleware skips, so we check env.
+
+    if os.getenv("FLOWYML_ENV") != "production":
+        # Local dev: always return default admin
+        return {"username": "developer", "role": "admin"}
+
+    # In production, if we reached here, AuthMiddleware validated us.
+    # We can inspect headers/cookies to determine *who* it is
+    # (if we had multi-user), but for now it's just Admin.
+    return {"username": ADMIN_USER, "role": "admin"}