llama-deploy-appserver 0.2.7a1__py3-none-any.whl

llama_deploy/appserver/routers/deployments.py

@@ -0,0 +1,433 @@
+import asyncio
+import json
+import logging
+from typing import Annotated, AsyncGenerator, List, Optional
+
+import httpx
+import websockets
+from fastapi import (
+    APIRouter,
+    Depends,
+    File,
+    HTTPException,
+    Request,
+    UploadFile,
+    WebSocket,
+)
+from fastapi.responses import JSONResponse, StreamingResponse
+from llama_deploy.appserver.deployment import Deployment
+from llama_deploy.appserver.deployment_config_parser import DeploymentConfig
+from llama_deploy.appserver.server import manager
+from llama_deploy.appserver.types import (
+    DeploymentDefinition,
+    EventDefinition,
+    SessionDefinition,
+    TaskDefinition,
+    TaskResult,
+    generate_id,
+)
+from starlette.background import BackgroundTask
+from workflows import Context
+from workflows.context import JsonSerializer
+from workflows.handler import WorkflowHandler
+
+deployments_router = APIRouter(
+    prefix="/deployments",
+)
+logger = logging.getLogger(__name__)
+
+
+def deployment(deployment_name: str) -> Deployment:
+    """FastAPI dependency to retrieve a Deployment instance"""
+    deployment = manager.get_deployment(deployment_name)
+    if deployment is None:
+        raise HTTPException(status_code=404, detail="Deployment not found")
+    return deployment
+
+
+@deployments_router.get("/")
+async def read_deployments() -> list[DeploymentDefinition]:
+    """Returns a list of active deployments."""
+    return [DeploymentDefinition(name=k) for k in manager._deployments.keys()]
+
+
+@deployments_router.get("/{deployment_name}")
+async def read_deployment(
+    deployment: Annotated[Deployment, Depends(deployment)],
+) -> DeploymentDefinition:
+    """Returns the details of a specific deployment."""
+
+    return DeploymentDefinition(name=deployment.name)
+
+
+@deployments_router.post("/create")
+async def create_deployment(
+    base_path: str = ".",
+    config_file: UploadFile = File(...),
+    reload: bool = False,
+    local: bool = False,
+) -> DeploymentDefinition:
+    """Creates a new deployment by uploading a configuration file."""
+    config = DeploymentConfig.from_yaml_bytes(await config_file.read())
+    await manager.deploy(config, base_path, reload, local)
+
+    return DeploymentDefinition(name=config.name)
+
+
+@deployments_router.post("/{deployment_name}/tasks/run")
+async def create_deployment_task(
+    deployment: Annotated[Deployment, Depends(deployment)],
+    task_definition: TaskDefinition,
+    session_id: str | None = None,
+) -> JSONResponse:
+    """Create a task for the deployment, wait for result and delete associated session."""
+
+    service_id = task_definition.service_id or deployment.default_service
+    if service_id is None:
+        raise HTTPException(
+            status_code=400,
+            detail="Service is None and deployment has no default service",
+        )
+
+    if service_id not in deployment.service_names:
+        raise HTTPException(
+            status_code=404,
+            detail=f"Service '{task_definition.service_id}' not found in deployment 'deployment_name'",
+        )
+
+    run_kwargs = json.loads(task_definition.input) if task_definition.input else {}
+    result = await deployment.run_workflow(
+        service_id=service_id, session_id=session_id, **run_kwargs
+    )
+    return JSONResponse(result)
+
+
+@deployments_router.post("/{deployment_name}/tasks/create")
+async def create_deployment_task_nowait(
+    deployment: Annotated[Deployment, Depends(deployment)],
+    task_definition: TaskDefinition,
+    session_id: str | None = None,
+) -> TaskDefinition:
+    """Create a task for the deployment but don't wait for result."""
+    service_id = task_definition.service_id or deployment.default_service
+    if service_id is None:
+        raise HTTPException(
+            status_code=400,
+            detail="Service is None and deployment has no default service",
+        )
+
+    if service_id not in deployment.service_names:
+        raise HTTPException(
+            status_code=404,
+            detail=f"Service '{task_definition.service_id}' not found in deployment 'deployment_name'",
+        )
+
+    run_kwargs = json.loads(task_definition.input) if task_definition.input else {}
+    handler_id, session_id = deployment.run_workflow_no_wait(
+        service_id=service_id, session_id=session_id, **run_kwargs
+    )
+
+    task_definition.session_id = session_id
+    task_definition.task_id = handler_id
+
+    return task_definition
+
+
+@deployments_router.post("/{deployment_name}/tasks/{task_id}/events")
+async def send_event(
+    deployment: Annotated[Deployment, Depends(deployment)],
+    task_id: str,
+    session_id: str,
+    event_def: EventDefinition,
+) -> EventDefinition:
+    """Send a human response event to a service for a specific task and session."""
+    ctx = deployment._contexts[session_id]
+    serializer = JsonSerializer()
+    event = serializer.deserialize(event_def.event_obj_str)
+    ctx.send_event(event)
+
+    return event_def
+
+
+@deployments_router.get("/{deployment_name}/tasks/{task_id}/events")
+async def get_events(
+    deployment: Annotated[Deployment, Depends(deployment)],
+    session_id: str,
+    task_id: str,
+    raw_event: bool = False,
+) -> StreamingResponse:
+    """
+    Get the stream of events from a given task and session.
+
+    Args:
+        raw_event (bool, default=False): Whether to return the raw event object
+            or just the event data.
+    """
+
+    async def event_stream(handler: WorkflowHandler) -> AsyncGenerator[str, None]:
+        serializer = JsonSerializer()
+        # this will hang indefinitely if done and queue is empty. Bail
+        if (
+            handler.is_done()
+            and handler.ctx is not None
+            and handler.ctx.streaming_queue.empty()
+        ):
+            return
+        async for event in handler.stream_events():
+            data = json.loads(serializer.serialize(event))
+            if raw_event:
+                yield json.dumps(data) + "\n"
+            else:
+                yield json.dumps(data.get("value")) + "\n"
+            await asyncio.sleep(0.01)
+        await handler
+
+    return StreamingResponse(
+        event_stream(deployment._handlers[task_id]),
+        media_type="application/x-ndjson",
+    )
+
+
+@deployments_router.get("/{deployment_name}/tasks/{task_id}/results")
+async def get_task_result(
+    deployment: Annotated[Deployment, Depends(deployment)],
+    session_id: str,
+    task_id: str,
+) -> TaskResult | None:
+    """Get the task result associated with a task and session."""
+
+    handler = deployment._handlers[task_id]
+    return TaskResult(task_id=task_id, history=[], result=await handler)
+
+
+@deployments_router.get("/{deployment_name}/tasks")
+async def get_tasks(
+    deployment: Annotated[Deployment, Depends(deployment)],
+) -> list[TaskDefinition]:
+    """Get all the tasks from all the sessions in a given deployment."""
+
+    tasks: list[TaskDefinition] = []
+    for task_id, handler in deployment._handlers.items():
+        if handler.is_done():
+            continue
+        tasks.append(
+            TaskDefinition(
+                task_id=task_id,
+                input=deployment._handler_inputs[task_id],
+            )
+        )
+
+    return tasks
+
+
+@deployments_router.get("/{deployment_name}/sessions")
+async def get_sessions(
+    deployment: Annotated[Deployment, Depends(deployment)],
+) -> list[SessionDefinition]:
+    """Get the active sessions in a deployment and service."""
+
+    return [SessionDefinition(session_id=k) for k in deployment._contexts.keys()]
+
+
+@deployments_router.get("/{deployment_name}/sessions/{session_id}")
+async def get_session(
+    deployment: Annotated[Deployment, Depends(deployment)], session_id: str
+) -> SessionDefinition:
+    """Get the definition of a session by ID."""
+
+    return SessionDefinition(session_id=session_id)
+
+
+@deployments_router.post("/{deployment_name}/sessions/create")
+async def create_session(
+    deployment: Annotated[Deployment, Depends(deployment)],
+) -> SessionDefinition:
+    """Create a new session for a deployment."""
+
+    workflow = deployment._workflow_services[deployment.default_service]
+    session_id = generate_id()
+    deployment._contexts[session_id] = Context(workflow)
+
+    return SessionDefinition(session_id=session_id)
+
+
+@deployments_router.post("/{deployment_name}/sessions/delete")
+async def delete_session(
+    deployment: Annotated[Deployment, Depends(deployment)], session_id: str
+) -> None:
+    """Get the active sessions in a deployment and service."""
+
+    deployment._contexts.pop(session_id)
+
+
+async def _ws_proxy(ws: WebSocket, upstream_url: str) -> None:
+    """Proxy WebSocket connection to upstream server."""
+    await ws.accept()
+
+    # Forward most headers except WebSocket-specific ones
+    header_blacklist = {
+        "host",
+        "connection",
+        "upgrade",
+        "sec-websocket-key",
+        "sec-websocket-version",
+        "sec-websocket-extensions",
+    }
+    hdrs = [(k, v) for k, v in ws.headers.items() if k.lower() not in header_blacklist]
+
+    try:
+        # Parse subprotocols if present
+        subprotocols: Optional[List[websockets.Subprotocol]] = None
+        if "sec-websocket-protocol" in ws.headers:
+            # Parse comma-separated subprotocols
+            subprotocols = [
+                websockets.Subprotocol(p.strip())
+                for p in ws.headers["sec-websocket-protocol"].split(",")
+            ]
+
+        # Open upstream WebSocket connection
+        async with websockets.connect(
+            upstream_url,
+            additional_headers=hdrs,
+            subprotocols=subprotocols,
+            open_timeout=None,
+            ping_interval=None,
+        ) as upstream:
+
+            async def client_to_upstream() -> None:
+                try:
+                    while True:
+                        msg = await ws.receive()
+                        if msg["type"] == "websocket.receive":
+                            if "text" in msg:
+                                await upstream.send(msg["text"])
+                            elif "bytes" in msg:
+                                await upstream.send(msg["bytes"])
+                        elif msg["type"] == "websocket.disconnect":
+                            break
+                except Exception as e:
+                    logger.debug(f"Client to upstream connection ended: {e}")
+
+            async def upstream_to_client() -> None:
+                try:
+                    async for message in upstream:
+                        if isinstance(message, str):
+                            await ws.send_text(message)
+                        else:
+                            await ws.send_bytes(message)
+                except Exception as e:
+                    logger.debug(f"Upstream to client connection ended: {e}")
+
+            # Pump both directions concurrently
+            await asyncio.gather(
+                client_to_upstream(), upstream_to_client(), return_exceptions=True
+            )
+
+    except Exception as e:
+        logger.error(f"WebSocket proxy error: {e}")
+    finally:
+        try:
+            await ws.close()
+        except Exception as e:
+            logger.debug(f"Error closing client connection: {e}")
+
+
+@deployments_router.websocket("/{deployment_name}/ui/{path:path}")
+@deployments_router.websocket("/{deployment_name}/ui")
+async def websocket_proxy(
+    websocket: WebSocket,
+    deployment: Annotated[Deployment, Depends(deployment)],
+    path: str | None = None,
+) -> None:
+    if deployment._config.ui is None:
+        raise HTTPException(status_code=404, detail="Deployment has no ui configured")
+
+    # Build the upstream WebSocket URL using FastAPI's extracted path parameter
+    slash_path = f"/{path}" if path else ""
+    upstream_path = f"/deployments/{deployment.name}/ui{slash_path}"
+
+    # Convert to WebSocket URL
+    upstream_url = f"ws://localhost:{deployment._config.ui.port}{upstream_path}"
+    if websocket.url.query:
+        upstream_url += f"?{websocket.url.query}"
+
+    logger.debug(f"Proxying WebSocket {websocket.url} -> {upstream_url}")
+
+    await _ws_proxy(websocket, upstream_url)
+
+
+@deployments_router.api_route(
+    "/{deployment_name}/ui/{path:path}",
+    methods=["GET", "POST", "PUT", "DELETE", "OPTIONS", "HEAD", "PATCH"],
+)
+@deployments_router.api_route(
+    "/{deployment_name}/ui",
+    methods=["GET", "POST", "PUT", "DELETE", "OPTIONS", "HEAD", "PATCH"],
+)
+async def proxy(
+    request: Request,
+    deployment: Annotated[Deployment, Depends(deployment)],
+    path: str | None = None,
+) -> StreamingResponse:
+    if deployment._config.ui is None:
+        raise HTTPException(status_code=404, detail="Deployment has no ui configured")
+
+    # Build the upstream URL using FastAPI's extracted path parameter
+    slash_path = f"/{path}" if path else ""
+    upstream_path = f"/deployments/{deployment.name}/ui{slash_path}"
+
+    upstream_url = httpx.URL(
+        f"http://localhost:{deployment._config.ui.port}{upstream_path}"
+    ).copy_with(params=request.query_params)
+
+    # Debug logging
+    logger.debug(f"Proxying {request.method} {request.url} -> {upstream_url}")
+
+    # Strip hop-by-hop headers + host
+    hop_by_hop = {
+        "connection",
+        "keep-alive",
+        "proxy-authenticate",
+        "proxy-authorization",
+        "te",  # codespell:ignore
+        "trailers",
+        "transfer-encoding",
+        "upgrade",
+        "host",
+    }
+    headers = {k: v for k, v in request.headers.items() if k.lower() not in hop_by_hop}
+
+    try:
+        client = httpx.AsyncClient(timeout=None)
+
+        req = client.build_request(
+            request.method,
+            upstream_url,
+            headers=headers,
+            content=request.stream(),  # stream uploads
+        )
+        upstream = await client.send(req, stream=True)
+
+        resp_headers = {
+            k: v for k, v in upstream.headers.items() if k.lower() not in hop_by_hop
+        }
+
+        # Close client when upstream response is done
+        async def cleanup() -> None:
+            await upstream.aclose()
+            await client.aclose()
+
+        return StreamingResponse(
+            upstream.aiter_raw(),  # stream downloads
+            status_code=upstream.status_code,
+            headers=resp_headers,
+            background=BackgroundTask(cleanup),  # tidy up when finished
+        )
+
+    except httpx.ConnectError:
+        raise HTTPException(status_code=502, detail="Upstream server unavailable")
+    except httpx.TimeoutException:
+        raise HTTPException(status_code=504, detail="Upstream server timeout")
+    except Exception as e:
+        logger.error(f"Proxy error: {e}")
+        raise HTTPException(status_code=502, detail="Proxy error")

llama_deploy/appserver/routers/status.py

@@ -0,0 +1,40 @@
+import httpx
+from fastapi import APIRouter
+from fastapi.exceptions import HTTPException
+from fastapi.responses import PlainTextResponse
+from llama_deploy.appserver.server import manager
+from llama_deploy.appserver.settings import settings
+from llama_deploy.appserver.types import Status, StatusEnum
+
+status_router = APIRouter(
+    prefix="/status",
+)
+
+
+@status_router.get("/")
+async def status() -> Status:
+    return Status(
+        status=StatusEnum.HEALTHY,
+        max_deployments=manager._max_deployments,
+        deployments=list(manager._deployments.keys()),
+        status_message="",
+    )
+
+
+@status_router.get("/metrics")
+async def metrics() -> PlainTextResponse:
+    """Proxies the Prometheus metrics endpoint through the API Server.
+
+    This endpoint is mostly used in serverless environments where the LlamaDeploy
+    container cannot expose more than one port (e.g. Knative, Google Cloud Run).
+    If Prometheus is not enabled, this endpoint returns an empty HTTP-204 response.
+    """
+    if not settings.prometheus_enabled:
+        return PlainTextResponse(status_code=204)
+
+    try:
+        async with httpx.AsyncClient() as client:
+            response = await client.get(f"http://127.0.0.1:{settings.prometheus_port}/")
+            return PlainTextResponse(content=response.text)
+    except httpx.RequestError as exc:
+        raise HTTPException(status_code=500, detail=str(exc))

llama_deploy/appserver/run_autodeploy.py

@@ -0,0 +1,141 @@
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+import uvicorn
+import yaml
+from prometheus_client import start_http_server
+
+from llama_deploy.appserver.settings import settings
+
+CLONED_REPO_FOLDER = Path("cloned_repo")
+RC_PATH = Path("/data")
+
+
+def run_process(args: list[str], cwd: str | None = None) -> None:
+    kwargs = {
+        "args": args,
+        "capture_output": True,
+        "text": True,
+        "check": False,
+    }
+    if cwd:
+        kwargs["cwd"] = cwd
+    process = subprocess.run(**kwargs)  # type: ignore
+    if process.returncode != 0:
+        stderr = process.stderr or ""
+        raise Exception(stderr)
+
+
+def setup_repo(
+    work_dir: Path, source: str, token: str | None = None, force: bool = False
+) -> None:
+    repo_url, ref_name = _parse_source(source, token)
+    dest_dir = work_dir / CLONED_REPO_FOLDER
+
+    # Remove existing repo if force=True
+    if dest_dir.exists() and force:
+        shutil.rmtree(dest_dir)
+
+    if not dest_dir.exists():
+        # need to do a full clone to resolve any kind of ref without exploding in
+        # complexity (tag, branch, commit, short commit)
+        clone_args = ["git", "clone", repo_url, str(dest_dir.absolute())]
+        run_process(clone_args, cwd=str(work_dir.absolute()))
+    else:
+        run_process(["git", "fetch", "origin"], cwd=str(dest_dir.absolute()))
+
+    # Checkout the ref (let git resolve it)
+    if ref_name:
+        run_process(["git", "checkout", ref_name], cwd=str(dest_dir.absolute()))
+    # If no ref specified, stay on whatever the clone gave us (default branch)
+
+
+def _is_valid_uri(uri: str) -> bool:
+    """Check if string looks like a valid URI"""
+    return "://" in uri and "/" in uri.split("://", 1)[1]
+
+
+def _parse_source(source: str, pat: str | None = None) -> tuple[str, str | None]:
+    """Accept Github urls like https://github.com/run-llama/llama_deploy.git@main
+    or https://user:token@github.com/run-llama/llama_deploy.git@v1.0.0
+    Returns the final URL (with auth if needed) and ref name (branch, tag, or commit SHA)"""
+
+    # Try splitting on last @ to see if we have a ref specifier
+    url = source
+    ref_name = None
+
+    if "@" in source:
+        potential_url, potential_ref = source.rsplit("@", 1)
+        if _is_valid_uri(potential_url):
+            url = potential_url
+            ref_name = potential_ref
+
+    # Inject PAT auth if provided and URL doesn't already have auth
+    if pat and "://" in url and "@" not in url:
+        url = url.replace("https://", f"https://{pat}@")
+
+    return url, ref_name
+
+
+def copy_sources(work_dir: Path, deployment_file_path: Path) -> None:
+    app_folder = deployment_file_path.parent
+    for item in app_folder.iterdir():
+        if item.is_dir():
+            # For directories, use copytree with dirs_exist_ok=True
+            shutil.copytree(
+                item, f"{work_dir.absolute()}/{item.name}", dirs_exist_ok=True
+            )
+        else:
+            # For files, use copy2 to preserve metadata
+            shutil.copy2(item, str(work_dir))
+
+
+if __name__ == "__main__":
+    if settings.prometheus_enabled:
+        start_http_server(settings.prometheus_port)
+
+    repo_url = os.environ.get("REPO_URL", "")
+    if not repo_url.startswith("https://") and not repo_url.startswith("http://"):
+        raise ValueError("Git remote must HTTP(S)")
+    repo_token = os.environ.get("GITHUB_PAT")
+    work_dir = Path(os.environ.get("WORK_DIR", RC_PATH))
+    work_dir.mkdir(exist_ok=True, parents=True)
+
+    setup_repo(work_dir, repo_url, repo_token)
+
+    if not settings.deployment_file_path:
+        # first fall back to none LLAMA_DEPLOY_APISERVER_ prefixed env var (settings requires the prefix)
+        settings.deployment_file_path = os.environ.get(
+            "DEPLOYMENT_FILE_PATH", "deployment.yml"
+        )
+    deployment_file_path = settings.deployment_file_path
+    deployment_file_abspath = work_dir / CLONED_REPO_FOLDER / deployment_file_path
+    if not deployment_file_abspath.exists():
+        raise ValueError(f"File {deployment_file_abspath} does not exist")
+
+    deployment_override_name = os.environ.get("DEPLOYMENT_NAME")
+    if deployment_override_name:
+        with open(deployment_file_abspath) as f:
+            # Replace deployment name with the overridden value
+            data = yaml.safe_load(f)
+
+        # Avoid failing here if the deployment config file has a wrong format,
+        # let's do nothing if there's no field `name`
+        if "name" in data:
+            data["name"] = deployment_override_name
+            with open(deployment_file_abspath, "w") as f:
+                yaml.safe_dump(data, f)
+
+    copy_sources(work_dir, deployment_file_abspath)
+    shutil.rmtree(work_dir / CLONED_REPO_FOLDER)
+
+    # update rc_path directly, as it has already been loaded, so setting the environment variable
+    # doesn't work
+    settings.rc_path = work_dir
+    uvicorn.run(
+        "llama_deploy.appserver.app:app",
+        host=settings.host,
+        port=settings.port,
+    )

llama_deploy/appserver/server.py

@@ -0,0 +1,60 @@
+import asyncio
+import logging
+from contextlib import asynccontextmanager
+from typing import Any, AsyncGenerator
+
+from fastapi import FastAPI
+
+from .deployment import Manager
+from .deployment_config_parser import DeploymentConfig
+from .settings import settings
+from .stats import apiserver_state
+
+logger = logging.getLogger("uvicorn.info")
+manager = Manager()
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI) -> AsyncGenerator[None, Any]:
+    apiserver_state.state("starting")
+
+    manager.set_deployments_path(settings.deployments_path)
+    t = asyncio.create_task(manager.serve())
+    await asyncio.sleep(0)
+
+    logger.info(f"deployments folder: {settings.deployments_path}")
+    logger.info(f"rc folder: {settings.rc_path}")
+
+    if settings.rc_path.exists():
+        if settings.deployment_file_path:
+            logger.info(
+                f"Browsing the rc folder {settings.rc_path} for deployment file {settings.deployment_file_path}"
+            )
+        else:
+            logger.info(
+                f"Browsing the rc folder {settings.rc_path} for deployments to start"
+            )
+
+        # if a deployment_file_path is provided, use it, otherwise glob all .yml/.yaml files
+        # q match both .yml and .yaml files with the glob
+        files = (
+            [settings.rc_path / settings.deployment_file_path]
+            if settings.deployment_file_path
+            else [
+                x for x in settings.rc_path.iterdir() if x.suffix in (".yml", ".yaml")
+            ]
+        )
+        for yaml_file in files:
+            try:
+                logger.info(f"Deploying startup configuration from {yaml_file}")
+                config = DeploymentConfig.from_yaml(yaml_file)
+                await manager.deploy(config, base_path=str(settings.rc_path))
+            except Exception as e:
+                logger.error(f"Failed to deploy {yaml_file}: {str(e)}")
+
+    apiserver_state.state("running")
+    yield
+
+    t.cancel()
+
+    apiserver_state.state("stopped")