dirigent-server 0.9.0__py3-none-any.whl

dirigent_server/__init__.py

@@ -0,0 +1,7 @@
+"""The dirigent API server: the REST surface, authentication, and the UI."""
+
+from dirigent_server.app import create_app
+from dirigent_server.logging import configure_logging, get_logger
+from dirigent_server.security import SESSION_COOKIE
+
+__all__ = ["SESSION_COOKIE", "configure_logging", "create_app", "get_logger"]

dirigent_server/app.py

@@ -0,0 +1,171 @@
+"""The FastAPI application factory."""
+
+import asyncio
+import contextlib
+from collections.abc import AsyncGenerator
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI
+
+from dirigent_core import __version__
+from dirigent_core.auth import BOOTSTRAP_PASSWORD_ENV, bootstrap_admin
+from dirigent_core.config import Settings, get_settings
+from dirigent_core.database import create_engine, create_session_factory, session_scope
+from dirigent_core.directory import apply_directory
+from dirigent_core.engine.services import EngineServices
+from dirigent_core.plugins import PluginHost, load_plugin_host
+from dirigent_core.scheduler import Scheduler
+from dirigent_core.telemetry import configure_telemetry, instrument_fastapi
+from dirigent_server.errors import install_error_handlers
+from dirigent_server.health import build_registry
+from dirigent_server.health import router as health_router
+from dirigent_server.logging import get_logger
+from dirigent_server.routes import TAGS, build_hooks_router, build_router
+from dirigent_server.security import install_cross_site_guard
+from dirigent_server.ui import mount_ui_assets, mount_ui_shell
+
+DESCRIPTION = """
+A generic pipeline orchestrator. Pipelines are data, composed from pluggable building
+blocks and executed as a DAG on a durable engine.
+
+Every endpoint under `/api/v1` requires authentication: a bearer token for automation, or
+the session cookie `POST /api/v1/auth/login` sets for a browser. The probes under `/health`
+and this document itself are the only exceptions. `POST /hooks/{token}` is outside the
+versioned API and authenticates with its own per-trigger token.
+"""
+
+_logger = get_logger("server")
+
+BOOTSTRAP_ENV = BOOTSTRAP_PASSWORD_ENV
+SCHEDULER_STOP_SECONDS = 10.0
+
+
+def create_app(
+    settings: Settings | None = None,
+    *,
+    host: PluginHost | None = None,
+    scheduler: bool | None = None,
+) -> FastAPI:
+    """Build the API application: health probes, the routers, the UI, and the embedded scheduler.
+
+    Scheduler leadership is an advisory lock, so running the scheduler here, in a dedicated
+    ``dg scheduler``, or in both at once is safe.
+
+    The UI mounts in two pieces around the routers: its fixed paths before them, its shell
+    after them, so an API route is matched before anything the bundle claims.
+    """
+    resolved = settings or get_settings()
+    embed = resolved.scheduler_enabled if scheduler is None else scheduler
+    configure_telemetry(resolved)
+
+    @asynccontextmanager
+    async def lifespan(app: FastAPI) -> AsyncGenerator[None]:
+        """Open the database engine for the process, and dispose of it on shutdown."""
+        engine = create_engine(resolved)
+        app.state.engine = engine
+        app.state.session_factory = create_session_factory(engine)
+        app.state.health_checks = build_registry(engine)
+        await _bootstrap(app)
+        await _apply_startup_directory(app)
+        clock = await _start_scheduler(app, embed=embed)
+        _logger.info(
+            "server starting",
+            database="sqlite" if resolved.is_sqlite else "postgresql",
+            blocks=len(app.state.services.host.block_ids),
+            scheduler=embed,
+        )
+        try:
+            yield
+        finally:
+            await _stop_scheduler(app, clock)
+            await engine.dispose()
+            _logger.info("server stopped")
+
+    app = FastAPI(
+        title="dirigent",
+        summary="A generic pipeline orchestrator.",
+        description=DESCRIPTION.strip(),
+        version=__version__,
+        openapi_tags=TAGS,
+        lifespan=lifespan,
+    )
+    app.state.settings = resolved
+    app.state.version = __version__
+    app.state.services = EngineServices.build(resolved, host or load_plugin_host())
+    install_cross_site_guard(app, resolved)
+    install_error_handlers(app)
+    mount_ui_assets(app, resolved)
+    app.include_router(health_router)
+    app.include_router(build_hooks_router())
+    app.include_router(build_router(), prefix=resolved.api_prefix)
+    mount_ui_shell(app, resolved)
+    instrument_fastapi(app)
+    return app
+
+
+async def _start_scheduler(app: FastAPI, *, embed: bool) -> asyncio.Task[None] | None:
+    """Start the embedded scheduler as a task in the API's own event loop."""
+    app.state.scheduler = None
+    if not embed:
+        return None
+    scheduler = Scheduler(app.state.session_factory, app.state.services)
+    app.state.scheduler = scheduler
+    return asyncio.create_task(scheduler.run())
+
+
+async def _stop_scheduler(app: FastAPI, task: asyncio.Task[None] | None) -> None:
+    """Ask the embedded scheduler to finish its tick and hand back leadership."""
+    scheduler = getattr(app.state, "scheduler", None)
+    if scheduler is not None:
+        scheduler.request_stop()
+    if task is not None:
+        with contextlib.suppress(asyncio.CancelledError, TimeoutError):
+            await asyncio.wait_for(task, timeout=SCHEDULER_STOP_SECONDS)
+
+
+async def _apply_startup_directory(app: FastAPI) -> None:
+    """Apply the configured directory of schemas and documents, before the scheduler starts firing.
+
+    The summary is one record; each refused file has already said its own warning. A
+    boot that found the lock held skips quietly: whoever holds it is doing this work.
+    """
+    settings: Settings = app.state.settings
+    if settings.apply_dir is None:
+        return
+    summary = await apply_directory(
+        app.state.session_factory,
+        app.state.services,
+        settings.apply_dir,
+        prune=settings.apply_prune,
+        lock_key=settings.apply_lock_key,
+    )
+    if summary.skipped:
+        return
+    _logger.info(
+        "startup directory applied",
+        directory=str(settings.apply_dir),
+        applied=len(summary.applied),
+        updated=len(summary.updated),
+        unchanged=len(summary.unchanged),
+        schemas=len(summary.schemas),
+        refused=[refusal.code or refusal.path for refusal in summary.refused],
+        pruned=summary.pruned,
+        trigger_documents_removed=summary.trigger_documents_removed,
+    )
+
+
+async def _bootstrap(app: FastAPI) -> None:
+    """Create the first admin when a container names a password and no account exists yet.
+
+    Does nothing once any account exists, so leaving the variable set on every deploy
+    cannot reset a live instance's admin password.
+    """
+    import os
+
+    password = os.environ.get(BOOTSTRAP_ENV)
+    if not password:
+        return
+    async with session_scope(app.state.session_factory) as session:
+        created = await bootstrap_admin(session, password)
+    if created is not None:
+        _logger.info("bootstrap admin created", username=created.username)

dirigent_server/dependencies.py

@@ -0,0 +1,51 @@
+"""FastAPI dependencies: the request session, the engine services, and the settings."""
+
+from collections.abc import AsyncGenerator
+from typing import Annotated
+
+from fastapi import Depends, Request
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from dirigent_core.config import Settings
+from dirigent_core.engine.services import EngineServices
+
+
+def get_settings(request: Request) -> Settings:
+    """Return the settings this application was built with."""
+    resolved: Settings = request.app.state.settings
+    return resolved
+
+
+def get_services(request: Request) -> EngineServices:
+    """Return the engine services: the plugin host, storage, and the secret box."""
+    services: EngineServices = request.app.state.services
+    return services
+
+
+def get_sessions(request: Request) -> async_sessionmaker[AsyncSession]:
+    """Return the session factory the API and the engine share."""
+    factory: async_sessionmaker[AsyncSession] = request.app.state.session_factory
+    return factory
+
+
+async def get_session(request: Request) -> AsyncGenerator[AsyncSession]:
+    """Open one transaction for one request, and hand it to the route to commit.
+
+    The exit of a dependency that yields runs after the response has gone out, so committing
+    here tells a client its write happened before it had: a client that reads straight back
+    can miss what it just wrote, and a commit that fails does so after a 2xx. ``Transactional``
+    commits while the response can still change. This only has to undo one that did not get
+    there.
+    """
+    async with get_sessions(request)() as session:
+        request.state.session = session
+        try:
+            yield session
+        except BaseException:
+            await session.rollback()
+            raise
+
+
+SessionDep = Annotated[AsyncSession, Depends(get_session)]
+ServicesDep = Annotated[EngineServices, Depends(get_services)]
+SettingsDep = Annotated[Settings, Depends(get_settings)]

dirigent_server/errors.py

@@ -0,0 +1,116 @@
+"""One error envelope for every refusal, in the shape RFC 9457 describes."""
+
+from collections.abc import Awaitable, Callable
+from http import HTTPStatus
+from typing import Any, cast
+
+from fastapi import FastAPI, HTTPException, Request, Response
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse
+from starlette.exceptions import HTTPException as StarletteHTTPException
+
+from dirigent_client.schemas import Problem
+from dirigent_core import __version__
+from dirigent_core.logging import redact_path
+from dirigent_server.logging import get_logger
+
+VERSION_HEADER = "X-Dirigent-Version"
+
+INTERNAL_DETAIL = "the server failed to handle this request; the server log has the detail"
+
+_logger = get_logger("errors")
+
+
+def render(status: int, detail: str, *, problems: list[str] | None = None, instance: str | None = None) -> Problem:
+    """Build the one problem shape, from whichever handler is answering."""
+    try:
+        title = HTTPStatus(status).phrase
+    except ValueError:  # pragma: no cover - a non-standard status from a plugin
+        title = "Error"
+    return Problem(status=status, title=title, detail=detail, problems=problems or [], instance=instance)
+
+
+def _problems_of(detail: Any) -> tuple[str, list[str]]:
+    """Split whatever was raised into one sentence and, when there is one, a list."""
+    if isinstance(detail, str):
+        return detail, []
+    if isinstance(detail, list):
+        rendered = [_one(item) for item in cast(list[object], detail)]
+        return "; ".join(rendered), rendered
+    return str(detail), []
+
+
+def _one(item: object) -> str:
+    """Render one entry of a problem list, whether it is a string or a pydantic error."""
+    if isinstance(item, str):
+        return item
+    if isinstance(item, dict):
+        mapping = cast(dict[str, Any], item)
+        location = ".".join(str(part) for part in mapping.get("loc", []))
+        message = str(mapping.get("msg", mapping))
+        return f"{location}: {message}" if location else message
+    return str(item)
+
+
+def _where(request: Request) -> str:
+    """Name the path that was asked for, never including a credential from it."""
+    return redact_path(request.url.path)
+
+
+def answer(problem: Problem) -> JSONResponse:
+    """Serialise a problem into a response."""
+    return JSONResponse(
+        status_code=problem.status,
+        content=problem.model_dump(mode="json"),
+        headers={VERSION_HEADER: __version__},
+    )
+
+
+async def _stamp_version(request: Request, call_next: Callable[[Request], Awaitable[Response]]) -> Response:
+    """Put the version header on every response."""
+    response = await call_next(request)
+    response.headers[VERSION_HEADER] = __version__
+    return response
+
+
+async def http_error(request: Request, error: Exception) -> JSONResponse:
+    """Render an HTTPException as a problem."""
+    if not isinstance(error, StarletteHTTPException):  # pragma: no cover - registered for this class
+        return await unhandled(request, error)
+    detail, problems = _problems_of(error.detail)
+    response = answer(render(error.status_code, detail, problems=problems, instance=_where(request)))
+    if isinstance(error, HTTPException) and error.headers:
+        response.headers.update(error.headers)
+    return response
+
+
+async def validation_error(request: Request, error: Exception) -> JSONResponse:
+    """Render a request-validation failure as a problem with its field list intact."""
+    if not isinstance(error, RequestValidationError):  # pragma: no cover - registered for this class
+        return await unhandled(request, error)
+    detail, problems = _problems_of(error.errors())
+    return answer(render(422, detail, problems=problems, instance=_where(request)))
+
+
+async def unhandled(request: Request, error: Exception) -> JSONResponse:
+    """Answer an unhandled exception in the same shape, and say nothing about it.
+
+    An unexpected exception's message may carry a DSN, a path, or a fragment of a payload,
+    so it goes to the log only and never to the caller.
+    """
+    _logger.error(
+        "unhandled exception",
+        path=_where(request),
+        method=request.method,
+        error=f"{type(error).__name__}: {error}",
+        exc_info=error,
+    )
+    return answer(render(500, INTERNAL_DETAIL, instance=_where(request)))
+
+
+def install_error_handlers(app: FastAPI) -> None:
+    """Install the error envelope and the version header on the application."""
+    app.middleware("http")(_stamp_version)
+    app.add_exception_handler(StarletteHTTPException, http_error)
+    app.add_exception_handler(RequestValidationError, validation_error)
+    app.add_exception_handler(Exception, unhandled)

dirigent_server/health.py

@@ -0,0 +1,97 @@
+"""The liveness and readiness probes, and the checks readiness runs."""
+
+import asyncio
+from abc import ABC, abstractmethod
+from collections.abc import Mapping
+from typing import ClassVar
+
+from fastapi import APIRouter, Request, Response, status
+from sqlalchemy.ext.asyncio import AsyncEngine
+
+from dirigent_client.schemas import CheckResult, CheckStatus, Health, Readiness
+from dirigent_core.database import ping
+
+router = APIRouter(tags=["health"])
+
+
+#: Aggregation takes the maximum, so a new status must sort worse than every status it
+#: outranks.
+_SEVERITY: dict[CheckStatus, int] = {
+    CheckStatus.HEALTHY: 0,
+    CheckStatus.DEGRADED: 1,
+    CheckStatus.UNHEALTHY: 2,
+}
+
+
+class HealthCheck(ABC):
+    """One named dependency the readiness probe verifies."""
+
+    name: ClassVar[str]
+
+    @abstractmethod
+    async def check(self) -> CheckResult:
+        """Verify the dependency, returning a result rather than raising."""
+        ...
+
+
+class DatabaseHealthCheck(HealthCheck):
+    """Verifies that the database answers a trivial query."""
+
+    name = "database"
+
+    def __init__(self, engine: AsyncEngine) -> None:
+        """Bind the check to the process's engine."""
+        self._engine = engine
+
+    async def check(self) -> CheckResult:
+        """Run SELECT 1 through the async engine."""
+        if await ping(self._engine):
+            return CheckResult(status=CheckStatus.HEALTHY)
+        return CheckResult(status=CheckStatus.UNHEALTHY, detail="database is unreachable")
+
+
+type HealthCheckRegistry = Mapping[str, HealthCheck]
+
+
+def build_registry(engine: AsyncEngine) -> dict[str, HealthCheck]:
+    """Assemble the checks this process runs."""
+    checks: list[HealthCheck] = [DatabaseHealthCheck(engine)]
+    return {check.name: check for check in checks}
+
+
+def aggregate(results: Mapping[str, CheckResult]) -> CheckStatus:
+    """Reduce a set of results to the worst status; no checks means healthy."""
+    if not results:
+        return CheckStatus.HEALTHY
+    return max(results.values(), key=lambda result: _SEVERITY[result.status]).status
+
+
+async def run_checks(registry: HealthCheckRegistry) -> dict[str, CheckResult]:
+    """Run every registered check concurrently, turning a raised error into unhealthy."""
+
+    async def run(check: HealthCheck) -> CheckResult:
+        try:
+            return await check.check()
+        except Exception as error:
+            return CheckResult(status=CheckStatus.UNHEALTHY, detail=f"{type(error).__name__}: {error}")
+
+    names = list(registry)
+    results = await asyncio.gather(*(run(registry[name]) for name in names))
+    return dict(zip(names, results, strict=True))
+
+
+@router.get("/health", operation_id="health", response_model=Health, summary="Liveness")
+async def health(request: Request) -> Health:
+    """Report that the process is alive, without touching any dependency."""
+    return Health(version=request.app.state.version)
+
+
+@router.get("/health/ready", operation_id="readiness", response_model=Readiness, summary="Readiness")
+async def health_ready(request: Request, response: Response) -> Readiness:
+    """Run every registered check and report the worst status, with 503 when unhealthy."""
+    registry: HealthCheckRegistry = request.app.state.health_checks
+    results = await run_checks(registry)
+    overall = aggregate(results)
+    if overall is CheckStatus.UNHEALTHY:
+        response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
+    return Readiness(status=overall, checks=results)

dirigent_server/logging.py

@@ -0,0 +1,5 @@
+"""Server-side re-export of the logging configuration from dirigent-core."""
+
+from dirigent_core.logging import PACKAGE_LOGGER, configure_logging, get_logger
+
+__all__ = ["PACKAGE_LOGGER", "configure_logging", "get_logger"]

dirigent_server/pagination.py

@@ -0,0 +1,56 @@
+"""One envelope for every listing, and the keyset walk behind it.
+
+Every listing answers ``{"items": [...], "next": <cursor or null>}``. ``limit`` bounds a page
+at 500 and defaults to 50; ``after`` carries the cursor a previous page's ``next`` gave out.
+A cursor is opaque to the caller and is the sort key of the last row returned, so a listing
+selects ``limit + 1`` rows past it in its own order, answers ``limit`` of them, and says
+``next`` only when the extra row existed. A cursor that does not parse is a 422.
+"""
+
+from collections.abc import Callable
+from typing import Annotated
+from uuid import UUID
+
+from fastapi import HTTPException, Query, status
+
+DEFAULT_PAGE = 50
+
+MAX_PAGE = 500
+
+LimitParam = Annotated[int, Query(ge=1, le=MAX_PAGE, description="How many rows to return.")]
+
+AfterParam = Annotated[str | None, Query(description="Continue from a previous page's next cursor.")]
+
+
+def clip[T](rows: list[T], limit: int, key: Callable[[T], object]) -> tuple[list[T], str | None]:
+    """Cut a ``limit + 1`` selection down to the page, and name the cursor that continues it."""
+    page = rows[:limit]
+    if len(rows) <= limit or not page:
+        return page, None
+    return page, str(key(page[-1]))
+
+
+def uuid_cursor(after: str | None) -> UUID | None:
+    """Read an id cursor, refusing anything that is not one."""
+    if after is None:
+        return None
+    try:
+        return UUID(after)
+    except ValueError as error:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
+            detail=f"after={after!r} is not a cursor this listing gave out",
+        ) from error
+
+
+def int_cursor(after: str | None, *, name: str = "after") -> int | None:
+    """Read a numeric cursor, refusing anything that is not one."""
+    if after is None:
+        return None
+    try:
+        return int(after)
+    except ValueError as error:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
+            detail=f"{name}={after!r} is not a cursor this listing gave out",
+        ) from error

dirigent_server/routes/__init__.py

@@ -0,0 +1,79 @@
+"""The versioned API, assembled from one router per resource.
+
+Every business router is mounted behind :func:`require_principal`, so authentication is a
+property of the mount. Only login is exempt. Webhook intake authenticates as the trigger
+rather than as a person, so it is mounted at the application root instead.
+"""
+
+from fastapi import APIRouter, Depends
+
+from dirigent_server.routes import (
+    alerts,
+    auth,
+    blocks,
+    connections,
+    hooks,
+    pipelines,
+    runs,
+    schema,
+    schemas,
+    system,
+    trigger_documents,
+    triggers,
+    users,
+    workers,
+)
+from dirigent_server.security import require_principal
+from dirigent_server.transactions import Transactional
+
+TAGS: list[dict[str, str]] = [
+    {"name": "auth", "description": "Logging in, and the tokens automation uses."},
+    {"name": "pipelines", "description": "Applying, exporting, and running definitions."},
+    {"name": "runs", "description": "Runs, their attempts, their logs, and their reports."},
+    {"name": "triggers", "description": "A pipeline's schedules and inbound webhooks."},
+    {"name": "alerts", "description": "Alert rules, and the queue they deliver through."},
+    {"name": "hooks", "description": "Webhook intake, authenticated by its own token."},
+    {"name": "connections", "description": "Named credential records of contributed kinds."},
+    {"name": "blocks", "description": "The catalog every plugin contributes to."},
+    {"name": "schema", "description": "The shape a document is written against."},
+    {"name": "schemas", "description": "Named JSON Schemas the instance holds."},
+    {"name": "workers", "description": "The worker registry."},
+    {"name": "users", "description": "Local accounts."},
+    {"name": "system", "description": "What this instance is, and whether it is well."},
+    {"name": "health", "description": "Liveness and readiness."},
+]
+
+
+def build_router() -> APIRouter:
+    """Assemble the versioned API: the login route, then everything behind authentication."""
+    router = APIRouter(route_class=Transactional)
+    router.include_router(auth.public_router)
+    guarded = APIRouter(dependencies=[Depends(require_principal)], route_class=Transactional)
+    for module in (
+        auth,
+        pipelines,
+        triggers,
+        trigger_documents,
+        runs,
+        alerts,
+        connections,
+        blocks,
+        schema,
+        schemas,
+        workers,
+        users,
+        system,
+    ):
+        guarded.include_router(module.router)
+    router.include_router(guarded)
+    return router
+
+
+def build_hooks_router() -> APIRouter:
+    """Assemble the unauthenticated webhook intake, mounted at the application root."""
+    router = APIRouter(route_class=Transactional)
+    router.include_router(hooks.router)
+    return router
+
+
+__all__ = ["TAGS", "build_hooks_router", "build_router"]