lorebase-keeper 0.2.3__py3-none-any.whl

lorebase_keeper/errors.py

@@ -0,0 +1,257 @@
+"""Problem details (RFC 9457): the single error format of the API.
+
+Every error response has the media type `application/problem+json` and the members `type`
+(stable URI), `title`, `status`, `detail`, `instance` (the request path), `code` (the last part
+of `type`), `request_id` and, when tracing is active, `trace_id`. Validation failures add
+`errors`, a list of `{pointer, message, code}`. Responses never contain stack traces and never
+echo request values.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from http import HTTPStatus
+from typing import Any
+
+from fastapi import FastAPI, Request
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse
+from opentelemetry import trace
+from pydantic import BaseModel, ConfigDict
+from starlette.exceptions import HTTPException
+
+PROBLEM_MEDIA_TYPE = "application/problem+json"
+
+
+@dataclass(frozen=True)
+class ProblemType:
+    """A kind of error: stable code, HTTP status and short title."""
+
+    code: str
+    status: int
+    title: str
+
+
+BAD_REQUEST = ProblemType("bad-request", 400, "Bad request")
+UNAUTHORIZED = ProblemType("unauthorized", 401, "Authentication required")
+FORBIDDEN = ProblemType("forbidden", 403, "Forbidden")
+NOT_FOUND = ProblemType("not-found", 404, "Not found")
+METHOD_NOT_ALLOWED = ProblemType("method-not-allowed", 405, "Method not allowed")
+CONFLICT = ProblemType("conflict", 409, "Conflict")
+PAYLOAD_TOO_LARGE = ProblemType("payload-too-large", 413, "Request body too large")
+VALIDATION_FAILED = ProblemType("validation-failed", 422, "Request validation failed")
+RATE_LIMITED = ProblemType("rate-limited", 429, "Too many requests")
+INTERNAL_ERROR = ProblemType("internal-error", 500, "Internal server error")
+BAD_GATEWAY = ProblemType("bad-gateway", 502, "Bad gateway")
+UNAVAILABLE = ProblemType("unavailable", 503, "Service unavailable")
+
+_BY_STATUS = {
+    p.status: p
+    for p in (
+        BAD_REQUEST,
+        UNAUTHORIZED,
+        FORBIDDEN,
+        NOT_FOUND,
+        METHOD_NOT_ALLOWED,
+        CONFLICT,
+        PAYLOAD_TOO_LARGE,
+        VALIDATION_FAILED,
+        RATE_LIMITED,
+        INTERNAL_ERROR,
+        BAD_GATEWAY,
+        UNAVAILABLE,
+    )
+}
+
+
+class ProblemError(HTTPException):
+    """Raise from any route or dependency to answer with a problem detail.
+
+    `extensions` adds members to the response body; `headers` adds response headers.
+    """
+
+    def __init__(
+        self,
+        problem: ProblemType,
+        detail: str | None = None,
+        *,
+        extensions: Mapping[str, Any] | None = None,
+        headers: Mapping[str, str] | None = None,
+    ) -> None:
+        super().__init__(
+            status_code=problem.status, detail=detail, headers=dict(headers) if headers else None
+        )
+        self.problem = problem
+        self.problem_detail = detail
+        self.extensions = dict(extensions or {})
+
+
+class FieldProblem(BaseModel):
+    """One invalid request member."""
+
+    pointer: str
+    message: str
+    code: str
+
+
+class Problem(BaseModel):
+    """Problem details document (schema of every error response)."""
+
+    model_config = ConfigDict(extra="allow")
+
+    type: str
+    title: str
+    status: int
+    detail: str | None = None
+    instance: str | None = None
+    code: str
+    request_id: str | None = None
+    trace_id: str | None = None
+
+
+class ValidationProblem(Problem):
+    """Problem details of a rejected request body, query or path."""
+
+    errors: list[FieldProblem]
+
+
+def current_trace_id() -> str | None:
+    """Trace id of the active span, if tracing is recording one."""
+    context = trace.get_current_span().get_span_context()
+    return f"{context.trace_id:032x}" if context.is_valid else None
+
+
+class ProblemRenderer:
+    """Builds problem responses with the configured `type` URI base."""
+
+    def __init__(self, type_base: str) -> None:
+        self._type_base = type_base
+
+    def type_uri(self, problem: ProblemType) -> str:
+        """Stable URI identifying the kind of error."""
+        return f"{self._type_base}{problem.code}"
+
+    def body(
+        self,
+        problem: ProblemType,
+        *,
+        detail: str | None = None,
+        instance: str | None = None,
+        request_id: str | None = None,
+        extensions: Mapping[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        """The JSON document of a problem."""
+        body: dict[str, Any] = {
+            "type": self.type_uri(problem),
+            "title": problem.title,
+            "status": problem.status,
+            "code": problem.code,
+        }
+        if detail:
+            body["detail"] = detail
+        if instance:
+            body["instance"] = instance
+        if request_id:
+            body["request_id"] = request_id
+        if trace_id := current_trace_id():
+            body["trace_id"] = trace_id
+        if extensions:
+            body.update(extensions)
+        return body
+
+    def response(
+        self,
+        problem: ProblemType,
+        *,
+        detail: str | None = None,
+        instance: str | None = None,
+        request_id: str | None = None,
+        extensions: Mapping[str, Any] | None = None,
+        headers: Mapping[str, str] | None = None,
+    ) -> JSONResponse:
+        """A complete problem response."""
+        return JSONResponse(
+            self.body(
+                problem,
+                detail=detail,
+                instance=instance,
+                request_id=request_id,
+                extensions=extensions,
+            ),
+            status_code=problem.status,
+            headers=dict(headers) if headers else None,
+            media_type=PROBLEM_MEDIA_TYPE,
+        )
+
+
+def request_id_of(scope_state: Mapping[str, Any] | None) -> str | None:
+    """The request id stored in an ASGI scope's state by the request context middleware."""
+    value = (scope_state or {}).get("request_id")
+    return value if isinstance(value, str) else None
+
+
+def _generic_problem(status: int) -> ProblemType:
+    known = _BY_STATUS.get(status)
+    if known is not None:
+        return known
+    return ProblemType(f"http-{status}", status, _phrase(status) or "Error")
+
+
+def install_error_handlers(app: FastAPI, renderer: ProblemRenderer) -> None:
+    """Route HTTP errors and request validation failures to problem details.
+
+    Unhandled exceptions are answered by the request context middleware, which sits outside
+    the framework's own error layer so that the response still carries the request id.
+    """
+
+    async def on_http_error(request: Request, exc: Exception) -> JSONResponse:
+        assert isinstance(exc, HTTPException)
+        request_id = request_id_of(request.scope.get("state"))
+        if isinstance(exc, ProblemError):
+            return renderer.response(
+                exc.problem,
+                detail=exc.problem_detail,
+                instance=request.url.path,
+                request_id=request_id,
+                extensions=exc.extensions,
+                headers=exc.headers,
+            )
+        problem = _generic_problem(exc.status_code)
+        raw: object = exc.detail  # FastAPI's subclass also allows non-string details
+        detail = raw if isinstance(raw, str) and raw != _phrase(exc.status_code) else None
+        return renderer.response(
+            problem,
+            detail=detail,
+            instance=request.url.path,
+            request_id=request_id,
+            headers=exc.headers,
+        )
+
+    async def on_validation_error(request: Request, exc: Exception) -> JSONResponse:
+        assert isinstance(exc, RequestValidationError)
+        errors = [
+            {
+                "pointer": "/" + "/".join(str(part) for part in error["loc"]),
+                "message": str(error["msg"]),
+                "code": str(error["type"]),
+            }
+            for error in exc.errors()
+        ]
+        return renderer.response(
+            VALIDATION_FAILED,
+            detail="One or more request members are invalid.",
+            instance=request.url.path,
+            request_id=request_id_of(request.scope.get("state")),
+            extensions={"errors": errors},
+        )
+
+    app.add_exception_handler(HTTPException, on_http_error)
+    app.add_exception_handler(RequestValidationError, on_validation_error)
+
+
+def _phrase(status: int) -> str:
+    try:
+        return HTTPStatus(status).phrase
+    except ValueError:
+        return ""

lorebase_keeper/health.py

@@ -0,0 +1,61 @@
+"""Liveness and readiness endpoints, served at the root (outside the API prefix)."""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from fastapi import APIRouter, Request
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+
+from lorebase_keeper.deps import KeeperSettingsDep
+from lorebase_keeper.readiness import ReadinessCheck, run_checks
+
+router = APIRouter(tags=["health"])
+
+
+class Liveness(BaseModel):
+    """The process is up."""
+
+    status: Literal["ok"] = "ok"
+
+
+class CheckReport(BaseModel):
+    """Outcome of one dependency check."""
+
+    status: Literal["ok", "degraded", "not_configured", "failed"]
+    detail: str | None = None
+
+
+class Readiness(BaseModel):
+    """Whether the service can take traffic, with the outcome of each check."""
+
+    status: Literal["ready", "not_ready"]
+    checks: dict[str, CheckReport]
+
+
+@router.get("/healthz", summary="Liveness", response_model=Liveness)
+async def healthz() -> Liveness:
+    """The process answers requests. Never touches dependencies."""
+    return Liveness()
+
+
+@router.get(
+    "/readyz",
+    summary="Readiness",
+    response_model=Readiness,
+    responses={503: {"model": Readiness, "description": "A dependency check failed."}},
+)
+async def readyz(request: Request, keeper: KeeperSettingsDep) -> JSONResponse:
+    """Dependencies are usable: the database is reachable and migrations are current."""
+    checks: list[ReadinessCheck] = request.app.state.readiness_checks
+    results = await run_checks(checks, keeper.readiness.timeout_s)
+    ready = all(result.passed for result in results.values())
+    report = Readiness(
+        status="ready" if ready else "not_ready",
+        checks={
+            name: CheckReport(status=result.status.value, detail=result.detail)
+            for name, result in results.items()
+        },
+    )
+    return JSONResponse(report.model_dump(), status_code=200 if ready else 503)

lorebase_keeper/kindhealth/__init__.py

@@ -0,0 +1,6 @@
+"""Health reports of a Domain and of its Kinds (`GET .../health`).
+
+`config` holds the settings (`read.health`), `models` the response shapes, `scan` the queries
+over the materialized head of a branch, `report` the assembly and `markdown` the rendering for
+agents.
+"""

lorebase_keeper/kindhealth/config.py

@@ -0,0 +1,51 @@
+"""Settings of health reports (`read.health`)."""
+
+from __future__ import annotations
+
+from pydantic import Field
+
+from lorebase_platform.settings import SectionModel
+
+
+class HealthSettings(SectionModel):
+    """Caps of `GET /domains/{d}/health` and `GET /domains/{d}/kinds/{kind}/health`.
+
+    A report is computed when asked for, from the materialized head of a branch, and kept in
+    the response cache under the Revision it describes. The caps keep one request affordable on
+    large Domains; what a cap cuts is reported (`sampled`, `skipped`, `truncated`).
+    """
+
+    max_records_validated: int = Field(
+        default=5000,
+        ge=1,
+        description="Records of one Kind re-validated against the current schema. A larger "
+        "Kind is sampled: a fixed, evenly spread subset of about this size, the same one for "
+        "the same data.",
+    )
+    examples_per_issue: int = Field(
+        default=3, ge=0, description="Record keys listed as examples of each kind of violation."
+    )
+    max_issue_groups: int = Field(
+        default=50, ge=1, description="Kinds of violation listed per Kind; the rest are cut."
+    )
+    duplicates_max_records: int = Field(
+        default=200_000,
+        ge=0,
+        description="Kinds with more Records are not scanned for duplicates (reported as "
+        "skipped). 0 turns the duplicate check off.",
+    )
+    max_duplicate_groups: int = Field(
+        default=20, ge=1, description="Duplicate groups listed per Attribute, most Records first."
+    )
+    duplicate_keys_per_group: int = Field(
+        default=5, ge=1, description="Record keys listed in a duplicate group."
+    )
+    orphan_examples: int = Field(
+        default=5, ge=0, description="Orphan Links and Records with a gap listed per Relation."
+    )
+    statement_timeout_ms: int = Field(
+        default=15_000,
+        ge=0,
+        description="statement_timeout of the scans, instead of `read.statement_timeout_ms`; "
+        "0 disables. A scan that exceeds it is answered 503.",
+    )

lorebase_keeper/kindhealth/markdown.py

@@ -0,0 +1,163 @@
+"""Markdown rendering of health reports, meant to be pasted into a prompt.
+
+Stable (same report, same bytes) and compact: plain ASCII, one table of Kinds, and a section
+only for what has something to report. Revisions are shortened to `SHORT_REVISION` digits.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from lorebase_keeper.kindhealth.models import (
+    DomainHealthOut,
+    KindHealth,
+    KindHealthOut,
+    RelationHealth,
+)
+
+SHORT_REVISION = 12
+
+
+def _row(cells: Sequence[object]) -> str:
+    return "| " + " | ".join(str(c).replace("|", "\\|") for c in cells) + " |"
+
+
+def _table(header: Sequence[str], rows: Sequence[Sequence[object]]) -> list[str]:
+    return [_row(header), "|" + " --- |" * len(header), *(_row(r) for r in rows)]
+
+
+def _verdict(ok: bool) -> str:
+    return "ok" if ok else "needs attention"
+
+
+def _kinds_table(kinds: Sequence[KindHealth]) -> list[str]:
+    rows = [
+        (
+            k.kind,
+            k.records,
+            k.violations.invalid_records,
+            k.duplicates.total_groups,
+            "stale" if k.derived is not None and k.derived.stale else "",
+            "" if k.last_change is None else k.last_change.revision[:SHORT_REVISION],
+        )
+        for k in kinds
+    ]
+    return _table(
+        ("kind", "records", "invalid", "duplicate groups", "derived", "last change"), rows
+    )
+
+
+def _kind_details(kind: KindHealth) -> list[str]:
+    lines: list[str] = []
+    violations = kind.violations
+    if violations.issues:
+        scope = f"{violations.checked} of {kind.records} Records checked" + (
+            " (sample)" if violations.sampled else ""
+        )
+        lines += [f"### {kind.kind}: violations ({scope})"]
+        for issue in violations.issues:
+            where = f" on {issue.attribute}" if issue.attribute else ""
+            examples = f" (e.g. {', '.join(issue.examples)})" if issue.examples else ""
+            lines.append(f"- {issue.code}{where}: {issue.count}{examples}")
+        if violations.issues_truncated:
+            lines.append("- (more kinds of violation omitted)")
+    duplicates = kind.duplicates
+    if duplicates.groups or duplicates.hidden:
+        lines += [f"### {kind.kind}: duplicates ({duplicates.total_groups} groups)"]
+        for group in duplicates.groups:
+            value = "" if group.value is None else f' "{group.value}"'
+            flag = " [unique]" if group.unique else ""
+            lines.append(
+                f"- {group.attribute}{value}{flag}: {', '.join(group.keys)} ({group.count})"
+            )
+        for withheld in duplicates.hidden:
+            flag = " [unique]" if withheld.unique else ""
+            lines.append(
+                f"- {withheld.attribute}{flag}: {withheld.groups} groups, {withheld.records} "
+                "Records (not listed: sensitive)"
+            )
+    elif duplicates.skipped:
+        lines.append(f"### {kind.kind}: duplicates not checked ({duplicates.skipped})")
+    if kind.derived is not None and kind.derived.stale:
+        names = ", ".join(kind.derived.attributes)
+        lines.append(
+            f"### {kind.kind}: {kind.derived.records_missing_value} Records lack derived "
+            f"values ({names})"
+        )
+    return lines
+
+
+def _relations(relations: Sequence[RelationHealth]) -> list[str]:
+    rows = [
+        (
+            r.relation,
+            f"{r.from_kind} -> {r.to_kind}",
+            r.links,
+            r.orphan_links.count,
+            "" if r.required_gaps is None else r.required_gaps.missing,
+            r.direction or "",
+        )
+        for r in relations
+    ]
+    lines = [
+        "## Relations",
+        *_table(("relation", "ends", "links", "orphans", "gaps", "side"), rows),
+    ]
+    for item in relations:
+        if item.orphan_links.examples:
+            pairs = ", ".join(f"{e.from_key}->{e.to_key}" for e in item.orphan_links.examples)
+            lines.append(f"- {item.relation} orphans: {pairs}")
+        if item.required_gaps is not None and item.required_gaps.missing:
+            lines.append(
+                f"- {item.relation} required, missing for: {', '.join(item.required_gaps.examples)}"
+            )
+    return lines
+
+
+def _summary(report: DomainHealthOut | KindHealthOut) -> list[str]:
+    s = report.summary
+    return [
+        f"{_verdict(s.ok)}: {s.kinds} kinds, {s.records} records, {s.links} links",
+        f"- invalid records: {s.invalid_records}; orphan links: {s.orphan_links}; "
+        f"required-relation gaps: {s.required_relation_gaps}; duplicate groups: "
+        f"{s.duplicate_groups}; kinds with stale derived values: {s.stale_kinds}",
+    ]
+
+
+def domain_markdown(report: DomainHealthOut) -> str:
+    """A Domain report as markdown."""
+    lines = [
+        f"# Health of {report.domain} on {report.branch}",
+        f"revision {report.revision[:SHORT_REVISION]}",
+        "",
+        *_summary(report),
+        "",
+        "## Kinds",
+        *_kinds_table(report.kinds),
+    ]
+    for kind in report.kinds:
+        details = _kind_details(kind)
+        if details:
+            lines += ["", *details]
+    if report.relations:
+        lines += ["", *_relations(report.relations)]
+    return "\n".join(lines) + "\n"
+
+
+def kind_markdown(report: KindHealthOut) -> str:
+    """A Kind report as markdown."""
+    kind = report.health
+    lines = [
+        f"# Health of {kind.kind} in {report.domain} on {report.branch}",
+        f"revision {report.revision[:SHORT_REVISION]}",
+        "",
+        *_summary(report),
+        "",
+        *_kinds_table([kind]),
+    ]
+    details = _kind_details(kind)
+    if details:
+        lines += ["", *details]
+    if report.relations:
+        lines += ["", *_relations(report.relations)]
+    return "\n".join(lines) + "\n"