lorebase-keeper 0.2.3__py3-none-any.whl

lorebase_keeper/routes/source_preview.py

@@ -0,0 +1,232 @@
+"""Preview of the expression mappings of a candidate Source (design 5.9, 11.1).
+
+| Route | Permission |
+|-------|------------|
+| `POST /domains/{d}/sources:preview` | `schema.read` on the branch or the Source's target Kind |
+
+The body is a schema change set, as for `POST /schema:validate` (`yaml`, or `patch` and
+`remove`; an empty `patch` previews the schema as saved), `source`, the api name of the Source in
+the schema it produces, and `rows`, sample rows of the Source as the caller read them (column
+name to value). The schema is validated like a schema change (`schema.expression_limits`
+included), then every Attribute mapped by `expression` is evaluated on each row, as Scribe
+evaluates it when it maps rows, and the value before the mapping's transforms is returned.
+Expressions are compiled once over every column the rows have: a column a row lacks is null.
+
+Rows come from the caller and nothing else is read: no Record, no connection, no secret. At
+most `read.source_preview_max_rows` rows of at most `read.source_preview_max_columns` columns
+are accepted, and at most `read.source_preview_max_cells` expressions (rows times Attributes
+mapped by an expression) are evaluated, off the event loop. A grant of `schema.read` narrowed
+to Kinds is enough when it names the Source's target Kind (in the candidate and, when the Kind
+exists there, in the schema read); otherwise the preview is refused (403) before the
+candidate is validated.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+from functools import partial
+from typing import Annotated, Any
+
+import anyio
+from fastapi import APIRouter, Body, Depends, Request
+from pydantic import BaseModel, Field, JsonValue
+
+from lorebase_core.model import Schema, Source
+from lorebase_keeper.auth.audit import annotate
+from lorebase_keeper.auth.deps import AuthDep, PrincipalDep
+from lorebase_keeper.auth.permissions import Permission
+from lorebase_keeper.context import StoreDep
+from lorebase_keeper.deps import KeeperSettingsDep
+from lorebase_keeper.errors import Problem, ValidationProblem
+from lorebase_keeper.reading.points import ReadPoint, resolve_point
+from lorebase_keeper.reading.problems import invalid
+from lorebase_keeper.reading.runtime import ReadRuntimeDep, ReadSessionDep
+from lorebase_keeper.reading.settings import ReadSettings
+from lorebase_keeper.routes.derivations import PreviewErrorOut
+from lorebase_keeper.routes.schema import ChangeSetBody
+from lorebase_keeper.services import source_preview as service
+from lorebase_keeper.services.derivation_preview import CandidateInvalidError
+from lorebase_keeper.services.errors import violations_problem
+from lorebase_keeper.services.reads import ReadGuard
+
+router = APIRouter(tags=["sources"])
+
+_ERRORS: dict[int | str, dict[str, Any]] = {
+    403: {
+        "model": Problem,
+        "description": "The caller lacks schema.read for the branch and for the Source's "
+        "target Kind.",
+    },
+    404: {"model": Problem, "description": "Domain, branch or Revision not found."},
+    422: {
+        "model": ValidationProblem,
+        "description": "Invalid request, or a candidate schema that fails validation (`issues` "
+        "locate why).",
+    },
+}
+
+
+class SourcePreviewRequest(ChangeSetBody):
+    """A candidate schema change set, the Source to preview in it and sample rows."""
+
+    source: str = Field(
+        min_length=1, description="Api name of the Source in the schema the change set produces."
+    )
+    rows: list[dict[str, JsonValue]] = Field(
+        min_length=1,
+        description="Sample rows: column name to value (text for CSV, native values for JSON). "
+        "At most `read.source_preview_max_rows`, each with at most "
+        "`read.source_preview_max_columns` columns.",
+    )
+    branch: str | None = Field(default=None, description="Default: the default branch.")
+    at: str | None = Field(default=None, description="Revision id, tag or branch name.")
+
+
+class ExpressionCellOut(BaseModel):
+    """The value of one expression mapping for one row, before the transforms."""
+
+    value: JsonValue = Field(description="Null when there is none or `error` is set.")
+    error: PreviewErrorOut | None = None
+
+
+class SourcePreviewOut(BaseModel):
+    """Values of the expression mappings of a candidate Source on sample rows."""
+
+    domain: str
+    branch: str | None
+    revision: str = Field(description="Revision whose schema the change set was applied to.")
+    source: str
+    target_kind: str
+    attributes: list[str] = Field(description="Attributes mapped by an expression, by api name.")
+    rows: list[dict[str, ExpressionCellOut]] = Field(
+        description="One entry per row sent: the cell of each Attribute in `attributes`."
+    )
+
+
+def schema_reader() -> Callable[..., Awaitable[ReadGuard]]:
+    """Dependency: the authenticated reader of the Domain named by the path; `schema.read` is
+    checked once the read point is resolved."""
+
+    async def dependency(
+        request: Request, principal: PrincipalDep, auth: AuthDep, keeper: KeeperSettingsDep
+    ) -> ReadGuard:
+        domain_id = str(request.path_params["domain_id"])
+        annotate(request, domain_id=domain_id, permission=Permission.SCHEMA_READ.value)
+        return ReadGuard(
+            authorizer=auth.authorizer,
+            principal=principal,
+            permissions=(Permission.SCHEMA_READ,),
+            domain_id=domain_id,
+            max_depth=keeper.reads.reachability_max_depth,
+        )
+
+    return dependency
+
+
+SchemaReader = Annotated[ReadGuard, Depends(schema_reader())]
+
+
+def _bound_rows(rows: list[dict[str, JsonValue]], expressions: int, read: ReadSettings) -> None:
+    """422 for more rows, columns or cells than the settings allow."""
+    if len(rows) > read.source_preview_max_rows:
+        raise invalid("/rows", f"must not exceed {read.source_preview_max_rows}", "limit_exceeded")
+    for index, row in enumerate(rows):
+        if len(row) > read.source_preview_max_columns:
+            raise invalid(
+                f"/rows/{index}",
+                f"must not have more than {read.source_preview_max_columns} columns",
+                "limit_exceeded",
+            )
+    if len(rows) * expressions > read.source_preview_max_cells:
+        raise invalid(
+            "/rows",
+            f"{len(rows)} rows times {expressions} expression mappings exceed "
+            f"{read.source_preview_max_cells} cells",
+            "limit_exceeded",
+        )
+
+
+def _target_kind_check(point: ReadPoint) -> Callable[[Schema, Source | None], None]:
+    """403 unless the caller may read the Source's target Kind: always with grants that are
+    not narrowed to Kinds; with Kind-narrowed grants, when they name it in the candidate and,
+    if the Kind exists there, in the schema read (a rename in the change set moves nothing)."""
+
+    def check(candidate: Schema, source: Source | None) -> None:
+        if point.scope.kinds is None:
+            return
+        kind = None if source is None else candidate.kinds.get(source.target_kind)
+        if kind is None:
+            raise point.scope.guard.denied()
+        names = {kind.api_name}
+        stored = point.schema.kinds.get(kind.id)
+        if stored is not None:
+            names.add(stored.api_name)
+        if point.scope.invisible and not all(point.scope.allows(n) for n in names):
+            # Like a Source that does not exist: nothing tells which Kind is missing.
+            raise point.scope.guard.denied()
+        point.scope.require(names)
+
+    return check
+
+
+@router.post(
+    "/domains/{domain_id}/sources:preview",
+    summary="Preview the expression mappings of a candidate Source",
+    response_model=SourcePreviewOut,
+    responses=_ERRORS,
+)
+async def preview_source(
+    domain_id: str,
+    payload: Annotated[SourcePreviewRequest, Body()],
+    access: SchemaReader,
+    store: StoreDep,
+    keeper: KeeperSettingsDep,
+    runtime: ReadRuntimeDep,
+    session: ReadSessionDep,
+) -> SourcePreviewOut:
+    """Evaluate the `expression` mappings of a Source, as edited and not yet saved, on sample
+    rows. Nothing is read from the Source and nothing is written."""
+    _bound_rows(payload.rows, 0, runtime.read)
+    point = await resolve_point(session, runtime, access, branch=payload.branch, at=payload.at)
+    try:
+        candidate = service.candidate_of(
+            point.schema,
+            payload.to_input(),
+            payload.source,
+            store.runtime.schema.rules(),
+            authorize=_target_kind_check(point),
+        )
+    except CandidateInvalidError as error:
+        raise violations_problem(error.issues, limit=keeper.schema_changes.max_issues) from None
+    attributes = service.expression_attributes(candidate)
+    _bound_rows(payload.rows, len(attributes), runtime.read)
+    cells = await anyio.to_thread.run_sync(
+        partial(
+            service.evaluate,
+            candidate,
+            payload.rows,
+            limits=store.runtime.schema.expression_limits,
+            now=store.clock(),
+        )
+    )
+    kind = candidate.schema.kinds[candidate.source.target_kind]
+    return SourcePreviewOut(
+        domain=domain_id,
+        branch=point.branch,
+        revision=point.revision_hex,
+        source=payload.source,
+        target_kind=kind.api_name,
+        attributes=list(attributes),
+        rows=[
+            {
+                attribute: ExpressionCellOut(
+                    value=cell.value,
+                    error=None
+                    if cell.error_code is None
+                    else PreviewErrorOut(code=cell.error_code, message=cell.error_message),
+                )
+                for attribute, cell in row.items()
+            }
+            for row in cells
+        ],
+    )

lorebase_keeper/routes/sources.py

@@ -0,0 +1,200 @@
+"""Structured Sources of a Domain (design 9.1): trigger a synchronization, list its runs.
+
+`POST /domains/{d}/sources/{s}/sync` queues a `source_sync` job that Scribe runs (permission
+`source.sync`: `editor`, `maintainer`, `admin`) and answers 202 with the job id; while an
+identical job is queued or running, that job is returned instead of a new one. The Source is
+named by id or api name and looked up in the schema of the Domain's default branch. Every
+trigger is audited (`source.sync`). Triggering a Source that maps vault Attributes also needs
+`pii.import` on the whole Domain (`services.pii_import`): a caller who could feed or re-run
+such a Source would learn whether a value was erased.
+
+`GET /domains/{d}/sources/{s}/runs` lists the runs recorded by Scribe, newest first, with their
+counts, Revision, Proposal and error. It needs `data.read` through a Domain-wide grant: run
+stats cover every Kind the Source writes and errors quote row values, so callers whose grants
+are narrowed to a Kind or a branch get 403, as for unknown Domains they may not be told about.
+Errors are shown only to callers cleared Domain-wide for every sensitivity tag (they may
+quote sensitive values of any branch's schema); others get `{"redacted": true}`.
+The trigger needs a Domain-wide `source.sync` grant and answers with the job only, never run
+data.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Annotated, Any
+
+from fastapi import APIRouter, Depends, Request, Response
+from pydantic import BaseModel, ConfigDict, Field
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from lorebase_core.model import Schema, Sensitivity, Source
+from lorebase_keeper.auth.audit import annotate
+from lorebase_keeper.auth.deps import Access, AuthDep, require
+from lorebase_keeper.auth.permissions import Permission
+from lorebase_keeper.context import StoreDep, require_domain
+from lorebase_keeper.errors import NOT_FOUND, VALIDATION_FAILED, Problem, ProblemError
+from lorebase_keeper.pagination import Page, PageParamsDep, encode_cursor
+from lorebase_keeper.reading.points import open_domain, read_guard
+from lorebase_keeper.reading.sensitivity import clearance_of
+from lorebase_keeper.services.pii_import import require_pii_import, source_vault_targets
+from lorebase_keeper.services.reads import ReadGuard
+from lorebase_store.commit.schema import load_compiled
+from lorebase_store.jobs import SOURCE_SYNC_JOB, enqueue_job, source_sync_payload
+from lorebase_store.repositories import (
+    MaterializedRepository,
+    SourceRunRepository,
+    SourceRunRow,
+)
+from lorebase_store.runtime import StoreRuntime
+
+router = APIRouter(tags=["sources"])
+
+BASE = "/domains/{domain_id}/sources/{source}"
+_ERRORS: dict[int | str, dict[str, Any]] = {
+    403: {"model": Problem, "description": "The caller lacks the permission."},
+    404: {"model": Problem, "description": "Domain or Source not found."},
+}
+_SYNC_ERRORS: dict[int | str, dict[str, Any]] = {
+    **_ERRORS,
+    403: {
+        "model": Problem,
+        "description": "The caller lacks source.sync, or pii.import for a Source mapping "
+        "vault Attributes.",
+    },
+}
+
+Sync = Annotated[Access, Depends(require(Permission.SOURCE_SYNC))]
+ReadRuns = Annotated[ReadGuard, Depends(read_guard())]
+
+
+class SyncRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    dry_run: bool = Field(
+        default=False, description="Compute and record the delta without writing to any branch."
+    )
+
+
+class SyncQueued(BaseModel):
+    job_id: int = Field(description="Id of the queued (or already pending) job.")
+    source: str = Field(description="Source id.")
+    api_name: str
+    dry_run: bool
+
+
+class SourceRunOut(BaseModel):
+    id: str
+    source: str
+    status: str
+    started_at: datetime | None = None
+    finished_at: datetime | None = None
+    stats: dict[str, Any]
+    revision: str | None = None
+    proposal_id: str | None = None
+    error: dict[str, Any] | None = None
+
+
+async def _source(
+    session: AsyncSession, domain_id: str, branch: str, reference: str, runtime: StoreRuntime
+) -> tuple[Schema, Source]:
+    """The Source named by id or api name in the schema of `branch`, with that schema."""
+    compiled = await MaterializedRepository(
+        session, domain_id, branch, writes=runtime.settings.writes
+    ).schema()
+    schema = None if compiled is None else load_compiled(compiled.compiled)
+    found = None
+    if schema is not None:
+        found = schema.sources.get(reference) or schema.source_by_api_name(reference)
+    if schema is None or found is None:
+        raise ProblemError(NOT_FOUND, f"Source {reference!r} not found.")
+    return schema, found
+
+
+REDACTED_ERROR: dict[str, Any] = {"redacted": True}
+
+
+def _run(row: SourceRunRow, *, errors: bool) -> SourceRunOut:
+    return SourceRunOut(
+        id=row.id,
+        source=row.source_id,
+        status=row.status.value,
+        started_at=row.started_at,
+        finished_at=row.finished_at,
+        stats=row.stats,
+        revision=None if row.revision_id is None else row.revision_id.hex(),
+        proposal_id=row.proposal_id,
+        error=row.error if errors or row.error is None else REDACTED_ERROR,
+    )
+
+
+@router.post(
+    f"{BASE}/sync",
+    status_code=202,
+    summary="Synchronize a Source",
+    response_model=SyncQueued,
+    responses=_SYNC_ERRORS,
+)
+async def sync_source(
+    domain_id: str,
+    source: str,
+    access: Sync,
+    auth: AuthDep,
+    store: StoreDep,
+    request: Request,
+    response: Response,
+    body: SyncRequest | None = None,
+) -> SyncQueued:
+    """Queue a synchronization of the Source for Scribe."""
+    dry_run = body is not None and body.dry_run
+    async with store.sessions() as session, session.begin():
+        domain = await require_domain(session, domain_id)
+        schema, found = await _source(
+            session, domain_id, domain.default_branch, source, store.runtime
+        )
+        if source_vault_targets(schema, found):
+            await require_pii_import(auth.authorizer, access.principal, domain_id)
+        job_id = await enqueue_job(
+            session,
+            SOURCE_SYNC_JOB,
+            domain_id=domain_id,
+            payload=source_sync_payload(found.id, dry_run=dry_run),
+            unique=True,
+        )
+    annotate(request, action="source.sync", target=found.id, job=job_id, dry_run=dry_run)
+    response.headers["Location"] = f"{request.url.path.rsplit('/', 1)[0]}/runs"
+    return SyncQueued(job_id=job_id, source=found.id, api_name=found.api_name, dry_run=dry_run)
+
+
+@router.get(f"{BASE}/runs", summary="List Source runs", responses=_ERRORS)
+async def list_source_runs(
+    domain_id: str, source: str, access: ReadRuns, store: StoreDep, params: PageParamsDep
+) -> Page[SourceRunOut]:
+    """Runs of the Source, newest first."""
+    after = None
+    if params.position is not None:
+        try:
+            after = (
+                datetime.fromisoformat(str(params.position["at"])),
+                str(params.position["id"]),
+            )
+        except (KeyError, ValueError):
+            raise ProblemError(VALIDATION_FAILED, "The cursor is not a run cursor.") from None
+    async with store.sessions() as session, session.begin():
+        domain = await open_domain(session, access)
+        await access.require_wide()
+        _, found = await _source(session, domain.id, domain.default_branch, source, store.runtime)
+        rows = await SourceRunRepository(session, domain_id).page(
+            found.id, limit=params.limit + 1, after=after
+        )
+    clearance = await clearance_of(access.authorizer, access.principal, domain_id, None)
+    errors = clearance.covers(Sensitivity, ())
+    more = len(rows) > params.limit
+    rows = rows[: params.limit]
+    last = rows[-1] if rows else None
+    next_cursor = (
+        encode_cursor({"at": last.started_at.isoformat(), "id": last.id})
+        if more and last is not None and last.started_at is not None
+        else None
+    )
+    items = [_run(r, errors=errors) for r in rows]
+    return Page[SourceRunOut](items=items, next_cursor=next_cursor)