lorebase-keeper 0.2.3__py3-none-any.whl

lorebase_keeper/routes/schema_suggestions.py

@@ -0,0 +1,328 @@
+"""Schema suggestions from document extraction (design 9.2 step 10, 12.3 Review).
+
+| Method and path | Permission | Notes |
+|-----------------|------------|-------|
+| `GET .../schema/suggestions` | `document.read` | `status`, `type`, `all` |
+| `POST .../schema/suggestions:decide` | `schema.write` | accepting: see below |
+| `POST .../schema/suggestions:merge` | `schema.write` | folds stored variants |
+
+Suggestions quote their documents, so listing them needs `document.read` (Domain-wide). Pending
+suggestions are listed once they reach `schema.suggestions.min_support` observations, the most
+supported first, at most `schema.suggestions.max_suggestions` in all.
+
+A decision rejects a suggestion, resets it to `pending`, or accepts it. All suggestions accepted
+by one request become one schema change set, committed on a new branch
+(`keeper.schema_suggestions.branch_prefix` and the first suggestion's id) and proposed into the
+default branch: the answer names the Proposal, which reviewers merge as any other. Accepting
+needs `schema.write` on that branch, `branch.create` and `proposal.create`; when the change set
+is invalid the answer is 422 with the issues and nothing is decided. An accepted suggestion
+cannot be decided again. Decisions are audited (`schema.suggestions.decide`).
+
+`:merge` folds the pending Attribute suggestions stored before variants were merged on arrival
+into the best supported one of each set (`schema.suggestions.merge_similarity`); it is safe to
+repeat and audited (`schema.suggestions.merge`).
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Annotated, Any, Literal
+
+from fastapi import APIRouter, Depends, Query, Request
+from pydantic import BaseModel, ConfigDict, Field, JsonValue
+
+from lorebase_keeper.auth.audit import annotate
+from lorebase_keeper.auth.deps import Access, AuthDep, PrincipalDep, authorize, require
+from lorebase_keeper.auth.permissions import Permission
+from lorebase_keeper.auth.principals import Principal
+from lorebase_keeper.auth.rbac import AccessContext
+from lorebase_keeper.auth.runtime import AuthRuntime
+from lorebase_keeper.context import StoreDep, require_domain
+from lorebase_keeper.deps import KeeperSettingsDep
+from lorebase_keeper.errors import Problem, ValidationProblem
+from lorebase_keeper.pagination import Page, PageParamsDep, encode_cursor
+from lorebase_keeper.reading.sensitivity import Masking, default_schema, masking_for
+from lorebase_keeper.routes.schema import ElementChangeOut
+from lorebase_keeper.services import schema_suggestions as service
+from lorebase_keeper.services.errors import commit_problem
+from lorebase_store.commit import CommitError
+
+router = APIRouter(tags=["schema"])
+
+SUGGESTIONS = "/domains/{domain_id}/schema/suggestions"
+StatusName = Literal["pending", "accepted", "rejected"]
+TypeName = Literal["kind", "attribute", "relation"]
+
+_ERRORS: dict[int | str, dict[str, Any]] = {
+    403: {"model": Problem, "description": "The caller lacks the permission."},
+    404: {"model": Problem, "description": "Domain or suggestion not found."},
+    409: {"model": Problem, "description": "A suggestion was accepted already."},
+    422: {
+        "model": ValidationProblem,
+        "description": "Invalid request, a suggestion the schema has meanwhile, or accepted "
+        "suggestions that do not form a valid schema.",
+    },
+    503: {"model": Problem, "description": "A branch is locked by another writer."},
+}
+
+Read = Annotated[Access, Depends(require(Permission.DOCUMENT_READ))]
+
+
+class TypeGuessOut(BaseModel):
+    type: str = Field(description="Attribute type inferred from the samples, e.g. `date`.")
+    constraints: dict[str, JsonValue]
+    confidence: float
+    reasons: list[str]
+
+
+class DuplicateOut(BaseModel):
+    section: Literal["kinds", "attributes", "relations"]
+    api_name: str
+
+
+class SuggestionOut(BaseModel):
+    id: str
+    type: TypeName
+    name: str = Field(description="Proposed api name.")
+    kind: str | None = Field(
+        default=None,
+        description="Owner Kind of an Attribute, from Kind of a Relation: an api name, or "
+        "`+Name` for a Kind another suggestion proposes.",
+    )
+    target: str | None = Field(default=None, description="To Kind of a Relation.")
+    labels: list[str] = Field(description="Labels as the documents wrote them.")
+    support: int = Field(description="Observations, over every document.")
+    documents: int = Field(description="Documents it was observed in.")
+    confidence: float = Field(description="Mean confidence of the observations.")
+    status: StatusName
+    type_guess: TypeGuessOut | None = Field(
+        default=None, description="For Attributes: the type the samples suggest."
+    )
+    samples: list[str]
+    examples: list[dict[str, Any]] = Field(
+        description="Evidence: document, chunk, offsets into the parsed text, quote."
+    )
+    sources: list[str] = Field(description="Techniques that observed it (rules, models).")
+    duplicate_of: DuplicateOut | None = Field(
+        default=None, description="The element of the current schema it duplicates."
+    )
+    withheld: bool = Field(
+        default=False,
+        description="Examples and samples are left out: it matches a sensitive Attribute the "
+        "caller may not read.",
+    )
+    proposal_id: str | None
+    branch: str | None
+    decided_by: str | None
+    decided_at: datetime | None
+    created_at: datetime | None
+    updated_at: datetime | None
+
+
+class SuggestionDecisionIn(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    suggestion: str = Field(min_length=1)
+    decision: StatusName
+    name: str | None = Field(
+        default=None, max_length=100, description="Api name to use instead (accepted only)."
+    )
+
+
+class DecideSuggestionsIn(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    decisions: list[SuggestionDecisionIn] = Field(min_length=1)
+
+
+class MergeSuggestionsOut(BaseModel):
+    merged: int = Field(description="Suggestions folded into another and removed.")
+    kept: int = Field(description="Suggestions that absorbed others.")
+
+
+class SuggestionProposalOut(BaseModel):
+    id: str
+    title: str
+    branch: str
+    changes: list[ElementChangeOut]
+
+
+class DecideSuggestionsOut(BaseModel):
+    suggestions: list[SuggestionOut]
+    proposal: SuggestionProposalOut | None = Field(
+        default=None, description="The Proposal opened for the accepted suggestions."
+    )
+
+
+def _out(view: service.SuggestionView) -> SuggestionOut:
+    row = view.row
+    guess = view.guess
+    return SuggestionOut(
+        id=row.id,
+        type=row.suggestion_type,
+        name=row.name,
+        kind=view.kind,
+        target=view.target,
+        labels=list(row.labels),
+        support=row.support,
+        documents=row.documents,
+        confidence=row.confidence,
+        status=row.status,
+        type_guess=None
+        if guess is None
+        else TypeGuessOut(
+            type=guess.type.describe(),
+            constraints=guess.constraints.model_dump(mode="json", exclude_none=True),
+            confidence=guess.confidence,
+            reasons=list(guess.reasons),
+        ),
+        samples=[] if view.withheld else list(row.samples),
+        examples=[] if view.withheld else list(row.examples),
+        sources=list(row.sources),
+        duplicate_of=None
+        if view.duplicate is None
+        else DuplicateOut(
+            section=view.duplicate.section,
+            api_name=view.duplicate.api_name,
+        ),
+        withheld=view.withheld,
+        proposal_id=row.proposal_id,
+        branch=row.branch,
+        decided_by=row.decided_by,
+        decided_at=row.decided_at,
+        created_at=row.created_at,
+        updated_at=row.updated_at,
+    )
+
+
+async def _masking(
+    store: StoreDep, auth: AuthRuntime, principal: Principal, domain_id: str, settings: Any
+) -> Masking:
+    async with store.sessions() as session, session.begin():
+        await require_domain(session, domain_id)
+        reference = await default_schema(session, domain_id, store.runtime)
+    masking = await masking_for(auth.authorizer, principal, domain_id, reference, settings)
+    return masking if reference is None else masking.over(reference)
+
+
+@router.get(SUGGESTIONS, summary="List schema suggestions", responses=_ERRORS)
+async def list_suggestions(
+    domain_id: str,
+    access: Read,
+    store: StoreDep,
+    params: PageParamsDep,
+    auth: AuthDep,
+    keeper: KeeperSettingsDep,
+    status: Annotated[list[StatusName] | None, Query()] = None,
+    type_: Annotated[list[TypeName] | None, Query(alias="type")] = None,
+    all_: Annotated[
+        bool,
+        Query(alias="all", description="Also those below `schema.suggestions.min_support`."),
+    ] = False,
+) -> Page[SuggestionOut]:
+    """Kinds, Attributes and Relations extraction found outside the schema, with support,
+    examples, samples and, for Attributes, the type their samples suggest."""
+    masking = await _masking(store, auth, access.principal, domain_id, keeper.reads.sensitivity)
+    found = await service.page(
+        store,
+        domain_id,
+        masking,
+        statuses=status or [],
+        types=type_ or [],
+        all_support=all_,
+        limit=params.limit,
+        position=params.position,
+    )
+    following = None if found.next_position is None else encode_cursor(found.next_position)
+    return Page[SuggestionOut](items=[_out(v) for v in found.views], next_cursor=following)
+
+
+@router.post(
+    SUGGESTIONS + ":decide", summary="Accept or reject schema suggestions", responses=_ERRORS
+)
+async def decide_suggestions(
+    domain_id: str,
+    body: DecideSuggestionsIn,
+    request: Request,
+    principal: PrincipalDep,
+    auth: AuthDep,
+    store: StoreDep,
+    keeper: KeeperSettingsDep,
+) -> DecideSuggestionsOut:
+    """Reject or reset suggestions; accept some as one schema change Proposal."""
+    settings = keeper.schema_suggestions
+    accepting = [d for d in body.decisions if d.decision == "accepted"]
+    if accepting:
+        branch = f"{settings.branch_prefix}{accepting[0].suggestion}"
+        for permission, context in (
+            (Permission.BRANCH_CREATE, AccessContext()),
+            (Permission.SCHEMA_WRITE, AccessContext(branch=branch)),
+            (Permission.PROPOSAL_CREATE, AccessContext()),
+        ):
+            await authorize(request, auth, principal, permission, domain_id, context)
+    else:
+        await authorize(request, auth, principal, Permission.SCHEMA_WRITE, domain_id)
+    masking = await _masking(store, auth, principal, domain_id, keeper.reads.sensitivity)
+    try:
+        outcome = await service.decide(
+            store,
+            domain_id,
+            [service.DecisionInput(d.suggestion, d.decision, d.name) for d in body.decisions],
+            principal,
+            settings=settings,
+            branches=keeper.branches,
+            proposals=keeper.proposals,
+            masking=masking,
+        )
+    except CommitError as error:
+        raise commit_problem(
+            error,
+            max_issues=keeper.schema_changes.max_issues,
+            retry_after_s=keeper.branches.lock_retry_after_s,
+        ) from None
+    annotate(
+        request,
+        action="schema.suggestions.decide",
+        target=domain_id,
+        accepted=len(accepting),
+        rejected=sum(1 for d in body.decisions if d.decision == "rejected"),
+        reset=sum(1 for d in body.decisions if d.decision == "pending"),
+        proposal=outcome.proposal_id,
+    )
+    proposal = None
+    if outcome.proposal_id is not None and outcome.branch is not None:
+        proposal = SuggestionProposalOut(
+            id=outcome.proposal_id,
+            title=outcome.proposal_title or "",
+            branch=outcome.branch,
+            changes=[
+                ElementChangeOut.model_validate(c, from_attributes=True) for c in outcome.changes
+            ],
+        )
+    return DecideSuggestionsOut(suggestions=[_out(v) for v in outcome.views], proposal=proposal)
+
+
+@router.post(
+    SUGGESTIONS + ":merge",
+    summary="Merge stored variants of schema suggestions",
+    responses={k: v for k, v in _ERRORS.items() if k in (403, 404)},
+)
+async def merge_suggestions(
+    domain_id: str,
+    request: Request,
+    principal: PrincipalDep,
+    auth: AuthDep,
+    store: StoreDep,
+) -> MergeSuggestionsOut:
+    """Fold pending Attribute suggestions that are variants of one another, with the same
+    similarity rules as new observations (idempotent)."""
+    await authorize(request, auth, principal, Permission.SCHEMA_WRITE, domain_id)
+    outcome = await service.merge_variants(store, domain_id)
+    annotate(
+        request,
+        action="schema.suggestions.merge",
+        target=domain_id,
+        merged=outcome.merged,
+        kept=outcome.kept,
+    )
+    return MergeSuggestionsOut(merged=outcome.merged, kept=outcome.kept)

lorebase_keeper/routes/sessions.py

@@ -0,0 +1,286 @@
+"""Browser sign-in for the web app: Keeper as backend-for-frontend (design 13.2).
+
+`GET /auth/login` sends the browser to the provider, `GET /auth/callback` turns the answer into
+a session cookie, `GET /auth/session` gives the app what it needs while signed in (who, until
+when, the CSRF token) and `POST /auth/logout` ends the session. The logic lives in
+`lorebase_keeper.auth.sessions`; every route answers 404 while `keeper.auth.sessions.enabled`
+is false.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime
+from typing import Annotated, Any, Literal
+from urllib.parse import urlencode, urlsplit
+
+from fastapi import APIRouter, Query, Request, Response
+from fastapi.responses import RedirectResponse
+from pydantic import BaseModel, Field
+
+from lorebase_keeper.auth.audit import annotate
+from lorebase_keeper.auth.config import SessionSettings, is_local_path, url_origin
+from lorebase_keeper.auth.deps import AuthDep, PrincipalDep
+from lorebase_keeper.auth.oidc import ProviderUnavailableError
+from lorebase_keeper.auth.runtime import AuthRuntime
+from lorebase_keeper.auth.sessions import (
+    CALLBACK_PATH,
+    LOGIN_PATH,
+    LOGOUT_PATH,
+    LoginError,
+    LoginErrorCode,
+    SessionManager,
+)
+from lorebase_keeper.errors import (
+    BAD_REQUEST,
+    FORBIDDEN,
+    NOT_FOUND,
+    UNAUTHORIZED,
+    UNAVAILABLE,
+    Problem,
+    ProblemError,
+)
+from lorebase_store.auth import NewAuditEntry
+
+logger = logging.getLogger("lorebase.keeper.auth.sessions")
+
+router = APIRouter(tags=["auth"])
+
+_NO_STORE = {"Cache-Control": "no-store"}
+_CALLBACK_PARAMS = ("code", "state", "error")
+_DISABLED: dict[int | str, dict[str, Any]] = {
+    404: {"model": Problem, "description": "Web sign-in is not enabled."}
+}
+
+
+class SessionInfo(BaseModel):
+    """The browser session of the caller."""
+
+    principal_id: str
+    display_name: str
+    csrf_header: str = Field(description="Header to send the CSRF token in on unsafe requests.")
+    csrf_token: str = Field(description="CSRF token of this session.")
+    expires_at: datetime = Field(description="End of the session's absolute lifetime.")
+    idle_timeout_s: int = Field(description="The session ends after this long without use.")
+
+
+class LogoutResult(BaseModel):
+    """Where to send the browser to sign out at the identity provider too."""
+
+    logout_url: str | None = Field(
+        description="The provider's end-session URL; null when it offers none or no session "
+        "was open."
+    )
+
+
+def _manager(auth: AuthRuntime) -> SessionManager:
+    if auth.sessions is None:
+        raise ProblemError(
+            NOT_FOUND, "Web sign-in is not enabled (keeper.auth.sessions.enabled is false)."
+        )
+    return auth.sessions
+
+
+def _set_cookie(
+    response: Response,
+    settings: SessionSettings,
+    name: str,
+    value: str,
+    *,
+    max_age: int,
+    same_site: Literal["lax", "strict"],
+) -> None:
+    response.set_cookie(
+        name,
+        value,
+        max_age=max_age,
+        path="/",
+        secure=settings.cookie_secure,
+        httponly=True,
+        samesite=same_site,
+    )
+
+
+def _clear_cookie(response: Response, settings: SessionSettings, name: str) -> None:
+    response.delete_cookie(
+        name, path="/", secure=settings.cookie_secure, httponly=True, samesite="lax"
+    )
+
+
+def _app_location(settings: SessionSettings, path: str) -> str:
+    return url_origin(settings.app_url or "") + path
+
+
+def _error_redirect(settings: SessionSettings, code: LoginErrorCode) -> RedirectResponse:
+    separator = "&" if "?" in settings.error_path else "?"
+    location = _app_location(settings, settings.error_path) + separator + urlencode({"error": code})
+    response = RedirectResponse(location, status_code=302, headers=_NO_STORE)
+    _clear_cookie(response, settings, settings.secure_login_cookie_name)
+    return response
+
+
+@router.get(
+    LOGIN_PATH,
+    summary="Start a browser sign-in",
+    status_code=302,
+    response_class=RedirectResponse,
+    responses={
+        302: {"description": "Redirect to the identity provider."},
+        400: {"model": Problem, "description": "return_to is not a path of the web app."},
+        **_DISABLED,
+        503: {"model": Problem, "description": "The identity provider is unavailable."},
+    },
+)
+async def login(
+    auth: AuthDep,
+    return_to: Annotated[
+        str | None,
+        Query(
+            max_length=2048,
+            description="Path on the web app's origin to return to after signing in.",
+        ),
+    ] = None,
+) -> RedirectResponse:
+    """Send the browser to the identity provider (authorization code flow with PKCE). The
+    sign-in state travels in a short-lived, encrypted, HttpOnly cookie."""
+    sessions = _manager(auth)
+    settings = sessions.settings
+    path = return_to or urlsplit(settings.app_url or "").path or "/"
+    if not is_local_path(path):
+        raise ProblemError(BAD_REQUEST, "return_to must be a path on the web app's origin.")
+    try:
+        start = await sessions.begin_login(path)
+    except ProviderUnavailableError:
+        raise ProblemError(UNAVAILABLE, "The identity provider is unavailable.") from None
+    response = RedirectResponse(start.url, status_code=302, headers=_NO_STORE)
+    # Lax whatever `same_site` says: the provider's redirect back is a cross-site navigation.
+    _set_cookie(
+        response,
+        settings,
+        settings.secure_login_cookie_name,
+        start.state_cookie,
+        max_age=settings.login_timeout_s,
+        same_site="lax",
+    )
+    return response
+
+
+@router.get(
+    CALLBACK_PATH,
+    summary="Finish a browser sign-in",
+    status_code=302,
+    response_class=RedirectResponse,
+    responses={
+        302: {
+            "description": "Signed in: redirect to the return path with the session cookie. "
+            "Failed: redirect to the app's error path with ?error=<code>."
+        },
+        **_DISABLED,
+    },
+)
+async def callback(request: Request, auth: AuthDep) -> RedirectResponse:
+    """The provider's redirect after sign-in. Checks state, redeems the code with the PKCE
+    verifier, verifies the ID token and its nonce, maps the user to a principal as for bearer
+    tokens, and starts a session with a new id."""
+    sessions = _manager(auth)
+    settings = sessions.settings
+    query = request.query_params
+    try:
+        if any(len(query.getlist(name)) > 1 for name in _CALLBACK_PARAMS):
+            raise LoginError("login_failed", "repeated callback parameter")
+        params = {name: query[name] for name in _CALLBACK_PARAMS if name in query}
+        result = await sessions.finish_login(
+            params, request.cookies.get(settings.secure_login_cookie_name)
+        )
+        try:
+            principal = await auth.authenticator.user_principal(result.claims)
+        except ProblemError as exc:
+            code: LoginErrorCode = "no_account" if exc.problem is FORBIDDEN else "login_failed"
+            raise LoginError(code, exc.problem_detail or exc.problem.code) from None
+    except LoginError as exc:
+        logger.info("sign-in failed (%s): %s", exc.code, exc)
+        return _error_redirect(settings, exc.code)
+    created = await sessions.create(
+        principal.id, result, replaces=request.cookies.get(settings.secure_cookie_name)
+    )
+    logger.info("session started", extra={"principal": principal.id})
+    annotate(request, actor=principal.id)
+    if auth.audit_writer is not None:
+        await auth.audit_writer.write(
+            NewAuditEntry(
+                actor=principal.id,
+                action="auth.session.create",
+                details={"outcome": "success", "method": "GET", "route": CALLBACK_PATH},
+            )
+        )
+    response = RedirectResponse(
+        _app_location(settings, result.return_to), status_code=302, headers=_NO_STORE
+    )
+    _set_cookie(
+        response,
+        settings,
+        settings.secure_cookie_name,
+        created.session_id,
+        max_age=settings.absolute_timeout_s,
+        same_site=settings.same_site,
+    )
+    _clear_cookie(response, settings, settings.secure_login_cookie_name)
+    return response
+
+
+@router.get(
+    "/auth/session",
+    summary="Describe the browser session",
+    responses={
+        401: {"model": Problem, "description": "No valid session cookie."},
+        **_DISABLED,
+    },
+)
+async def session_info(
+    request: Request, response: Response, principal: PrincipalDep, auth: AuthDep
+) -> SessionInfo:
+    """Who is signed in, until when, and the CSRF token the web app sends with unsafe
+    requests. Requests authenticated by an API key or bearer token get 401."""
+    sessions = _manager(auth)
+    session_id = auth.authenticator.session_id(request.headers)
+    if principal.method != "session" or session_id is None or not principal.session_expires_at:
+        raise ProblemError(UNAUTHORIZED, "No browser session: sign in through /auth/login.")
+    response.headers.update(_NO_STORE)
+    return SessionInfo(
+        principal_id=principal.id,
+        display_name=principal.display_name,
+        csrf_header=sessions.settings.csrf_header,
+        csrf_token=sessions.csrf_token(session_id),
+        expires_at=principal.session_expires_at,
+        idle_timeout_s=sessions.settings.idle_timeout_s,
+    )
+
+
+@router.post(
+    LOGOUT_PATH,
+    summary="Sign out of the browser session",
+    response_model=LogoutResult,
+    responses={
+        403: {"model": Problem, "description": "CSRF check failed."},
+        **_DISABLED,
+    },
+)
+async def logout(request: Request, auth: AuthDep) -> Response:
+    """End the session of the cookie (CSRF token required), clear the cookie and return the
+    provider's end-session URL for RP-initiated logout. Without a session it only clears the
+    cookie."""
+    sessions = _manager(auth)
+    settings = sessions.settings
+    session_id = auth.authenticator.session_id(request.headers)
+    logout_url: str | None = None
+    if session_id is not None:
+        sessions.check_request(session_id, request.method, request.headers)
+        ended = await sessions.end(session_id)
+        if ended.principal_id is not None:
+            annotate(request, actor=ended.principal_id, action="auth.session.delete")
+            logger.info("session ended by sign-out", extra={"principal": ended.principal_id})
+        logout_url = ended.logout_url
+    body = LogoutResult(logout_url=logout_url).model_dump_json()
+    response = Response(body, media_type="application/json", headers=_NO_STORE)
+    _clear_cookie(response, settings, settings.secure_cookie_name)
+    return response