lorebase-keeper 0.2.3__py3-none-any.whl

lorebase_keeper/services/derivation_preview.py

@@ -0,0 +1,279 @@
+"""Preview of a candidate Derivation on sample Records (design 5.8), without writing.
+
+The candidate is a schema change set (`services.schema.ChangeSetInput`) naming the Derivation
+to evaluate: the schema it produces is validated with the rules of schema changes
+(`schema.expression_limits` included), then the Derivation is evaluated on a few Records of its
+Kind at the branch head, from the stored and derived values of the branch, with the evaluator
+the materializer uses (`lorebase_store.materialize.derived`). Nothing is stored.
+
+What the caller may see follows Record reads: the Kinds the Derivation touches (its own, the
+far Kind of an aggregate, the Kinds reached through `links.<relation>`) must be readable, and
+the value is masked when the computed Attribute is sensitive in the schema read, in the default
+branch's or in the candidate, which carries the tags of everything the Derivation may read
+(`lorebase_core.expr.sensitivity`), unless the caller is cleared for each tag on the
+Derivation's Kind and on every Kind the tag comes from (`tag_origins`: a value computed from a
+sensitive Attribute of a far Kind tells about its Records). A masked value only says whether
+it failed: no error code, no message.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+
+from pydantic import JsonValue
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from lorebase_core.expr import (
+    CompiledDerivation,
+    ExprLimits,
+    compile_derivation,
+    derivation_reads,
+    links_read,
+)
+from lorebase_core.expr.sensitivity import schema_tags, tag_origins
+from lorebase_core.expr.visibility import element_ids
+from lorebase_core.model import (
+    AggregateDerivation,
+    Attribute,
+    Derivation,
+    ExpressionDerivation,
+    Kind,
+    LinkDirection,
+    Schema,
+    Sensitivity,
+)
+from lorebase_core.validate import SchemaRules
+from lorebase_keeper.auth.read_audit import note_sensitive
+from lorebase_keeper.reading.points import ReadPoint
+from lorebase_keeper.reading.runtime import ReadRuntime
+from lorebase_keeper.services.errors import Finding
+from lorebase_keeper.services.schema import (
+    ChangeSetInput,
+    ChangeSetInvalidError,
+    static_issues,
+    target_schema,
+)
+from lorebase_store.materialize.derived import DerivedValues
+from lorebase_store.repositories import MaterializedRepository
+
+DERIVATION_NOT_FOUND = "derivation.not_found"
+DERIVATION_UNSUPPORTED = "derivation.unsupported"
+DERIVATION_ID_UNAVAILABLE = "derivation.id_unavailable"
+
+
+class CandidateInvalidError(Exception):
+    """The candidate cannot be evaluated; `issues` locate why."""
+
+    def __init__(self, issues: Sequence[Finding]) -> None:
+        super().__init__(f"{len(issues)} issue(s)")
+        self.issues = list(issues)
+
+
+@dataclass(frozen=True)
+class Candidate:
+    """A validated candidate: the schema the change set asks for and the Derivation in it."""
+
+    schema: Schema
+    derivation: Derivation
+    kind: Kind
+    attribute: Attribute
+    compiled: CompiledDerivation
+    limits: ExprLimits
+
+    @property
+    def touched(self) -> frozenset[str]:
+        """Ids of the Kinds the Derivation reads Records of, its own included."""
+        return touched_kinds(self.schema, self.derivation, self.compiled, self.limits)
+
+
+@dataclass(frozen=True)
+class PreviewItem:
+    """One sample Record: its key and title, and the value or the error masked for the caller.
+
+    A masked item carries the masked value (none with `keeper.reads.sensitivity.omit`) and
+    whether the value failed (`failed`), without the error code or message.
+    """
+
+    key: str
+    title: str | None
+    value: JsonValue
+    masked: bool
+    failed: bool = False
+    error_code: str | None = None
+    error_message: str | None = None
+
+
+def _issue(code: str, message: str, path: tuple[str | int, ...]) -> Finding:
+    return Finding(code, message, path)
+
+
+def candidate_of(
+    current: Schema, change: ChangeSetInput, name: str, rules: SchemaRules
+) -> Candidate:
+    """The Derivation `name` of the schema `change` asks for, once the schema is valid.
+
+    Raises `CandidateInvalidError` when the change set cannot be read, fails static validation
+    (the issues of a schema change) or `name` is not an `expression` or `aggregate` Derivation
+    of it. Path Derivations compose Links and compute no values.
+    """
+    try:
+        schema = target_schema(current, change)
+    except ChangeSetInvalidError as invalid:
+        raise CandidateInvalidError(invalid.issues) from None
+    issues = static_issues(schema, rules)
+    if issues:
+        raise CandidateInvalidError(issues)
+    derivation = schema.derivation_by_api_name(name)
+    if derivation is None:
+        raise CandidateInvalidError(
+            [
+                _issue(
+                    DERIVATION_NOT_FOUND, f"No Derivation {name!r} in the schema.", ("derivations",)
+                )
+            ]
+        )
+    definition = derivation.definition
+    if not isinstance(definition, ExpressionDerivation | AggregateDerivation):
+        raise CandidateInvalidError(
+            [
+                _issue(
+                    DERIVATION_UNSUPPORTED,
+                    "Only expression and aggregate Derivations compute values; a path "
+                    "Derivation composes Links.",
+                    ("derivations", name, "definition", "variant"),
+                )
+            ]
+        )
+    kind = schema.kinds[definition.kind]
+    attribute = kind.attributes[definition.attribute]
+    limits = rules.expression_limits or ExprLimits()
+    compiled = compile_derivation(schema, derivation, limits)
+    return Candidate(schema, derivation, kind, attribute, compiled, limits)
+
+
+def require_new_ids(candidate: Candidate, hidden: frozenset[str]) -> None:
+    """Refuse a candidate built on a projected schema (`ReadPoint.hidden`) that reuses an id
+    the projection left out: Records and values stored under it are not the caller's to read
+    (design 13.3)."""
+    reused = sorted(element_ids(candidate.schema) & hidden)
+    if reused:
+        raise CandidateInvalidError(
+            [_issue(DERIVATION_ID_UNAVAILABLE, f"Id {reused[0]} cannot be used here.", ())]
+        )
+
+
+def touched_kinds(
+    schema: Schema,
+    derivation: Derivation,
+    compiled: CompiledDerivation,
+    limits: ExprLimits | None = None,
+) -> frozenset[str]:
+    """Ids of the Kinds whose Records the Derivation reads: its own, the Kinds of the
+    Attributes it may read and the far Kinds of the Relations it follows (every Kind a path
+    Derivation crosses)."""
+    definition = derivation.definition
+    if not isinstance(definition, ExpressionDerivation | AggregateDerivation):
+        return frozenset()
+    paths = schema.path_relations()
+    kinds = {definition.kind}
+    kinds.update(kind_id for kind_id, _ in derivation_reads(schema, derivation, limits))
+    followed: list[tuple[str, LinkDirection]] = []
+    if isinstance(definition, AggregateDerivation):
+        followed.append((definition.relation, definition.direction))
+    for program in (compiled.expression, compiled.filter):
+        if program is not None:
+            followed.extend((f.relation.id, f.direction) for f in links_read(program))
+    for relation_id, direction in followed:
+        relation = schema.link_relation(relation_id)
+        if relation is not None:
+            kinds.add(relation.far_endpoint(direction))
+        if relation_id in paths:
+            kinds.update(paths[relation_id].kinds)
+    return frozenset(kinds)
+
+
+def kind_names(schema: Schema, kind_ids: frozenset[str]) -> list[str]:
+    """Api names of Kinds, sorted."""
+    return sorted(schema.kinds[k].api_name for k in kind_ids if k in schema.kinds)
+
+
+async def preview(
+    session: AsyncSession,
+    runtime: ReadRuntime,
+    point: ReadPoint,
+    candidate: Candidate,
+    *,
+    keys: Sequence[str] | None,
+    limit: int,
+) -> list[PreviewItem]:
+    """Evaluate the candidate on `keys` (the first `limit` Records of its Kind by key when
+    `None`), at the branch head `point` names, in the read transaction of `session`.
+
+    The caller has checked the Kinds the candidate touches against the point's scope.
+    """
+    store = runtime.store
+    repository = MaterializedRepository(
+        session,
+        point.domain.id,
+        point.branch,
+        writes=store.settings.writes,
+        layout=point.layout,
+    )
+    materialize = store.settings.materialize
+    values = DerivedValues(
+        repository,
+        candidate.schema,
+        limits=store.schema.expression_limits,
+        now=point.revision.created_at,
+        batch=materialize.read_batch_rows,
+        max_incremental=materialize.derivation_max_incremental_records,
+        fanout=store.settings.traverse.path_fanout,
+    )
+    masking = point.masking.over(point.schema, candidate.schema)
+    origins = tag_origins(
+        candidate.schema,
+        candidate.attribute.id,
+        _united(masking.reference, schema_tags(point.schema)),
+        candidate.limits,
+    )
+    masked = masking.hidden_with_origins(candidate.attribute, (candidate.kind.id,), origins)
+    items: list[PreviewItem] = []
+    for found in await values.preview(candidate.attribute.id, keys=keys, limit=limit):
+        title = masking.title(candidate.schema, candidate.kind.id, found.title)
+        if masked:
+            note_sensitive("masked")
+            items.append(
+                PreviewItem(
+                    found.key,
+                    title,
+                    None if masking.settings.omit else masking.masked,
+                    True,
+                    found.error is not None,
+                )
+            )
+            continue
+        if masking.sensitive(candidate.attribute):
+            note_sensitive("shown")
+        error = found.error
+        items.append(
+            PreviewItem(
+                found.key,
+                title,
+                found.value,
+                False,
+                error is not None,
+                None if error is None else error.code.value,
+                None if error is None else error.describe(),
+            )
+        )
+    return items
+
+
+def _united(*tag_maps: Mapping[str, frozenset[Sensitivity]]) -> dict[str, frozenset[Sensitivity]]:
+    """Attribute id -> tags, united over `tag_maps`."""
+    united: dict[str, frozenset[Sensitivity]] = {}
+    for tags in tag_maps:
+        for attribute_id, found in tags.items():
+            united[attribute_id] = united.get(attribute_id, frozenset()) | found
+    return united

lorebase_keeper/services/document_urls.py

@@ -0,0 +1,114 @@
+"""Document intake by URL: the guarded fetch behind `POST /domains/{d}/documents` with a URL.
+
+The URL needs a scheme in `keeper.documents.url_schemes` and a host in `url_hosts` (exact
+match). The host is then resolved once and every address must be global unicast, the check of
+webhook targets and HTTP JSON Sources (`lorebase_store.webhooks.delivery.check_target`), unless
+`url_allow_private_targets`. The request connects to those checked addresses only, never to a
+new lookup, so a DNS answer that changes after the check (rebinding) cannot reach an internal
+service; the `Host` header and the TLS server name keep the host name. Redirects are never
+followed, proxies from the environment are ignored and the body is read in bounded steps.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import httpx
+
+from lorebase_keeper.services.config import DocumentSettings
+from lorebase_store.webhooks import WebhookSettings
+from lorebase_store.webhooks.delivery import Resolver, check_target, resolve_host
+
+
+class UrlRefusedError(ValueError):
+    """The URL may not be fetched (scheme, host or the addresses it resolves to)."""
+
+
+class UrlUnavailableError(RuntimeError):
+    """The URL could not be fetched now (resolution, connection or an answer other than 200)."""
+
+
+class DocumentTooLargeError(ValueError):
+    """The body exceeds `keeper.documents.max_bytes`."""
+
+
+def pin(url: httpx.URL, address: str) -> tuple[httpx.URL, dict[str, str], dict[str, Any]]:
+    """`url` rewritten to connect to the checked `address`, with the `Host` header and request
+    extensions that keep the host name for the server and for TLS (as Scribe's webhook sender
+    and Source connectors do)."""
+    return (
+        url.copy_with(host=address.split("%", 1)[0]),
+        {"Host": url.netloc.decode("ascii")},
+        {"sni_hostname": url.host} if url.scheme == "https" else {},
+    )
+
+
+async def checked_addresses(
+    url: httpx.URL, settings: DocumentSettings, resolve: Resolver
+) -> tuple[str, ...]:
+    """Addresses a fetch of `url` may connect to; empty when private targets are allowed (the
+    client then resolves the host itself)."""
+    if url.scheme not in settings.url_schemes:
+        raise UrlRefusedError(f"scheme must be one of {', '.join(settings.url_schemes)}")
+    allowed = {host.lower() for host in settings.url_hosts}
+    if not url.host or url.host.lower() not in allowed:
+        raise UrlRefusedError("the host is not in keeper.documents.url_hosts")
+    checks = WebhookSettings(
+        allowed_schemes=settings.url_schemes,
+        allow_private_targets=settings.url_allow_private_targets,
+    )
+    target = await check_target(str(url), checks, resolve)
+    if target.problem is None:
+        return target.addresses
+    if target.problem.startswith("cannot resolve"):
+        raise UrlUnavailableError("the URL host cannot be resolved")
+    # `check_target` problems never contain the host name or its addresses.
+    raise UrlRefusedError(
+        f"the URL host {target.problem}, refused unless keeper.documents.url_allow_private_targets"
+    )
+
+
+async def fetch(
+    url: str,
+    settings: DocumentSettings,
+    *,
+    resolve: Resolver = resolve_host,
+    transport: httpx.AsyncBaseTransport | None = None,
+) -> bytes:
+    """The body of a 200 answer to GET `url`, at most `settings.max_bytes` long."""
+    target = httpx.URL(url)
+    addresses = await checked_addresses(target, settings, resolve)
+    attempts = [pin(target, address) for address in addresses] or [(target, {}, {})]
+    async with httpx.AsyncClient(
+        follow_redirects=False,
+        timeout=settings.url_timeout_s,
+        trust_env=False,
+        transport=transport,
+    ) as client:
+        failure: httpx.HTTPError | None = None
+        for pinned, headers, extensions in attempts:
+            try:
+                return await _get(client, pinned, headers, extensions, settings.max_bytes)
+            except (httpx.ConnectError, httpx.ConnectTimeout) as error:
+                failure = error  # the next checked address, never a new lookup
+            except httpx.HTTPError as error:
+                raise UrlUnavailableError(f"the URL could not be fetched: {error}") from None
+    raise UrlUnavailableError(f"the URL could not be fetched: {failure}")
+
+
+async def _get(
+    client: httpx.AsyncClient,
+    url: httpx.URL,
+    headers: dict[str, str],
+    extensions: dict[str, Any],
+    limit: int,
+) -> bytes:
+    data = bytearray()
+    async with client.stream("GET", url, headers=headers, extensions=extensions) as response:
+        if response.status_code != 200:
+            raise UrlUnavailableError(f"the URL answered {response.status_code}")
+        async for block in response.aiter_bytes():
+            data.extend(block)
+            if len(data) > limit:
+                raise DocumentTooLargeError(f"the document exceeds {limit} bytes")
+    return bytes(data)

lorebase_keeper/services/domains.py

@@ -0,0 +1,118 @@
+"""Domain use cases: create (root Revision and default branch), read, update, delete."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from sqlalchemy.exc import IntegrityError
+
+from lorebase_core.model import Domain
+from lorebase_keeper.auth.principals import Principal
+from lorebase_keeper.context import StoreAccess, author_of, require_domain
+from lorebase_keeper.errors import NOT_FOUND, ProblemError
+from lorebase_keeper.services.config import DomainSettings
+from lorebase_keeper.services.errors import ALREADY_EXISTS
+from lorebase_store.auth import GrantRepository
+from lorebase_store.commit import create_domain
+from lorebase_store.repositories import DomainRepository, DomainRow, RefRepository
+
+
+@dataclass(frozen=True)
+class CreatedDomain:
+    """A Domain and the head of its default branch; `replayed` when it already existed."""
+
+    domain: DomainRow
+    head: bytes
+    replayed: bool = False
+
+
+async def create(
+    store: StoreAccess,
+    domain: Domain,
+    principal: Principal,
+    settings: DomainSettings,
+    *,
+    idempotency_key: str | None,
+) -> CreatedDomain:
+    """Create a Domain: row, root Revision, protected default branch and the creator's grant.
+
+    With an idempotency key, creating a Domain that the same principal already created
+    returns it (`replayed`); otherwise an existing id is a conflict.
+    """
+    async with store.sessions() as session:
+        existing = await DomainRepository(session).get(domain.id)
+    if existing is not None:
+        return await _existing(store, existing, principal, idempotency_key)
+    try:
+        async with store.sessions() as session, session.begin():
+            result = await create_domain(
+                session,
+                domain,
+                author=author_of(principal),
+                leaf_size=settings.leaf_size,
+                runtime=store.runtime,
+                clock=store.clock,
+            )
+            if not settings.protect_default_branch:
+                await RefRepository(session, domain.id).set_protected(domain.default_branch, False)
+            if settings.creator_role and principal.method != "none":
+                await GrantRepository(session).add(
+                    domain_id=domain.id,
+                    subject=principal.id,
+                    role=settings.creator_role,
+                    created_by=principal.id,
+                )
+            row = await DomainRepository(session).require(domain.id)
+    except IntegrityError:
+        async with store.sessions() as session:
+            found = await DomainRepository(session).get(domain.id)
+        if found is None:
+            raise
+        return await _existing(store, found, principal, idempotency_key)
+    return CreatedDomain(domain=row, head=result.revision_id)
+
+
+async def _existing(
+    store: StoreAccess, row: DomainRow, principal: Principal, idempotency_key: str | None
+) -> CreatedDomain:
+    if idempotency_key is None or row.created_by != principal.id:
+        raise ProblemError(ALREADY_EXISTS, f"Domain {row.id!r} already exists.")
+    async with store.sessions() as session, session.begin():
+        await require_domain(session, row.id)
+        ref = await RefRepository(session, row.id).require(row.default_branch)
+    return CreatedDomain(domain=row, head=ref.head, replayed=True)
+
+
+async def get(store: StoreAccess, domain_id: str) -> tuple[DomainRow, bytes | None]:
+    """The Domain and the head of its default branch."""
+    async with store.sessions() as session, session.begin():
+        domain = await require_domain(session, domain_id)
+        ref = await RefRepository(session, domain_id).get(domain.default_branch)
+    return domain, None if ref is None else ref.head
+
+
+async def page(store: StoreAccess, *, limit: int, after: str | None) -> list[DomainRow]:
+    """Domains ordered by id."""
+    async with store.sessions() as session:
+        return await DomainRepository(session).list(limit=limit, after=after)
+
+
+async def update(
+    store: StoreAccess, domain_id: str, *, display_name: str | None, description: str | None
+) -> DomainRow:
+    """Change display metadata (not versioned: the Domain element of the schema is unchanged)."""
+    async with store.sessions() as session, session.begin():
+        await require_domain(session, domain_id)
+        row = await DomainRepository(session).update(
+            domain_id, display_name=display_name, description=description
+        )
+    if row is None:  # pragma: no cover - deleted concurrently
+        raise ProblemError(NOT_FOUND, f"Domain {domain_id!r} not found.")
+    return row
+
+
+async def delete(store: StoreAccess, domain_id: str) -> None:
+    """Delete a Domain and everything scoped to it. The audit log keeps its entries."""
+    async with store.sessions() as session, session.begin():
+        await require_domain(session, domain_id)
+        await DomainRepository(session).delete(domain_id)