lorebase-store 0.2.3__py3-none-any.whl

lorebase_store/commit/schema.py

@@ -0,0 +1,123 @@
+"""The `schema` partition: schema elements keyed by id, the Domain element keyed by Domain id.
+
+The materialized head keeps the compiled schema (`Schema.content()`) next to the schema
+partition hash, so commits read the schema with one row.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping, Sequence
+from typing import Any
+
+from lorebase_core.canonical import ObjectType
+from lorebase_core.model import (
+    Derivation,
+    Domain,
+    Kind,
+    Operation,
+    Relation,
+    Schema,
+    Source,
+    Trait,
+)
+from lorebase_store.commit.changes import SchemaChange, SchemaDelete, SchemaElementModel
+from lorebase_store.commit.errors import SchemaChangeError
+
+ELEMENT_MODELS: Mapping[ObjectType, type[SchemaElementModel]] = {
+    ObjectType.SCHEMA_DOMAIN: Domain,
+    ObjectType.SCHEMA_KIND: Kind,
+    ObjectType.SCHEMA_RELATION: Relation,
+    ObjectType.SCHEMA_TRAIT: Trait,
+    ObjectType.SCHEMA_OPERATION: Operation,
+    ObjectType.SCHEMA_DERIVATION: Derivation,
+    ObjectType.SCHEMA_SOURCE: Source,
+}
+
+_COLLECTIONS: Mapping[ObjectType, str] = {
+    ObjectType.SCHEMA_KIND: "kinds",
+    ObjectType.SCHEMA_RELATION: "relations",
+    ObjectType.SCHEMA_TRAIT: "traits",
+    ObjectType.SCHEMA_OPERATION: "operations",
+    ObjectType.SCHEMA_DERIVATION: "derivations",
+    ObjectType.SCHEMA_SOURCE: "sources",
+}
+
+
+def object_type_of(element: SchemaElementModel) -> ObjectType:
+    return ObjectType(element.object_type)
+
+
+def element_key(element: SchemaElementModel) -> str:
+    """Key of an element in the `schema` partition."""
+    return element.id
+
+
+def schema_elements(schema: Schema) -> dict[str, SchemaElementModel]:
+    """Every element of a Schema by partition key, the Domain included."""
+    elements: dict[str, SchemaElementModel] = {schema.domain.id: schema.domain}
+    for element in schema.elements():
+        elements[element.id] = element  # type: ignore[assignment]
+    return elements
+
+
+def schema_from_elements(elements: Iterable[SchemaElementModel]) -> Schema:
+    """Assemble a Schema; exactly one Domain element is required."""
+    domain: Domain | None = None
+    collections: dict[str, dict[str, Any]] = {name: {} for name in _COLLECTIONS.values()}
+    for element in elements:
+        if isinstance(element, Domain):
+            if domain is not None:
+                raise SchemaChangeError("a schema holds exactly one Domain element")
+            domain = element
+            continue
+        collections[_COLLECTIONS[object_type_of(element)]][element.id] = element
+    if domain is None:
+        raise SchemaChangeError("a schema needs its Domain element")
+    return Schema(domain=domain, **collections)
+
+
+def element_from_body(object_type: ObjectType, body: Mapping[str, Any]) -> SchemaElementModel:
+    """Parse a stored schema object."""
+    return ELEMENT_MODELS[object_type].model_validate(body)
+
+
+def compile_schema(schema: Schema) -> dict[str, Any]:
+    """Document stored in `mat_schema.compiled`."""
+    return schema.content()
+
+
+def load_compiled(compiled: Mapping[str, Any]) -> Schema:
+    """Inverse of `compile_schema`."""
+    return Schema.model_validate(compiled)
+
+
+def apply_schema_changes(
+    schema: Schema, changes: Sequence[SchemaChange], domain_id: str
+) -> tuple[Schema, list[SchemaElementModel], list[str]]:
+    """New schema plus the upserted elements and the removed element ids."""
+    elements = schema_elements(schema)
+    upserted: list[SchemaElementModel] = []
+    removed: list[str] = []
+    seen: set[str] = set()
+    for change in changes:
+        key = change.element_id if isinstance(change, SchemaDelete) else change.element.id
+        if key in seen:
+            raise SchemaChangeError(f"schema element {key} is changed more than once")
+        seen.add(key)
+        if isinstance(change, SchemaDelete):
+            if key == domain_id:
+                raise SchemaChangeError("the Domain element cannot be removed")
+            if key not in elements:
+                raise SchemaChangeError(f"unknown schema element {key}")
+            del elements[key]
+            removed.append(key)
+            continue
+        element = change.element
+        if isinstance(element, Domain) and element.id != domain_id:
+            raise SchemaChangeError(f"Domain element {element.id} in Domain {domain_id}")
+        current = elements.get(key)
+        if current is not None and object_type_of(current) is not object_type_of(element):
+            raise SchemaChangeError(f"schema element {key} cannot change its type")
+        elements[key] = element
+        upserted.append(element)
+    return schema_from_elements(elements.values()), upserted, removed

lorebase_store/db.py

@@ -0,0 +1,58 @@
+"""Async engine factory built from the `store` settings."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from sqlalchemy import URL, make_url
+from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
+from sqlalchemy.pool import NullPool
+
+from lorebase_store.settings import DatabaseSettings
+
+ASYNC_DRIVER = "postgresql+asyncpg"
+_POSTGRES_SCHEMES = frozenset({"postgres", "postgresql", ASYNC_DRIVER})
+
+
+def async_url(url: str | URL) -> URL:
+    """Parse a Postgres URL and switch it to the asyncpg driver."""
+    parsed = make_url(url)
+    if parsed.drivername not in _POSTGRES_SCHEMES:
+        raise ValueError(f"unsupported database driver {parsed.drivername!r}; use postgresql://")
+    return parsed.set(drivername=ASYNC_DRIVER)
+
+
+def create_engine(
+    database: DatabaseSettings,
+    *,
+    url: str | URL | None = None,
+    pooled: bool = True,
+    statement_timeout_ms: int | None = None,
+) -> AsyncEngine:
+    """Engine for `database`; `url` overrides the configured DSN, `pooled=False` uses no pool."""
+    timeout = (
+        database.statement_timeout_ms if statement_timeout_ms is None else statement_timeout_ms
+    )
+    connect_args: dict[str, Any] = {
+        "server_settings": {
+            "application_name": database.application_name,
+            "statement_timeout": str(timeout),
+            # `indexes.sql_literal` writes standard literals, where a backslash is plain text.
+            "standard_conforming_strings": "on",
+        },
+        # Statements are prepared once per connection and SQL text, then reused.
+        "prepared_statement_cache_size": database.prepared_statement_cache_size,
+    }
+    pool_args: dict[str, Any] = (
+        {
+            "pool_size": database.pool_size,
+            "max_overflow": database.max_overflow,
+            "pool_timeout": database.pool_timeout_s,
+            "pool_recycle": database.pool_recycle_s,
+            "pool_pre_ping": True,
+        }
+        if pooled
+        else {"poolclass": NullPool}
+    )
+    target = async_url(url if url is not None else database.dsn.get_secret_value())
+    return create_async_engine(target, connect_args=connect_args, **pool_args)

lorebase_store/definer.py

@@ -0,0 +1,50 @@
+"""Grants on the security definer read functions of revisions 0005, 0010, 0018, 0035 and 0037
+(store schema notes).
+
+The migration revokes EXECUTE from PUBLIC and grants it to `database.application_role`.
+`grant_read_functions` grants it to another role later: one created after the migration, or a
+new application role.
+"""
+
+from __future__ import annotations
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncConnection
+
+READ_FUNCTIONS = (
+    "lb_search",
+    "lb_links_in",
+    "lb_neighborhood",
+    "lb_query_by_attribute",
+    "lb_search_overlay",
+    "lb_links_in_overlay",
+    "lb_neighborhood_overlay",
+    "lb_query_by_attribute_overlay",
+    "lb_search_corrections",
+    "lb_search_corrected",
+    "lb_search_corrected_overlay",
+    "lb_search_bounded",
+    "lb_search_bounded_overlay",
+)
+"""Functions only the store read primitives call, with a Domain already authorized."""
+BM25_FUNCTIONS = ("lb_search_bm25", "lb_search_bm25_overlay")
+"""The BM25 search of revision 0037, called the same way; they reach the BM25 index only
+where `pg_search` is installed (`lorebase_store.bm25`)."""
+
+_SIGNATURES = text(
+    "SELECT CAST(CAST(p.oid AS regprocedure) AS text) FROM pg_proc p "
+    "WHERE p.proname = ANY(CAST(:names AS text[])) "
+    "AND p.pronamespace = CAST(current_schema() AS regnamespace) ORDER BY 1"
+)
+_QUOTED = text("SELECT quote_ident(:role)")
+
+
+async def grant_read_functions(connection: AsyncConnection, role: str) -> list[str]:
+    """Grant EXECUTE on the read functions to `role`; returns the signatures granted."""
+    quoted = (await connection.execute(_QUOTED, {"role": role})).scalar_one()
+    names = [*READ_FUNCTIONS, *BM25_FUNCTIONS]
+    found = await connection.execute(_SIGNATURES, {"names": names})
+    signatures = list(found.scalars())
+    for signature in signatures:
+        await connection.exec_driver_sql(f"GRANT EXECUTE ON FUNCTION {signature} TO {quoted}")
+    return signatures

lorebase_store/digests.py

@@ -0,0 +1,99 @@
+"""Keyed digests of inputs that may hold personal data (design 13.5).
+
+Lorebase stores digests of some inputs it must not keep: the parameters of Operation invocations
+(idempotency, provenance) and the Record keys named by erasures in the audit log. A plain hash of
+such an input confirms a guessed value, so every digest is an HMAC-SHA256 under a key of the
+installation, chosen in this order:
+
+1. `vault.digest_key`;
+2. the vault's current token key, when the vault is configured (the digests made before this
+   module existed are unchanged);
+3. the random secret revision `0027` stored in `installation_secrets` (name `digest`): it keeps
+   the digests unconfirmable for whoever reads the API or the audit log, not for whoever reads
+   that table. Configure one of the keys above for that.
+
+The key of case 3 is read from the database once per process (`ready`). In case 2 the
+earlier token keys (`vault.previous_token_keys`) still recognize the digests made under them
+(`all_digests`), so a token key rotation does not break idempotent replays.
+"""
+
+from __future__ import annotations
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from lorebase_store.keyring import Keyring
+from lorebase_store.models import metadata
+from lorebase_store.vault.keys import NAMESPACE, VaultKeys, framed
+from lorebase_store.vault.settings import VaultSettings
+
+SECRET_NAME = "digest"
+_SECRETS = metadata.tables["installation_secrets"]
+
+
+class DigestKeysNotReadyError(RuntimeError):
+    """The installation secret has not been read yet (`DigestKeys.ready`)."""
+
+
+class DigestKeys:
+    """Makes the keyed digests of the installation (module doc)."""
+
+    def __init__(
+        self,
+        keyring: Keyring | None = None,
+        *,
+        source: str = "installation",
+        earlier: tuple[Keyring, ...] = (),
+    ) -> None:
+        self._keyring = keyring
+        self._source = source
+        self._earlier = earlier
+
+    @classmethod
+    def from_settings(cls, settings: VaultSettings, vault: VaultKeys | None) -> DigestKeys:
+        if settings.digest_key is not None:
+            return cls(
+                Keyring(settings.digest_key.get_secret_value(), namespace=NAMESPACE),
+                source="vault.digest_key",
+            )
+        if vault is not None:
+            return cls(
+                vault.digest_keyring(),
+                source="vault.token_key",
+                earlier=vault.earlier_digest_keyrings(),
+            )
+        return cls()
+
+    @property
+    def source(self) -> str:
+        """Where the key comes from: `vault.digest_key`, `vault.token_key` or `installation`."""
+        return self._source
+
+    async def ready(self, session: AsyncSession) -> DigestKeys:
+        """These keys, after reading the installation secret if they need it."""
+        if self._keyring is None:
+            secret = (
+                await session.execute(
+                    select(_SECRETS.c.secret).where(_SECRETS.c.name == SECRET_NAME)
+                )
+            ).scalar_one_or_none()
+            if secret is None:
+                raise DigestKeysNotReadyError(
+                    "installation_secrets has no digest secret: run the migrations (revision "
+                    "0027) or set vault.digest_key"
+                )
+            self._keyring = Keyring(secret, namespace=NAMESPACE)
+        return self
+
+    def digest(self, domain_id: str, purpose: str, payload: bytes) -> str:
+        """Keyed digest (hex) of `payload` for `purpose` in a Domain."""
+        if self._keyring is None:
+            raise DigestKeysNotReadyError("call DigestKeys.ready first")
+        return self._keyring.mac("digest/" + purpose, framed(domain_id, payload)).hex()
+
+    def all_digests(self, domain_id: str, purpose: str, payload: bytes) -> frozenset[str]:
+        """The digest under the current key and under the earlier token keys."""
+        found = {self.digest(domain_id, purpose, payload)}
+        for keyring in self._earlier:
+            found.add(keyring.mac("digest/" + purpose, framed(domain_id, payload)).hex())
+        return frozenset(found)