lorebase-store 0.2.3__py3-none-any.whl

lorebase_store/vault/keys.py

@@ -0,0 +1,216 @@
+"""Tokens, fingerprints and encryption of vault values (design 13.5).
+
+- **Token** of a value: `vlt_` + the first 128 bits of HMAC-SHA256 (key derived from
+  `vault.token_key`) over the Domain, Kind, Record key, Attribute, generation and the value's
+  canonical JSON. The same value of the same Attribute of the same Record gets the same token,
+  so writing it again changes nothing and merges compare tokens as they would compare values.
+  Two Records holding the same value get different tokens: erasing one data subject never
+  erases another's value. The generation starts at 0 and moves on only when the token of a
+  lower one was erased, so writing an erased value again stores it under a new token and the
+  Revisions that held the erased one keep reading `erased`.
+- **Fingerprint** of a value: HMAC-SHA256 over the Domain, Kind, Attribute and the value's
+  matching form (`lorebase_core.types.matching`: case, compatibility characters and white space
+  ignored), without the Record key; erasure by value finds the rows of every Record holding it,
+  whichever way it was written. Rows written before matching forms existed hold the fingerprint
+  of the canonical value (scheme 1); the rotation job moves them to the current scheme.
+- **Encryption**: AES-256-GCM under a key derived from `vault.encryption_key` (or one of
+  `previous_keys`, by the row's `key_id`), with the Domain and token as associated data, so a
+  ciphertext copied to another row does not open.
+
+Both keys rotate. A new `encryption_key` seals new values; `lorebase_store.vault.rotation`
+seals the stored ones again. A new `token_key` (with its own `token_key_id`, the old one kept in
+`previous_token_keys`) mints new tokens: a value written again first looks for the token it got
+under each earlier key, so unchanged values stay unchanged, and fingerprints are matched under
+every configured token key until the rotation job has made them again under the current one.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+
+from pydantic import JsonValue
+
+from lorebase_core.canonical import jcs_bytes
+from lorebase_core.model.vault import TOKEN_PREFIX
+from lorebase_core.types import matching_form
+from lorebase_store.keyring import Keyring
+from lorebase_store.vault.settings import VaultSettings
+
+NAMESPACE = "vault"
+_TOKEN = "token"
+_FINGERPRINT = "fingerprint"
+_MATCHING = "fingerprint/2"
+FINGERPRINT_SCHEME = 2
+"""Current fingerprint scheme: HMAC of the matching form. Scheme 1 hashed the canonical value."""
+_VALUE = "value"
+_TOKEN_BYTES = 16
+
+
+def framed(*parts: str | bytes) -> bytes:
+    """Length-prefixed concatenation: no two part lists give the same bytes."""
+    out = bytearray()
+    for part in parts:
+        data = part.encode() if isinstance(part, str) else part
+        out += len(data).to_bytes(4, "big") + data
+    return bytes(out)
+
+
+class VaultKeys:
+    """Derives tokens and fingerprints and seals values under the configured keys."""
+
+    def __init__(
+        self,
+        token_key: str,
+        key_id: str,
+        current: str,
+        previous: Mapping[str, str],
+        *,
+        token_key_id: str = "1",
+        previous_token_keys: Mapping[str, str] | None = None,
+    ):
+        self._token_key_id = token_key_id
+        self._token_keys = {token_key_id: Keyring(token_key, namespace=NAMESPACE)}
+        for previous_id, secret in (previous_token_keys or {}).items():
+            self._token_keys[previous_id] = Keyring(secret, namespace=NAMESPACE)
+        self._tokens = self._token_keys[token_key_id]
+        self._key_id = key_id
+        self._keyrings = {key_id: Keyring(current, namespace=NAMESPACE)}
+        for previous_id, secret in previous.items():
+            self._keyrings[previous_id] = Keyring(secret, namespace=NAMESPACE)
+
+    @classmethod
+    def from_settings(cls, settings: VaultSettings) -> VaultKeys | None:
+        """Keys of the settings; None when the vault is not configured."""
+        if settings.token_key is None or settings.encryption_key is None:
+            return None
+        return cls(
+            settings.token_key.get_secret_value(),
+            settings.key_id,
+            settings.encryption_key.get_secret_value(),
+            {k: v.get_secret_value() for k, v in settings.previous_keys.items()},
+            token_key_id=settings.token_key_id,
+            previous_token_keys={
+                k: v.get_secret_value() for k, v in settings.previous_token_keys.items()
+            },
+        )
+
+    @property
+    def key_id(self) -> str:
+        """Id of the key new values are sealed under."""
+        return self._key_id
+
+    @property
+    def key_ids(self) -> frozenset[str]:
+        """Ids of every configured encryption key."""
+        return frozenset(self._keyrings)
+
+    @property
+    def token_key_id(self) -> str:
+        """Id of the key new tokens and fingerprints are made under."""
+        return self._token_key_id
+
+    @property
+    def token_key_ids(self) -> frozenset[str]:
+        """Ids of every configured token key."""
+        return frozenset(self._token_keys)
+
+    @property
+    def fingerprint_key(self) -> str:
+        """What a row stores of how its fingerprint was made: `<token_key_id>/<scheme>`."""
+        return f"{self._token_key_id}/{FINGERPRINT_SCHEME}"
+
+    def token(
+        self,
+        domain_id: str,
+        kind_id: str,
+        key: str,
+        attribute_id: str,
+        value: JsonValue,
+        generation: int = 0,
+    ) -> str:
+        """The token of a value under the current token key."""
+        return _token(self._tokens, domain_id, kind_id, key, attribute_id, value, generation)
+
+    def earlier_tokens(
+        self,
+        domain_id: str,
+        kind_id: str,
+        key: str,
+        attribute_id: str,
+        value: JsonValue,
+        generation: int = 0,
+    ) -> list[str]:
+        """The tokens the value had under the previous token keys."""
+        return [
+            _token(keyring, domain_id, kind_id, key, attribute_id, value, generation)
+            for key_id, keyring in self._token_keys.items()
+            if key_id != self._token_key_id
+        ]
+
+    def fingerprint(
+        self, domain_id: str, kind_id: str, attribute_id: str, value: JsonValue
+    ) -> bytes:
+        """Fingerprint of a canonical value under the current token key and scheme."""
+        return _matching(self._tokens, domain_id, kind_id, attribute_id, value)
+
+    def fingerprints(
+        self, domain_id: str, kind_id: str, attribute_id: str, value: JsonValue
+    ) -> list[bytes]:
+        """Every fingerprint a row holding the value (or a case variant of it, for rows of the
+        current scheme) may carry: each configured token key, current and earlier scheme."""
+        found: list[bytes] = []
+        for keyring in self._token_keys.values():
+            found.append(_matching(keyring, domain_id, kind_id, attribute_id, value))
+            found.append(
+                keyring.mac(
+                    _FINGERPRINT, framed(domain_id, kind_id, attribute_id, jcs_bytes(value))
+                )
+            )
+        return found
+
+    def digest_keyring(self) -> Keyring:
+        """The keyring of the keyed digests when no `vault.digest_key` is set."""
+        return self._tokens
+
+    def earlier_digest_keyrings(self) -> tuple[Keyring, ...]:
+        """The keyrings of the previous token keys, which made earlier digests."""
+        return tuple(
+            keyring for key_id, keyring in self._token_keys.items() if key_id != self._token_key_id
+        )
+
+    def seal(self, domain_id: str, token: str, value: JsonValue) -> bytes:
+        return self._keyrings[self._key_id].seal(
+            _VALUE, jcs_bytes(value), context=framed(domain_id, token)
+        )
+
+    def open(self, domain_id: str, token: str, key_id: str, sealed: bytes) -> bytes | None:
+        """Canonical JSON of the value; None when the key id is not configured or the
+        ciphertext does not open."""
+        keyring = self._keyrings.get(key_id)
+        if keyring is None:
+            return None
+        return keyring.open(_VALUE, sealed, context=framed(domain_id, token))
+
+
+def _token(
+    keyring: Keyring,
+    domain_id: str,
+    kind_id: str,
+    key: str,
+    attribute_id: str,
+    value: JsonValue,
+    generation: int,
+) -> str:
+    message = framed(domain_id, kind_id, key, attribute_id, str(generation), jcs_bytes(value))
+    return TOKEN_PREFIX + keyring.mac(_TOKEN, message)[:_TOKEN_BYTES].hex()
+
+
+def _matching(
+    keyring: Keyring,
+    domain_id: str,
+    kind_id: str,
+    attribute_id: str,
+    value: JsonValue,
+) -> bytes:
+    form = jcs_bytes(matching_form(value))
+    return keyring.mac(_MATCHING, framed(domain_id, kind_id, attribute_id, form))

lorebase_store/vault/quotes.py

@@ -0,0 +1,133 @@
+"""Which texts hold a vaulted value, without opening any (extraction evidence, design 9.2).
+
+Word sequences of a text are encoded as the Attribute's type canonicalizes values and
+fingerprinted like stored values (`VaultKeys.fingerprints`: the matching form, under every
+configured token key and fingerprint scheme, so case, compatibility characters and white space
+do not matter); a text matches when a live vault row of that Kind and Attribute has one of the
+fingerprints, whichever Record holds it. Nothing is decrypted. A value written differently from
+its canonical form in the text (a date spelled out, a reordered name) is not found.
+
+The check fails closed: a text that exceeds its limits counts as a match.
+"""
+
+from __future__ import annotations
+
+import re
+from collections import defaultdict
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from lorebase_core.model import Schema
+from lorebase_core.types import AttributeType, EncodingError, encode_value
+from lorebase_store.vault.keys import VaultKeys
+from lorebase_store.vault.store import VAULT, vault_attributes
+
+_UNQUOTED_TYPES = frozenset(
+    {AttributeType.JSON, AttributeType.ARRAY, AttributeType.GEO_POINT, AttributeType.BOOLEAN}
+)
+_WORD = re.compile(r"\S+")
+_EDGE = " \t\r\n.,;:!?()[]{}<>\"'`*_|"
+_POSSESSIVE = re.compile(r"['\N{RIGHT SINGLE QUOTATION MARK}]s$", re.IGNORECASE)
+_LOOKUP_BATCH = 500
+
+
+@dataclass(frozen=True, slots=True)
+class QuoteLimits:
+    """Bounds of one `quoted_values` call; a text that exceeds one is reported as a match."""
+
+    max_words: int
+    """Longest word sequence of a text compared with the stored values."""
+    max_text_words: int
+    """A text with more words is not checked."""
+    max_text_chars: int
+    """A text with more characters is not checked."""
+    max_candidates: int
+    """Distinct word sequences of all the texts together."""
+
+
+def _candidates(text: str, max_words: int) -> set[str]:
+    """Word sequences of `text` (at most `max_words` words, single-spaced), as written, with
+    the punctuation at their edges stripped, and without a closing possessive."""
+    words = _WORD.findall(text)
+    found: set[str] = set()
+    for start in range(len(words)):
+        for end in range(start + 1, min(len(words), start + max_words) + 1):
+            candidate = " ".join(words[start:end])
+            found.add(candidate)
+            stripped = candidate.strip(_EDGE)
+            if stripped:
+                found.add(stripped)
+                bare = _POSSESSIVE.sub("", stripped)
+                if bare:
+                    found.add(bare)
+    return found
+
+
+async def has_vault_values(session: AsyncSession, domain_id: str) -> bool:
+    """Whether the Domain has a live vault row."""
+    v = VAULT
+    found = await session.scalar(
+        select(v.c.id).where(v.c.domain_id == domain_id, v.c.erased_at.is_(None)).limit(1)
+    )
+    return found is not None
+
+
+async def quoted_values(
+    session: AsyncSession,
+    keys: VaultKeys,
+    domain_id: str,
+    schema: Schema,
+    texts: Sequence[str],
+    limits: QuoteLimits,
+) -> set[int]:
+    """Indexes of the `texts` to withhold: those that hold a live vault value of any vault
+    Attribute of the schema (the whole Domain), and those that could not be checked within
+    `limits`."""
+    vaulted = vault_attributes(schema)
+    hits: set[int] = set()
+    owners: dict[str, set[int]] = defaultdict(set)
+    for index, text in enumerate(texts):
+        if len(text) > limits.max_text_chars or len(text.split()) > limits.max_text_words:
+            hits.add(index)
+            continue
+        found = _candidates(text, limits.max_words)
+        if len(owners) + len(found - owners.keys()) > limits.max_candidates:
+            hits.update(range(index, len(texts)))
+            break
+        for candidate in found:
+            owners[candidate].add(index)
+    if not vaulted or not owners:
+        return hits
+    v = VAULT
+    for kind_id, attribute_ids in vaulted.items():
+        for attribute_id in attribute_ids:
+            spec = schema.kinds[kind_id].attributes[attribute_id].type
+            if spec.name in _UNQUOTED_TYPES:
+                continue
+            by_fingerprint: dict[bytes, set[int]] = defaultdict(set)
+            for candidate, indexes in owners.items():
+                try:
+                    canonical = encode_value(spec, candidate)
+                except EncodingError:
+                    continue
+                for fingerprint in keys.fingerprints(domain_id, kind_id, attribute_id, canonical):
+                    by_fingerprint[fingerprint] |= indexes
+            wanted = list(by_fingerprint)
+            for offset in range(0, len(wanted), _LOOKUP_BATCH):
+                rows = await session.scalars(
+                    select(v.c.fingerprint)
+                    .where(
+                        v.c.domain_id == domain_id,
+                        v.c.kind_id == kind_id,
+                        v.c.attribute_id == attribute_id,
+                        v.c.erased_at.is_(None),
+                        v.c.fingerprint.in_(wanted[offset : offset + _LOOKUP_BATCH]),
+                    )
+                    .distinct()
+                )
+                for fingerprint in rows:
+                    hits |= by_fingerprint.get(bytes(fingerprint), set())
+    return hits

lorebase_store/vault/rotation.py

@@ -0,0 +1,209 @@
+"""Vault key rotation (design 13.5): rows sealed or fingerprinted under earlier keys are made
+again under the current ones.
+
+A row is outdated when its value is sealed under another key than `vault.key_id`, or its
+fingerprint was made under another token key or scheme than the current ones
+(`VaultKeys.fingerprint_key`). `rotate` walks the live rows of every Domain in `(domain_id, id)`
+order, `vault.rekey_batch` rows per transaction, opens each outdated one, seals it again under
+the current key with the same token (Record objects do not change) and makes its fingerprint
+again. The position it stopped at is returned, so a job continues where the last one stopped;
+rows locked by a concurrent erasure are skipped and found by the next pass. A row whose key is
+no longer configured cannot be opened: it is counted as `unopenable` and left as it is.
+
+`usage` counts live rows by encryption key and fingerprint key and tombstones by fingerprint
+key: an encryption key can be removed from `previous_keys` once no live row uses it, a token key
+from `previous_token_keys` once no live row and no tombstone uses it (tombstones cannot be made
+again: they hold no value; dropping a token key retires the tombstones made under it).
+"""
+
+from __future__ import annotations
+
+import json
+from collections import Counter
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from typing import Any
+
+from sqlalchemy import func, or_, select, tuple_, update
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from lorebase_store.auth.scope import rls_scope
+from lorebase_store.vault.keys import VaultKeys
+from lorebase_store.vault.store import TOMBSTONES, VAULT
+
+
+@dataclass(slots=True)
+class RotationReport:
+    """What a rotation pass did; `after` is where the next pass starts (None: done)."""
+
+    rows: int = 0
+    resealed: int = 0
+    fingerprinted: int = 0
+    unopenable: int = 0
+    after: tuple[str, str] | None = None
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "rows": self.rows,
+            "resealed": self.resealed,
+            "fingerprinted": self.fingerprinted,
+            "unopenable": self.unopenable,
+            "after": None if self.after is None else list(self.after),
+            "done": self.after is None,
+        }
+
+
+@dataclass(frozen=True, slots=True)
+class KeyUsage:
+    """Live rows by encryption key id and by fingerprint key, tombstones by fingerprint key
+    (`<token_key_id>/<scheme>`, `legacy` for rows written before rotation)."""
+
+    encryption_keys: dict[str, int] = field(default_factory=dict)
+    fingerprint_keys: dict[str, int] = field(default_factory=dict)
+    tombstone_keys: dict[str, int] = field(default_factory=dict)
+
+    def outdated(self, keys: VaultKeys) -> int:
+        """Live rows the rotation still has to make again (upper bound: a row can be both)."""
+        sealed = sum(n for k, n in self.encryption_keys.items() if k != keys.key_id)
+        fingerprinted = sum(
+            n for k, n in self.fingerprint_keys.items() if k != keys.fingerprint_key
+        )
+        return max(sealed, fingerprinted)
+
+
+LEGACY = "legacy"
+
+
+def _outdated(keys: VaultKeys) -> Any:
+    v = VAULT
+    return or_(
+        v.c.key_id != keys.key_id,
+        v.c.fingerprint_key.is_(None),
+        v.c.fingerprint_key != keys.fingerprint_key,
+    )
+
+
+async def refresh_rows(
+    session: AsyncSession,
+    keys: VaultKeys,
+    condition: Sequence[Any],
+    *,
+    limit: int | None = None,
+    after: tuple[str, str] | None = None,
+    skip_locked: bool = False,
+) -> RotationReport:
+    """Make the outdated live rows matching `condition` again (module doc), in the caller's
+    transaction; at most `limit` rows after `after`."""
+    v = VAULT
+    query = (
+        select(
+            v.c.domain_id,
+            v.c.id,
+            v.c.kind_id,
+            v.c.attribute_id,
+            v.c.key_id,
+            v.c.value_ciphertext,
+            v.c.fingerprint_key,
+        )
+        .where(*condition, v.c.value_ciphertext.is_not(None), _outdated(keys))
+        .order_by(v.c.domain_id, v.c.id)
+        .with_for_update(skip_locked=skip_locked)
+    )
+    if after is not None:
+        query = query.where(tuple_(v.c.domain_id, v.c.id) > tuple_(*after))
+    if limit is not None:
+        query = query.limit(limit)
+    report = RotationReport()
+    for row in (await session.execute(query)).all():
+        report.rows += 1
+        report.after = (row.domain_id, row.id)
+        opened = keys.open(row.domain_id, row.id, row.key_id, bytes(row.value_ciphertext))
+        if opened is None:
+            report.unopenable += 1
+            continue
+        value = json.loads(opened)
+        changes: dict[str, Any] = {}
+        if row.key_id != keys.key_id:
+            changes["value_ciphertext"] = keys.seal(row.domain_id, row.id, value)
+            changes["key_id"] = keys.key_id
+            report.resealed += 1
+        if row.fingerprint_key != keys.fingerprint_key:
+            changes["fingerprint"] = keys.fingerprint(
+                row.domain_id, row.kind_id, row.attribute_id, value
+            )
+            changes["fingerprint_key"] = keys.fingerprint_key
+            report.fingerprinted += 1
+        await session.execute(
+            update(v).where(v.c.domain_id == row.domain_id, v.c.id == row.id).values(**changes)
+        )
+    return report
+
+
+async def rotate(
+    sessions: async_sessionmaker[AsyncSession],
+    keys: VaultKeys,
+    *,
+    batch: int,
+    max_rows: int,
+    after: tuple[str, str] | None = None,
+) -> RotationReport:
+    """Make outdated rows of every Domain again, `batch` per transaction, until `max_rows` rows
+    were looked at or none is left (module doc)."""
+    total = RotationReport(after=after)
+    position = after
+    while total.rows < max_rows:
+        async with sessions() as session, session.begin(), rls_scope(session, bypass=True):
+            report = await refresh_rows(
+                session,
+                keys,
+                [],
+                limit=min(batch, max_rows - total.rows),
+                after=position,
+                skip_locked=True,
+            )
+        total.rows += report.rows
+        total.resealed += report.resealed
+        total.fingerprinted += report.fingerprinted
+        total.unopenable += report.unopenable
+        if report.after is None:
+            total.after = None
+            return total
+        position = total.after = report.after
+    return total
+
+
+async def usage(session: AsyncSession) -> KeyUsage:
+    """Key usage of every Domain (module doc); runs with the row-level security bypass."""
+    v, t = VAULT, TOMBSTONES
+    async with rls_scope(session, bypass=True):
+        sealed = (
+            await session.execute(
+                select(v.c.key_id, func.count())
+                .where(v.c.value_ciphertext.is_not(None))
+                .group_by(v.c.key_id)
+            )
+        ).all()
+        fingerprinted = (
+            await session.execute(
+                select(v.c.fingerprint_key, func.count())
+                .where(v.c.value_ciphertext.is_not(None))
+                .group_by(v.c.fingerprint_key)
+            )
+        ).all()
+        tombstones = (
+            await session.execute(
+                select(t.c.fingerprint_key, func.count()).group_by(t.c.fingerprint_key)
+            )
+        ).all()
+    return KeyUsage(
+        encryption_keys=_counts(sealed),
+        fingerprint_keys=_counts(fingerprinted),
+        tombstone_keys=_counts(tombstones),
+    )
+
+
+def _counts(rows: Sequence[Any]) -> dict[str, int]:
+    counts: Counter[str] = Counter()
+    for key, count in rows:
+        counts[key or LEGACY] += int(count)
+    return dict(sorted(counts.items()))