lorebase-store 0.2.3__py3-none-any.whl

lorebase_store/keyring.py

@@ -0,0 +1,72 @@
+"""Keys derived from one configured secret, one per namespace and purpose.
+
+HKDF-SHA256 turns the secret into independent 256-bit keys (info `lorebase/<namespace>/<purpose>`),
+so a value sealed for one purpose never opens for another. Sealing is AES-256-GCM with a random
+96-bit nonce and the purpose (plus an optional context) as associated data; MACs are
+HMAC-SHA256. Keeper's web sessions (`keeper.auth.sessions.encryption_key`, namespace `keeper`),
+webhook signing keys at rest (`webhooks.encryption_key`, namespace `webhooks`) and the
+personal data vault (`vault.encryption_key` and `vault.token_key`, namespace `vault`) use it.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import os
+
+from cryptography.exceptions import InvalidTag
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+from cryptography.hazmat.primitives.kdf.hkdf import HKDF
+
+_NONCE_BYTES = 12
+
+
+class Keyring:
+    """Seals, opens and authenticates values under keys derived from one secret."""
+
+    def __init__(self, secret: str, *, namespace: str) -> None:
+        self._secret = secret.encode()
+        self._info = f"lorebase/{namespace}/".encode()
+        self._keys: dict[str, bytes] = {}
+
+    def _key(self, purpose: str) -> bytes:
+        key = self._keys.get(purpose)
+        if key is None:
+            key = HKDF(
+                algorithm=hashes.SHA256(),
+                length=32,
+                salt=None,
+                info=self._info + purpose.encode(),
+            ).derive(self._secret)
+            self._keys[purpose] = key
+        return key
+
+    def seal(self, purpose: str, plaintext: bytes, *, context: bytes = b"") -> bytes:
+        """Encrypt and authenticate `plaintext` for `purpose`. A non-empty `context` (such as
+        the row a value is stored in) is authenticated too: the value opens only with it."""
+        nonce = os.urandom(_NONCE_BYTES)
+        aad = _associated(purpose, context)
+        return nonce + AESGCM(self._key(purpose)).encrypt(nonce, plaintext, aad)
+
+    def open(self, purpose: str, sealed: bytes, *, context: bytes = b"") -> bytes | None:
+        """The plaintext of a value sealed for `purpose` (and `context`); None when it was
+        altered, sealed for another purpose or context, or under another secret."""
+        if len(sealed) <= _NONCE_BYTES:
+            return None
+        nonce, ciphertext = sealed[:_NONCE_BYTES], sealed[_NONCE_BYTES:]
+        try:
+            return AESGCM(self._key(purpose)).decrypt(
+                nonce, ciphertext, _associated(purpose, context)
+            )
+        except InvalidTag:
+            return None
+
+    def mac(self, purpose: str, message: bytes) -> bytes:
+        """HMAC-SHA256 of `message` under the key of `purpose`."""
+        return hmac.new(self._key(purpose), message, hashlib.sha256).digest()
+
+
+def _associated(purpose: str, context: bytes) -> bytes:
+    """Associated data: the purpose alone (as before contexts existed), or purpose and context."""
+    return purpose.encode() if not context else purpose.encode() + b"\x00" + context

lorebase_store/layout.py

@@ -0,0 +1,335 @@
+"""Where the materialized rows of a branch live: its own rows, or an overlay on a base branch.
+
+A `full` branch has every row of its head in `mat_records`, `mat_links` and `mat_history`.
+An `overlay` branch (design 6.11, AC-16) stores only the rows it changed, tombstones
+(`deleted`) included, and reads the rows of its base branch as they were at the fork point:
+
+    overlay rows (not deleted)
+    UNION ALL
+    base rows with revision_generation <= base_generation that the overlay does not shadow
+
+A base commit that changes a row an overlay still reads copies the previous row into the
+overlay first (copy on write, `MaterializedRepository`), so the base rows an overlay reads
+never changed after its fork; the generation filter hides the rows the base wrote since.
+
+This module renders that row set once, as SQL text with the Domain, branch, base branch and
+generation as literals (per-Attribute partial indexes of the base match, see
+`lorebase_store.indexes`), and offers it both as text (`records_from`) and as a SQLAlchemy
+`FromClause` with the table's columns (`records`). Every row of the set reports the overlay
+as its `branch`, so predicates written for a table (`branch = :branch`) keep working. For a
+full branch both return the plain table, so its statements are unchanged.
+
+The base half also checks that the ref is still an overlay forked at that generation: a
+reader that resolved the layout before a promotion or a re-fork (READ COMMITTED) then reads
+only the overlay rows, which a promotion has completed. Revision `0010` renders the same text
+inside the security definer read functions (`lb_overlay_rows`).
+
+For lookups by identity (`records_lookup`, `links_lookup`) one index scan reads both
+branches and the overlay's row of an identity wins; revision `0012` renders the same text
+(`lb_overlay_lookup`).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from sqlalchemy import FromClause, Label, Table, func, text
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import Session
+
+from lorebase_store.models import metadata
+from lorebase_store.models.enums import Materialization
+
+MAT_RECORDS: Table = metadata.tables["mat_records"]
+MAT_LINKS: Table = metadata.tables["mat_links"]
+MAT_HISTORY: Table = metadata.tables["mat_history"]
+
+RECORD_COLUMNS = tuple(c.name for c in MAT_RECORDS.columns)
+LINK_COLUMNS = tuple(c.name for c in MAT_LINKS.columns)
+HISTORY_COLUMNS = tuple(c.name for c in MAT_HISTORY.columns)
+
+
+def lineage_column(rows: FromClause, column: str = "lineage") -> Label[Any]:
+    """A map column of Record or Link `rows` (`lineage`, `lineage_revisions`) as the map it
+    stands for: with shared lineage maps (revision 0036) the column may hold a reference."""
+    return func.lb_lineage(rows.c.domain_id, rows.c[column], type_=JSONB).label(column)
+
+
+_LAYOUT = text(
+    "SELECT CAST(materialization AS text), base_branch, base_generation FROM refs "
+    "WHERE domain_id = :domain AND name = :branch"
+)
+
+
+@dataclass(frozen=True, slots=True)
+class BranchLayout:
+    """The rows a branch reads: its own (`base` is None) or an overlay on `base`."""
+
+    domain_id: str
+    branch: str
+    base: str | None = None
+    base_generation: int | None = None
+    building: bool = field(default=False, compare=False)
+    """A `building` branch: its own rows are still being written from its tree by a
+    background job (`materialize.background`); commits wait for it."""
+
+    def __post_init__(self) -> None:
+        if (self.base is None) != (self.base_generation is None):
+            raise ValueError("an overlay needs both its base branch and its base generation")
+
+    @property
+    def overlay(self) -> bool:
+        return self.base is not None
+
+    # SQL text -----------------------------------------------------------------------
+
+    def records_sql(self, *, probe: bool = True) -> str:
+        """The resolved Record rows as one statement (all `mat_records` columns).
+
+        `probe` keeps the anti-join a primary-key probe of the overlay per base row read,
+        right for lookups by key and short pages; false lets the planner hash the overlay's
+        keys instead, which reads that match many base rows (search) prefer.
+        """
+        return self._resolved("mat_records", RECORD_COLUMNS, ("kind_id", "key"), probe=probe)
+
+    def links_sql(self) -> str:
+        """The resolved Link rows as one statement (all `mat_links` columns)."""
+        return self._resolved("mat_links", LINK_COLUMNS, ("relation_id", "from_key", "to_key"))
+
+    def history_sql(self) -> str:
+        """The branch history: the overlay's own entries and the base's up to the fork that
+        it does not hold itself (a background copy, `materialize.background`, copies them in
+        batches before the overlay becomes a full copy)."""
+        own = ", ".join(f"h.{_quoted(c)}" for c in HISTORY_COLUMNS)
+        base = ", ".join(
+            f"CAST({_literal(self.branch)} AS text)" if c == "branch" else f"b.{_quoted(c)}"
+            for c in HISTORY_COLUMNS
+        )
+        domain = _literal(self.domain_id)
+        return (
+            f"SELECT {own} FROM mat_history h WHERE h.domain_id = {domain} "
+            f"AND h.branch = {_literal(self.branch)} UNION ALL SELECT {base} FROM mat_history b "
+            f"WHERE b.domain_id = {domain} AND b.branch = {_literal(self._base())} "
+            f"AND EXISTS (SELECT 1 FROM revisions r WHERE r.domain_id = {domain} "
+            f"AND r.id = b.revision_id AND r.generation <= {self._generation()} OFFSET 0) "
+            f"AND {self._guard()} AND NOT EXISTS (SELECT 1 FROM mat_history x "
+            f"WHERE x.domain_id = {domain} AND x.branch = {_literal(self.branch)} "
+            f"AND x.partition = b.partition AND x.element_key = b.element_key "
+            f"AND x.revision_id = b.revision_id OFFSET 0)"
+        )
+
+    def records_from(self, alias: str | None = None, *, probe: bool = True) -> str:
+        """A FROM item for text statements: `mat_records [alias]` or the resolved rows
+        (`probe` as in `records_sql`)."""
+        return self._from_item("mat_records", alias, lambda: self.records_sql(probe=probe))
+
+    def links_from(self, alias: str | None = None) -> str:
+        return self._from_item("mat_links", alias, self.links_sql)
+
+    def history_from(self, alias: str | None = None) -> str:
+        return self._from_item("mat_history", alias, self.history_sql)
+
+    def records_lookup(self, where: str) -> str:
+        """The resolved Record rows of an overlay that match `where`, as a parenthesized
+        subquery for a lateral lookup by identity (conditions on alias `x` that pin one
+        identity or a few, without bind parameters).
+
+        One index scan reads the overlay's and the base's rows together (`branch IN
+        (overlay, base)`), and the overlay's row of an identity, tombstone included, wins
+        over the base's (`DISTINCT ON`): one scan per lookup instead of one per half plus a
+        primary-key probe of the overlay per base row. Revision `0012` renders the same
+        text inside the read functions (`lb_overlay_lookup`).
+        """
+        return self._lookup("mat_records", RECORD_COLUMNS, ("kind_id", "key"), where)
+
+    def links_lookup(self, where: str) -> str:
+        """The resolved Link rows of an overlay that match `where` (see `records_lookup`)."""
+        key = ("relation_id", "from_key", "to_key")
+        return self._lookup("mat_links", LINK_COLUMNS, key, where)
+
+    # SQLAlchemy ---------------------------------------------------------------------
+
+    def records(self, name: str | None = None) -> FromClause:
+        """`mat_records` (aliased when `name` is given) or the resolved rows of an overlay."""
+        return self._from_clause(MAT_RECORDS, name)
+
+    def links(self, name: str | None = None) -> FromClause:
+        return self._from_clause(MAT_LINKS, name)
+
+    def history(self, name: str | None = None) -> FromClause:
+        return self._from_clause(MAT_HISTORY, name)
+
+    def records_alias(self) -> FromClause:
+        """An anonymous alias of the rows (for self-joins)."""
+        return self._resolved_clause(MAT_RECORDS) if self.overlay else MAT_RECORDS.alias()
+
+    def links_alias(self) -> FromClause:
+        return self._resolved_clause(MAT_LINKS) if self.overlay else MAT_LINKS.alias()
+
+    # Rendering ----------------------------------------------------------------------
+
+    def _resolved(
+        self, table: str, columns: tuple[str, ...], key: tuple[str, ...], *, probe: bool = True
+    ) -> str:
+        own, base = self._halves(table, columns, key, own="o", base="s", probe=probe)
+        return f"{own} UNION ALL {base}"
+
+    def _halves(
+        self,
+        table: str,
+        columns: tuple[str, ...],
+        key: tuple[str, ...],
+        *,
+        own: str,
+        base: str,
+        probe: bool = True,
+    ) -> tuple[str, str]:
+        """The overlay half and the base half of the resolved rows."""
+        domain, branch = _literal(self.domain_id), _literal(self.branch)
+        own_columns = ", ".join(f"{own}.{_quoted(c)}" for c in columns)
+        base_columns = ", ".join(
+            f"CAST({branch} AS text)" if c == "branch" else f"{base}.{_quoted(c)}" for c in columns
+        )
+        shadow = " AND ".join(f"x.{c} = {base}.{c}" for c in key)
+        # Postgres does not flatten a UNION ALL whose members have WHERE clauses: the halves
+        # stay subqueries; restriction quals and lateral references are pushed into each,
+        # ordering and LIMIT are not (`records_arms` repeats them per half). OFFSET 0 keeps
+        # the anti-join a primary-key probe per base row read (`probe`).
+        return (
+            f"SELECT {own_columns} FROM {table} {own} WHERE {own}.domain_id = {domain} "
+            f"AND {own}.branch = {branch} AND NOT {own}.deleted",
+            f"SELECT {base_columns} FROM {table} {base} WHERE {base}.domain_id = {domain} "
+            f"AND {base}.branch = {_literal(self._base())} "
+            f"AND {base}.revision_generation <= {self._generation()} AND {self._guard()} "
+            f"AND NOT EXISTS (SELECT 1 FROM {table} x WHERE x.domain_id = {domain} "
+            f"AND x.branch = {branch} AND {shadow}{' OFFSET 0' if probe else ''})",
+        )
+
+    def records_arms(self, tail: str, alias: str = "m") -> str:
+        """A FROM item with the resolved Record rows of an overlay where `tail` is applied
+        inside each half: conditions (starting with ` AND`) on alias `m`, then `ORDER BY` and
+        `LIMIT`, so that a keyset page reads at most `LIMIT` rows of each half. `tail` may hold
+        bind parameters and must not test `branch` (the halves read different branches)."""
+        if not self.overlay:
+            raise ValueError("records_arms applies to overlay branches")
+        own, base = (
+            escape_colons(half)
+            for half in self._halves(
+                "mat_records", RECORD_COLUMNS, ("kind_id", "key"), own="m", base="m"
+            )
+        )
+        return f"(({own}{tail}) UNION ALL ({base}{tail})) AS {alias}"
+
+    def _lookup(
+        self, table: str, columns: tuple[str, ...], key: tuple[str, ...], where: str
+    ) -> str:
+        if not self.overlay:
+            raise ValueError("lookups over both branches apply to overlay branches")
+        domain, branch, base = (
+            _literal(self.domain_id),
+            _literal(self.branch),
+            _literal(self._base()),
+        )
+        select = ", ".join(
+            f"CAST({branch} AS text) AS branch" if c == "branch" else f"x.{_quoted(c)}"
+            for c in columns
+        )
+        identity = ", ".join(f"x.{c}" for c in key)
+        return escape_colons(
+            f"(SELECT {select} FROM (SELECT DISTINCT ON ({identity}) * FROM {table} x "
+            f"WHERE x.domain_id = {domain} AND x.branch IN ({branch}, {base}) "
+            f"AND (x.branch = {branch} OR (x.revision_generation <= {self._generation()} "
+            f"AND {self._guard()})) AND {where} "
+            f"ORDER BY {identity}, x.branch = {branch} DESC) x WHERE NOT x.deleted)"
+        )
+
+    def _guard(self) -> str:
+        return (
+            f"EXISTS (SELECT 1 FROM refs f WHERE f.domain_id = {_literal(self.domain_id)} "
+            f"AND f.name = {_literal(self.branch)} AND f.base_branch = {_literal(self._base())} "
+            f"AND f.base_generation = {self._generation()} OFFSET 0)"
+        )
+
+    def _base(self) -> str:
+        assert self.base is not None
+        return self.base
+
+    def _generation(self) -> int:
+        assert self.base_generation is not None
+        return int(self.base_generation)
+
+    def _from_item(self, table: str, alias: str | None, resolved: Any) -> str:
+        if not self.overlay:
+            return table if alias is None else f"{table} {alias}"
+        return f"({escape_colons(resolved())}) AS {alias or table}"
+
+    def _from_clause(self, table: Table, name: str | None) -> FromClause:
+        if not self.overlay:
+            return table.alias(name) if name else table
+        return self._resolved_clause(table, name or table.name)
+
+    def _resolved_clause(self, table: Table, name: str | None = None) -> FromClause:
+        sql = {
+            MAT_RECORDS.name: self.records_sql,
+            MAT_LINKS.name: self.links_sql,
+            MAT_HISTORY.name: self.history_sql,
+        }[table.name]()
+        return text(escape_colons(sql)).columns(*table.columns).subquery(name)
+
+
+def escape_colons(fragment: str) -> str:
+    """A literal-holding SQL fragment made safe for `text()`, where `:name` binds."""
+    return fragment.replace(":", r"\:")
+
+
+def full_layout(domain_id: str, branch: str) -> BranchLayout:
+    return BranchLayout(domain_id, branch)
+
+
+def layout_from(
+    domain_id: str,
+    branch: str,
+    materialization: Materialization | str | None,
+    base: str | None,
+    generation: int | None,
+) -> BranchLayout:
+    """The layout of a ref row's materialization columns."""
+    if materialization == Materialization.OVERLAY and base is not None:
+        return BranchLayout(domain_id, branch, base, generation)
+    return BranchLayout(domain_id, branch, building=materialization == Materialization.BUILDING)
+
+
+async def load_layout(session: AsyncSession, domain_id: str, branch: str) -> BranchLayout:
+    """The layout of a branch (one primary-key read); a missing ref reads as full."""
+    found = (await session.execute(_LAYOUT, {"domain": domain_id, "branch": branch})).one_or_none()
+    if found is None:
+        return BranchLayout(domain_id, branch)
+    return layout_from(domain_id, branch, found[0], found[1], found[2])
+
+
+def load_layout_sync(session: Session, domain_id: str, branch: str) -> BranchLayout:
+    found = session.execute(_LAYOUT, {"domain": domain_id, "branch": branch}).one_or_none()
+    if found is None:
+        return BranchLayout(domain_id, branch)
+    return layout_from(domain_id, branch, found[0], found[1], found[2])
+
+
+async def resolve_layout(
+    session: AsyncSession, domain_id: str, branch: str | BranchLayout
+) -> BranchLayout:
+    """`branch` itself when it is a layout, else the layout read from its ref."""
+    if isinstance(branch, BranchLayout):
+        return branch
+    return await load_layout(session, domain_id, branch)
+
+
+def _literal(value: str) -> str:
+    if "\x00" in value:
+        raise ValueError("SQL literals cannot contain NUL")
+    return "'" + value.replace("'", "''") + "'"
+
+
+def _quoted(column: str) -> str:
+    return f'"{column}"'

lorebase_store/materialize/__init__.py

@@ -0,0 +1,103 @@
+"""Materialized heads (design 6.11): incremental apply, full rebuild, branch copy or overlay,
+promotion of overlays, background copies and deletions of large branches, rewrite of rows
+materialized before a sensitivity tag, backfill of lineage revisions, verify."""
+
+from lorebase_store.jobs import (
+    BRANCH_DELETE_JOB,
+    LINEAGE_BACKFILL_JOB,
+    MATERIALIZE_JOB,
+    SENSITIVITY_REFRESH_JOB,
+)
+from lorebase_store.materialize.apply import MaterializedChange, apply_changeset, refresh_kind
+from lorebase_store.materialize.background import (
+    BranchDeletion,
+    delete_branch,
+    queue_materialization,
+    run_branch_delete_job,
+    run_materialize_job,
+)
+from lorebase_store.materialize.branch import (
+    BranchCreation,
+    BranchDeletingError,
+    BranchExistsError,
+    BranchMode,
+    MaterializationMode,
+    create_branch,
+)
+from lorebase_store.materialize.lineage_backfill import (
+    LineageBackfill,
+    backfill_lineage_revisions,
+    queue_lineage_sweep,
+    run_lineage_backfill_job,
+)
+from lorebase_store.materialize.overlay import (
+    OverlayError,
+    promote,
+    promote_overlays_of,
+    refork,
+)
+from lorebase_store.materialize.paths import (
+    refresh_path_fanout,
+    refresh_stale_paths,
+    run_path_fanout_sweep,
+)
+from lorebase_store.materialize.rebuild import RebuildReport, materialize_revision, rebuild
+from lorebase_store.materialize.stale import (
+    StaleRefresh,
+    queue_after_tagging,
+    queue_stale_refresh,
+    refresh_stale_rows,
+    run_sensitivity_refresh_job,
+    stale_kinds,
+)
+from lorebase_store.materialize.verify import (
+    Divergence,
+    DivergenceReason,
+    VerifyReport,
+    verify,
+)
+
+__all__ = [
+    "BRANCH_DELETE_JOB",
+    "LINEAGE_BACKFILL_JOB",
+    "MATERIALIZE_JOB",
+    "SENSITIVITY_REFRESH_JOB",
+    "BranchCreation",
+    "BranchDeletingError",
+    "BranchDeletion",
+    "BranchExistsError",
+    "BranchMode",
+    "Divergence",
+    "DivergenceReason",
+    "LineageBackfill",
+    "MaterializationMode",
+    "MaterializedChange",
+    "OverlayError",
+    "RebuildReport",
+    "StaleRefresh",
+    "VerifyReport",
+    "apply_changeset",
+    "backfill_lineage_revisions",
+    "create_branch",
+    "delete_branch",
+    "materialize_revision",
+    "promote",
+    "promote_overlays_of",
+    "queue_after_tagging",
+    "queue_lineage_sweep",
+    "queue_materialization",
+    "queue_stale_refresh",
+    "rebuild",
+    "refork",
+    "refresh_kind",
+    "refresh_path_fanout",
+    "refresh_stale_paths",
+    "refresh_stale_rows",
+    "run_branch_delete_job",
+    "run_lineage_backfill_job",
+    "run_materialize_job",
+    "run_path_fanout_sweep",
+    "run_sensitivity_refresh_job",
+    "stale_kinds",
+    "verify",
+]