lorebase-store 0.2.3__py3-none-any.whl

lorebase_store/materialize/links.py

@@ -0,0 +1,266 @@
+"""Links of a branch by endpoint, for Relations and path Derivations (design 5.8).
+
+`links_of` reads the Links of a Relation from near keys: stored Links, or the Records holding
+the key of a foreign-key Relation. `path_links_of` reads the Links of a path Derivation the
+same way, composed from the Links of its steps (`lorebase_core.model.paths`: every Link of
+the first step, at most `fanout` Links, smallest far keys first, per intermediate Record
+later). Read from its far end, a path is walked backwards and each intermediate Record is
+checked to contribute the far key within its bound, so both directions give the same pairs.
+
+`path_affected` lists the Records whose pairs a changeset can change, from the steps followed
+from them: the start Records for the forward direction, the end Records for the inverse one
+(`reverse_steps`, where a changed Link of an intermediate Record also marks the far keys it
+can push in or out of that Record's bound).
+
+All reads go through `MaterializedRepository`, so overlays read their resolved rows.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from collections.abc import Mapping, Sequence
+from typing import TYPE_CHECKING, Any
+
+from lorebase_core.model import LinkDirection, PathRelation, Relation, RelationBacking
+from lorebase_core.model.paths import PathHop, bounded
+from lorebase_store.repositories.materialized import MaterializedRepository
+
+if TYPE_CHECKING:
+    from lorebase_store.materialize.derived import AppliedChanges
+
+
+def reverse(direction: LinkDirection) -> LinkDirection:
+    return LinkDirection.INVERSE if direction is LinkDirection.FORWARD else LinkDirection.FORWARD
+
+
+def reverse_steps(steps: Sequence[PathHop]) -> tuple[PathHop, ...]:
+    """The steps of a path walked from its far end."""
+    return tuple(PathHop(step.relation, reverse(step.direction)) for step in reversed(steps))
+
+
+async def links_of(
+    repository: MaterializedRepository,
+    relation: Relation,
+    direction: LinkDirection,
+    keys: Sequence[str],
+    *,
+    limit: int | None = None,
+) -> dict[str, list[tuple[str, Any]]]:
+    """`near key -> [(far key, link values)]` of `relation` followed in `direction` from
+    `keys`: stored Links, or the Records holding the key of a foreign-key Relation; at most
+    `limit` Links in all when set."""
+    forward = direction is LinkDirection.FORWARD
+    out: dict[str, list[tuple[str, Any]]] = defaultdict(list)
+    if relation.backing is RelationBacking.FOREIGN_KEY:
+        attribute = relation.foreign_key_attribute
+        if attribute is None:
+            return out
+        pairs = await repository.foreign_key_links(
+            relation.from_kind, attribute, keys=keys, by_target=not forward, limit=limit
+        )
+        for from_key, to_key in pairs:
+            near, far = (from_key, to_key) if forward else (to_key, from_key)
+            out[near].append((far, {}))
+        return out
+    found = await repository.linked_keys(
+        relation.id, near="from_key" if forward else "to_key", keys=keys, limit=limit
+    )
+    for near, far, values in found:
+        out[near].append((far, values))
+    return out
+
+
+class PathCapExceededError(Exception):
+    """The first step of a path read with a cap (`path_links_of`) has that many Links."""
+
+
+async def _fars(
+    repository: MaterializedRepository,
+    step: PathHop,
+    keys: Sequence[str],
+    cap: int | None = None,
+) -> dict[str, list[str]]:
+    """Far keys of `keys` along a step; with `cap`, raises `PathCapExceededError` when they have
+    `cap` Links or more (reading no more than that)."""
+    found = (
+        await links_of(repository, step.relation, step.direction, keys, limit=cap) if keys else {}
+    )
+    if cap is not None and sum(len(items) for items in found.values()) >= cap:
+        raise PathCapExceededError(cap)
+    return {near: [far for far, _ in items] for near, items in found.items()}
+
+
+async def path_pairs_from(
+    repository: MaterializedRepository,
+    path: PathRelation,
+    starts: Sequence[str],
+    fanout: int,
+    cap: int | None = None,
+) -> dict[str, set[str]]:
+    """Far keys reached from each start Record (`lorebase_core.model.paths`). With `cap`,
+    the first step is read up to `cap` Links (`_fars`)."""
+    reached: dict[str, set[str]] = {start: {start} for start in starts}
+    for index, step in enumerate(path.steps):
+        found = await _fars(
+            repository,
+            step,
+            sorted({k for keys in reached.values() for k in keys}),
+            cap if index == 0 else None,
+        )
+        nexts = {
+            near: (fars if index == 0 else bounded(fars, fanout)) for near, fars in found.items()
+        }
+        reached = {
+            start: {far for near in keys for far in nexts.get(near, ())}
+            for start, keys in reached.items()
+        }
+    return reached
+
+
+async def path_pairs_to(
+    repository: MaterializedRepository,
+    path: PathRelation,
+    ends: Sequence[str],
+    fanout: int,
+    cap: int | None = None,
+) -> dict[str, set[str]]:
+    """Start keys reaching each end Record: the steps walked backwards, keeping an
+    intermediate Record only when the key it leads to is within its bound. With `cap`, the
+    first step walked is read up to `cap` Links (`_fars`)."""
+    reached: dict[str, set[str]] = {end: {end} for end in ends}
+    last = len(path.steps) - 1
+    for index in reversed(range(len(path.steps))):
+        step = path.steps[index]
+        frontier = sorted({k for keys in reached.values() for k in keys})
+        back = await _fars(
+            repository,
+            PathHop(step.relation, reverse(step.direction)),
+            frontier,
+            cap if index == last else None,
+        )
+        if index > 0:
+            nears = sorted({near for nears in back.values() for near in nears})
+            allowed = {
+                near: set(bounded(fars, fanout))
+                for near, fars in (await _fars(repository, step, nears)).items()
+            }
+            back = {
+                far: [near for near in nears if far in allowed.get(near, ())]
+                for far, nears in back.items()
+            }
+        reached = {
+            end: {near for far in keys for near in back.get(far, ())}
+            for end, keys in reached.items()
+        }
+    return reached
+
+
+async def path_links_of(
+    repository: MaterializedRepository,
+    path: PathRelation,
+    direction: LinkDirection,
+    keys: Sequence[str],
+    fanout: int,
+    *,
+    cap: int | None = None,
+) -> dict[str, list[tuple[str, Any]]]:
+    """`near key -> [(far key, {})]` of a path Derivation followed in `direction` from
+    `keys`, far keys in order (the shape of `links_of`). With `cap`, raises
+    `PathCapExceededError` when the first step walked has `cap` Links or more, read no further."""
+    if direction is LinkDirection.FORWARD:
+        reached = await path_pairs_from(repository, path, keys, fanout, cap)
+    else:
+        reached = await path_pairs_to(repository, path, keys, fanout, cap)
+    return {near: [(far, {}) for far in sorted(fars)] for near, fars in reached.items() if fars}
+
+
+async def path_affected(
+    repository: MaterializedRepository,
+    steps: Sequence[PathHop],
+    applied: AppliedChanges,
+    *,
+    fanout: int | None = None,
+    inverse: bool = False,
+) -> set[str]:
+    """Records at the start of `steps` whose pairs a changeset can change (a superset): a
+    Link of step `i` that changed marks its near Record at level `i`; walking the steps
+    backwards without bound gives the start Records reaching a marked Record.
+
+    `inverse` says `steps` are a path walked from its far end (`reverse_steps`): the bound
+    of an intermediate Record then sits at the far side of a step, and a Link added or
+    removed there can push another far key in or out of that Record's bounded set
+    (`_displaced`), which marks it too."""
+    starts: set[str] = set()
+    for level, step in enumerate(steps):
+        marked = await _marked(repository, step, applied)
+        if inverse and fanout is not None and level < len(steps) - 1:
+            marked |= await _displaced(repository, step, applied, fanout)
+        for back in reversed(steps[:level]):
+            if not marked:
+                break
+            found = await links_of(
+                repository, back.relation, reverse(back.direction), sorted(marked)
+            )
+            marked = {far for items in found.values() for far, _ in items}
+        starts |= marked
+    return starts
+
+
+async def _displaced(
+    repository: MaterializedRepository, step: PathHop, applied: AppliedChanges, fanout: int
+) -> set[str]:
+    """Near Records of a step walked backwards whose membership in the bounded set of an
+    intermediate Record (at the step's far side) a changeset can change. With `n` changed
+    Links at an intermediate Record holding `c` Links now, its bounded set before and after
+    the change lies within its `fanout + n` smallest far keys once `c + n` exceeds
+    `fanout`: those are marked (the changed ones are marked by `_marked`)."""
+    hop = PathHop(step.relation, reverse(step.direction))
+    changes = await _changes(repository, hop, applied)
+    if not changes:
+        return set()
+    current = await _fars(repository, hop, sorted(changes))
+    displaced: set[str] = set()
+    for near, count in changes.items():
+        fars = sorted(set(current.get(near, ())))
+        if len(fars) + count > fanout:
+            displaced.update(fars[: fanout + count])
+    return displaced
+
+
+async def _marked(
+    repository: MaterializedRepository, step: PathHop, applied: AppliedChanges
+) -> set[str]:
+    """Near Records of a step whose Links along it changed."""
+    return set(await _changes(repository, step, applied))
+
+
+async def _changes(
+    repository: MaterializedRepository, step: PathHop, applied: AppliedChanges
+) -> dict[str, int]:
+    """Near Records of a step whose Links along it changed, with how many of their Links
+    may have changed (an upper bound)."""
+    relation = step.relation
+    forward = step.direction is LinkDirection.FORWARD
+    counts: dict[str, int] = defaultdict(int)
+    for f, t in applied.links.get(relation.id, ()):
+        counts[f if forward else t] += 1
+    attribute = relation.foreign_key_attribute
+    if relation.backing is not RelationBacking.FOREIGN_KEY or attribute is None:
+        return counts
+    source = relation.from_kind
+    holders = set(applied.written.get(source, ())) | set(applied.removed.get(source, ()))
+    if forward:
+        for key in holders:
+            counts[key] += 1
+        return counts
+    previous: Mapping[tuple[str, str], Mapping[str, Any] | None] = applied.previous
+    for key in holders:  # targets before the change, then after it
+        before = previous.get((source, key))
+        if before is not None and isinstance(before.get(attribute), str):
+            counts[before[attribute]] += 1
+    if holders:
+        now = await links_of(repository, relation, LinkDirection.FORWARD, sorted(holders))
+        for items in now.values():
+            for far, _ in items:
+                counts[far] += 1
+    return counts

lorebase_store/materialize/overlay.py

@@ -0,0 +1,292 @@
+"""Overlay branches (design 6.11 mode `overlay`, AC-16; store schema notes).
+
+An overlay starts with no Record or Link rows: creating it inserts its ref (base branch, fork
+Revision and generation) and a copy of the base's compiled schema row, and queues its
+per-Attribute indexes (`lorebase_store.indexes`). Its commits write only the rows they change
+(`MaterializedRepository`), and reads resolve it over the base (`lorebase_store.layout`).
+
+An overlay is promoted to a full copy ("materialized"):
+- when a commit leaves it with more than `store.materialize.overlay_max_rows` rows of its own,
+  or when its base has moved more than `overlay_max_base_lag_revisions` generations past its
+  fork point (checked by its own commits, unless a background copy of it is queued or running;
+  a base commit that pushes an overlay past the lag queues a `store.materialize` job for it,
+  which copies it in batches, `materialize.background`). Over a base holding more than
+  `async_copy_min_rows` rows, the overlay's own commit queues that job too;
+- before a commit or merge that changes its schema (whole-Kind rewrites and index changes
+  belong to full branches): inline over a small base; over a large one the copy job takes
+  the change over (`apply.defer_schema_change`);
+- before its base is deleted or rebuilt (`promote_overlays_of`): inline over a small base;
+  over a large one it is detached and rebuilt from its tree in the background;
+- on request (`promote`).
+
+After an overlay is merged into its base, `refork` moves it to the base head and empties it,
+which costs as much as creating it.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Callable
+from datetime import UTC, datetime
+
+from sqlalchemy import delete
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from lorebase_store.indexes import BranchIndexes
+from lorebase_store.jobs import MATERIALIZE_JOB, enqueue_job
+from lorebase_store.layout import MAT_HISTORY, MAT_LINKS, MAT_RECORDS
+from lorebase_store.models.enums import Materialization
+from lorebase_store.repositories.materializations import (
+    BranchMaterializationRepository,
+    copy_fence,
+)
+from lorebase_store.repositories.materialized import MaterializedRepository
+from lorebase_store.repositories.refs import RefMovedError, RefRepository, RefRow
+from lorebase_store.repositories.revisions import RevisionRepository
+from lorebase_store.repositories.scope import scope_to_domain
+from lorebase_store.runtime import StoreRuntime, default_runtime
+from lorebase_store.settings import MaterializeSettings
+from lorebase_store.storage import constraints_only
+
+log = logging.getLogger(__name__)
+
+REFORK_REASON = "branch.refork"
+
+
+class OverlayError(Exception):
+    """An overlay operation does not apply to the branch."""
+
+
+def wants_overlay(name: str, settings: MaterializeSettings) -> bool:
+    """Whether a branch created without an explicit mode defaults to an overlay."""
+    if settings.branch_mode == "overlay":
+        return True
+    return any(name.startswith(prefix) for prefix in settings.overlay_branch_prefixes)
+
+
+async def promote(
+    session: AsyncSession,
+    domain_id: str,
+    branch: str,
+    *,
+    runtime: StoreRuntime | None = None,
+    locked: bool = False,
+) -> bool:
+    """Turn the overlay `branch` into a full copy; `False` when it is not an overlay.
+
+    Takes the branch lock unless the caller holds it (`locked`). The head and ref version do
+    not change, so a commit of the caller's transaction still moves the ref afterwards. A
+    background copy of the branch (`materialize.background`) is marked done.
+    """
+    runtime = runtime or default_runtime()
+    refs = RefRepository(session, domain_id)
+    if not locked:
+        await refs.lock(branch, timeout_ms=runtime.settings.commit.lock_timeout_ms)
+    ref = await refs.get(branch)
+    if ref is None or ref.materialization is not Materialization.OVERLAY:
+        return False
+    await copy_fence(session, domain_id, branch)
+    repository = MaterializedRepository(
+        session, domain_id, branch, writes=runtime.settings.writes, layout=ref.layout
+    )
+    await repository.complete_overlay()
+    await refs.set_materialization(branch, Materialization.FULL)
+    states = BranchMaterializationRepository(session, domain_id)
+    if await states.get(branch) is not None:
+        await states.finish(branch, at=datetime.now(UTC))
+    schema = await repository.schema()
+    if schema is not None:
+        # Imported here: the commit package imports the materializer.
+        from lorebase_store.commit.schema import load_compiled
+
+        await BranchIndexes(
+            session,
+            domain_id,
+            branch,
+            settings=runtime.settings.indexes,
+            constraints_only=constraints_only(runtime.settings),
+            maintenance=runtime.index_maintenance,
+        ).sync(load_compiled(schema.compiled))
+    log.info("promoted overlay branch %s of Domain %s to a full copy", branch, domain_id)
+    return True
+
+
+async def base_is_large(
+    session: AsyncSession, domain_id: str, base: str, *, runtime: StoreRuntime
+) -> bool:
+    """Whether copying the rows of `base` into an overlay is a job for the background: it
+    holds more than `store.materialize.async_copy_min_rows` rows."""
+    limit = runtime.settings.materialize.async_copy_min_rows
+    rows = MaterializedRepository(session, domain_id, base, writes=runtime.settings.writes)
+    return await rows.own_rows(limit=limit) > limit
+
+
+async def promote_overlays_of(
+    session: AsyncSession, domain_id: str, base: str, *, runtime: StoreRuntime | None = None
+) -> list[str]:
+    """Make every overlay based on `base` independent of it (before deleting or rebuilding
+    it): promoted to a full copy in this transaction when `base` is small, otherwise
+    detached and rebuilt from its tree by a background job (`background.detach_to_build`).
+    The caller holds the lock of `base`; each overlay's lock is taken after it. Returns the
+    overlays handled."""
+    runtime = runtime or default_runtime()
+    refs = RefRepository(session, domain_id)
+    overlays = await refs.overlays_of(base)
+    if not overlays:
+        return []
+    large = await base_is_large(session, domain_id, base, runtime=runtime)
+    handled = []
+    for ref in overlays:
+        if large:
+            # Imported here: the background jobs import this module.
+            from lorebase_store.materialize.background import detach_to_build
+
+            done = await detach_to_build(
+                session, domain_id, ref.name, runtime=runtime, at=datetime.now(UTC)
+            )
+        else:
+            done = await promote(session, domain_id, ref.name, runtime=runtime)
+        if done:
+            handled.append(ref.name)
+    return handled
+
+
+async def check_overlay(
+    repository: MaterializedRepository, *, generation: int, runtime: StoreRuntime
+) -> bool:
+    """After a commit wrote `repository`'s branch at `generation`: promote it when it is an
+    overlay over its row or lag limits; queue promotions of overlays based on it that the
+    commit pushed past the lag limit. Returns whether the branch itself was promoted."""
+    settings = runtime.settings.materialize
+    layout = await repository.layout()
+    session = repository.session
+    if layout.overlay:
+        assert layout.base is not None
+        if await BranchMaterializationRepository(session, layout.domain_id).active(layout.branch):
+            # Its background copy makes it a full copy without a long transaction.
+            return False
+        refs = RefRepository(session, layout.domain_id)
+        base = await refs.get(layout.base)
+        base_generation = None
+        if base is not None:
+            found = await RevisionRepository(session, layout.domain_id).generations([base.head])
+            base_generation = found.get(base.head)
+        lagging = (
+            base_generation is not None
+            and layout.base_generation is not None
+            and base_generation - layout.base_generation > settings.overlay_max_base_lag_revisions
+        )
+        if lagging or await repository.own_rows(limit=settings.overlay_max_rows) > (
+            settings.overlay_max_rows
+        ):
+            if await base_is_large(session, layout.domain_id, layout.base, runtime=runtime):
+                # Imported here: the background jobs import this module.
+                from lorebase_store.materialize.background import queue_materialization
+
+                ref = await refs.get(layout.branch)
+                if ref is not None:
+                    await queue_materialization(
+                        session, layout.domain_id, ref, at=datetime.now(UTC)
+                    )
+                return False
+            promoted = await promote(
+                session, layout.domain_id, layout.branch, runtime=runtime, locked=True
+            )
+            repository.forget_layout()
+            return promoted
+        return False
+    for name, base_generation in await repository.dependents():
+        if generation - base_generation > settings.overlay_max_base_lag_revisions:
+            await enqueue_job(
+                session,
+                MATERIALIZE_JOB,
+                domain_id=layout.domain_id,
+                payload={"branch": name},
+                unique=True,
+            )
+    return False
+
+
+async def refork(
+    session: AsyncSession,
+    domain_id: str,
+    branch: str,
+    *,
+    actor: str,
+    base: str | None = None,
+    runtime: StoreRuntime | None = None,
+    clock: Callable[[], datetime] | None = None,
+) -> RefRow:
+    """Move `branch` to the head of its base (default: its current base) as an empty overlay.
+
+    The branch head must be an ancestor of the base head (merged into it), so the move is a
+    fast-forward and loses nothing. Rows of the branch are dropped; the base is locked first,
+    then the branch.
+    """
+    runtime = runtime or default_runtime()
+    now = clock or (lambda: datetime.now(UTC))
+    timeout = runtime.settings.commit.lock_timeout_ms
+    await scope_to_domain(session, domain_id)
+    refs = RefRepository(session, domain_id)
+    ref = await refs.require(branch)
+    base_name = base or ref.base_branch
+    if base_name is None:
+        raise OverlayError(f"branch {branch!r} has no base; name one")
+    await refs.lock(base_name, timeout_ms=timeout)
+    await refs.lock(branch, timeout_ms=timeout)
+    await copy_fence(session, domain_id, branch)
+    ref = await refs.require(branch)
+    base_ref = await refs.require(base_name)
+    if base_ref.materialization is not Materialization.FULL:
+        raise OverlayError(f"base branch {base_name!r} is not fully materialized")
+    if await refs.overlays_of(branch):
+        raise OverlayError(f"branch {branch!r} is the base of other overlays")
+    revisions = RevisionRepository(session, domain_id)
+    ahead, _ = await revisions.ahead_behind(ref.head, base_ref.head)
+    if ahead:
+        raise OverlayError(
+            f"branch {branch!r} has {ahead} Revisions that {base_name!r} does not contain; "
+            "merge it first"
+        )
+    head = await revisions.require(base_ref.head)
+    writes = runtime.settings.writes
+    for table in (MAT_RECORDS, MAT_LINKS, MAT_HISTORY):
+        await session.execute(
+            delete(table).where(table.c.domain_id == domain_id, table.c.branch == branch)
+        )
+    schema = await MaterializedRepository(session, domain_id, base_name, writes=writes).schema()
+    if schema is not None:
+        await MaterializedRepository(session, domain_id, branch, writes=writes).set_schema(schema)
+    if ref.head != head.id:
+        try:
+            await refs.compare_and_swap(
+                branch,
+                expected_head=ref.head,
+                expected_version=ref.version,
+                new_head=head.id,
+                actor=actor,
+                at=now(),
+                reason=REFORK_REASON,
+            )
+        except RefMovedError as error:
+            raise OverlayError(f"branch {branch!r} moved meanwhile") from error
+    await refs.set_materialization(
+        branch,
+        Materialization.OVERLAY,
+        base_branch=base_name,
+        base_revision=head.id,
+        base_generation=head.generation,
+    )
+    if schema is not None:
+        # Now an overlay: the indexes of the base's schema, none unique, queued.
+        from lorebase_store.commit.schema import load_compiled
+
+        await BranchIndexes(
+            session,
+            domain_id,
+            branch,
+            settings=runtime.settings.indexes,
+            constraints_only=constraints_only(runtime.settings),
+            maintenance=runtime.index_maintenance,
+        ).sync(load_compiled(schema.compiled))
+    return await refs.require(branch)