lorebase-store 0.2.3__py3-none-any.whl

lorebase_store/materialize/rows.py

@@ -0,0 +1,153 @@
+"""Pure mapping from tree content to materialized rows, shared by incremental apply and rebuild.
+
+Both paths must produce identical rows; everything that decides a column value lives here.
+
+Lineage of a row maps each Attribute holding a value to the provenance entry of its last
+change: the provenance maps of the element's changes are folded oldest first and the result
+is restricted to the Attributes present in the current values. Changes that touch no
+Attribute (deletions, schema elements) are attributed under `ELEMENT_PROVENANCE_KEY`.
+The Revisions of those changes are folded the same way (`next_lineage_revisions`), with the
+Revision that created the element under `CREATED_KEY`.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping
+from dataclasses import dataclass
+from enum import StrEnum
+from typing import Any
+
+from lorebase_core.model import Kind, SearchWeight
+from lorebase_core.tree import LINK_KEY_SEPARATOR, PARTITION_SCHEMA
+from lorebase_core.types import AttributeType, to_text
+
+ELEMENT_PROVENANCE_KEY = "$element"
+"""Provenance map key of changes that touch no Attribute value."""
+
+CREATED_KEY = "$created"
+"""Lineage revisions key of the Revision that last created the element."""
+
+DEFAULT_SEARCH_WEIGHT = SearchWeight.B
+"""Weight of searchable Attributes without an explicit `search_weight` (titles use A)."""
+
+_KIND_PREFIX = "k:"
+_RELATION_PREFIX = "r:"
+
+
+class PartitionType(StrEnum):
+    SCHEMA = "schema"
+    KIND = "kind"
+    RELATION = "relation"
+
+
+@dataclass(frozen=True, slots=True)
+class PartitionId:
+    """A parsed partition id: its type and the Kind or Relation id it holds."""
+
+    type: PartitionType
+    element_id: str
+
+
+def parse_partition(partition: str) -> PartitionId:
+    """Inverse of `kind_partition`, `relation_partition` and `PARTITION_SCHEMA`."""
+    if partition == PARTITION_SCHEMA:
+        return PartitionId(PartitionType.SCHEMA, "")
+    if partition.startswith(_KIND_PREFIX):
+        return PartitionId(PartitionType.KIND, partition[len(_KIND_PREFIX) :])
+    if partition.startswith(_RELATION_PREFIX):
+        return PartitionId(PartitionType.RELATION, partition[len(_RELATION_PREFIX) :])
+    raise ValueError(f"unknown partition {partition!r}")
+
+
+def split_link_key(element_key: str) -> tuple[str, str]:
+    """`(from_key, to_key)` of a Link element key."""
+    from_key, separator, to_key = element_key.partition(LINK_KEY_SEPARATOR)
+    if not separator:
+        raise ValueError(f"not a Link key: {element_key!r}")
+    return from_key, to_key
+
+
+def changed_attributes(old: Mapping[str, Any] | None, new: Mapping[str, Any]) -> list[str]:
+    """Attributes whose value differs between two value maps (absent counts as a value)."""
+    previous = old or {}
+    return sorted(
+        attribute
+        for attribute in previous.keys() | new.keys()
+        if previous.get(attribute) != new.get(attribute)
+    )
+
+
+def changed_provenance(
+    provenance: Mapping[str, str], old: Mapping[str, Any] | None, new: Mapping[str, Any]
+) -> dict[str, str]:
+    """`provenance` restricted to the Attributes whose value differs between `old` and `new`
+    (and the element key): what a change actually attributes. Merges bring the whole Lineage
+    of an element taken from the source; only the values they change are theirs."""
+    kept = {*changed_attributes(old, new), ELEMENT_PROVENANCE_KEY}
+    return {attribute: entry for attribute, entry in provenance.items() if attribute in kept}
+
+
+def next_lineage(
+    previous: Mapping[str, str] | None,
+    provenance: Mapping[str, str] | None,
+    values: Mapping[str, Any],
+) -> dict[str, str]:
+    """Lineage after a change: previous lineage updated with the change's provenance map."""
+    merged = dict(previous or {})
+    merged.update(provenance or {})
+    return {attribute: entry for attribute, entry in sorted(merged.items()) if attribute in values}
+
+
+def next_lineage_revisions(
+    previous: Mapping[str, str] | None,
+    provenance: Mapping[str, str] | None,
+    revision: bytes,
+    values: Mapping[str, Any],
+    *,
+    created: bool,
+) -> dict[str, str]:
+    """Lineage revisions after a change made by `revision` (hex values, see module doc).
+
+    Attributes the change attributes take `revision`; the others keep their previous entry.
+    `previous` is None when unknown (rows written before the revisions were kept): only the
+    change's own entries are known then, and readers resolve the rest from the history.
+    """
+    merged = {} if created else dict(previous or {})
+    merged.update(dict.fromkeys(provenance or {}, revision.hex()))
+    if created:
+        merged[CREATED_KEY] = revision.hex()
+    return {
+        attribute: entry
+        for attribute, entry in sorted(merged.items())
+        if attribute in values or attribute == CREATED_KEY
+    }
+
+
+def record_title(kind: Kind | None, values: Mapping[str, Any]) -> str | None:
+    """Text of the Kind's title Attribute, if any. A sensitive title Attribute gives no title:
+    titles are shown and searched without the clearance its values need (design 10.2)."""
+    if kind is None or kind.title_attribute is None:
+        return None
+    value = values.get(kind.title_attribute)
+    attribute = kind.attributes.get(kind.title_attribute)
+    if value is None or attribute is None or attribute.is_sensitive:
+        return None
+    if attribute.type.name in (AttributeType.JSON, AttributeType.ARRAY):
+        return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+    return to_text(attribute.type, value)
+
+
+def search_weights(kind: Kind | None) -> list[tuple[str, str]]:
+    """`(attribute_id, weight)` of the Kind's searchable Attributes, by Attribute id.
+
+    Sensitive Attributes are left out, like from embedding templates: the full-text index and
+    its snippets serve every reader of the Kind, cleared for sensitive values or not.
+    """
+    if kind is None:
+        return []
+    return [
+        (attribute.id, str(attribute.search_weight or DEFAULT_SEARCH_WEIGHT))
+        for attribute in sorted(kind.attributes.values(), key=lambda a: a.id)
+        if attribute.searchable and not attribute.is_sensitive
+    ]

lorebase_store/materialize/sensitivity.py

@@ -0,0 +1,66 @@
+"""Sensitivity tags of the default branch, for what every branch materializes (design 10.2).
+
+Titles and full-text vectors leave sensitive Attributes out. A branch forked before the
+default branch tagged an Attribute still has it untagged in its own schema, so the
+materializer, the rebuild and the verification compute titles and search vectors with the
+effective tags (`tagged_schema`): the default branch's tags added, and carried through the
+branch's Derivations, as reads see them.
+The default branch's schema comes from the store's schema cache, keyed by its head Revision.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from lorebase_core.expr.limits import ExprLimits
+from lorebase_core.expr.sensitivity import Tags, rebased_tags, schema_tags, with_tags
+from lorebase_core.model import Schema
+from lorebase_store.reads import head_schema
+from lorebase_store.repositories.domains import DomainRepository
+from lorebase_store.runtime import StoreRuntime
+
+
+async def default_branch_schema(
+    session: AsyncSession, domain_id: str, *, runtime: StoreRuntime
+) -> Schema | None:
+    """The schema at the head of the Domain's default branch (cached by head Revision); None
+    when the Domain or its materialized head is missing."""
+    domain = await DomainRepository(session).get(domain_id)
+    if domain is None:
+        return None
+    found = await head_schema(session, domain_id, domain.default_branch, runtime=runtime)
+    return None if found is None else found.schema
+
+
+async def reference_tags(
+    session: AsyncSession, domain_id: str, branch: str, *, runtime: StoreRuntime
+) -> dict[str, Tags]:
+    """Tags of the default branch's schema by Attribute id; empty on the default branch
+    itself (its own schema decides) or when it is not materialized."""
+    domain = await DomainRepository(session).get(domain_id)
+    if domain is None or branch == domain.default_branch:
+        return {}
+    found = await head_schema(session, domain_id, domain.default_branch, runtime=runtime)
+    return {} if found is None else schema_tags(found.schema)
+
+
+def tagged_schema(
+    schema: Schema, reference: Mapping[str, Tags], limits: ExprLimits | None = None
+) -> Schema:
+    """`schema` with the effective tags of its Attributes (`rebased_tags`): the tags of
+    `reference` (the default branch's) added, and carried to the derived Attributes reading
+    them. A schema that already has every tag of `reference` is returned as it is: its
+    validation made its derived Attributes carry what they read."""
+    if not reference or with_tags(schema, reference) == schema:
+        return schema
+    return rebased_tags(schema, reference, limits)
+
+
+async def branch_tagged_schema(
+    session: AsyncSession, domain_id: str, branch: str, schema: Schema, *, runtime: StoreRuntime
+) -> Schema:
+    """`tagged_schema` of the schema of `branch` with the default branch's tags."""
+    reference = await reference_tags(session, domain_id, branch, runtime=runtime)
+    return tagged_schema(schema, reference, runtime.schema.expression_limits)

lorebase_store/materialize/snapshot.py

@@ -0,0 +1,205 @@
+"""Snapshots: the materialized rows of a past Revision, for reads at it (design 6.12).
+
+A read at a Revision other than a branch head is served from the tree; filters, ordering,
+includes, aggregates, Link summaries, traversal, search, history and health need materialized
+rows. A snapshot provides them: a ref of kind `snapshot` named after the Revision
+(`snapshot_name`), whose rows `materialize_revision` writes like for a branch rebuilt at that
+Revision (content, history of its first-parent chain, lineage, derived values, path Links and
+search vectors) with the sensitivity tags of the Revision's own schema, without per-Attribute
+indexes and without a ref log entry. Branch listings, bundles, embeddings, index plans and the
+garbage collector's branch guard ignore snapshot refs; the collector keeps their Revisions.
+
+Snapshots are shared by every read of the Revision in the Domain:
+
+- Readers take a shared transaction-scoped advisory lock on a snapshot before using it
+  (`hold_snapshot`, also before it exists), so it is not dropped while they read it.
+- `drop_unused` (its own short transaction) deletes the oldest snapshots beyond `keep` that no
+  reader holds and that are older than a grace period, through `delete_branch` (large ones are
+  deleted by the background job).
+- `build_snapshot` creates one under the Domain's snapshot lock, which serializes the builds
+  of a Domain and their count against the cap: refs being deleted count too, and a full Domain
+  raises `SnapshotBusyError` instead of exceeding it.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+
+from sqlalchemy import func, select, text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from lorebase_core.tree import Tree
+from lorebase_store.materialize.background import delete_branch
+from lorebase_store.materialize.rebuild import materialize_revision
+from lorebase_store.materialize.rows import PartitionType, parse_partition
+from lorebase_store.models.enums import Materialization, RefKind
+from lorebase_store.repositories.refs import REFS, RefRepository
+from lorebase_store.repositories.revisions import RevisionRepository, RevisionRow
+from lorebase_store.runtime import StoreRuntime
+
+SNAPSHOT_PREFIX = "~snapshot/"
+"""Prefix of snapshot ref names; branch and tag names may not start with it."""
+
+SNAPSHOT_CREATOR = "lorebase:time-travel"
+"""`created_by` of snapshot refs."""
+
+_DOMAIN_LOCK = "~snapshot"
+"""Ref lock name serializing the snapshot builds and drops of a Domain."""
+
+_USE_KEY_SCOPE = b"lb1\x00snapshot-use\x00"
+
+Clock = Callable[[], datetime]
+
+
+class SnapshotBusyError(Exception):
+    """No snapshot can be built now: the Domain holds its maximum (in use, recent or being
+    deleted), or the snapshot of the Revision is still being deleted."""
+
+
+def snapshot_name(revision: bytes) -> str:
+    """Ref name of the snapshot of `revision`."""
+    return SNAPSHOT_PREFIX + revision.hex()
+
+
+def snapshot_entries(tree: Tree) -> int:
+    """Records and Links a snapshot of `tree` holds (O(1), from the partition counts)."""
+    return sum(
+        ref.count
+        for partition, ref in tree.partitions.items()
+        if parse_partition(partition).type is not PartitionType.SCHEMA
+    )
+
+
+def _use_key(domain_id: str, name: str) -> int:
+    digest = hashlib.sha256(
+        _USE_KEY_SCOPE + domain_id.encode("utf-8") + b"\x00" + name.encode("utf-8")
+    ).digest()
+    return int.from_bytes(digest[:8], "big", signed=True)
+
+
+def _now() -> datetime:
+    return datetime.now(UTC)
+
+
+async def hold_snapshot(session: AsyncSession, domain_id: str, revision: bytes) -> bool:
+    """Take the shared lock of the snapshot of `revision` for the session's transaction (it
+    is not dropped while held, even when it is built later) and say whether the snapshot
+    exists. Works in read-only transactions."""
+    name = snapshot_name(revision)
+    await session.execute(
+        text("SELECT pg_advisory_xact_lock_shared(:key)"), {"key": _use_key(domain_id, name)}
+    )
+    found = await session.scalar(
+        select(REFS.c.name).where(
+            REFS.c.domain_id == domain_id,
+            REFS.c.name == name,
+            REFS.c.kind == RefKind.SNAPSHOT,
+            REFS.c.deleted_at.is_(None),
+        )
+    )
+    return found is not None
+
+
+async def drop_unused(
+    session: AsyncSession,
+    domain_id: str,
+    *,
+    keep: int,
+    min_age_s: float,
+    runtime: StoreRuntime,
+    lock_timeout_ms: int,
+    clock: Clock | None = None,
+) -> int:
+    """Delete the snapshots of the Domain beyond the `keep` newest that no reader holds and
+    that were built at least `min_age_s` ago, in the caller's (short) transaction; returns how
+    many were deleted or scheduled for deletion."""
+    now = (clock or _now)()
+    await RefRepository(session, domain_id).lock(_DOMAIN_LOCK, timeout_ms=lock_timeout_ms)
+    names = (
+        await session.scalars(
+            select(REFS.c.name)
+            .where(
+                REFS.c.domain_id == domain_id,
+                REFS.c.kind == RefKind.SNAPSHOT,
+                REFS.c.deleted_at.is_(None),
+                REFS.c.created_at <= now - timedelta(seconds=min_age_s),
+            )
+            .order_by(REFS.c.created_at.desc(), REFS.c.name)
+        )
+    ).all()
+    young = await _count(session, domain_id, live=True) - len(names)
+    dropped = 0
+    for name in names[max(keep - young, 0) :]:
+        free = await session.scalar(
+            text("SELECT pg_try_advisory_xact_lock(:key)"), {"key": _use_key(domain_id, name)}
+        )
+        if free:
+            await delete_branch(session, domain_id, name, runtime=runtime, clock=lambda: now)
+            dropped += 1
+    return dropped
+
+
+async def build_snapshot(
+    session: AsyncSession,
+    domain_id: str,
+    revision: RevisionRow,
+    *,
+    max_snapshots: int,
+    runtime: StoreRuntime,
+    lock_timeout_ms: int,
+    clock: Clock | None = None,
+) -> bool:
+    """Create the snapshot of `revision` in the caller's transaction (scoped to the Domain);
+    `False` when it already exists. Raises `SnapshotBusyError` when the Domain already holds
+    `max_snapshots` snapshots (being deleted included) or this one is being deleted, and
+    `LockTimeoutError` when another build of the Domain holds the lock longer than
+    `lock_timeout_ms`."""
+    now = (clock or _now)()
+    name = snapshot_name(revision.id)
+    refs = RefRepository(session, domain_id)
+    await refs.lock(_DOMAIN_LOCK, timeout_ms=lock_timeout_ms)
+    if await refs.get(name) is not None:
+        return False
+    if await refs.get_deleting(name) is not None:
+        raise SnapshotBusyError(f"the snapshot {name!r} of Domain {domain_id!r} is being deleted")
+    held = await _count(session, domain_id, live=False)
+    if held >= max_snapshots:
+        raise SnapshotBusyError(
+            f"Domain {domain_id!r} holds {held} snapshots, the most allowed ({max_snapshots})"
+        )
+    await refs.create(
+        name,
+        revision.id,
+        created_at=now,
+        created_by=SNAPSHOT_CREATOR,
+        kind=RefKind.SNAPSHOT,
+        materialization=Materialization.FULL,
+        reason="snapshot",
+        log=False,
+    )
+    # The ref now roots the Revision for the garbage collector; one that removed the Revision
+    # before (it was unreachable) leaves nothing to build.
+    if await RevisionRepository(session, domain_id).get(revision.id) is None:
+        raise LookupError(f"Revision {revision.id.hex()} of Domain {domain_id!r} is gone")
+    await materialize_revision(
+        session,
+        domain_id,
+        name,
+        revision,
+        runtime=runtime,
+        indexes=False,
+        default_branch_tags=False,
+    )
+    return True
+
+
+async def _count(session: AsyncSession, domain_id: str, *, live: bool) -> int:
+    """Snapshot refs of the Domain: live ones only, or those being deleted too."""
+    query = select(func.count()).where(
+        REFS.c.domain_id == domain_id, REFS.c.kind == RefKind.SNAPSHOT
+    )
+    if live:
+        query = query.where(REFS.c.deleted_at.is_(None))
+    return int(await session.scalar(query) or 0)