lorebase-keeper 0.2.3__py3-none-any.whl

lorebase_keeper/reading/settings.py

@@ -0,0 +1,339 @@
+"""The `read` and `search` settings sections (design 8, 16), registered with the platform.
+
+`read` bounds every read: page sizes, query complexity, traversal caps, time travel, diff and
+log pages, and the statement timeout of read transactions. `search` holds the ranking knobs of
+lexical, semantic and hybrid search. The text search configuration is the one the vectors were
+built with (`store.materialize.text_search_config`) and the vector dimension is
+`store.embeddings.dims`, so neither is repeated here.
+"""
+
+from __future__ import annotations
+
+from typing import Self
+
+from pydantic import Field, model_validator
+
+from lorebase_core.query import QueryLimits
+from lorebase_keeper.kindhealth.config import HealthSettings
+from lorebase_platform.settings import SectionModel, Settings, register_section
+from lorebase_store.bm25 import SearchBackend
+
+READ_SECTION = "read"
+SEARCH_SECTION = "search"
+
+
+class Bounds(SectionModel):
+    """A page size default and its maximum."""
+
+    default_limit: int = Field(default=50, ge=1)
+    max_limit: int = Field(default=500, ge=1)
+
+    @model_validator(mode="after")
+    def _default_within_max(self) -> Self:
+        if self.default_limit > self.max_limit:
+            raise ValueError("default_limit cannot exceed max_limit")
+        return self
+
+
+class QuerySettings(SectionModel):
+    """Complexity limits of the query DSL."""
+
+    max_via_depth: int = Field(default=3, ge=1, description="Nesting of via/count predicates.")
+    max_predicates: int = Field(default=64, ge=1, description="Predicates per query.")
+    max_in_values: int = Field(default=1000, ge=1, description="Operands of one `in`.")
+    max_order_terms: int = Field(default=4, ge=1, description="Terms of `order_by`.")
+    max_includes: int = Field(default=5, ge=0, description="Relations in `include`.")
+    include_default_limit: int = Field(default=10, ge=1, description="Neighbors per Record.")
+    include_max_limit: int = Field(default=50, ge=1)
+    max_aggregate_groups: int = Field(
+        default=1000, ge=1, description="Groups returned by `aggregate`; more are cut."
+    )
+
+
+class TraverseSettings(SectionModel):
+    """`POST /traverse` and `POST /neighborhood` (design 8.3)."""
+
+    max_hops: int = Field(default=5, ge=1, description="Relations in one traversal path.")
+    max_nodes_per_hop: int = Field(default=1000, ge=1, description="Records reached per hop.")
+    max_seeds: int = Field(default=100, ge=1, description="Seed Records of a neighborhood.")
+    max_depth: int = Field(default=3, ge=1, description="Depth of a neighborhood.")
+    max_nodes: int = Field(default=2000, ge=1, description="Records in a neighborhood.")
+    max_edges: int = Field(default=5000, ge=1, description="Links in a neighborhood.")
+
+
+class TimeTravelSettings(SectionModel):
+    """Reads at an arbitrary Revision, served from the tree (design 6.12)."""
+
+    max_scan_entries: int = Field(
+        default=100_000,
+        ge=1,
+        description="Largest Kind or Relation partition a list read at a Revision scans; "
+        "larger ones are refused (read the materialized branch instead).",
+    )
+    snapshot_max_entries: int = Field(
+        default=100_000,
+        ge=0,
+        description="Most Records and Links a Revision may hold for reads at it that need "
+        "materialized rows (filters, ordering, includes, aggregates, Link summaries, "
+        "traversal, search, history, health, context packs); they are served from a snapshot "
+        "of the Revision, and refused above this size. 0 refuses them all.",
+    )
+    snapshot_max_history_revisions: int = Field(
+        default=100_000,
+        ge=1,
+        description="Most Revisions in the history of a Revision (its generation + 1, a bound "
+        "on its first-parent chain, whose changes a snapshot copies); deeper ones are refused.",
+    )
+    snapshot_max_per_domain: int = Field(
+        default=4,
+        ge=1,
+        description="Snapshots a Domain holds, those being deleted included. Building another "
+        "first drops the oldest that no read holds and that are older than "
+        "`snapshot_min_age_s`; when none can go, the read answers 503 with Retry-After.",
+    )
+    snapshot_min_age_s: int = Field(
+        default=60,
+        ge=0,
+        description="Grace period of a new snapshot: younger ones are not dropped for others.",
+    )
+    snapshot_max_concurrent_builds: int = Field(
+        default=2,
+        ge=1,
+        description="Snapshot builds running at once in this process. Each holds a second "
+        "database connection besides its read's, so keep it well below the pool size; builds "
+        "of one Domain also run one at a time.",
+    )
+    snapshot_build_wait_ms: int = Field(
+        default=5000,
+        ge=0,
+        description="How long a read waits for a build slot or for the Domain's running build "
+        "before answering 503 with Retry-After.",
+    )
+    snapshot_builds_per_minute: float = Field(
+        default=6,
+        gt=0,
+        description="Snapshot builds one principal may start per minute in this process "
+        "(token bucket); beyond it reads that need a new snapshot answer 429.",
+    )
+    snapshot_build_burst: int = Field(
+        default=3, ge=1, description="Snapshot builds one principal may start at once."
+    )
+
+
+class DiffSettings(SectionModel):
+    """Diff of two Revisions (design 6.7)."""
+
+    max_changes: int = Field(
+        default=100_000, ge=1, description="Changed elements a diff may hold; larger ones fail."
+    )
+    page: Bounds = Field(default_factory=lambda: Bounds(default_limit=100, max_limit=1000))
+
+
+class HistorySettings(Bounds):
+    """Per-Record history and lineage (`.../records/{key}/history` and `/lineage`)."""
+
+    lineage_scan_changes: int = Field(
+        default=1000,
+        ge=1,
+        description="Newest changes of a Record lineage searches for the Revision behind a "
+        "value, or the change that created the Record, when the materialized row does not name "
+        "them (rows written before migration 0032 and not yet backfilled); older ones name no "
+        "Revision.",
+    )
+    lineage_count_max: int = Field(
+        default=10_000,
+        ge=1,
+        description="Changes of a Record lineage counts; beyond, `changes` is this bound and "
+        "`changes_truncated` is true.",
+    )
+
+
+class ReadSettings(SectionModel):
+    """Bounds of the read path."""
+
+    records: Bounds = Field(default_factory=Bounds, description="Record and Link pages.")
+    log: Bounds = Field(default_factory=Bounds, description="Pages of the Revision log.")
+    history: HistorySettings = Field(
+        default_factory=HistorySettings, description="Per-Record history and lineage."
+    )
+    statement_timeout_ms: int = Field(
+        default=2000, ge=0, description="statement_timeout of read transactions; 0 disables."
+    )
+    schema_cache_entries: int = Field(
+        default=64, ge=1, description="Compiled schemas kept in memory, by schema hash."
+    )
+    derivation_preview: Bounds = Field(
+        default_factory=lambda: Bounds(default_limit=5, max_limit=25),
+        description="Records `POST /derivations:preview` evaluates a candidate Derivation on: "
+        "the default and the most one request may ask for.",
+    )
+    derivation_preview_max_key_length: int = Field(
+        default=1024,
+        ge=1,
+        description="Characters of each Record key `POST /derivations:preview` accepts in `keys`.",
+    )
+    source_preview_max_rows: int = Field(
+        default=100,
+        ge=1,
+        description="Sample rows `POST /sources:preview` evaluates the expression mappings of a "
+        "candidate Source on.",
+    )
+    source_preview_max_columns: int = Field(
+        default=200,
+        ge=1,
+        description="Columns of one sample row `POST /sources:preview` accepts.",
+    )
+    source_preview_max_cells: int = Field(
+        default=1000,
+        ge=1,
+        description="Expressions evaluated by one `POST /sources:preview`: sample rows times "
+        "Attributes mapped by an expression.",
+    )
+    query: QuerySettings = Field(default_factory=QuerySettings)
+    traverse: TraverseSettings = Field(default_factory=TraverseSettings)
+    time_travel: TimeTravelSettings = Field(default_factory=TimeTravelSettings)
+    diff: DiffSettings = Field(default_factory=DiffSettings)
+    health: HealthSettings = Field(default_factory=HealthSettings)
+
+    def query_limits(self) -> QueryLimits:
+        """Limits handed to the DSL resolver."""
+        query = self.query
+        return QueryLimits(
+            default_limit=self.records.default_limit,
+            max_limit=self.records.max_limit,
+            max_via_depth=query.max_via_depth,
+            max_predicates=query.max_predicates,
+            max_in_values=query.max_in_values,
+            include_default_limit=query.include_default_limit,
+            include_max_limit=query.include_max_limit,
+            max_includes=query.max_includes,
+            max_order_terms=query.max_order_terms,
+        )
+
+
+class SemanticSettings(SectionModel):
+    """Vector search over `embeddings`. Query vectors always come from the caller."""
+
+    model: str = Field(
+        default="BAAI/bge-small-en-v1.5",
+        min_length=1,
+        description="Embedding model whose vectors are searched; equals "
+        "inference.embeddings.model. The dimension is store.embeddings.dims.",
+    )
+
+
+class HybridSettings(SectionModel):
+    """Reciprocal rank fusion of the lexical and semantic lists (design 8.4)."""
+
+    rrf_k: int = Field(default=60, ge=1, description="Rank offset of reciprocal rank fusion.")
+    lexical_weight: float = Field(default=1.0, ge=0)
+    semantic_weight: float = Field(default=1.0, ge=0)
+
+
+class PartialMatchSettings(SectionModel):
+    """Ranked partial matching of lexical search: descriptive queries rarely have every word
+    in one Record, so when the query as written (every term) finds too few, Records holding
+    some of its terms follow, after every full match."""
+
+    min_results: int = Field(
+        default=1,
+        ge=0,
+        description="Full matches below which partial matches are added; 0 turns partial "
+        "matching off.",
+    )
+    weight: float = Field(
+        default=0.5,
+        ge=0,
+        description="Partial matches score weight * (ts_rank_cd of the terms joined by OR + "
+        "term_bonus * terms matched).",
+    )
+    term_bonus: float = Field(
+        default=1.0, ge=0, description="Score added for each query term a partial match holds."
+    )
+    candidates: int = Field(
+        default=100,
+        ge=1,
+        description="Partial matches ranked per search, best by ts_rank_cd first; at most "
+        "store.reads.max_search_rows without filters, search.candidates with them.",
+    )
+    stop_words: str | None = Field(
+        default="english",
+        min_length=1,
+        description="Text search configuration whose stop words are left out of the terms of "
+        "partial matching (the, for, that...): the search configuration, simple by default, "
+        "keeps every word. null leaves only the search configuration's own stop words out.",
+    )
+
+
+class SearchSettings(SectionModel):
+    """Ranking of `POST /search`."""
+
+    results: Bounds = Field(default_factory=lambda: Bounds(default_limit=20, max_limit=100))
+    backend: SearchBackend = Field(
+        default="postgres",
+        description="Ranking of lexical search without filters: postgres (ts_rank_cd over the "
+        "GIN index, the default) or bm25 (the BM25 index of ParadeDB pg_search, an optional "
+        "AGPL-3.0 extension; store.storage.bm25_index, docs/search.md#bm25-ranking). Without "
+        "the extension or a valid index, search keeps postgres and /readyz reports it.",
+    )
+    candidates: int = Field(
+        default=200, ge=1, description="Rows of each ranked list before fusion and paging."
+    )
+    trigram_threshold: float = Field(
+        default=0.3,
+        gt=0,
+        le=1,
+        description="Least trigram similarity between a query word the branch does not hold "
+        "and a word of the branch that corrects it.",
+    )
+    trigram_weight: float = Field(
+        default=0.5,
+        ge=0,
+        description="Weight of ts_rank_cd for hits found only through a corrected word; they "
+        "always rank after the hits of the query as written.",
+    )
+    typo_candidates: int = Field(
+        default=3,
+        ge=0,
+        description="Corrections tried per unknown query word, most similar first; 0 turns "
+        "typo tolerance off.",
+    )
+    typo_min_length: int = Field(
+        default=4, ge=1, description="Shortest query word (in characters) that is corrected."
+    )
+    common_term_docs: int = Field(
+        default=2000,
+        ge=0,
+        description="Lexical search without filters ranks every match when the word lexicon "
+        "bounds the matches of q within this many Records; when every term of q is more "
+        "common, it ranks only the first search.common_term_candidates matches in index order "
+        "and the response says truncated_candidates. 0 turns this off: "
+        "store.reads.search_candidates matches are ranked, whatever the terms.",
+    )
+    common_term_candidates: int = Field(
+        default=1000,
+        ge=1,
+        description="Matches ranked when every term of q is above search.common_term_docs.",
+    )
+    headline_options: str = Field(
+        default="MaxWords=30, MinWords=10, MaxFragments=2",
+        min_length=1,
+        description="ts_headline options of result snippets.",
+    )
+    partial: PartialMatchSettings = Field(default_factory=PartialMatchSettings)
+    semantic: SemanticSettings = Field(default_factory=SemanticSettings)
+    hybrid: HybridSettings = Field(default_factory=HybridSettings)
+
+
+register_section(READ_SECTION, ReadSettings)
+register_section(SEARCH_SECTION, SearchSettings)
+
+
+def read_settings(settings: Settings) -> ReadSettings:
+    """The validated `read` section."""
+    return settings.section(READ_SECTION, ReadSettings)
+
+
+def search_settings(settings: Settings) -> SearchSettings:
+    """The validated `search` section."""
+    return settings.section(SEARCH_SECTION, SearchSettings)

lorebase_keeper/reading/snapshots.py

@@ -0,0 +1,148 @@
+"""Materialized rows for reads at a past Revision (design 6.12).
+
+Key reads and listings ordered by key at a Revision other than the branch head read the tree.
+Reads that need materialized rows (filters, ordering, includes, aggregates, Link and lineage
+summaries, traversal, neighborhoods, search, Record history, health, context packs and
+explanations) read a snapshot of the Revision instead (`lorebase_store.materialize.snapshot`)
+with the statements of a head read, so they answer what the head answered when the Revision
+was the head. Callers validate and authorize the request before asking for the snapshot.
+
+The snapshot is built on first use and shared by later reads of the Revision. The read holds
+the snapshot's shared lock from before the build, so nothing drops it between the build and
+the read. Building is bounded (`read.time_travel`):
+
+- Revisions holding more than `snapshot_max_entries` Records and Links, or more than
+  `snapshot_max_history_revisions` Revisions of history, are refused (422 `time_travel`);
+- each principal may start `snapshot_builds_per_minute` builds (burst `snapshot_build_burst`)
+  in this process, then gets 429 with Retry-After;
+- at most `snapshot_max_concurrent_builds` builds run in this process, each on a second
+  database connection; builds of one Domain run one at a time; a read waits
+  `snapshot_build_wait_ms` for its turn, then gets 503 with Retry-After;
+- unused snapshots older than `snapshot_min_age_s` are dropped in a short transaction before
+  a build; a Domain holding `snapshot_max_per_domain` snapshots none of which can go answers
+  503 with Retry-After.
+
+Authorization and masking are those of the read point: a snapshot point differs only in where
+its rows come from (`ReadPoint.layout`).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import math
+from dataclasses import replace
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from lorebase_keeper.errors import NOT_FOUND, RATE_LIMITED, UNAVAILABLE, ProblemError
+from lorebase_keeper.ratelimit.bucket import BucketSpec
+from lorebase_keeper.reading.points import ReadPoint
+from lorebase_keeper.reading.problems import snapshot_refused
+from lorebase_keeper.reading.runtime import ReadRuntime
+from lorebase_keeper.reading.tree import TreeReader
+from lorebase_store.layout import BranchLayout
+from lorebase_store.materialize.snapshot import (
+    SnapshotBusyError,
+    build_snapshot,
+    drop_unused,
+    hold_snapshot,
+    snapshot_entries,
+    snapshot_name,
+)
+from lorebase_store.repositories import scope_to_domain
+from lorebase_store.repositories.refs import LockTimeoutError
+
+
+async def with_rows(
+    session: AsyncSession, runtime: ReadRuntime, point: ReadPoint, what: str
+) -> ReadPoint:
+    """`point` when materialized rows serve it, otherwise the same point read from a snapshot
+    of its Revision (built when missing). `what` names the feature in a refusal."""
+    if point.materialized:
+        return point
+    domain_id, revision = point.domain.id, point.revision
+    settings = runtime.read.time_travel
+    tree = await TreeReader(session, runtime, domain_id, revision.root).tree()
+    entries = snapshot_entries(tree)
+    if settings.snapshot_max_entries == 0 or entries > settings.snapshot_max_entries:
+        raise snapshot_refused(
+            what,
+            f"this one holds {entries} Records and Links, more than "
+            f"read.time_travel.snapshot_max_entries ({settings.snapshot_max_entries})",
+        )
+    history = revision.generation + 1
+    if history > settings.snapshot_max_history_revisions:
+        raise snapshot_refused(
+            what,
+            f"its history holds up to {history} Revisions, more than "
+            f"read.time_travel.snapshot_max_history_revisions "
+            f"({settings.snapshot_max_history_revisions})",
+        )
+    # Held from here on: a snapshot built below cannot be dropped before this read ends.
+    if not await hold_snapshot(session, domain_id, revision.id):
+        await _build(runtime, point)
+        if not await hold_snapshot(session, domain_id, revision.id):
+            raise _busy(runtime)
+    layout = BranchLayout(domain_id, snapshot_name(revision.id))
+    return replace(point, layout=layout, snapshot=True)
+
+
+async def _build(runtime: ReadRuntime, point: ReadPoint) -> None:
+    """Build the snapshot of the point's Revision within the build bounds (module doc)."""
+    settings = runtime.read.time_travel
+    spec = BucketSpec(
+        capacity=settings.snapshot_build_burst,
+        refill_per_s=settings.snapshot_builds_per_minute / 60,
+    )
+    principal = point.scope.guard.principal.id
+    decision = await runtime.snapshot_rates.take(f"{principal}\x00snapshot", spec)
+    if not decision.allowed:
+        raise ProblemError(
+            RATE_LIMITED,
+            "Too many snapshot builds for reads at past Revisions; retry later.",
+            headers={"Retry-After": str(decision.retry_after_s)},
+        )
+    wait_ms = settings.snapshot_build_wait_ms
+    try:
+        await asyncio.wait_for(runtime.snapshot_builds.acquire(), timeout=wait_ms / 1000)
+    except TimeoutError:
+        raise _busy(runtime) from None
+    domain_id = point.domain.id
+    try:
+        async with runtime.sessions() as writer:
+            async with writer.begin():
+                await scope_to_domain(writer, domain_id)
+                await drop_unused(
+                    writer,
+                    domain_id,
+                    keep=settings.snapshot_max_per_domain - 1,
+                    min_age_s=settings.snapshot_min_age_s,
+                    runtime=runtime.store,
+                    lock_timeout_ms=wait_ms,
+                )
+            async with writer.begin():
+                await scope_to_domain(writer, domain_id)
+                await build_snapshot(
+                    writer,
+                    domain_id,
+                    point.revision,
+                    max_snapshots=settings.snapshot_max_per_domain,
+                    runtime=runtime.store,
+                    lock_timeout_ms=wait_ms,
+                )
+    except (SnapshotBusyError, LockTimeoutError):
+        raise _busy(runtime) from None
+    except LookupError:
+        raise ProblemError(NOT_FOUND, f"Revision {point.revision_hex} not found.") from None
+    finally:
+        runtime.snapshot_builds.release()
+
+
+def _busy(runtime: ReadRuntime) -> ProblemError:
+    retry = max(1, math.ceil(runtime.read.time_travel.snapshot_build_wait_ms / 1000))
+    return ProblemError(
+        UNAVAILABLE,
+        "No snapshot of this Revision can be built right now (other builds or snapshots in "
+        "use); retry shortly.",
+        headers={"Retry-After": str(retry)},
+    )

lorebase_keeper/reading/tree.py

@@ -0,0 +1,98 @@
+"""Reads from a Revision tree (time travel, design 6.12).
+
+Key lookups walk the trie (prefetched path, then a cached node read per level); listings scan
+the whole Kind or Relation partition, sort its keys and page through them, so their cost grows
+with the partition. Partitions above `read.time_travel.max_scan_entries` are refused.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from typing import Any
+
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import Session
+
+from lorebase_core.tree import Tree, tree_get
+from lorebase_keeper.errors import VALIDATION_FAILED, ProblemError
+from lorebase_keeper.reading.runtime import ReadRuntime
+from lorebase_store.nodestore import NodeRepository, PgNodeStore
+from lorebase_store.objectstore import PgObjectStore
+
+
+@dataclass(frozen=True, slots=True)
+class TreeEntry:
+    key: str
+    object_hash: bytes
+
+
+class TreeReader:
+    """Reads of one Revision tree in one Domain."""
+
+    def __init__(
+        self, session: AsyncSession, runtime: ReadRuntime, domain_id: str, root: bytes
+    ) -> None:
+        settings = runtime.store_settings
+        self._session = session
+        self._runtime = runtime
+        self._domain = domain_id
+        self._root = root
+        self._nodes = NodeRepository(
+            session,
+            domain_id,
+            writes=settings.writes,
+            cache=runtime.store.node_cache,
+            batch=settings.nodes.prefetch_batch,
+        )
+        self._objects = PgObjectStore(
+            session, domain_id, writes=settings.writes, batch=settings.materialize.read_batch_rows
+        )
+        self._tree: Tree | None = None
+
+    async def tree(self) -> Tree:
+        if self._tree is None:
+            self._tree = await self._nodes.load_tree(self._root)
+        return self._tree
+
+    async def count(self, partition: str) -> int:
+        ref = (await self.tree()).partitions.get(partition)
+        return 0 if ref is None else ref.count
+
+    async def get(self, partition: str, key: str) -> bytes | None:
+        """Object hash stored under `key`, or None."""
+        settings = self._runtime.store_settings
+        cache = self._runtime.store.node_cache
+        domain, root = self._domain, self._root
+
+        def lookup(sync: Session) -> bytes | None:
+            store = PgNodeStore(
+                sync, domain, cache=cache, prefetch_batch=settings.nodes.prefetch_batch
+            )
+            store.prefetch_paths(root, [(partition, key)])
+            return tree_get(store, root, partition, key)
+
+        return await self._session.run_sync(lookup)
+
+    async def entries(self, partition: str) -> list[TreeEntry]:
+        """Every entry of a partition sorted by key; refused above the configured size."""
+        limit = self._runtime.read.time_travel.max_scan_entries
+        ref = (await self.tree()).partitions.get(partition)
+        if ref is None:
+            return []
+        if ref.count > limit:
+            raise ProblemError(
+                VALIDATION_FAILED,
+                f"Listing at a Revision scans the whole partition; this one holds {ref.count} "
+                f"entries, more than the limit of {limit}. Read the materialized branch instead.",
+            )
+        found: list[TreeEntry] = []
+        async for batch in self._nodes.scan(ref.root):
+            found.extend(TreeEntry(key, digest) for key, digest in batch)
+        found.sort(key=lambda entry: entry.key)
+        return found
+
+    async def bodies(self, hashes: Iterable[bytes]) -> dict[bytes, Mapping[str, Any]]:
+        """Stored object bodies by hash."""
+        stored = await self._objects.get_many(hashes)
+        return {digest: body.body for digest, body in stored.items()}