pyoq-sql 1.0.2__py3-none-any.whl

pyoq/relations/batching.py

@@ -0,0 +1,219 @@
+"""Collecting the keys a relation needs, so one query serves many parents.
+
+An N+1 access asks for one parent's children at a time. The fix is to gather
+the keys first and ask once, which a dialect's parameter limit turns into
+asking a bounded number of times rather than once per parent.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Iterator
+from dataclasses import dataclass
+
+from pyoq.errors import ParameterLimitError, QueryValidationError
+from pyoq.relations.model import TypedRelation
+
+DEFAULT_KEY_LIMIT = 10_000
+
+
+@dataclass(frozen=True, slots=True)
+class KeyBatch:
+    """Keys for one relation that fit in one statement."""
+
+    relation: TypedRelation
+    keys: tuple[tuple[object, ...], ...]
+
+    @property
+    def parameters(self) -> int:
+        """How many bound values this batch costs.
+
+        A key spanning several columns costs one value per column, which is what
+        a limit counts rather than the number of keys.
+        """
+        return sum(len(key) for key in self.keys)
+
+
+def plan_key_batches(
+    relation: TypedRelation,
+    keys: Iterable[tuple[object, ...]],
+    /,
+    *,
+    maximum_parameters: int,
+) -> tuple[KeyBatch, ...]:
+    """Split keys into the fewest statements a parameter limit allows."""
+    _require_positive(maximum_parameters, "maximum parameters")
+    ordered = tuple(keys)
+    if not ordered:
+        return ()
+    width = _uniform_width(relation, ordered)
+    per_batch = maximum_parameters // width
+    if per_batch < 1:
+        message = (
+            f"one key of {width} columns exceeds the {maximum_parameters} "
+            f"parameters a statement is allowed"
+        )
+        raise ParameterLimitError(message)
+    return tuple(
+        KeyBatch(relation, ordered[start : start + per_batch])
+        for start in range(0, len(ordered), per_batch)
+    )
+
+
+class RelationBatch:
+    """The keys one relation is waiting on, in order and without repeats.
+
+    A parent whose key is null reaches no children, so such a key is counted
+    rather than collected: asking for it would return nothing and cost a
+    parameter to do so.
+    """
+
+    __slots__ = ("_keys", "_limit", "_relation", "_skipped")
+
+    def __init__(
+        self,
+        relation: TypedRelation,
+        *,
+        key_limit: int = DEFAULT_KEY_LIMIT,
+    ) -> None:
+        _require_positive(key_limit, "key limit")
+        self._relation = relation
+        self._limit = key_limit
+        self._keys: dict[tuple[object, ...], None] = {}
+        self._skipped = 0
+
+    @property
+    def relation(self) -> TypedRelation:
+        return self._relation
+
+    @property
+    def skipped_keys(self) -> int:
+        """Keys not collected because they reach nothing."""
+        return self._skipped
+
+    def __len__(self) -> int:
+        return len(self._keys)
+
+    def __iter__(self) -> Iterator[tuple[object, ...]]:
+        return iter(self._keys)
+
+    def add(self, key: tuple[object, ...], /) -> bool:
+        """Collect a key, reporting whether it will be asked about."""
+        _require_width(self._relation, key)
+        if any(value is None for value in key):
+            self._skipped += 1
+            return False
+        if key in self._keys:
+            return True
+        if len(self._keys) >= self._limit:
+            message = (
+                f"a relation batch holds {self._limit} keys, which is all it "
+                f"was allowed; drain it before collecting more"
+            )
+            raise QueryValidationError(message)
+        self._keys[key] = None
+        return True
+
+    def batches(self, *, maximum_parameters: int) -> tuple[KeyBatch, ...]:
+        return plan_key_batches(
+            self._relation,
+            self._keys,
+            maximum_parameters=maximum_parameters,
+        )
+
+    def drain(self) -> tuple[tuple[object, ...], ...]:
+        """Take the keys collected so far and start again."""
+        keys = tuple(self._keys)
+        self._keys = {}
+        self._skipped = 0
+        return keys
+
+
+class RelationBatchLoader:
+    """Every relation a scope is waiting on, each bounded on its own.
+
+    A loader belongs to one scope. Keys gathered for one request answer that
+    request, so a loader is not shared the way an observer is.
+    """
+
+    __slots__ = ("_batches", "_key_limit")
+
+    def __init__(self, *, key_limit: int = DEFAULT_KEY_LIMIT) -> None:
+        _require_positive(key_limit, "key limit")
+        self._key_limit = key_limit
+        self._batches: dict[TypedRelation, RelationBatch] = {}
+
+    @property
+    def relations(self) -> tuple[TypedRelation, ...]:
+        return tuple(self._batches)
+
+    def __len__(self) -> int:
+        return sum(len(batch) for batch in self._batches.values())
+
+    def enqueue(self, relation: TypedRelation, key: tuple[object, ...], /) -> bool:
+        return self._batch_for(relation).add(key)
+
+    def batch_for(self, relation: TypedRelation, /) -> RelationBatch:
+        return self._batch_for(relation)
+
+    def batches(self, *, maximum_parameters: int) -> tuple[KeyBatch, ...]:
+        """Every statement this scope now owes, across every relation."""
+        return tuple(
+            batch
+            for pending in self._batches.values()
+            for batch in pending.batches(maximum_parameters=maximum_parameters)
+        )
+
+    def drain(self, relation: TypedRelation, /) -> tuple[tuple[object, ...], ...]:
+        """Take one relation's keys and start it again."""
+        return self._batch_for(relation).drain()
+
+    def drain_all(self) -> tuple[KeyBatch, ...]:
+        """Take every relation's keys at once, as one statement each."""
+        drained = tuple(
+            KeyBatch(relation, batch.drain())
+            for relation, batch in self._batches.items()
+        )
+        return tuple(batch for batch in drained if batch.keys)
+
+    def _batch_for(self, relation: TypedRelation) -> RelationBatch:
+        batch = self._batches.get(relation)
+        if batch is None:
+            batch = RelationBatch(relation, key_limit=self._key_limit)
+            self._batches[relation] = batch
+        return batch
+
+
+def _uniform_width(
+    relation: TypedRelation,
+    keys: tuple[tuple[object, ...], ...],
+) -> int:
+    width = len(relation.target.columns)
+    for key in keys:
+        _require_width(relation, key)
+    return width
+
+
+def _require_width(relation: TypedRelation, key: tuple[object, ...]) -> None:
+    width = len(relation.target.columns)
+    if len(key) == width:
+        return
+    message = (
+        f"a key for {relation.target.table.name.value!r} spans {width} columns, "
+        f"and this one holds {len(key)}"
+    )
+    raise QueryValidationError(message)
+
+
+def _require_positive(value: int, label: str) -> None:
+    if isinstance(value, bool) or value < 1:
+        message = f"{label} must be a positive integer"
+        raise QueryValidationError(message)
+
+
+__all__ = (
+    "DEFAULT_KEY_LIMIT",
+    "KeyBatch",
+    "RelationBatch",
+    "RelationBatchLoader",
+    "plan_key_batches",
+)

pyoq/relations/derivation.py

@@ -0,0 +1,111 @@
+"""Deriving both navigable directions of every foreign key."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+
+from pyoq.relations.model import (
+    RelationCardinality,
+    RelationDirection,
+    RelationEndpoint,
+    TypedRelation,
+)
+from pyoq.schema import (
+    Catalog,
+    Identifier,
+    ObjectReference,
+    Relation,
+    Schema,
+    SchemaSnapshot,
+    Table,
+)
+
+
+def derive_relations(snapshot: SchemaSnapshot) -> tuple[TypedRelation, ...]:
+    """Read every foreign key in a snapshot from both of its sides."""
+    return tuple(
+        relation
+        for catalog in snapshot.catalogs
+        for schema in catalog.schemas
+        for table in schema.tables
+        for relation in table_relations(catalog, schema, table)
+    )
+
+
+def table_relations(
+    catalog: Catalog,
+    schema: Schema,
+    table: Table,
+) -> Iterator[TypedRelation]:
+    """Read the foreign keys one table declares from both of their sides."""
+    reference = ObjectReference(table.name, schema.name, catalog.name)
+    for relation in table.relations:
+        yield _forward(reference, table, relation)
+        yield _reverse(reference, table, relation)
+
+
+def _forward(
+    reference: ObjectReference,
+    table: Table,
+    relation: Relation,
+) -> TypedRelation:
+    """A foreign key names at most one row of the table it points at."""
+    return TypedRelation(
+        RelationDirection.FORWARD,
+        RelationCardinality.TO_ONE,
+        RelationEndpoint(reference, relation.columns),
+        RelationEndpoint(relation.target, relation.target_columns),
+        optional=_any_nullable(table, relation.columns),
+        constraint=relation.name,
+        on_update=relation.on_update,
+        on_delete=relation.on_delete,
+    )
+
+
+def _reverse(
+    reference: ObjectReference,
+    table: Table,
+    relation: Relation,
+) -> TypedRelation:
+    """The referenced row owns however many rows point back at it.
+
+    That is one row when the referencing columns are themselves unique, and any
+    number of rows otherwise. A referenced row may own none either way, so the
+    unique case is optional and the collection case is not.
+    """
+    unique = _uniquely_constrained(table, relation.columns)
+    cardinality = RelationCardinality.TO_ONE if unique else RelationCardinality.TO_MANY
+    return TypedRelation(
+        RelationDirection.REVERSE,
+        cardinality,
+        RelationEndpoint(relation.target, relation.target_columns),
+        RelationEndpoint(reference, relation.columns),
+        optional=unique,
+        constraint=relation.name,
+        on_update=relation.on_update,
+        on_delete=relation.on_delete,
+    )
+
+
+def _any_nullable(table: Table, columns: tuple[Identifier, ...]) -> bool:
+    """A foreign key with any null part references nothing at all."""
+    names = frozenset(column.value for column in columns)
+    return any(
+        column.nullable for column in table.columns if column.name.value in names
+    )
+
+
+def _uniquely_constrained(table: Table, columns: tuple[Identifier, ...]) -> bool:
+    """Report whether these columns can repeat within the table.
+
+    A key covering a subset of the columns is enough, because a subset that is
+    already unique makes the wider set unique too.
+    """
+    covered = frozenset(column.value for column in columns)
+    return any(
+        frozenset(column.value for column in key.columns) <= covered
+        for key in table.keys
+    )
+
+
+__all__ = ("derive_relations", "table_relations")

pyoq/relations/fetching.py

@@ -0,0 +1,355 @@
+"""Saying which relations a query fetches, and how, before it runs."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import StrEnum
+
+from pyoq.errors import FetchPlanError, RelationResolutionError
+from pyoq.relations.graph import RelationGraph
+from pyoq.relations.model import RelationCardinality, TypedRelation
+from pyoq.schema import Identifier, ObjectReference
+
+MAXIMUM_FETCH_DEPTH = 8
+
+
+class FetchStrategy(StrEnum):
+    """How a relation's rows are to be brought back.
+
+    ``AUTO`` is a request rather than an instruction. It is resolved against a
+    dialect's capabilities when the query is planned, so a plan can be written
+    once and still take the best route each database offers.
+    """
+
+    JOINED = "joined"
+    NESTED = "nested"
+    SELECT_IN = "select-in"
+    AUTO = "auto"
+
+
+@dataclass(frozen=True, slots=True)
+class FetchOrder:
+    """One term a collection is ordered by, and which way it runs.
+
+    A column is named, so a plan can check the related table has it. An
+    expression is named too, but only named: what it is finally written as
+    belongs to the layer that writes queries, and a plan does not reach up
+    into one. Naming it is still enough to plan with, because whether a
+    collection is ordered at all is what decides how it can be fetched.
+    """
+
+    name: str
+    column: Identifier | None = None
+    descending: bool = False
+
+    def __post_init__(self) -> None:
+        if not self.name:
+            message = "a term a collection is ordered by needs a name"
+            raise FetchPlanError(message)
+
+    def describe(self) -> str:
+        """How this term reads when a plan explains itself."""
+        return f"{self.name} descending" if self.descending else self.name
+
+
+def by_column(name: str, /, *, descending: bool = False) -> FetchOrder:
+    """Order a collection by one of the related table's own columns."""
+    return FetchOrder(name, Identifier(name), descending=descending)
+
+
+def by_expression(name: str, /, *, descending: bool = False) -> FetchOrder:
+    """Order a collection by an expression, under the name a plan reports.
+
+    The plan carries the name and not the expression, so it stays checkable
+    against a schema and readable when it explains itself. The expression is
+    given to whatever carries the plan out, which is the layer that knows how
+    to write one.
+    """
+    return FetchOrder(name, descending=descending)
+
+
+@dataclass(frozen=True, slots=True)
+class FetchJoin:
+    """A table a collection reads alongside the one its relation names.
+
+    The relation still decides which rows there are. A join decides what each
+    of them carries, which is why it is named here by the relation that
+    reaches it rather than by a table and a condition a caller writes out.
+    """
+
+    relation: TypedRelation
+    carrying: tuple[Identifier, ...]
+    """Columns of the joined table each row of the collection carries."""
+
+    def __post_init__(self) -> None:
+        if not self.carrying:
+            message = "a joined table carries at least one column"
+            raise FetchPlanError(message)
+        if not self.relation.to_one:
+            message = (
+                "a collection joins a table one of its rows points at, and "
+                f"{self.relation.target.table.name.value!r} is reached through "
+                "a to-many relation; fetch it as a collection of its own"
+            )
+            raise FetchPlanError(message)
+
+
+@dataclass(frozen=True, slots=True)
+class FetchRequest:
+    """One relation to fetch, and whatever is to be fetched through it.
+
+    ``order_by`` is the terms the collection is ordered by, each a column of
+    the related table or an expression over it. A collection whose order
+    matters cannot be fetched the same way on every dialect, so saying whether
+    it matters is what lets a plan be resolved rather than guessed.
+    """
+
+    relation: TypedRelation
+    strategy: FetchStrategy = FetchStrategy.AUTO
+    nested: tuple[FetchRequest, ...] = ()
+    order_by: tuple[FetchOrder, ...] = ()
+    joins: tuple[FetchJoin, ...] = ()
+    limit: int | None = None
+    """How many rows of the collection to keep, where only the first few matter.
+
+    The rows kept are the ones the order put first, so a limit without an
+    order would keep an arbitrary few. Asking for one is refused here rather
+    than answered with rows nobody chose.
+    """
+
+    def __post_init__(self) -> None:
+        if self.limit is None:
+            return
+        if self.limit < 1:
+            message = "a collection keeps at least one row or is not limited"
+            raise FetchPlanError(message)
+        if not self.order_by:
+            message = (
+                "a limited collection is ordered, so the rows it keeps are "
+                "the ones the order put first"
+            )
+            raise FetchPlanError(message)
+
+    @property
+    def table(self) -> ObjectReference:
+        return self.relation.target.table
+
+    @property
+    def depth(self) -> int:
+        """How many levels this request reaches, counted without recursion.
+
+        A key that points back at its own table lets a plan nest as far as the
+        caller cares to build, and a depth that could not be measured without
+        exhausting the stack would leave the depth limit unable to refuse it.
+        """
+        deepest = 0
+        pending: list[tuple[FetchRequest, int]] = [(self, 1)]
+        while pending:
+            request, level = pending.pop()
+            deepest = max(deepest, level)
+            pending.extend((nested, level + 1) for nested in request.nested)
+        return deepest
+
+
+@dataclass(frozen=True, slots=True)
+class FetchPlan:
+    """Everything one query fetches beyond its own rows."""
+
+    root: ObjectReference
+    requests: tuple[FetchRequest, ...] = field(default_factory=tuple)
+
+    @property
+    def depth(self) -> int:
+        return max((request.depth for request in self.requests), default=0)
+
+    def describe(self) -> tuple[str, ...]:
+        """Read the plan back as lines, so it can be seen before it runs."""
+        lines: list[str] = []
+        pending = [(request, 0) for request in reversed(self.requests)]
+        while pending:
+            request, level = pending.pop()
+            lines.append(_describe(request, level))
+            pending.extend((nested, level + 1) for nested in reversed(request.nested))
+        return tuple(lines)
+
+
+def validate_fetch_plan(
+    plan: FetchPlan,
+    graph: RelationGraph,
+    *,
+    maximum_depth: int = MAXIMUM_FETCH_DEPTH,
+) -> None:
+    """Refuse a plan that describes a fetch the schema cannot perform.
+
+    A plan is checked once, before any query runs, so a mistake is reported
+    against the schema rather than as a failure part-way through a fetch.
+    """
+    if maximum_depth < 1:
+        message = "a fetch plan must be allowed at least one level"
+        raise FetchPlanError(message)
+    pending = [(plan.requests, graph.resolve(plan.root), 1)]
+    while pending:
+        requests, table, level = pending.pop()
+        _require_within_depth(level, maximum_depth)
+        pending.extend(_validate_level(requests, table, graph, level))
+
+
+def _validate_level(
+    requests: tuple[FetchRequest, ...],
+    table: ObjectReference,
+    graph: RelationGraph,
+    level: int,
+) -> list[tuple[tuple[FetchRequest, ...], ObjectReference, int]]:
+    available = graph.relations_from(table)
+    seen: set[TypedRelation] = set()
+    deeper: list[tuple[tuple[FetchRequest, ...], ObjectReference, int]] = []
+    for request in requests:
+        _require_available(request, table, available)
+        _require_unrepeated(request, seen)
+        _require_order_columns(request, graph)
+        _require_joins(request, graph)
+        if request.nested:
+            deeper.append((request.nested, graph.resolve(request.table), level + 1))
+    return deeper
+
+
+def _require_within_depth(level: int, maximum_depth: int) -> None:
+    if level <= maximum_depth:
+        return
+    message = f"fetch plan nests beyond the {maximum_depth} levels allowed"
+    raise FetchPlanError(message)
+
+
+def _require_available(
+    request: FetchRequest,
+    table: ObjectReference,
+    available: tuple[TypedRelation, ...],
+) -> None:
+    if request.relation in available:
+        return
+    message = (
+        f"{_describe_table(table)} has no relation to "
+        f"{_describe_table(request.table)} matching this request"
+    )
+    raise FetchPlanError(message)
+
+
+def _require_order_columns(request: FetchRequest, graph: RelationGraph) -> None:
+    """An order naming a column the table has not got is a mistake in the plan.
+
+    A term that only names an expression names no column to check.
+    """
+    named = tuple(term.column for term in request.order_by if term.column is not None)
+    _require_columns(request.table, named, graph, "is ordered by")
+
+
+def _require_joins(request: FetchRequest, graph: RelationGraph) -> None:
+    """A join reaches its table from the one being fetched, and carries columns of it.
+
+    Both are checked here rather than left to the query that is finally
+    written, because a join the schema cannot support is a mistake in the plan
+    and reads far better against the schema than as SQL a server rejects.
+    """
+    available = graph.relations_from(request.table)
+    for join in request.joins:
+        _require_reachable(request, join, available)
+        _require_columns(join.relation.target.table, join.carrying, graph, "carries")
+
+
+def _require_reachable(
+    request: FetchRequest,
+    join: FetchJoin,
+    available: tuple[TypedRelation, ...],
+) -> None:
+    if join.relation in available:
+        return
+    message = (
+        f"{_describe_table(request.table)} has no relation to "
+        f"{_describe_table(join.relation.target.table)} to join it by"
+    )
+    raise FetchPlanError(message)
+
+
+def _require_columns(
+    table: ObjectReference,
+    named: tuple[Identifier, ...],
+    graph: RelationGraph,
+    verb: str,
+) -> None:
+    """A table the snapshot does not describe cannot be checked.
+
+    That is the same answer the graph gives everywhere else about a table it
+    does not hold.
+    """
+    if not named:
+        return
+    try:
+        described = graph.table(table)
+    except RelationResolutionError:
+        return
+    known = frozenset(column.name for column in described.columns)
+    missing = tuple(column for column in named if column not in known)
+    if not missing:
+        return
+    names = ", ".join(column.value for column in missing)
+    message = f"{table.name.value!r} {verb} {names}, which it has not got"
+    raise FetchPlanError(message)
+
+
+def _require_unrepeated(request: FetchRequest, seen: set[TypedRelation]) -> None:
+    if request.relation in seen:
+        message = (
+            f"the relation to {_describe_table(request.table)} is fetched twice "
+            f"at the same level"
+        )
+        raise FetchPlanError(message)
+    seen.add(request.relation)
+
+
+def _describe(request: FetchRequest, level: int) -> str:
+    collection = (
+        " []" if request.relation.cardinality is RelationCardinality.TO_MANY else ""
+    )
+    return (
+        f"{'  ' * level}{_describe_table(request.table)}{collection} "
+        f"via {request.strategy.value}"
+        f"{_describe_joins(request)}{_describe_limit(request)}"
+    )
+
+
+def _describe_joins(request: FetchRequest) -> str:
+    """A plan explains what widens each row, not only which rows there are."""
+    if not request.joins:
+        return ""
+    joined = ", ".join(
+        _describe_table(join.relation.target.table) for join in request.joins
+    )
+    return f" joining {joined}"
+
+
+def _describe_limit(request: FetchRequest) -> str:
+    if request.limit is None:
+        return ""
+    return f" keeping the first {request.limit}"
+
+
+def _describe_table(reference: ObjectReference) -> str:
+    parts = tuple(
+        part.value
+        for part in (reference.catalog, reference.schema, reference.name)
+        if part is not None
+    )
+    return ".".join(parts)
+
+
+__all__ = (
+    "MAXIMUM_FETCH_DEPTH",
+    "FetchJoin",
+    "FetchOrder",
+    "FetchPlan",
+    "FetchRequest",
+    "FetchStrategy",
+    "by_column",
+    "by_expression",
+    "validate_fetch_plan",
+)