pyoq-sql 1.0.2__py3-none-any.whl

pyoq/relations/graph.py

@@ -0,0 +1,245 @@
+"""Navigating a schema's relations from any table."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from types import MappingProxyType
+
+from pyoq.errors import RelationResolutionError, SchemaValidationError
+from pyoq.relations.derivation import table_relations
+from pyoq.relations.model import (
+    RelationCardinality,
+    RelationEndpoint,
+    TypedRelation,
+)
+from pyoq.schema import Identifier, ObjectReference, SchemaSnapshot, Table
+
+
+@dataclass(frozen=True, slots=True, eq=False)
+class RelationGraph:
+    """Every relation in a snapshot, indexed by the table it starts from.
+
+    Endpoints and lookups are both resolved to the canonical reference of the
+    table they name, so a partly qualified name reaches the table it means
+    without the caller having to know how the snapshot qualified it.
+
+    A graph is an index over a snapshot rather than a value, so it compares and
+    hashes by identity. Comparing two indexes of the same snapshot is not a
+    question worth asking, and claiming a hash it could not produce would be
+    worse than not offering one.
+    """
+
+    _tables: Mapping[ObjectReference, Table]
+    _relations: Mapping[ObjectReference, tuple[TypedRelation, ...]]
+    _unresolved: tuple[ObjectReference, ...]
+
+    @property
+    def tables(self) -> tuple[ObjectReference, ...]:
+        return tuple(self._tables)
+
+    @property
+    def unresolved_targets(self) -> tuple[ObjectReference, ...]:
+        """Tables a relation names that this snapshot does not describe.
+
+        A snapshot covers one database, and a foreign key may point outside it,
+        so these are reported rather than treated as corruption.
+        """
+        return self._unresolved
+
+    def table(self, reference: ObjectReference, /) -> Table:
+        table = self._tables.get(self.resolve(reference))
+        if table is None:
+            raise RelationResolutionError(_missing_table_message(reference))
+        return table
+
+    def resolve(self, reference: ObjectReference, /) -> ObjectReference:
+        """Name a table the way this snapshot names it."""
+        return _canonical_reference(reference, self._tables)
+
+    def relations_from(
+        self,
+        reference: ObjectReference,
+        /,
+    ) -> tuple[TypedRelation, ...]:
+        return self._relations.get(self.resolve(reference), ())
+
+    def to_one_from(
+        self,
+        reference: ObjectReference,
+        /,
+    ) -> tuple[TypedRelation, ...]:
+        return self._with_cardinality(reference, RelationCardinality.TO_ONE)
+
+    def to_many_from(
+        self,
+        reference: ObjectReference,
+        /,
+    ) -> tuple[TypedRelation, ...]:
+        return self._with_cardinality(reference, RelationCardinality.TO_MANY)
+
+    def target_table(self, relation: TypedRelation, /) -> Table:
+        return self.table(relation.target.table)
+
+    def _with_cardinality(
+        self,
+        reference: ObjectReference,
+        cardinality: RelationCardinality,
+    ) -> tuple[TypedRelation, ...]:
+        return tuple(
+            relation
+            for relation in self.relations_from(reference)
+            if relation.cardinality is cardinality
+        )
+
+
+def build_relation_graph(snapshot: SchemaSnapshot) -> RelationGraph:
+    tables = _table_index(snapshot)
+    relations = tuple(
+        _canonical(relation, tables)
+        for catalog in snapshot.catalogs
+        for schema in catalog.schemas
+        for table in schema.tables
+        for relation in table_relations(catalog, schema, table)
+    )
+    _require_known_columns(relations, tables)
+    return RelationGraph(
+        MappingProxyType(dict(tables)),
+        MappingProxyType(_grouped(relations)),
+        _unresolved_targets(relations, tables),
+    )
+
+
+def _table_index(snapshot: SchemaSnapshot) -> dict[ObjectReference, Table]:
+    return {
+        ObjectReference(table.name, schema.name, catalog.name): table
+        for catalog in snapshot.catalogs
+        for schema in catalog.schemas
+        for table in schema.tables
+    }
+
+
+def _canonical(
+    relation: TypedRelation,
+    tables: Mapping[ObjectReference, Table],
+) -> TypedRelation:
+    source = _canonical_endpoint(relation.source, tables)
+    target = _canonical_endpoint(relation.target, tables)
+    if source is relation.source and target is relation.target:
+        return relation
+    return TypedRelation(
+        relation.direction,
+        relation.cardinality,
+        source,
+        target,
+        optional=relation.optional,
+        constraint=relation.constraint,
+        on_update=relation.on_update,
+        on_delete=relation.on_delete,
+    )
+
+
+def _canonical_endpoint(
+    endpoint: RelationEndpoint,
+    tables: Mapping[ObjectReference, Table],
+) -> RelationEndpoint:
+    reference = _canonical_reference(endpoint.table, tables)
+    if reference is endpoint.table:
+        return endpoint
+    return RelationEndpoint(reference, endpoint.columns)
+
+
+def _canonical_reference(
+    reference: ObjectReference,
+    tables: Mapping[ObjectReference, Table],
+) -> ObjectReference:
+    if reference in tables:
+        return reference
+    matches = tuple(
+        candidate
+        for candidate in tables
+        if candidate.name == reference.name
+        and _qualification_matches(reference, candidate)
+    )
+    return matches[0] if len(matches) == 1 else reference
+
+
+def _qualification_matches(
+    reference: ObjectReference,
+    candidate: ObjectReference,
+) -> bool:
+    """An unqualified name means whichever table carries it, if only one does."""
+    return (reference.schema is None or reference.schema == candidate.schema) and (
+        reference.catalog is None or reference.catalog == candidate.catalog
+    )
+
+
+def _require_known_columns(
+    relations: Iterable[TypedRelation],
+    tables: Mapping[ObjectReference, Table],
+) -> None:
+    for relation in relations:
+        for endpoint in (relation.source, relation.target):
+            table = tables.get(endpoint.table)
+            if table is not None:
+                _require_columns(table, endpoint)
+
+
+def _require_columns(table: Table, endpoint: RelationEndpoint) -> None:
+    known = frozenset(column.name for column in table.columns)
+    missing = tuple(column for column in endpoint.columns if column not in known)
+    if missing:
+        raise SchemaValidationError(_missing_column_message(table.name, missing))
+
+
+def _unresolved_targets(
+    relations: Iterable[TypedRelation],
+    tables: Mapping[ObjectReference, Table],
+) -> tuple[ObjectReference, ...]:
+    unresolved = {
+        relation.target.table
+        for relation in relations
+        if relation.target.table not in tables
+    }
+    return tuple(sorted(unresolved, key=_reference_order))
+
+
+def _grouped(
+    relations: Iterable[TypedRelation],
+) -> dict[ObjectReference, tuple[TypedRelation, ...]]:
+    grouped: dict[ObjectReference, list[TypedRelation]] = {}
+    for relation in relations:
+        grouped.setdefault(relation.source.table, []).append(relation)
+    return {reference: tuple(items) for reference, items in grouped.items()}
+
+
+def _reference_order(reference: ObjectReference) -> tuple[str, str, str]:
+    return (
+        reference.catalog.value if reference.catalog is not None else "",
+        reference.schema.value if reference.schema is not None else "",
+        reference.name.value,
+    )
+
+
+def _missing_table_message(reference: ObjectReference) -> str:
+    return f"the relation graph holds no table named {_describe(reference)}"
+
+
+def _missing_column_message(
+    table: Identifier,
+    missing: tuple[Identifier, ...],
+) -> str:
+    names = ", ".join(column.value for column in missing)
+    return f"table {table.value!r} has no column named {names}"
+
+
+def _describe(reference: ObjectReference) -> str:
+    parts = tuple(
+        part.value
+        for part in (reference.catalog, reference.schema, reference.name)
+        if part is not None
+    )
+    return ".".join(parts)
+
+
+__all__ = ("RelationGraph", "build_relation_graph")

pyoq/relations/loading.py

@@ -0,0 +1,74 @@
+"""What a relation holds after a fetch, including having not been fetched."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import StrEnum
+from typing import Generic, TypeVar
+
+from pyoq.errors import RelationNotLoadedError
+
+Value = TypeVar("Value")
+Loadable = TypeVar("Loadable")
+
+
+class RelationState(StrEnum):
+    LOADED = "loaded"
+    ABSENT = "absent"
+    UNLOADED = "unloaded"
+
+
+@dataclass(frozen=True, slots=True)
+class RelationValue(Generic[Value]):
+    """A relation is loaded, known to be absent, or was never fetched.
+
+    Rows are detached values, so reading a relation never reaches a database.
+    That makes the third state necessary: without it, a relation nobody asked
+    for would be indistinguishable from one that resolved to nothing, and the
+    caller would read absence where it should read ignorance.
+    """
+
+    state: RelationState
+    _value: Value | None = None
+
+    @staticmethod
+    def loaded(value: Loadable, /) -> RelationValue[Loadable]:
+        return RelationValue(RelationState.LOADED, value)
+
+    @staticmethod
+    def absent() -> RelationValue[Value]:
+        return RelationValue(RelationState.ABSENT)
+
+    @staticmethod
+    def unloaded() -> RelationValue[Value]:
+        return RelationValue(RelationState.UNLOADED)
+
+    @property
+    def is_loaded(self) -> bool:
+        return self.state is RelationState.LOADED
+
+    @property
+    def is_absent(self) -> bool:
+        return self.state is RelationState.ABSENT
+
+    @property
+    def is_unloaded(self) -> bool:
+        return self.state is RelationState.UNLOADED
+
+    @property
+    def value(self) -> Value | None:
+        """The related value, or nothing when the relation resolved to nothing.
+
+        Reading a relation nobody fetched raises rather than answering, because
+        any answer would be a guess and no answer can be produced without I/O.
+        """
+        if self.state is RelationState.UNLOADED:
+            message = (
+                "this relation was not fetched; include it in the query's fetch "
+                "plan to read it"
+            )
+            raise RelationNotLoadedError(message)
+        return self._value
+
+
+__all__ = ("RelationState", "RelationValue")

pyoq/relations/model.py

@@ -0,0 +1,75 @@
+"""Typed relation facts derived from a schema."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import StrEnum
+
+from pyoq.errors import SchemaValidationError
+from pyoq.schema import Identifier, ObjectReference, ReferentialAction
+
+
+class RelationCardinality(StrEnum):
+    TO_ONE = "to-one"
+    TO_MANY = "to-many"
+
+
+class RelationDirection(StrEnum):
+    """Which way a foreign key is being followed.
+
+    A foreign key is one fact that can be read from either side, so both
+    directions are derived from it rather than declared separately.
+    """
+
+    FORWARD = "forward"
+    REVERSE = "reverse"
+
+
+@dataclass(frozen=True, slots=True)
+class RelationEndpoint:
+    table: ObjectReference
+    columns: tuple[Identifier, ...]
+
+    def __post_init__(self) -> None:
+        if not self.columns:
+            message = "relation endpoint requires at least one column"
+            raise SchemaValidationError(message)
+
+
+@dataclass(frozen=True, slots=True)
+class TypedRelation:
+    """One navigable direction of one foreign key.
+
+    ``optional`` describes whether a to-one relation may resolve to nothing. A
+    to-many relation is never optional, because the absence of children is an
+    empty collection rather than a missing value.
+    """
+
+    direction: RelationDirection
+    cardinality: RelationCardinality
+    source: RelationEndpoint
+    target: RelationEndpoint
+    optional: bool = False
+    constraint: Identifier | None = None
+    on_update: ReferentialAction = ReferentialAction.NO_ACTION
+    on_delete: ReferentialAction = ReferentialAction.NO_ACTION
+
+    def __post_init__(self) -> None:
+        if len(self.source.columns) != len(self.target.columns):
+            message = "relation source and target column counts must match"
+            raise SchemaValidationError(message)
+        if self.cardinality is RelationCardinality.TO_MANY and self.optional:
+            message = "a to-many relation cannot be optional"
+            raise SchemaValidationError(message)
+
+    @property
+    def to_one(self) -> bool:
+        return self.cardinality is RelationCardinality.TO_ONE
+
+
+__all__ = (
+    "RelationCardinality",
+    "RelationDirection",
+    "RelationEndpoint",
+    "TypedRelation",
+)

pyoq/relations/planning.py

@@ -0,0 +1,206 @@
+"""Choosing how a relation is fetched, and saying why.
+
+``AUTO`` is a request rather than an instruction, so a plan carries intent and
+this turns it into a decision against what a dialect can actually do. The
+decision is recorded with its reason, because a plan that cannot be read back
+is a plan nobody can review.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from pyoq.errors import FetchPlanError
+from pyoq.relations.fetching import (
+    FetchJoin,
+    FetchOrder,
+    FetchPlan,
+    FetchRequest,
+    FetchStrategy,
+)
+from pyoq.relations.model import RelationCardinality, TypedRelation
+from pyoq.schema import ObjectReference
+
+
+@dataclass(frozen=True, slots=True)
+class FetchCapabilities:
+    """What one dialect can do when fetching a collection in one query.
+
+    Ordering is separate from aggregation because a dialect can offer the second
+    without the first, and a collection aggregated in the wrong order is worse
+    than one fetched in a second query.
+    """
+
+    nested_collections: bool = False
+    ordered_nested_collections: bool = False
+
+    def __post_init__(self) -> None:
+        if self.ordered_nested_collections and not self.nested_collections:
+            message = "a dialect that cannot aggregate a collection cannot order one"
+            raise FetchPlanError(message)
+
+
+@dataclass(frozen=True, slots=True)
+class ResolvedFetch:
+    """One relation, the way it will actually be fetched, and why."""
+
+    relation: TypedRelation
+    strategy: FetchStrategy
+    reason: str
+    order_by: tuple[FetchOrder, ...] = ()
+    joins: tuple[FetchJoin, ...] = ()
+    limit: int | None = None
+    nested: tuple[ResolvedFetch, ...] = field(default_factory=tuple)
+
+    @property
+    def table(self) -> ObjectReference:
+        return self.relation.target.table
+
+
+@dataclass(frozen=True, slots=True)
+class ResolvedFetchPlan:
+    root: ObjectReference
+    fetches: tuple[ResolvedFetch, ...] = field(default_factory=tuple)
+
+    def explain(self) -> tuple[str, ...]:
+        """Read the decision back, one line per relation."""
+        lines: list[str] = []
+        pending = [(fetch, 0) for fetch in reversed(self.fetches)]
+        while pending:
+            fetch, level = pending.pop()
+            lines.append(_explain(fetch, level))
+            pending.extend((nested, level + 1) for nested in reversed(fetch.nested))
+        return tuple(lines)
+
+
+def resolve_fetch_plan(
+    plan: FetchPlan,
+    capabilities: FetchCapabilities,
+    /,
+) -> ResolvedFetchPlan:
+    return ResolvedFetchPlan(
+        plan.root,
+        tuple(_resolve(request, capabilities) for request in plan.requests),
+    )
+
+
+def _resolve(
+    request: FetchRequest,
+    capabilities: FetchCapabilities,
+) -> ResolvedFetch:
+    strategy, reason = _decide(request, capabilities)
+    return ResolvedFetch(
+        request.relation,
+        strategy,
+        reason,
+        request.order_by,
+        request.joins,
+        request.limit,
+        tuple(_resolve(nested, capabilities) for nested in request.nested),
+    )
+
+
+def _decide(
+    request: FetchRequest,
+    capabilities: FetchCapabilities,
+) -> tuple[FetchStrategy, str]:
+    if request.relation.cardinality is RelationCardinality.TO_ONE:
+        return _decide_to_one(request)
+    if request.strategy is FetchStrategy.AUTO:
+        return _decide_collection(request, capabilities)
+    _require_possible(request, capabilities)
+    return request.strategy, "the plan asked for it"
+
+
+def _decide_to_one(request: FetchRequest) -> tuple[FetchStrategy, str]:
+    """A to-one relation is one row, which a join brings back for free."""
+    if request.strategy in (FetchStrategy.AUTO, FetchStrategy.JOINED):
+        return FetchStrategy.JOINED, "a to-one relation is one row of the join"
+    return request.strategy, "the plan asked for it"
+
+
+def _decide_collection(
+    request: FetchRequest,
+    capabilities: FetchCapabilities,
+) -> tuple[FetchStrategy, str]:
+    if request.nested:
+        # A collection comes back as JSON holding the columns that were asked
+        # for. Fetching through its rows needs their keys, and a second query
+        # keyed on them is how they are had.
+        return (
+            FetchStrategy.SELECT_IN,
+            "this collection is fetched through, and its own rows are what "
+            "the deeper fetch is keyed on",
+        )
+    if not capabilities.nested_collections:
+        return (
+            FetchStrategy.SELECT_IN,
+            "this dialect cannot aggregate a collection in one query",
+        )
+    if request.order_by and not capabilities.ordered_nested_collections:
+        # A limited collection is always ordered, so this covers one too: the
+        # rows a limit keeps are the ones the order put first.
+        return (
+            FetchStrategy.SELECT_IN,
+            "this dialect cannot order inside an aggregate, and this "
+            "collection's order matters",
+        )
+    return FetchStrategy.NESTED, "one query returns the collection with its parent"
+
+
+def _require_possible(
+    request: FetchRequest,
+    capabilities: FetchCapabilities,
+) -> None:
+    """Refuse a collection this plan cannot be carried out as asked.
+
+    Falling back quietly would be worse: a caller who asked for an ordered
+    nested collection and received an unordered one has no way to notice.
+    """
+    table = request.relation.target.table.name.value
+    if request.strategy is FetchStrategy.JOINED:
+        message = (
+            f"the collection of {table!r} cannot be joined into its parent's "
+            f"row, because a parent with several would arrive as several "
+            f"parents; fetch it nested or with select-in"
+        )
+        raise FetchPlanError(message)
+    if request.strategy is not FetchStrategy.NESTED:
+        return
+    if request.nested:
+        message = (
+            f"the collection of {table!r} is fetched through, and a nested "
+            f"collection comes back as the columns asked for rather than as "
+            f"rows to key from; fetch it with select-in instead"
+        )
+        raise FetchPlanError(message)
+    if not capabilities.nested_collections:
+        message = (
+            f"this dialect cannot aggregate the collection of {table!r} in one "
+            f"query; fetch it with select-in instead"
+        )
+        raise FetchPlanError(message)
+    if request.order_by and not capabilities.ordered_nested_collections:
+        message = (
+            f"this dialect cannot order inside an aggregate, so the collection "
+            f"of {table!r} cannot be nested in the order asked for; fetch it "
+            f"with select-in instead"
+        )
+        raise FetchPlanError(message)
+
+
+def _explain(fetch: ResolvedFetch, level: int) -> str:
+    names = ", ".join(term.describe() for term in fetch.order_by)
+    order = f" ordered by {names}" if names else ""
+    return (
+        f"{'  ' * level}{fetch.table.name.value}{order} "
+        f"via {fetch.strategy.value}: {fetch.reason}"
+    )
+
+
+__all__ = (
+    "FetchCapabilities",
+    "ResolvedFetch",
+    "ResolvedFetchPlan",
+    "resolve_fetch_plan",
+)

pyoq/runtime/__init__.py

@@ -0,0 +1,9 @@
+"""Runtime engine information."""
+
+from os import getenv
+
+from pyoq.runtime.selection import RuntimeEngine, select_runtime
+
+engine = select_runtime(getenv("PYOQ_RUNTIME"))
+
+__all__ = ("RuntimeEngine", "engine")

pyoq/runtime/kernels.py

@@ -0,0 +1,25 @@
+"""Batch-oriented runtime kernel facade."""
+
+from collections.abc import Sequence
+
+from pyoq import runtime
+from pyoq.runtime.selection import (
+    RuntimeBackend,
+    RuntimeEngine,
+    load_native_runtime,
+    load_python_runtime,
+)
+
+_backend: RuntimeBackend = (
+    load_native_runtime()
+    if runtime.engine is RuntimeEngine.NATIVE
+    else load_python_runtime()
+)
+
+
+def fingerprint_fragments(fragments: Sequence[bytes]) -> int:
+    return _backend.fingerprint_fragments(list(fragments))
+
+
+def group_integer_keys(keys: Sequence[int | None]) -> list[tuple[int, int]]:
+    return _backend.group_integer_keys(list(keys))

pyoq/runtime/python.py

@@ -0,0 +1,43 @@
+"""Portable runtime kernel implementations."""
+
+from collections.abc import Sequence
+
+_FINGERPRINT_OFFSET = 0xCBF29CE484222325
+_FINGERPRINT_PRIME = 0x100000001B3
+_FINGERPRINT_MASK = (1 << 64) - 1
+
+
+def runtime_marker() -> str:
+    return "python"
+
+
+def fingerprint_fragments(fragments: Sequence[bytes]) -> int:
+    fingerprint = _FINGERPRINT_OFFSET
+    for fragment in fragments:
+        fingerprint = _fingerprint_fragment(fingerprint, fragment)
+    return fingerprint
+
+
+def group_integer_keys(keys: Sequence[int | None]) -> list[tuple[int, int]]:
+    group_indexes: dict[int, int] = {}
+    groups: list[tuple[int, int]] = []
+    for key in keys:
+        if key is None:
+            continue
+        group_index = group_indexes.get(key)
+        if group_index is None:
+            group_indexes[key] = len(groups)
+            groups.append((key, 1))
+            continue
+        grouped_key, count = groups[group_index]
+        groups[group_index] = (grouped_key, count + 1)
+    return groups
+
+
+def _fingerprint_fragment(fingerprint: int, fragment: bytes) -> int:
+    length = len(fragment).to_bytes(8, byteorder="little")
+    for data in (length, fragment):
+        for byte in data:
+            fingerprint ^= byte
+            fingerprint = (fingerprint * _FINGERPRINT_PRIME) & _FINGERPRINT_MASK
+    return fingerprint