pyoq-sql 1.0.2__py3-none-any.whl

pyoq/generation/writer.py

@@ -0,0 +1,184 @@
+"""Atomic generated package replacement."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import tempfile
+from collections.abc import Callable
+from dataclasses import dataclass
+from hashlib import sha256
+from pathlib import Path, PurePosixPath
+
+from pyoq.errors import GenerationOwnershipError, GenerationWriteError
+from pyoq.generation.cleanup import ManifestOwnedCleanup
+from pyoq.generation.contracts import (
+    MANIFEST_NAME,
+    GenerationPlan,
+    ManifestStore,
+    OwnedFile,
+    OwnershipManifest,
+    output_path,
+)
+
+DirectoryReplacer = Callable[[Path, Path], None]
+
+
+@dataclass(frozen=True, slots=True)
+class AtomicGenerationWriter:
+    manifest_store: ManifestStore
+    cleanup: ManifestOwnedCleanup
+    replace_directory: DirectoryReplacer = os.replace
+
+    def write(
+        self,
+        package_root: Path,
+        plan: GenerationPlan,
+        current_manifest: OwnershipManifest | None,
+        desired_manifest: OwnershipManifest,
+    ) -> None:
+        parent = package_root.parent
+        parent.mkdir(parents=True, exist_ok=True)
+        if package_root.is_symlink() or (
+            package_root.exists() and not package_root.is_dir()
+        ):
+            message = f"generated package path is not a directory: {package_root}"
+            raise GenerationWriteError(message)
+        staging_root = Path(
+            tempfile.mkdtemp(prefix=f".{package_root.name}.pyoq-stage-", dir=parent)
+        )
+        try:
+            _copy_existing_package(package_root, staging_root)
+            _write_planned_files(staging_root, plan, current_manifest)
+            retained_paths = frozenset(file.path for file in plan.files)
+            self.cleanup.remove_stale(
+                staging_root,
+                current_manifest,
+                retained_paths,
+            )
+            manifest_path = staging_root / MANIFEST_NAME
+            manifest_path.write_bytes(self.manifest_store.encode(desired_manifest))
+            _replace_package(
+                package_root,
+                staging_root,
+                self.replace_directory,
+            )
+        except (GenerationOwnershipError, GenerationWriteError):
+            raise
+        except OSError as error:
+            message = f"unable to write generated package: {error}"
+            raise GenerationWriteError(message) from error
+        finally:
+            if staging_root.exists():
+                shutil.rmtree(staging_root, ignore_errors=True)
+
+
+def _copy_existing_package(package_root: Path, staging_root: Path) -> None:
+    if package_root.exists():
+        shutil.copytree(package_root, staging_root, dirs_exist_ok=True, symlinks=True)
+
+
+def _write_planned_files(
+    staging_root: Path,
+    plan: GenerationPlan,
+    current_manifest: OwnershipManifest | None,
+) -> None:
+    owned_files = _owned_file_map(current_manifest)
+    for generated_file in plan.files:
+        destination = output_path(staging_root, generated_file.path)
+        _verify_destination(
+            staging_root,
+            destination,
+            generated_file.path,
+            owned_files.get(generated_file.path),
+        )
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        destination.write_bytes(generated_file.encoded_content)
+
+
+def _owned_file_map(
+    manifest: OwnershipManifest | None,
+) -> dict[PurePosixPath, OwnedFile]:
+    if manifest is None:
+        return {}
+    return {owned_file.path: owned_file for owned_file in manifest.files}
+
+
+def _verify_destination(
+    staging_root: Path,
+    destination: Path,
+    relative_path: PurePosixPath,
+    owned_file: OwnedFile | None,
+) -> None:
+    if not destination.exists() and not destination.is_symlink():
+        _verify_parent_chain(staging_root, destination.parent, relative_path)
+        return
+    if owned_file is None:
+        message = f"refusing to replace unowned file: {relative_path}"
+        raise GenerationOwnershipError(message)
+    valid_owned_file = (
+        destination.is_file()
+        and not destination.is_symlink()
+        and sha256(destination.read_bytes()).hexdigest() == owned_file.digest
+    )
+    if not valid_owned_file:
+        message = f"refusing to replace modified owned file: {relative_path}"
+        raise GenerationOwnershipError(message)
+
+
+def _verify_parent_chain(
+    staging_root: Path,
+    parent: Path,
+    relative_path: PurePosixPath,
+) -> None:
+    current = parent
+    while current != staging_root:
+        if not current.exists():
+            current = current.parent
+            continue
+        if current.is_symlink() or not current.is_dir():
+            message = f"unsafe parent for generated file: {relative_path}"
+            raise GenerationOwnershipError(message)
+        current = current.parent
+
+
+def _replace_package(
+    package_root: Path,
+    staging_root: Path,
+    replace_directory: DirectoryReplacer,
+) -> None:
+    if not package_root.exists():
+        replace_directory(staging_root, package_root)
+        return
+    backup_root = _unused_backup_path(package_root)
+    replace_directory(package_root, backup_root)
+    try:
+        replace_directory(staging_root, package_root)
+    except OSError as replacement_error:
+        _restore_package(backup_root, package_root, replace_directory)
+        message = f"package replacement failed and was rolled back: {replacement_error}"
+        raise GenerationWriteError(message) from replacement_error
+    shutil.rmtree(backup_root, ignore_errors=True)
+
+
+def _restore_package(
+    backup_root: Path,
+    package_root: Path,
+    replace_directory: DirectoryReplacer,
+) -> None:
+    try:
+        replace_directory(backup_root, package_root)
+    except OSError:
+        shutil.copytree(backup_root, package_root, symlinks=True)
+        shutil.rmtree(backup_root, ignore_errors=True)
+
+
+def _unused_backup_path(package_root: Path) -> Path:
+    backup_root = Path(
+        tempfile.mkdtemp(
+            prefix=f".{package_root.name}.pyoq-backup-",
+            dir=package_root.parent,
+        )
+    )
+    backup_root.rmdir()
+    return backup_root

pyoq/hydration/__init__.py

@@ -0,0 +1,24 @@
+"""Turning result rows into typed values."""
+
+from pyoq.hydration.engine import hydrate
+from pyoq.hydration.identity import IdentityMap, RowIdentity, RowKey, row_key_for
+from pyoq.hydration.plan import (
+    HydrationNode,
+    RelationValues,
+    RowConstructor,
+    TableProjection,
+    validate_hydration_node,
+)
+
+__all__ = (
+    "HydrationNode",
+    "IdentityMap",
+    "RelationValues",
+    "RowConstructor",
+    "RowIdentity",
+    "RowKey",
+    "TableProjection",
+    "hydrate",
+    "row_key_for",
+    "validate_hydration_node",
+)

pyoq/hydration/engine.py

@@ -0,0 +1,155 @@
+"""Turning a flat result into rows that carry their children.
+
+A join hands back one row per combination, so the same parent arrives once per
+child and the same child can arrive under several parents. Rebuilding the shape
+means reading the result twice: once to group it, once to construct it, because
+a frozen row cannot be given its children after it exists.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass
+from typing import cast
+
+from pyoq.errors import HydrationError
+from pyoq.hydration.identity import IdentityMap, RowIdentity
+from pyoq.hydration.plan import HydrationNode, validate_hydration_node
+from pyoq.relations import (
+    MAXIMUM_FETCH_DEPTH,
+    RelationCardinality,
+    RelationValue,
+    TypedRelation,
+)
+
+
+def hydrate(
+    node: HydrationNode,
+    rows: Iterable[Sequence[object]],
+    /,
+    *,
+    maximum_depth: int = MAXIMUM_FETCH_DEPTH,
+) -> tuple[object, ...]:
+    """Read a result into root rows, each carrying whatever the plan asked for.
+
+    Rows come back in the order the result first mentioned them, and a row the
+    result mentions twice is built once.
+    """
+    validate_hydration_node(node, maximum_depth=maximum_depth)
+    state = _NodeState(node)
+    for row in rows:
+        state.absorb(row)
+    return state.build_all()
+
+
+@dataclass(frozen=True, slots=True)
+class _Edge:
+    parent: RowIdentity
+    child_index: int
+
+
+class _NodeState:
+    """What one table of a result has been seen to hold.
+
+    There is one of these per table in the plan rather than one per parent, so
+    a child that several parents share is built once. It is not shared across
+    plan positions: the same table read at two places carries different
+    children, and one row cannot hold both.
+    """
+
+    __slots__ = ("_children", "_edges", "_node", "_order", "_rows", "_values")
+
+    def __init__(self, node: HydrationNode) -> None:
+        self._node = node
+        self._children = tuple(_NodeState(child) for child in node.children)
+        self._order: list[RowIdentity] = []
+        self._values: dict[RowIdentity, tuple[object, ...]] = {}
+        self._edges: dict[_Edge, dict[RowIdentity, None]] = {}
+        self._rows = IdentityMap()
+
+    def absorb(self, row: Sequence[object], /) -> RowIdentity | None:
+        identity = self._node.projection.key.identify(row)
+        if identity is None:
+            return None
+        if identity not in self._values:
+            self._order.append(identity)
+            self._values[identity] = self._node.projection.select(row)
+        for index, child in enumerate(self._children):
+            child_identity = child.absorb(row)
+            if child_identity is not None:
+                self._record(identity, index, child_identity)
+        return identity
+
+    def build_all(self) -> tuple[object, ...]:
+        return tuple(self.build(identity) for identity in self._order)
+
+    def build(self, identity: RowIdentity, /) -> object:
+        return self._rows.resolve(identity, lambda: self._construct(identity))
+
+    def _construct(self, identity: RowIdentity) -> object:
+        relations = {
+            _require_relation(child): self._value_for(child, identity, index)
+            for index, child in enumerate(self._children)
+        }
+        return self._node.construct(self._values[identity], relations)
+
+    def _value_for(
+        self,
+        child: _NodeState,
+        parent: RowIdentity,
+        index: int,
+    ) -> RelationValue[object]:
+        """Read this parent's children, which only this parent recorded."""
+        identities = tuple(self._edges.get(_Edge(parent, index), {}))
+        rows = tuple(child.build(identity) for identity in identities)
+        relation = _require_relation(child)
+        if relation.cardinality is RelationCardinality.TO_MANY:
+            return RelationValue.loaded(rows)
+        _require_at_most_one(relation, rows)
+        if not rows:
+            return RelationValue[object].absent()
+        return RelationValue.loaded(rows[0])
+
+    def _record(
+        self,
+        parent: RowIdentity,
+        index: int,
+        child: RowIdentity,
+    ) -> None:
+        """Remember the order children arrived in, without repeating any."""
+        self._edges.setdefault(_Edge(parent, index), {})[child] = None
+
+    @property
+    def relation(self) -> TypedRelation | None:
+        return self._node.relation
+
+
+def _require_relation(state: _NodeState) -> TypedRelation:
+    """Every table below the root is reached by a relation.
+
+    The plan is validated before a single row is read, so this holds by the
+    time any row is built. Checking it again here would be code no result could
+    reach.
+    """
+    return cast("TypedRelation", state.relation)
+
+
+def _require_at_most_one(
+    relation: TypedRelation,
+    rows: tuple[object, ...],
+) -> None:
+    """A to-one relation that returned several rows is not a to-one relation.
+
+    The schema said these columns were unique, and the result disagreed, so the
+    disagreement is reported rather than resolved by discarding rows.
+    """
+    if len(rows) <= 1:
+        return
+    message = (
+        f"the relation to {relation.target.table.name.value!r} is to-one, but "
+        f"the result holds {len(rows)} distinct rows for one parent"
+    )
+    raise HydrationError(message)
+
+
+__all__ = ("hydrate",)

pyoq/hydration/identity.py

@@ -0,0 +1,194 @@
+"""Deciding which result rows are the same row.
+
+A join repeats a parent row once per child, and an outer join invents a child
+row made entirely of nulls. Both are answered by asking what identifies a row,
+which is why identity comes before hydration rather than inside it.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+from typing import Final, TypeVar, cast
+
+from pyoq.errors import RowIdentityError
+from pyoq.schema import Identifier, KeyKind, ObjectReference, Table
+
+Row = TypeVar("Row")
+
+_MISSING: Final = object()
+
+
+@dataclass(frozen=True, slots=True)
+class RowIdentity:
+    """What makes one row of one table distinct from another."""
+
+    table: ObjectReference
+    values: tuple[object, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class RowKey:
+    """Where a row's identifying values sit in a result row.
+
+    The columns must be ones the database cannot leave null, because a null
+    among them is read as the row not being there at all.
+    """
+
+    table: ObjectReference
+    positions: tuple[int, ...]
+    columns: tuple[Identifier, ...] = ()
+
+    def __post_init__(self) -> None:
+        if not self.positions:
+            message = "a row key needs at least one column"
+            raise RowIdentityError(message)
+        if any(position < 0 for position in self.positions):
+            message = "row key positions cannot be negative"
+            raise RowIdentityError(message)
+
+    def identify(self, row: Sequence[object], /) -> RowIdentity | None:
+        """Identify a row, or report that the result holds no row here.
+
+        An outer join fills an absent row with nulls, so a null among the
+        identifying values means there is nothing to identify rather than a row
+        whose identity happens to be null.
+        """
+        values: list[object] = []
+        for position in self.positions:
+            value = _require_position(self, row, position)
+            if value is None:
+                return None
+            values.append(_hashable(value))
+        return RowIdentity(self.table, tuple(values))
+
+
+def row_key_for(
+    table: Table,
+    reference: ObjectReference,
+    positions: Callable[[Identifier], int],
+) -> RowKey:
+    """Choose what identifies a table's rows in a result.
+
+    A primary key is preferred, then a unique key no part of which is nullable.
+    A table offering neither is identified by everything projected from it,
+    which is the only sound answer left: rows it cannot tell apart are rows
+    nobody can tell apart.
+    """
+    columns = _identifying_columns(table)
+    return RowKey(
+        reference,
+        tuple(_require_projected(table, column, positions) for column in columns),
+        columns,
+    )
+
+
+def _require_projected(
+    table: Table,
+    column: Identifier,
+    positions: Callable[[Identifier], int],
+) -> int:
+    """A row cannot be identified by a column the query did not select."""
+    try:
+        return positions(column)
+    except KeyError as error:
+        message = (
+            f"{table.name.value!r} is identified by {column.value!r}, which this "
+            f"query does not select"
+        )
+        raise RowIdentityError(message) from error
+
+
+def _identifying_columns(table: Table) -> tuple[Identifier, ...]:
+    nullable = frozenset(column.name for column in table.columns if column.nullable)
+    for kind in (KeyKind.PRIMARY, KeyKind.UNIQUE):
+        for key in table.keys:
+            if key.kind is kind and not nullable.intersection(key.columns):
+                return key.columns
+    return tuple(column.name for column in table.columns)
+
+
+class IdentityMap:
+    """One fetch's view of which rows it has already built.
+
+    It lives for one fetch and is discarded with it, so a row built here can
+    never be a stale row from an earlier one.
+    """
+
+    __slots__ = ("_rows",)
+
+    def __init__(self) -> None:
+        self._rows: dict[RowIdentity, object] = {}
+
+    def __len__(self) -> int:
+        return len(self._rows)
+
+    def resolve(
+        self,
+        identity: RowIdentity,
+        build: Callable[[], Row],
+        /,
+    ) -> Row:
+        """Return the row already built for this identity, or build it once.
+
+        The identity names a table, so every row stored under one identity is a
+        row of one type, which is what the cast rests on.
+        """
+        existing = _lookup(self._rows, identity)
+        if existing is not _MISSING:
+            return cast("Row", existing)
+        built = build()
+        self._rows[identity] = built
+        return built
+
+    def contains(self, identity: RowIdentity, /) -> bool:
+        return _lookup(self._rows, identity) is not _MISSING
+
+
+def _lookup(rows: dict[RowIdentity, object], identity: RowIdentity) -> object:
+    """Find a row, reporting a value no dictionary can hold.
+
+    A driver can hand back a value that cannot be hashed, and a raw type error
+    from deep inside a fetch says nothing about which row caused it.
+    """
+    try:
+        return rows.get(identity, _MISSING)
+    except TypeError as error:
+        message = (
+            f"a row of {identity.table.name.value!r} is identified by a value "
+            f"that cannot be compared: {error}"
+        )
+        raise RowIdentityError(message) from error
+
+
+def _require_position(
+    key: RowKey,
+    row: Sequence[object],
+    position: int,
+) -> object:
+    if position < len(row):
+        return row[position]
+    message = (
+        f"identifying column {position} of {key.table.name.value!r} is beyond "
+        f"a result row of {len(row)} values"
+    )
+    raise RowIdentityError(message)
+
+
+def _hashable(value: object) -> object:
+    """Identify by value, however a driver chose to spell it.
+
+    Buffers and arrays carry their meaning in their contents rather than in the
+    object holding them, so two rows spelled differently by one driver still
+    identify as one row.
+    """
+    if isinstance(value, memoryview):
+        return value.tobytes()
+    if isinstance(value, bytearray):
+        return bytes(value)
+    if isinstance(value, list):
+        return tuple(_hashable(item) for item in cast("list[object]", value))
+    return value
+
+
+__all__ = ("IdentityMap", "RowIdentity", "RowKey", "row_key_for")

pyoq/hydration/plan.py

@@ -0,0 +1,116 @@
+"""Describing what a result holds, so hydration reads rather than guesses."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass, field
+from typing import TypeAlias
+
+from pyoq.errors import HydrationError
+from pyoq.hydration.identity import RowKey
+from pyoq.relations import MAXIMUM_FETCH_DEPTH, RelationValue, TypedRelation
+
+RelationValues: TypeAlias = Mapping[TypedRelation, RelationValue[object]]
+RowConstructor: TypeAlias = Callable[
+    [tuple[object, ...], RelationValues],
+    object,
+]
+
+
+@dataclass(frozen=True, slots=True)
+class TableProjection:
+    """Where one table's values sit in a result row."""
+
+    key: RowKey
+    positions: tuple[int, ...]
+
+    def __post_init__(self) -> None:
+        if not self.positions:
+            message = "a table projection needs at least one column"
+            raise HydrationError(message)
+
+    def select(self, row: Sequence[object], /) -> tuple[object, ...]:
+        """Take this table's values out of a result row.
+
+        A projection describing more columns than the result carries is a
+        mistake in the plan, and a bare index error from inside a fetch says
+        nothing about which table was being read.
+        """
+        try:
+            return tuple(row[position] for position in self.positions)
+        except IndexError as error:
+            message = (
+                f"the projection for {self.key.table.name.value!r} reads beyond "
+                f"a result row of {len(row)} values"
+            )
+            raise HydrationError(message) from error
+
+
+@dataclass(frozen=True, slots=True)
+class HydrationNode:
+    """One table in a result, and whatever hangs off it.
+
+    ``relation`` says how this node reaches its parent, which is what decides
+    whether the parent receives one row or a collection. The root has none.
+    """
+
+    projection: TableProjection
+    construct: RowConstructor
+    relation: TypedRelation | None = None
+    children: tuple[HydrationNode, ...] = field(default_factory=tuple)
+
+    @property
+    def depth(self) -> int:
+        deepest = 0
+        pending: list[tuple[HydrationNode, int]] = [(self, 1)]
+        while pending:
+            node, level = pending.pop()
+            deepest = max(deepest, level)
+            pending.extend((child, level + 1) for child in node.children)
+        return deepest
+
+
+def validate_hydration_node(
+    node: HydrationNode,
+    *,
+    maximum_depth: int = MAXIMUM_FETCH_DEPTH,
+) -> None:
+    """Refuse a shape hydration could not carry out.
+
+    Checked before a single row is read, so a mistake is reported against the
+    plan rather than part-way through a result.
+    """
+    if maximum_depth < 1:
+        message = "hydration must be allowed at least one level"
+        raise HydrationError(message)
+    if node.depth > maximum_depth:
+        message = f"hydration nests beyond the {maximum_depth} levels allowed"
+        raise HydrationError(message)
+    _require_root_without_relation(node)
+    _require_children_carry_relations(node)
+
+
+def _require_root_without_relation(node: HydrationNode) -> None:
+    if node.relation is None:
+        return
+    message = "the root of a result is reached by no relation"
+    raise HydrationError(message)
+
+
+def _require_children_carry_relations(root: HydrationNode) -> None:
+    pending = list(root.children)
+    while pending:
+        node = pending.pop()
+        if node.relation is None:
+            message = "every table below the root is reached by a relation"
+            raise HydrationError(message)
+        pending.extend(node.children)
+
+
+__all__ = (
+    "HydrationNode",
+    "RelationValues",
+    "RowConstructor",
+    "TableProjection",
+    "validate_hydration_node",
+)

pyoq/migrations/__init__.py

@@ -0,0 +1,9 @@
+"""Bringing generated code back in step after a migration."""
+
+from pyoq.migrations.hooks import (
+    MigrationOutcome,
+    after_migration,
+    after_migration_at,
+)
+
+__all__ = ("MigrationOutcome", "after_migration", "after_migration_at")