pyoq-sql 1.0.2__py3-none-any.whl

pyoq/snapshots/drift.py

@@ -0,0 +1,312 @@
+"""Comparing what a snapshot recorded against what a database now holds.
+
+Generated code drifts when the schema moves under it. Comparing two snapshots
+says what moved in terms of the schema rather than of the files, so the answer
+names the table or the column that changed rather than the line that differs.
+
+Named things are matched by name and then by value, so a renamed table reads
+as one removed and one added rather than as a change to something that is no
+longer there.
+
+Everything else a container holds is compared by standing its named children
+aside and comparing what is left. A comment, a capability, a format version
+and any fact the schema model gains later are all compared that way, so a new
+fact is compared from the day it exists rather than from the day someone
+remembers to add it here.
+
+The children are stood aside rather than emptied, because a snapshot without
+a catalog and a table without a column are both refused by the model. One
+fixed stand-in is put in their place, the same one on both sides, so what
+remains is the container's own facts and nothing else.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass, replace
+from enum import StrEnum
+from typing import TYPE_CHECKING, Any, Protocol, TypeVar
+
+from pyoq.schema.models import (
+    Catalog,
+    Column,
+    Identifier,
+    Schema,
+    SqlType,
+    TypeKind,
+)
+
+if TYPE_CHECKING:
+    from pyoq.schema.models import SchemaSnapshot, Table
+
+
+class HasName(Protocol):
+    """Anything a schema names, which is everything compared here."""
+
+    @property
+    def name(self) -> Identifier | None: ...
+
+
+Named = TypeVar("Named", bound=HasName)
+
+
+class SchemaChange(StrEnum):
+    """What happened to one named thing between two snapshots."""
+
+    ADDED = "added"
+    REMOVED = "removed"
+    CHANGED = "changed"
+
+
+@dataclass(frozen=True, slots=True)
+class SchemaDifference:
+    """One named thing that is not the same in both snapshots."""
+
+    change: SchemaChange
+    kind: str
+    name: str
+
+    def describe(self) -> str:
+        """What changed, and which one, where there is a which one to name.
+
+        The snapshot itself has no name to give, so it is described by what it
+        is rather than followed by nothing.
+        """
+        if not self.name:
+            return f"{self.change.value} {self.kind}"
+        return f"{self.change.value} {self.kind} {self.name}"
+
+
+@dataclass(frozen=True, slots=True)
+class SchemaDriftReport:
+    """Everything that moved between two snapshots, and whether anything did."""
+
+    differences: tuple[SchemaDifference, ...]
+
+    @property
+    def is_current(self) -> bool:
+        return not self.differences
+
+    def summary(self) -> str:
+        if self.is_current:
+            return "the recorded schema matches the database"
+        written = ", ".join(item.describe() for item in self.differences)
+        return f"{len(self.differences)} schema difference(s): {written}"
+
+
+def compare_snapshots(
+    recorded: SchemaSnapshot,
+    observed: SchemaSnapshot,
+    /,
+) -> SchemaDriftReport:
+    """What changed between the snapshot on file and the one just read."""
+    differences: list[SchemaDifference] = []
+    if recorded.dialect is not observed.dialect:
+        differences.append(
+            SchemaDifference(SchemaChange.CHANGED, "dialect", observed.dialect.value)
+        )
+    _compare_own(
+        differences,
+        "snapshot",
+        "",
+        replace(recorded, catalogs=_STAND_IN_CATALOGS, dialect=observed.dialect),
+        replace(observed, catalogs=_STAND_IN_CATALOGS),
+    )
+    left = _by_name(recorded.catalogs, "")
+    right = _by_name(observed.catalogs, "")
+    _record_missing(differences, "catalog", left, right)
+    for name in _shared(left, right):
+        _compare_catalog(differences, name, left[name], right[name])
+    return SchemaDriftReport(tuple(differences))
+
+
+def _compare_catalog(
+    differences: list[SchemaDifference],
+    prefix: str,
+    recorded: Catalog,
+    observed: Catalog,
+) -> None:
+    if recorded == observed:
+        return
+    _compare_own(
+        differences,
+        "catalog",
+        prefix,
+        replace(recorded, schemas=_STAND_IN_SCHEMAS),
+        replace(observed, schemas=_STAND_IN_SCHEMAS),
+    )
+    left = _by_name(recorded.schemas, prefix)
+    right = _by_name(observed.schemas, prefix)
+    _record_missing(differences, "schema", left, right)
+    for name in _shared(left, right):
+        _compare_schema(differences, name, left[name], right[name])
+
+
+def _compare_schema(
+    differences: list[SchemaDifference],
+    prefix: str,
+    recorded: Schema,
+    observed: Schema,
+) -> None:
+    if recorded == observed:
+        return
+    _compare_own(
+        differences,
+        "schema",
+        prefix,
+        replace(recorded, **_NO_SCHEMA_CHILDREN),
+        replace(observed, **_NO_SCHEMA_CHILDREN),
+    )
+    _compare_tables(differences, prefix, recorded.tables, observed.tables)
+    _compare_leaves(differences, "view", prefix, recorded.views, observed.views)
+    _compare_leaves(differences, "enum", prefix, recorded.enums, observed.enums)
+    _compare_leaves(differences, "domain", prefix, recorded.domains, observed.domains)
+    _compare_leaves(
+        differences, "routine", prefix, recorded.routines, observed.routines
+    )
+
+
+def _compare_tables(
+    differences: list[SchemaDifference],
+    prefix: str,
+    recorded: Sequence[Table],
+    observed: Sequence[Table],
+) -> None:
+    left = _by_name(recorded, prefix)
+    right = _by_name(observed, prefix)
+    _record_missing(differences, "table", left, right)
+    for name in _shared(left, right):
+        if left[name] == right[name]:
+            continue
+        _compare_table(differences, name, left[name], right[name])
+
+
+def _compare_table(
+    differences: list[SchemaDifference],
+    prefix: str,
+    recorded: Table,
+    observed: Table,
+) -> None:
+    """A table's named parts by name, and everything else as the table itself."""
+    _compare_own(
+        differences,
+        "table",
+        prefix,
+        replace(recorded, **_NO_TABLE_CHILDREN),
+        replace(observed, **_NO_TABLE_CHILDREN),
+    )
+    for kind, held in _TABLE_CHILDREN.items():
+        _compare_leaves(differences, kind, prefix, held(recorded), held(observed))
+
+
+def _compare_own(
+    differences: list[SchemaDifference],
+    kind: str,
+    name: str,
+    recorded: object,
+    observed: object,
+) -> None:
+    """What a container holds itself, once its named children are emptied.
+
+    Comparing the remainder rather than a written list of attributes is what
+    keeps this exhaustive: a fact added to the schema model is compared
+    without being named here.
+    """
+    if recorded == observed:
+        return
+    differences.append(SchemaDifference(SchemaChange.CHANGED, kind, name))
+
+
+def _compare_leaves(
+    differences: list[SchemaDifference],
+    kind: str,
+    prefix: str,
+    recorded: Sequence[Named],
+    observed: Sequence[Named],
+) -> None:
+    """Things compared whole, because nothing inside them is named separately."""
+    left = _by_name(recorded, prefix)
+    right = _by_name(observed, prefix)
+    _record_missing(differences, kind, left, right)
+    for name in _shared(left, right):
+        if left[name] != right[name]:
+            differences.append(SchemaDifference(SchemaChange.CHANGED, kind, name))
+
+
+_STAND_IN_COLUMNS: tuple[Column, ...] = (
+    Column(Identifier("pyoq_stand_in"), SqlType(TypeKind.OTHER, "pyoq"), nullable=True),
+)
+_STAND_IN_SCHEMAS: tuple[Schema, ...] = (Schema(Identifier("pyoq_stand_in")),)
+_STAND_IN_CATALOGS: tuple[Catalog, ...] = (
+    Catalog(Identifier("pyoq_stand_in"), _STAND_IN_SCHEMAS),
+)
+"""What stands in for a container's children while its own facts are compared.
+
+The same value on both sides, so it cancels out and only the container's own
+facts remain.
+"""
+
+_NO_SCHEMA_CHILDREN: dict[str, Any] = {
+    "tables": (),
+    "views": (),
+    "enums": (),
+    "domains": (),
+    "routines": (),
+}
+_NO_TABLE_CHILDREN: dict[str, Any] = {
+    "columns": _STAND_IN_COLUMNS,
+    "keys": (),
+    "relations": (),
+    "indexes": (),
+    "checks": (),
+}
+_TABLE_CHILDREN: dict[str, Callable[[Table], Sequence[HasName]]] = {
+    "column": lambda table: table.columns,
+    "key": lambda table: table.keys,
+    "relation": lambda table: table.relations,
+    "index": lambda table: table.indexes,
+    "check": lambda table: table.checks,
+}
+"""Everything a table names, so each is reported as itself rather than as the
+table it sits in."""
+
+
+def _record_missing(
+    differences: list[SchemaDifference],
+    kind: str,
+    recorded: Mapping[str, Named],
+    observed: Mapping[str, Named],
+) -> None:
+    for name in sorted(set(recorded) - set(observed)):
+        differences.append(SchemaDifference(SchemaChange.REMOVED, kind, name))
+    for name in sorted(set(observed) - set(recorded)):
+        differences.append(SchemaDifference(SchemaChange.ADDED, kind, name))
+
+
+def _shared(
+    recorded: Mapping[str, Named],
+    observed: Mapping[str, Named],
+) -> tuple[str, ...]:
+    return tuple(sorted(set(recorded) & set(observed)))
+
+
+def _by_name(items: Sequence[Named], prefix: str) -> dict[str, Named]:
+    """Named things by their written name, qualified by whatever holds them.
+
+    A catalog or a schema may be unnamed, and an unnamed one is still a
+    position, so it is written as the position it holds.
+    """
+    named: dict[str, Named] = {}
+    for index, item in enumerate(items):
+        written = f"[{index}]" if item.name is None else item.name.value
+        named[f"{prefix}.{written}" if prefix else written] = item
+    return named
+
+
+__all__ = (
+    "HasName",
+    "SchemaChange",
+    "SchemaDifference",
+    "SchemaDriftReport",
+    "compare_snapshots",
+)

pyoq/snapshots/files.py

@@ -0,0 +1,96 @@
+"""Reading and writing a schema snapshot as a file.
+
+A snapshot is the one canonical record of what a database holds. Written to a
+file it becomes reviewable in a diff and usable without a database, which is
+what lets generation and drift checks run where no credentials exist.
+
+The bytes are the codec's, so a snapshot written twice from one database is
+byte for byte the same file and an unrelated change shows as no change.
+"""
+
+from __future__ import annotations
+
+import os
+from contextlib import suppress
+from pathlib import Path
+from tempfile import NamedTemporaryFile
+
+from pyoq.errors import SchemaSerializationError
+from pyoq.schema.codec import decode_snapshot, encode_snapshot
+from pyoq.schema.models import SchemaSnapshot
+
+_ENCODING = "utf-8"
+
+
+def write_snapshot(snapshot: SchemaSnapshot, path: Path, /) -> Path:
+    """Write a snapshot where a reader and a reviewer can both reach it.
+
+    Written beside its destination and moved into place, so a reader never
+    sees half a snapshot and a failed write leaves the previous one intact.
+
+    A write can fail while the staged file is being written, while it is being
+    closed, or as it is moved. Whichever it is, the staged file goes, so a
+    failed recording leaves the directory as it found it rather than beside a
+    partial file nobody will claim.
+    """
+    destination = path.resolve()
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    staged = _staged(destination, encode_snapshot(snapshot))
+    try:
+        os.replace(staged, destination)
+    except BaseException:
+        _discard(staged)
+        raise
+    return destination
+
+
+def _staged(destination: Path, written: str, /) -> Path:
+    """The snapshot written beside where it is going, ready to be moved.
+
+    Closed inside the guard rather than after it, because a write is only
+    known to have reached the file once the file is closed, and a failure
+    there leaves as much behind as a failure while writing.
+    """
+    handle = NamedTemporaryFile(  # noqa: SIM115
+        "w",
+        encoding=_ENCODING,
+        newline="\n",
+        dir=destination.parent,
+        prefix=f".{destination.name}.",
+        suffix=".partial",
+        delete=False,
+    )
+    staged = Path(handle.name)
+    try:
+        with handle:
+            handle.write(written)
+            handle.write("\n")
+    except BaseException:
+        _discard(staged)
+        raise
+    return staged
+
+
+def _discard(staged: Path, /) -> None:
+    """Take the staged file away, and say nothing about having done so.
+
+    What went wrong is what the caller is told. A failure to clean up after a
+    failure would replace the reason with a second one, so it is not allowed
+    to.
+    """
+    with suppress(OSError):
+        staged.unlink(missing_ok=True)
+
+
+def read_snapshot(path: Path, /) -> SchemaSnapshot:
+    """The snapshot a file holds, or a precise error saying why it does not."""
+    location = path.resolve()
+    try:
+        written = location.read_text(encoding=_ENCODING)
+    except OSError as error:
+        message = f"cannot read the schema snapshot at {location}: {error}"
+        raise SchemaSerializationError(message) from error
+    return decode_snapshot(written)
+
+
+__all__ = ("read_snapshot", "write_snapshot")

pyoq/snapshots/routing.py

@@ -0,0 +1,40 @@
+"""Choosing where a schema is read from.
+
+Generation asks for a snapshot and does not care where it came from. A project
+that records its schema in a file is read from that file, and one that does not
+is read from its database, so the same pipeline serves both and neither knows
+which it got.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from pyoq.schema.registry import DialectRoutingSchemaSource
+from pyoq.snapshots.source import SnapshotSchemaSource
+
+if TYPE_CHECKING:
+    from pyoq.config import Configuration
+    from pyoq.schema.models import SchemaSnapshot
+    from pyoq.schema.source import SchemaSource
+
+
+_LIVE = DialectRoutingSchemaSource()
+"""The database itself, which is what a project without a snapshot reads."""
+
+
+@dataclass(frozen=True, slots=True)
+class ConfiguredSchemaSource:
+    """The recorded schema where a project keeps one, the database otherwise."""
+
+    live: SchemaSource = _LIVE
+
+    def load(self, configuration: Configuration) -> SchemaSnapshot:
+        recorded = configuration.schema_snapshot_path
+        if recorded is None:
+            return self.live.load(configuration)
+        return SnapshotSchemaSource(recorded).load(configuration)
+
+
+__all__ = ("ConfiguredSchemaSource",)

pyoq/snapshots/source.py

@@ -0,0 +1,33 @@
+"""Reading a schema from a file rather than from a database.
+
+Generation asks a source for a snapshot and does not care where it came from.
+A snapshot on disk is therefore all that generation needs, which is what lets
+a check run where there are no credentials and nothing to connect to.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from pyoq.snapshots.files import read_snapshot
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from pyoq.config import Configuration
+    from pyoq.schema.models import SchemaSnapshot
+
+
+@dataclass(frozen=True, slots=True)
+class SnapshotSchemaSource:
+    """A schema source that opens a file and never opens a connection."""
+
+    path: Path
+
+    def load(self, configuration: Configuration) -> SchemaSnapshot:
+        del configuration
+        return read_snapshot(self.path)
+
+
+__all__ = ("SnapshotSchemaSource",)

pyoq/tracing/__init__.py

@@ -0,0 +1,5 @@
+"""Reporting PyOQ's statements to OpenTelemetry."""
+
+from pyoq.tracing.spans import INSTRUMENTATION_NAME, TracingSink
+
+__all__ = ("INSTRUMENTATION_NAME", "TracingSink")

pyoq/tracing/spans.py

@@ -0,0 +1,89 @@
+"""Turning statement events into spans.
+
+The span carries the shape of a statement and its counts, which is what the
+event carries. Nothing a query was asked about reaches a trace unless the policy
+that produced the event already allowed it, so the decision stays in one place.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from opentelemetry.trace import SpanKind, Status, StatusCode, get_tracer
+
+from pyoq.diagnostics.events import (
+    StatementFailed,
+    StatementFinished,
+    StatementStarted,
+)
+
+if TYPE_CHECKING:
+    from opentelemetry.trace import Span, Tracer
+
+    from pyoq.diagnostics.events import StatementEvent
+
+INSTRUMENTATION_NAME = "pyoq"
+
+_NANOSECONDS = 1_000_000_000
+
+
+class TracingSink:
+    """Records each finished statement as a span of its own.
+
+    A span is opened and closed when the statement is answered rather than when
+    it starts, because the event that says how long it took is the one that can
+    give the span its duration. A statement that never finished has no duration
+    to report and would leave a span open for as long as the process runs.
+    """
+
+    __slots__ = ("_tracer",)
+
+    def __init__(self, tracer: Tracer | None = None) -> None:
+        self._tracer = tracer or get_tracer(INSTRUMENTATION_NAME)
+
+    def record(self, event: StatementEvent, /) -> None:
+        if isinstance(event, StatementStarted):
+            return
+        span = self._tracer.start_span(
+            _name(event),
+            kind=SpanKind.CLIENT,
+            start_time=_started(event),
+        )
+        _describe(span, event)
+        span.end()
+
+
+def _name(event: StatementFinished | StatementFailed) -> str:
+    return f"pyoq {event.kind.value}"
+
+
+def _started(event: StatementFinished | StatementFailed) -> int:
+    from time import time_ns
+
+    return time_ns() - int(event.duration * _NANOSECONDS)
+
+
+def _describe(span: Span, event: StatementFinished | StatementFailed) -> None:
+    span.set_attribute("db.system", "sql")
+    span.set_attribute("db.statement", event.shape.sql)
+    span.set_attribute("pyoq.shape", event.shape.digest)
+    span.set_attribute("pyoq.parameters", event.parameters)
+    span.set_attribute("pyoq.sensitive_parameters", event.sensitive_parameters)
+    if isinstance(event, StatementFinished):
+        _describe_finished(span, event)
+        return
+    span.set_attribute("pyoq.failure", event.failure)
+    if event.detail is not None:
+        span.set_attribute("pyoq.failure_detail", event.detail)
+    span.set_status(Status(StatusCode.ERROR, event.failure))
+
+
+def _describe_finished(span: Span, event: StatementFinished) -> None:
+    if event.rows is not None:
+        span.set_attribute("pyoq.rows", event.rows)
+    if event.slow:
+        span.set_attribute("pyoq.slow", value=True)
+    span.set_status(Status(StatusCode.OK))
+
+
+__all__ = ("INSTRUMENTATION_NAME", "TracingSink")

pyoq/unset.py

@@ -0,0 +1,14 @@
+"""Explicit unset marker shared by write values and generated builders."""
+
+from __future__ import annotations
+
+from enum import Enum
+
+
+class UnsetType(Enum):
+    UNSET = "unset"
+
+
+UNSET = UnsetType.UNSET
+
+__all__ = ("UNSET", "UnsetType")