pyoq-sql 1.0.0__py3-none-any.whl

pyoq/schema/sqlite/types.py

@@ -0,0 +1,74 @@
+"""SQLite declared-type normalization."""
+
+from __future__ import annotations
+
+import re
+
+from pyoq.schema import SqlType, TypeKind
+
+_TYPE_ARGUMENTS = re.compile(r"^\s*([^()]*)\s*(?:\(([^()]*)\))?\s*$")
+
+
+def sqlite_type(declared_type: str) -> SqlType:
+    source_name = declared_type.strip() or "ANY"
+    match = _TYPE_ARGUMENTS.fullmatch(source_name)
+    base_name = source_name if match is None else match.group(1).strip()
+    arguments = () if match is None else _numeric_arguments(match.group(2))
+    kind = _type_kind(base_name.upper())
+    if kind is TypeKind.STRING and len(arguments) == 1:
+        return SqlType(kind, source_name, length=arguments[0])
+    if kind is TypeKind.DECIMAL and arguments:
+        precision = arguments[0]
+        scale = arguments[1] if len(arguments) == 2 else None
+        return SqlType(kind, source_name, precision=precision, scale=scale)
+    return SqlType(kind, source_name)
+
+
+def _numeric_arguments(value: str | None) -> tuple[int, ...]:
+    if value is None:
+        return ()
+    parts = tuple(part.strip() for part in value.split(","))
+    if not 1 <= len(parts) <= 2 or any(not part.isdecimal() for part in parts):
+        return ()
+    return tuple(int(part) for part in parts)
+
+
+_EXACT_KINDS: dict[str, TypeKind] = {
+    "DATE": TypeKind.DATE,
+    "TIME": TypeKind.TIME,
+}
+"""Names that say what they are only when they are the whole name."""
+
+_CONTAINED_KINDS: tuple[tuple[tuple[str, ...], TypeKind], ...] = (
+    (("BOOL",), TypeKind.BOOLEAN),
+    (("BIGINT",), TypeKind.BIG_INTEGER),
+    (("SMALLINT", "TINYINT"), TypeKind.SMALL_INTEGER),
+    (("INT",), TypeKind.INTEGER),
+    (("CHAR", "CLOB", "TEXT"), TypeKind.STRING),
+    (("BLOB",), TypeKind.BINARY),
+    (("DOUBLE",), TypeKind.DOUBLE),
+    (("REAL", "FLOA"), TypeKind.REAL),
+    (("DATETIME", "TIMESTAMP"), TypeKind.DATETIME),
+    (("JSON",), TypeKind.JSON),
+    (("NUM", "DEC", "MONEY"), TypeKind.DECIMAL),
+)
+"""What a declared name holds, read in the order SQLite reads it.
+
+The order is the rule rather than an arrangement of one: a name holding
+`BIGINT` also holds `INT`, and one holding `SMALLINT` holds both, so the
+narrower token has to be asked about first.
+"""
+
+
+def _type_kind(name: str) -> TypeKind:
+    """What a declared type name holds, or that it says nothing recognised."""
+    exact = _EXACT_KINDS.get(name)
+    if exact is not None:
+        return exact
+    for tokens, kind in _CONTAINED_KINDS:
+        if any(token in name for token in tokens):
+            return kind
+    return TypeKind.OTHER
+
+
+__all__ = ("sqlite_type",)

pyoq/serving/__init__.py

@@ -0,0 +1,19 @@
+"""Databases held by an application that serves requests."""
+
+from pyoq.serving.databases import (
+    DEFAULT,
+    AsyncDatabase,
+    Databases,
+    SupportsAsyncTransaction,
+    SupportsTransaction,
+    SyncDatabase,
+)
+
+__all__ = (
+    "DEFAULT",
+    "AsyncDatabase",
+    "Databases",
+    "SupportsAsyncTransaction",
+    "SupportsTransaction",
+    "SyncDatabase",
+)

pyoq/serving/databases.py

@@ -0,0 +1,107 @@
+"""What a serving application opens once and closes once.
+
+A pool belongs to the application, not to a request. Naming the close alongside
+the operations keeps the two together, so a database cannot be registered
+without saying how it is given back.
+
+Nothing here is specific to one web framework. The adapters differ in where they
+put these and when they run them, not in what they are.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Protocol, TypeVar, runtime_checkable
+
+if TYPE_CHECKING:
+    from collections.abc import Awaitable, Callable
+    from collections.abc import Set as AbstractSet
+    from contextlib import AbstractAsyncContextManager, AbstractContextManager
+
+    from pyoq.query.execution import QueryOperations
+    from pyoq.query.execution.aio import AsyncQueryOperations
+
+DEFAULT = "default"
+
+
+@runtime_checkable
+class SupportsTransaction(Protocol):
+    """Operations that can open a transaction of their own."""
+
+    def transaction(self) -> AbstractContextManager[QueryOperations]: ...
+
+
+@runtime_checkable
+class SupportsAsyncTransaction(Protocol):
+    def transaction(self) -> AbstractAsyncContextManager[AsyncQueryOperations]: ...
+
+
+@dataclass(frozen=True, slots=True)
+class SyncDatabase:
+    """A database and the way to give it back."""
+
+    operations: QueryOperations
+    close: Callable[[], None]
+
+
+@dataclass(frozen=True, slots=True)
+class AsyncDatabase:
+    operations: AsyncQueryOperations
+    close: Callable[[], Awaitable[None]]
+
+
+DatabaseType = TypeVar("DatabaseType", SyncDatabase, AsyncDatabase)
+
+
+class Databases:
+    """Every database this application opened, by name."""
+
+    __slots__ = ("_asynchronous", "_synchronous")
+
+    def __init__(
+        self,
+        synchronous: dict[str, SyncDatabase],
+        asynchronous: dict[str, AsyncDatabase],
+    ) -> None:
+        self._synchronous = synchronous
+        self._asynchronous = asynchronous
+
+    @property
+    def names(self) -> AbstractSet[str]:
+        return self._synchronous.keys() | self._asynchronous.keys()
+
+    @property
+    def opened(self) -> tuple[dict[str, SyncDatabase], dict[str, AsyncDatabase]]:
+        """What was opened, for whoever has to give it back."""
+        return self._synchronous, self._asynchronous
+
+    def synchronous(self, name: str, /) -> QueryOperations:
+        return _require(self._synchronous, name, "synchronous").operations
+
+    def asynchronous(self, name: str, /) -> AsyncQueryOperations:
+        return _require(self._asynchronous, name, "asynchronous").operations
+
+
+def _require(
+    databases: dict[str, DatabaseType],
+    name: str,
+    kind: str,
+) -> DatabaseType:
+    found = databases.get(name)
+    if found is None:
+        known = ", ".join(sorted(databases)) or "none"
+        message = (
+            f"no {kind} database named {name!r} was opened; {kind} databases: {known}"
+        )
+        raise KeyError(message)
+    return found
+
+
+__all__ = (
+    "DEFAULT",
+    "AsyncDatabase",
+    "Databases",
+    "SupportsAsyncTransaction",
+    "SupportsTransaction",
+    "SyncDatabase",
+)

pyoq/snapshots/__init__.py

@@ -0,0 +1,20 @@
+"""Schema snapshots as files, and what changed between two of them."""
+
+from pyoq.snapshots.drift import (
+    SchemaChange,
+    SchemaDifference,
+    SchemaDriftReport,
+    compare_snapshots,
+)
+from pyoq.snapshots.files import read_snapshot, write_snapshot
+from pyoq.snapshots.source import SnapshotSchemaSource
+
+__all__ = (
+    "SchemaChange",
+    "SchemaDifference",
+    "SchemaDriftReport",
+    "SnapshotSchemaSource",
+    "compare_snapshots",
+    "read_snapshot",
+    "write_snapshot",
+)

pyoq/snapshots/drift.py

@@ -0,0 +1,312 @@
+"""Comparing what a snapshot recorded against what a database now holds.
+
+Generated code drifts when the schema moves under it. Comparing two snapshots
+says what moved in terms of the schema rather than of the files, so the answer
+names the table or the column that changed rather than the line that differs.
+
+Named things are matched by name and then by value, so a renamed table reads
+as one removed and one added rather than as a change to something that is no
+longer there.
+
+Everything else a container holds is compared by standing its named children
+aside and comparing what is left. A comment, a capability, a format version
+and any fact the schema model gains later are all compared that way, so a new
+fact is compared from the day it exists rather than from the day someone
+remembers to add it here.
+
+The children are stood aside rather than emptied, because a snapshot without
+a catalog and a table without a column are both refused by the model. One
+fixed stand-in is put in their place, the same one on both sides, so what
+remains is the container's own facts and nothing else.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass, replace
+from enum import StrEnum
+from typing import TYPE_CHECKING, Any, Protocol, TypeVar
+
+from pyoq.schema.models import (
+    Catalog,
+    Column,
+    Identifier,
+    Schema,
+    SqlType,
+    TypeKind,
+)
+
+if TYPE_CHECKING:
+    from pyoq.schema.models import SchemaSnapshot, Table
+
+
+class HasName(Protocol):
+    """Anything a schema names, which is everything compared here."""
+
+    @property
+    def name(self) -> Identifier | None: ...
+
+
+Named = TypeVar("Named", bound=HasName)
+
+
+class SchemaChange(StrEnum):
+    """What happened to one named thing between two snapshots."""
+
+    ADDED = "added"
+    REMOVED = "removed"
+    CHANGED = "changed"
+
+
+@dataclass(frozen=True, slots=True)
+class SchemaDifference:
+    """One named thing that is not the same in both snapshots."""
+
+    change: SchemaChange
+    kind: str
+    name: str
+
+    def describe(self) -> str:
+        """What changed, and which one, where there is a which one to name.
+
+        The snapshot itself has no name to give, so it is described by what it
+        is rather than followed by nothing.
+        """
+        if not self.name:
+            return f"{self.change.value} {self.kind}"
+        return f"{self.change.value} {self.kind} {self.name}"
+
+
+@dataclass(frozen=True, slots=True)
+class SchemaDriftReport:
+    """Everything that moved between two snapshots, and whether anything did."""
+
+    differences: tuple[SchemaDifference, ...]
+
+    @property
+    def is_current(self) -> bool:
+        return not self.differences
+
+    def summary(self) -> str:
+        if self.is_current:
+            return "the recorded schema matches the database"
+        written = ", ".join(item.describe() for item in self.differences)
+        return f"{len(self.differences)} schema difference(s): {written}"
+
+
+def compare_snapshots(
+    recorded: SchemaSnapshot,
+    observed: SchemaSnapshot,
+    /,
+) -> SchemaDriftReport:
+    """What changed between the snapshot on file and the one just read."""
+    differences: list[SchemaDifference] = []
+    if recorded.dialect is not observed.dialect:
+        differences.append(
+            SchemaDifference(SchemaChange.CHANGED, "dialect", observed.dialect.value)
+        )
+    _compare_own(
+        differences,
+        "snapshot",
+        "",
+        replace(recorded, catalogs=_STAND_IN_CATALOGS, dialect=observed.dialect),
+        replace(observed, catalogs=_STAND_IN_CATALOGS),
+    )
+    left = _by_name(recorded.catalogs, "")
+    right = _by_name(observed.catalogs, "")
+    _record_missing(differences, "catalog", left, right)
+    for name in _shared(left, right):
+        _compare_catalog(differences, name, left[name], right[name])
+    return SchemaDriftReport(tuple(differences))
+
+
+def _compare_catalog(
+    differences: list[SchemaDifference],
+    prefix: str,
+    recorded: Catalog,
+    observed: Catalog,
+) -> None:
+    if recorded == observed:
+        return
+    _compare_own(
+        differences,
+        "catalog",
+        prefix,
+        replace(recorded, schemas=_STAND_IN_SCHEMAS),
+        replace(observed, schemas=_STAND_IN_SCHEMAS),
+    )
+    left = _by_name(recorded.schemas, prefix)
+    right = _by_name(observed.schemas, prefix)
+    _record_missing(differences, "schema", left, right)
+    for name in _shared(left, right):
+        _compare_schema(differences, name, left[name], right[name])
+
+
+def _compare_schema(
+    differences: list[SchemaDifference],
+    prefix: str,
+    recorded: Schema,
+    observed: Schema,
+) -> None:
+    if recorded == observed:
+        return
+    _compare_own(
+        differences,
+        "schema",
+        prefix,
+        replace(recorded, **_NO_SCHEMA_CHILDREN),
+        replace(observed, **_NO_SCHEMA_CHILDREN),
+    )
+    _compare_tables(differences, prefix, recorded.tables, observed.tables)
+    _compare_leaves(differences, "view", prefix, recorded.views, observed.views)
+    _compare_leaves(differences, "enum", prefix, recorded.enums, observed.enums)
+    _compare_leaves(differences, "domain", prefix, recorded.domains, observed.domains)
+    _compare_leaves(
+        differences, "routine", prefix, recorded.routines, observed.routines
+    )
+
+
+def _compare_tables(
+    differences: list[SchemaDifference],
+    prefix: str,
+    recorded: Sequence[Table],
+    observed: Sequence[Table],
+) -> None:
+    left = _by_name(recorded, prefix)
+    right = _by_name(observed, prefix)
+    _record_missing(differences, "table", left, right)
+    for name in _shared(left, right):
+        if left[name] == right[name]:
+            continue
+        _compare_table(differences, name, left[name], right[name])
+
+
+def _compare_table(
+    differences: list[SchemaDifference],
+    prefix: str,
+    recorded: Table,
+    observed: Table,
+) -> None:
+    """A table's named parts by name, and everything else as the table itself."""
+    _compare_own(
+        differences,
+        "table",
+        prefix,
+        replace(recorded, **_NO_TABLE_CHILDREN),
+        replace(observed, **_NO_TABLE_CHILDREN),
+    )
+    for kind, held in _TABLE_CHILDREN.items():
+        _compare_leaves(differences, kind, prefix, held(recorded), held(observed))
+
+
+def _compare_own(
+    differences: list[SchemaDifference],
+    kind: str,
+    name: str,
+    recorded: object,
+    observed: object,
+) -> None:
+    """What a container holds itself, once its named children are emptied.
+
+    Comparing the remainder rather than a written list of attributes is what
+    keeps this exhaustive: a fact added to the schema model is compared
+    without being named here.
+    """
+    if recorded == observed:
+        return
+    differences.append(SchemaDifference(SchemaChange.CHANGED, kind, name))
+
+
+def _compare_leaves(
+    differences: list[SchemaDifference],
+    kind: str,
+    prefix: str,
+    recorded: Sequence[Named],
+    observed: Sequence[Named],
+) -> None:
+    """Things compared whole, because nothing inside them is named separately."""
+    left = _by_name(recorded, prefix)
+    right = _by_name(observed, prefix)
+    _record_missing(differences, kind, left, right)
+    for name in _shared(left, right):
+        if left[name] != right[name]:
+            differences.append(SchemaDifference(SchemaChange.CHANGED, kind, name))
+
+
+_STAND_IN_COLUMNS: tuple[Column, ...] = (
+    Column(Identifier("pyoq_stand_in"), SqlType(TypeKind.OTHER, "pyoq"), nullable=True),
+)
+_STAND_IN_SCHEMAS: tuple[Schema, ...] = (Schema(Identifier("pyoq_stand_in")),)
+_STAND_IN_CATALOGS: tuple[Catalog, ...] = (
+    Catalog(Identifier("pyoq_stand_in"), _STAND_IN_SCHEMAS),
+)
+"""What stands in for a container's children while its own facts are compared.
+
+The same value on both sides, so it cancels out and only the container's own
+facts remain.
+"""
+
+_NO_SCHEMA_CHILDREN: dict[str, Any] = {
+    "tables": (),
+    "views": (),
+    "enums": (),
+    "domains": (),
+    "routines": (),
+}
+_NO_TABLE_CHILDREN: dict[str, Any] = {
+    "columns": _STAND_IN_COLUMNS,
+    "keys": (),
+    "relations": (),
+    "indexes": (),
+    "checks": (),
+}
+_TABLE_CHILDREN: dict[str, Callable[[Table], Sequence[HasName]]] = {
+    "column": lambda table: table.columns,
+    "key": lambda table: table.keys,
+    "relation": lambda table: table.relations,
+    "index": lambda table: table.indexes,
+    "check": lambda table: table.checks,
+}
+"""Everything a table names, so each is reported as itself rather than as the
+table it sits in."""
+
+
+def _record_missing(
+    differences: list[SchemaDifference],
+    kind: str,
+    recorded: Mapping[str, Named],
+    observed: Mapping[str, Named],
+) -> None:
+    for name in sorted(set(recorded) - set(observed)):
+        differences.append(SchemaDifference(SchemaChange.REMOVED, kind, name))
+    for name in sorted(set(observed) - set(recorded)):
+        differences.append(SchemaDifference(SchemaChange.ADDED, kind, name))
+
+
+def _shared(
+    recorded: Mapping[str, Named],
+    observed: Mapping[str, Named],
+) -> tuple[str, ...]:
+    return tuple(sorted(set(recorded) & set(observed)))
+
+
+def _by_name(items: Sequence[Named], prefix: str) -> dict[str, Named]:
+    """Named things by their written name, qualified by whatever holds them.
+
+    A catalog or a schema may be unnamed, and an unnamed one is still a
+    position, so it is written as the position it holds.
+    """
+    named: dict[str, Named] = {}
+    for index, item in enumerate(items):
+        written = f"[{index}]" if item.name is None else item.name.value
+        named[f"{prefix}.{written}" if prefix else written] = item
+    return named
+
+
+__all__ = (
+    "HasName",
+    "SchemaChange",
+    "SchemaDifference",
+    "SchemaDriftReport",
+    "compare_snapshots",
+)

pyoq/snapshots/files.py

@@ -0,0 +1,96 @@
+"""Reading and writing a schema snapshot as a file.
+
+A snapshot is the one canonical record of what a database holds. Written to a
+file it becomes reviewable in a diff and usable without a database, which is
+what lets generation and drift checks run where no credentials exist.
+
+The bytes are the codec's, so a snapshot written twice from one database is
+byte for byte the same file and an unrelated change shows as no change.
+"""
+
+from __future__ import annotations
+
+import os
+from contextlib import suppress
+from pathlib import Path
+from tempfile import NamedTemporaryFile
+
+from pyoq.errors import SchemaSerializationError
+from pyoq.schema.codec import decode_snapshot, encode_snapshot
+from pyoq.schema.models import SchemaSnapshot
+
+_ENCODING = "utf-8"
+
+
+def write_snapshot(snapshot: SchemaSnapshot, path: Path, /) -> Path:
+    """Write a snapshot where a reader and a reviewer can both reach it.
+
+    Written beside its destination and moved into place, so a reader never
+    sees half a snapshot and a failed write leaves the previous one intact.
+
+    A write can fail while the staged file is being written, while it is being
+    closed, or as it is moved. Whichever it is, the staged file goes, so a
+    failed recording leaves the directory as it found it rather than beside a
+    partial file nobody will claim.
+    """
+    destination = path.resolve()
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    staged = _staged(destination, encode_snapshot(snapshot))
+    try:
+        os.replace(staged, destination)
+    except BaseException:
+        _discard(staged)
+        raise
+    return destination
+
+
+def _staged(destination: Path, written: str, /) -> Path:
+    """The snapshot written beside where it is going, ready to be moved.
+
+    Closed inside the guard rather than after it, because a write is only
+    known to have reached the file once the file is closed, and a failure
+    there leaves as much behind as a failure while writing.
+    """
+    handle = NamedTemporaryFile(  # noqa: SIM115
+        "w",
+        encoding=_ENCODING,
+        newline="\n",
+        dir=destination.parent,
+        prefix=f".{destination.name}.",
+        suffix=".partial",
+        delete=False,
+    )
+    staged = Path(handle.name)
+    try:
+        with handle:
+            handle.write(written)
+            handle.write("\n")
+    except BaseException:
+        _discard(staged)
+        raise
+    return staged
+
+
+def _discard(staged: Path, /) -> None:
+    """Take the staged file away, and say nothing about having done so.
+
+    What went wrong is what the caller is told. A failure to clean up after a
+    failure would replace the reason with a second one, so it is not allowed
+    to.
+    """
+    with suppress(OSError):
+        staged.unlink(missing_ok=True)
+
+
+def read_snapshot(path: Path, /) -> SchemaSnapshot:
+    """The snapshot a file holds, or a precise error saying why it does not."""
+    location = path.resolve()
+    try:
+        written = location.read_text(encoding=_ENCODING)
+    except OSError as error:
+        message = f"cannot read the schema snapshot at {location}: {error}"
+        raise SchemaSerializationError(message) from error
+    return decode_snapshot(written)
+
+
+__all__ = ("read_snapshot", "write_snapshot")

pyoq/snapshots/routing.py

@@ -0,0 +1,40 @@
+"""Choosing where a schema is read from.
+
+Generation asks for a snapshot and does not care where it came from. A project
+that records its schema in a file is read from that file, and one that does not
+is read from its database, so the same pipeline serves both and neither knows
+which it got.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from pyoq.schema.registry import DialectRoutingSchemaSource
+from pyoq.snapshots.source import SnapshotSchemaSource
+
+if TYPE_CHECKING:
+    from pyoq.config import Configuration
+    from pyoq.schema.models import SchemaSnapshot
+    from pyoq.schema.source import SchemaSource
+
+
+_LIVE = DialectRoutingSchemaSource()
+"""The database itself, which is what a project without a snapshot reads."""
+
+
+@dataclass(frozen=True, slots=True)
+class ConfiguredSchemaSource:
+    """The recorded schema where a project keeps one, the database otherwise."""
+
+    live: SchemaSource = _LIVE
+
+    def load(self, configuration: Configuration) -> SchemaSnapshot:
+        recorded = configuration.schema_snapshot_path
+        if recorded is None:
+            return self.live.load(configuration)
+        return SnapshotSchemaSource(recorded).load(configuration)
+
+
+__all__ = ("ConfiguredSchemaSource",)