pyoq-sql 1.0.0__py3-none-any.whl

pyoq/django/schema.py

@@ -0,0 +1,379 @@
+"""The schema a Django project's migrations describe.
+
+Read from migration state rather than from a database, because migrations are
+the authority on what the schema will be and a developer generating types has
+usually not applied them yet. Nothing here opens a connection: Django answers
+what type a field takes on a backend without consulting the server.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, replace
+from typing import TYPE_CHECKING, Protocol
+
+from django.db import DEFAULT_DB_ALIAS, connections
+from django.db.migrations.loader import MigrationLoader
+from django.db.models import CheckConstraint as DjangoCheckConstraint
+from django.db.models import ForeignKey
+
+from pyoq.config import Configuration, DatabaseDialect
+from pyoq.errors import OperationUnavailableError
+from pyoq.schema import (
+    Catalog,
+    CheckConstraint,
+    Column,
+    GeneratedKind,
+    GeneratedValue,
+    Identifier,
+    Index,
+    IndexTerm,
+    Key,
+    KeyKind,
+    MetadataEntry,
+    ObjectReference,
+    Relation,
+    Schema,
+    SchemaSnapshot,
+    SqlExpression,
+    SqlType,
+    Table,
+    TypeKind,
+)
+from pyoq.schema.mysql.types import mysql_type
+from pyoq.schema.postgres.types import postgres_type
+from pyoq.schema.sqlite.types import sqlite_type
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Sequence
+
+    from django.db.backends.base.base import BaseDatabaseWrapper
+    from django.db.models import Field, Model
+    from django.db.models.options import Options
+
+
+class TypeParser(Protocol):
+    def __call__(self, declared_type: str, /) -> SqlType: ...
+
+
+_PARSERS: dict[str, tuple[DatabaseDialect, TypeParser]] = {
+    "sqlite": (DatabaseDialect.SQLITE, sqlite_type),
+    "postgresql": (DatabaseDialect.POSTGRES, postgres_type),
+    "mysql": (DatabaseDialect.MYSQL, mysql_type),
+}
+
+_AUTO_FIELDS = frozenset({"AutoField", "BigAutoField", "SmallAutoField"})
+
+
+@dataclass(frozen=True, slots=True)
+class MigrationStateSchemaSource:
+    """Every table the migrations on disk would create, and nothing else."""
+
+    alias: str = DEFAULT_DB_ALIAS
+    app_labels: tuple[str, ...] = ()
+
+    def load(self, configuration: Configuration) -> SchemaSnapshot:
+        del configuration
+        connection = connections[self.alias]
+        dialect, parse = _parser_for(connection)
+        models = _selected_models(self.app_labels)
+        tables = [_table(model, connection, parse) for model in models]
+        tables.extend(_join_tables(models, connection, parse))
+        schema = Schema(name=None, tables=tuple(_ordered(tables)))
+        return SchemaSnapshot(
+            dialect=dialect,
+            catalogs=(Catalog(name=None, schemas=(schema,)),),
+        )
+
+
+def _parser_for(
+    connection: BaseDatabaseWrapper,
+) -> tuple[DatabaseDialect, TypeParser]:
+    parser = _PARSERS.get(connection.vendor)
+    if parser is None:
+        message = (
+            f"PyOQ cannot read a schema for the Django backend "
+            f"{connection.vendor!r}; supported backends: "
+            f"{', '.join(sorted(_PARSERS))}"
+        )
+        raise OperationUnavailableError(message)
+    return parser
+
+
+def _selected_models(app_labels: tuple[str, ...]) -> tuple[type[Model], ...]:
+    """The models the migrations describe, with no database consulted.
+
+    A loader given no connection reads the migration files alone and never asks
+    which of them have been applied, which is the question that needs a server.
+    """
+    state = MigrationLoader(None).project_state()
+    wanted = frozenset(app_labels)
+    chosen = [
+        state.apps.get_model(label, name)
+        for label, name in sorted(state.models)
+        if not wanted or label in wanted
+    ]
+    _require_known_labels(wanted, frozenset(label for label, _ in state.models))
+    return tuple(model for model in chosen if model._meta.managed)
+
+
+def _require_known_labels(
+    wanted: frozenset[str],
+    available: frozenset[str],
+) -> None:
+    missing = sorted(wanted - available)
+    if not missing:
+        return
+    message = (
+        f"no migrations describe the app {', '.join(missing)!r}; "
+        f"apps with migrations: {', '.join(sorted(available))}"
+    )
+    raise OperationUnavailableError(message)
+
+
+def _table(
+    model: type[Model],
+    connection: BaseDatabaseWrapper,
+    parse: TypeParser,
+) -> Table:
+    meta = model._meta
+    columns = tuple(
+        _column(field, declared, parse)
+        for field, declared in _declared_types(meta, connection)
+    )
+    return Table(
+        name=Identifier(meta.db_table),
+        columns=columns,
+        keys=_keys(meta),
+        relations=_relations(meta),
+        indexes=_indexes(meta),
+        checks=_checks(meta),
+    )
+
+
+def _declared_types(
+    meta: Options[Model],
+    connection: BaseDatabaseWrapper,
+) -> Iterable[tuple[Field[object, object], str]]:
+    """A field without a type of its own has no column to describe."""
+    for field in meta.local_fields:
+        declared = field.db_type(connection)
+        if declared is not None:
+            yield field, declared
+
+
+_MEANT_KINDS: dict[str, TypeKind] = {
+    "JSONField": TypeKind.JSON,
+    "UUIDField": TypeKind.UUID,
+    "DurationField": TypeKind.INTERVAL,
+}
+"""What a field means, where the column it asks for does not say so.
+
+Django declares a JSON column as text, a UUID as a fixed width string, and a
+duration as an integer, so reading the column back would hand a caller the
+stored form rather than the value. The field is the one that knows.
+"""
+
+
+def _data_type(
+    field: Field[object, object],
+    declared: str,
+    parse: TypeParser,
+) -> SqlType:
+    """The declared column, named by what the field puts in it."""
+    stored = parse(declared)
+    meant = _MEANT_KINDS.get(field.get_internal_type())
+    if meant is None or stored.kind is meant:
+        return stored
+    return replace(stored, kind=meant, element_type=None, named_type=None)
+
+
+def _column(
+    field: Field[object, object],
+    declared: str,
+    parse: TypeParser,
+) -> Column:
+    return Column(
+        name=Identifier(_column_of(field)),
+        data_type=_data_type(field, declared, parse),
+        nullable=bool(field.null),
+        default=None,
+        generated=_generated(field),
+        metadata=_column_metadata(field),
+    )
+
+
+def _generated(field: Field[object, object]) -> GeneratedValue | None:
+    """Only a field the database fills in counts as generated here.
+
+    A Django default is applied in Python before the row is sent, so it is not
+    a fact about the column. Recording it as one would tell a caller they may
+    omit a value that the database will then refuse.
+    """
+    if type(field).__name__ not in _AUTO_FIELDS:
+        return None
+    return GeneratedValue(kind=GeneratedKind.IDENTITY_BY_DEFAULT)
+
+
+def _column_metadata(
+    field: Field[object, object],
+) -> tuple[MetadataEntry, ...]:
+    entries = [MetadataEntry(key="django-field", value=type(field).__name__)]
+    if field.has_default():
+        entries.append(MetadataEntry(key="django-default", value="python"))
+    if isinstance(field, ForeignKey):
+        entries.append(MetadataEntry(key="django-on-delete", value=_on_delete(field)))
+    return tuple(entries)
+
+
+def _on_delete(field: ForeignKey[object]) -> str:
+    """What the model asked for, beside what the database was told.
+
+    Django deletes related rows itself, so the constraint carries no action at
+    all. Keeping the model's choice visible costs nothing and stops a reader
+    concluding the database will do it.
+    """
+    return str(getattr(field.remote_field.on_delete, "__name__", "unknown"))
+
+
+def _keys(meta: Options[Model]) -> tuple[Key, ...]:
+    keys: list[Key] = []
+    keys.append(Key(kind=KeyKind.PRIMARY, columns=(Identifier(_column_of(meta.pk)),)))
+    keys.extend(
+        Key(kind=KeyKind.UNIQUE, columns=(Identifier(_column_of(field)),))
+        for field in meta.local_fields
+        if field.unique and not field.primary_key
+    )
+    keys.extend(
+        Key(kind=KeyKind.UNIQUE, columns=_columns(meta, group))
+        for group in meta.unique_together
+    )
+    keys.extend(_constraint_keys(meta))
+    return tuple(keys)
+
+
+def _constraint_keys(meta: Options[Model]) -> Iterable[Key]:
+    """A unique constraint counts only when it holds for every row.
+
+    A constraint carrying a condition is unique among the rows it matches and
+    silently not unique among the rest. Treating one as a key would let a
+    relation be described as reaching a single row when it reaches many.
+    """
+    for constraint in meta.constraints:
+        fields = getattr(constraint, "fields", None)
+        if not fields or _is_partial(constraint):
+            continue
+        yield Key(
+            kind=KeyKind.UNIQUE,
+            columns=_columns(meta, fields),
+            name=Identifier(constraint.name),
+        )
+
+
+def _is_partial(constraint: object) -> bool:
+    return any(
+        getattr(constraint, attribute, None)
+        for attribute in ("condition", "expressions", "nulls_distinct")
+    )
+
+
+def _relations(meta: Options[Model]) -> tuple[Relation, ...]:
+    """Django enforces deletion in Python, so the database is told nothing.
+
+    The referential action recorded here is the one the database will actually
+    take. What the model asked for is kept beside it as a fact about Django.
+    """
+    return tuple(
+        Relation(
+            columns=(Identifier(_column_of(field)),),
+            target=ObjectReference(name=Identifier(_target_table(field))),
+            target_columns=(Identifier(_column_of(field.target_field)),),
+        )
+        for field in _foreign_keys(meta)
+    )
+
+
+def _foreign_keys(meta: Options[Model]) -> Iterable[ForeignKey[object]]:
+    for field in meta.local_fields:
+        if isinstance(field, ForeignKey):
+            yield field
+
+
+def _target_table(field: ForeignKey[object]) -> str:
+    return str(field.related_model._meta.db_table)
+
+
+def _column_of(field: Field[object, object]) -> str:
+    return str(field.column)
+
+
+def _indexes(meta: Options[Model]) -> tuple[Index, ...]:
+    """Only indexes the project named itself.
+
+    Django names the indexes it adds for itself inside a schema editor, which
+    reaches the server. Guessing those names here would put a name in the
+    schema that the database may not agree with.
+    """
+    return tuple(
+        Index(
+            name=Identifier(index.name),
+            terms=tuple(_terms(meta, index.fields)),
+        )
+        for index in meta.indexes
+        if index.fields
+    )
+
+
+def _terms(meta: Options[Model], fields: Sequence[str]) -> Iterable[IndexTerm]:
+    for name in fields:
+        descending = name.startswith("-")
+        column = _column_name(meta, name.lstrip("-"))
+        yield IndexTerm(column=Identifier(column), descending=descending)
+
+
+def _checks(meta: Options[Model]) -> tuple[CheckConstraint, ...]:
+    return tuple(
+        CheckConstraint(
+            expression=SqlExpression(text=str(constraint.condition)),
+            name=Identifier(constraint.name),
+        )
+        for constraint in meta.constraints
+        if isinstance(constraint, DjangoCheckConstraint)
+    )
+
+
+def _columns(
+    meta: Options[Model],
+    names: Iterable[str],
+) -> tuple[Identifier, ...]:
+    return tuple(Identifier(_column_name(meta, name)) for name in names)
+
+
+def _column_name(meta: Options[Model], name: str) -> str:
+    field = meta.get_field(name)
+    column = getattr(field, "column", None)
+    return column if isinstance(column, str) else name
+
+
+def _join_tables(
+    models: tuple[type[Model], ...],
+    connection: BaseDatabaseWrapper,
+    parse: TypeParser,
+) -> Iterable[Table]:
+    """A many-to-many field owns a table that migration state does not list.
+
+    Django creates it from the field rather than from a model of its own, so it
+    has to be described here or the schema would be missing a real table. A
+    through model the project wrote is already a model and is left alone.
+    """
+    for model in models:
+        for field in model._meta.local_many_to_many:
+            through = field.remote_field.through
+            if through is not None and through._meta.auto_created:
+                yield _table(through, connection, parse)
+
+
+def _ordered(tables: Iterable[Table]) -> list[Table]:
+    return sorted(tables, key=lambda table: table.name.value)
+
+
+__all__ = ("MigrationStateSchemaSource",)

pyoq/django/settings.py

@@ -0,0 +1,87 @@
+"""PyOQ's own settings, read from the Django project that owns them."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Final, cast
+
+from django.conf import settings
+
+from pyoq.config import Configuration, DatabaseDialect, DatabaseProfile
+from pyoq.errors import OperationUnavailableError
+
+SETTING: Final = "PYOQ"
+
+_PROFILE: Final = "django"
+
+
+def pyoq_settings() -> dict[str, object]:
+    """The project's PyOQ settings, which it has to have written itself.
+
+    There is no default output location worth guessing. A package written to
+    the wrong place is worse than being told where to put it.
+    """
+    configured: object = getattr(settings, SETTING, None)
+    if not isinstance(configured, Mapping):
+        message = (
+            f"Django settings define no {SETTING} mapping; add "
+            f'{SETTING} = {{"codegen_directory": ..., '
+            f'"codegen_package": ...}}'
+        )
+        raise OperationUnavailableError(message)
+    # Django settings carry no types of their own, so this is the one place
+    # that says what the mapping holds. Every read below re-checks the value.
+    entries = cast("Mapping[str, object]", configured)
+    return dict(entries)
+
+
+def django_configuration(dialect: DatabaseDialect, /) -> Configuration:
+    """Where generated output goes, with no database named.
+
+    The schema comes from migration files, so a profile here carries the
+    dialect and nothing that would open a connection.
+    """
+    configured = pyoq_settings()
+    return Configuration(
+        project_root=_root(configured),
+        codegen_directory=Path(_required(configured, "codegen_directory")),
+        codegen_package=_required(configured, "codegen_package"),
+        selected_profile=_PROFILE,
+        profiles=(DatabaseProfile(name=_PROFILE, dialect=dialect),),
+    )
+
+
+def codegen_after_makemigrations() -> bool:
+    """Overriding a command a project already runs has to be asked for."""
+    return bool(pyoq_settings().get("codegen_after_makemigrations", False))
+
+
+def _root(configured: dict[str, object]) -> Path:
+    named = configured.get("project_root")
+    if named is not None:
+        return Path(str(named))
+    base = getattr(settings, "BASE_DIR", None)
+    if base is None:
+        message = (
+            f"neither {SETTING}['project_root'] nor BASE_DIR is set, so there "
+            f"is nowhere to write generated code relative to"
+        )
+        raise OperationUnavailableError(message)
+    return Path(str(base))
+
+
+def _required(configured: dict[str, object], key: str) -> str:
+    value = configured.get(key)
+    if not isinstance(value, str) or not value:
+        message = f"{SETTING}[{key!r}] must be a non-empty string"
+        raise OperationUnavailableError(message)
+    return value
+
+
+__all__ = (
+    "SETTING",
+    "codegen_after_makemigrations",
+    "django_configuration",
+    "pyoq_settings",
+)

pyoq/django/timeouts.py

@@ -0,0 +1,105 @@
+"""Bounding a statement on a connection PyOQ does not own.
+
+A timeout is session state, and the session belongs to Django, so it has to be
+put back exactly as it was found. Inside a transaction the database does that
+itself; outside one, the previous value is read first and restored after.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Generator
+from contextlib import contextmanager
+from dataclasses import dataclass
+
+from django.db import DatabaseError
+from django.db.backends.base.base import BaseDatabaseWrapper
+
+MILLISECONDS = 1000
+
+
+@dataclass(frozen=True, slots=True)
+class TimeoutSetting:
+    """The session setting one backend uses to bound a statement."""
+
+    name: str
+    reads: bool = True
+    writes: bool = True
+
+    def read(self, connection: BaseDatabaseWrapper, /) -> str:
+        with connection.cursor() as cursor:
+            cursor.execute(f"SHOW {self.name}")
+            row = cursor.fetchone()
+        return "" if row is None else str(row[0])
+
+    def assign(
+        self,
+        connection: BaseDatabaseWrapper,
+        value: str,
+        /,
+        *,
+        local: bool,
+    ) -> None:
+        scope = "LOCAL " if local else ""
+        with connection.cursor() as cursor:
+            cursor.execute(f"SET {scope}{self.name} = {value}")
+
+
+POSTGRES_TIMEOUT = TimeoutSetting("statement_timeout")
+MYSQL_TIMEOUT = TimeoutSetting("@@max_execution_time", writes=False)
+
+
+@contextmanager
+def applied(
+    setting: TimeoutSetting,
+    connection: BaseDatabaseWrapper,
+    seconds: float,
+    /,
+) -> Generator[None]:
+    """Bound statements for the length of this block, and leave no trace.
+
+    Inside a transaction the setting is local, so the database restores it when
+    the transaction ends. Outside one there is nothing to restore it, so the
+    previous value is read first and put back afterwards.
+    """
+    milliseconds = max(int(seconds * MILLISECONDS), 1)
+    if connection.in_atomic_block:
+        setting.assign(connection, str(milliseconds), local=True)
+        yield
+        return
+    previous = setting.read(connection)
+    setting.assign(connection, str(milliseconds), local=False)
+    try:
+        yield
+    finally:
+        _restore(setting, connection, previous)
+
+
+def _restore(
+    setting: TimeoutSetting,
+    connection: BaseDatabaseWrapper,
+    previous: str,
+) -> None:
+    """Put the setting back without ever replacing the reason we are here.
+
+    A statement that failed can leave a connection unable to accept the next
+    one, and a restore that raised there would hide the failure that caused it.
+    Django owns the connection and will deal with it.
+    """
+    try:
+        setting.assign(connection, _quoted(previous), local=False)
+    except DatabaseError:
+        return
+
+
+def _quoted(previous: str) -> str:
+    """A read setting comes back with its unit, which has to go back quoted."""
+    return previous if previous.isdigit() else f"'{previous}'"
+
+
+__all__ = (
+    "MILLISECONDS",
+    "MYSQL_TIMEOUT",
+    "POSTGRES_TIMEOUT",
+    "TimeoutSetting",
+    "applied",
+)

pyoq/dsl/__init__.py

@@ -0,0 +1,64 @@
+"""Building a statement and running it in one chain.
+
+`using(database)` gives a context to build against. The builders in
+`pyoq.query` are unchanged and still cannot reach a database on their own.
+"""
+
+from pyoq.dsl.aio import (
+    AsyncBoundConflictUpdate,
+    AsyncBoundDelete,
+    AsyncBoundInsert,
+    AsyncBoundJoinStep,
+    AsyncBoundMutation,
+    AsyncBoundQuery,
+    AsyncBoundReturning,
+    AsyncBoundSelect,
+    AsyncBoundSemiJoinStep,
+    AsyncBoundUpdate,
+    AsyncDslContext,
+)
+from pyoq.dsl.context import DslContext
+from pyoq.dsl.entry import using
+from pyoq.dsl.queries import (
+    BoundJoinStep,
+    BoundQuery,
+    BoundSelect,
+    BoundSemiJoinStep,
+)
+from pyoq.dsl.result import Result, Row
+from pyoq.dsl.writes import (
+    BoundConflictUpdate,
+    BoundDelete,
+    BoundInsert,
+    BoundMutation,
+    BoundReturning,
+    BoundUpdate,
+)
+
+__all__ = (
+    "AsyncBoundConflictUpdate",
+    "AsyncBoundDelete",
+    "AsyncBoundInsert",
+    "AsyncBoundJoinStep",
+    "AsyncBoundMutation",
+    "AsyncBoundQuery",
+    "AsyncBoundReturning",
+    "AsyncBoundSelect",
+    "AsyncBoundSemiJoinStep",
+    "AsyncBoundUpdate",
+    "AsyncDslContext",
+    "BoundConflictUpdate",
+    "BoundDelete",
+    "BoundInsert",
+    "BoundJoinStep",
+    "BoundMutation",
+    "BoundQuery",
+    "BoundReturning",
+    "BoundSelect",
+    "BoundSemiJoinStep",
+    "BoundUpdate",
+    "DslContext",
+    "Result",
+    "Row",
+    "using",
+)

pyoq/dsl/aio/__init__.py

@@ -0,0 +1,31 @@
+"""The asynchronous counterpart of the bound builders."""
+
+from pyoq.dsl.aio.context import AsyncDslContext
+from pyoq.dsl.aio.queries import (
+    AsyncBoundJoinStep,
+    AsyncBoundQuery,
+    AsyncBoundSelect,
+    AsyncBoundSemiJoinStep,
+)
+from pyoq.dsl.aio.writes import (
+    AsyncBoundConflictUpdate,
+    AsyncBoundDelete,
+    AsyncBoundInsert,
+    AsyncBoundMutation,
+    AsyncBoundReturning,
+    AsyncBoundUpdate,
+)
+
+__all__ = (
+    "AsyncBoundConflictUpdate",
+    "AsyncBoundDelete",
+    "AsyncBoundInsert",
+    "AsyncBoundJoinStep",
+    "AsyncBoundMutation",
+    "AsyncBoundQuery",
+    "AsyncBoundReturning",
+    "AsyncBoundSelect",
+    "AsyncBoundSemiJoinStep",
+    "AsyncBoundUpdate",
+    "AsyncDslContext",
+)