pyoq-sql 1.0.2__py3-none-any.whl

pyoq/django/generation.py

@@ -0,0 +1,59 @@
+"""Running the shared generation pipeline for a Django project."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from django.db import DEFAULT_DB_ALIAS, connections
+
+from pyoq.cli.defaults import default_generation_pipeline
+from pyoq.config import DatabaseDialect
+from pyoq.django.schema import MigrationStateSchemaSource
+from pyoq.django.settings import django_configuration
+from pyoq.errors import OperationUnavailableError
+from pyoq.generation import GenerationMode
+
+_DIALECTS = {
+    "sqlite": "sqlite",
+    "postgresql": "postgres",
+    "mysql": "mysql",
+}
+
+
+@dataclass(frozen=True, slots=True)
+class GenerationRequest:
+    """One run of generation, described before anything is written."""
+
+    alias: str = DEFAULT_DB_ALIAS
+    app_labels: tuple[str, ...] = ()
+    mode: GenerationMode = GenerationMode.WRITE
+
+
+def generate(request: GenerationRequest, /) -> str:
+    """The pipeline every other entry point uses, pointed at Django.
+
+    Generation is not reimplemented here. Staging, validation, the manifest,
+    drift, and the atomic replacement all belong to the shared pipeline, and a
+    second copy of them would be a second set of answers.
+    """
+    dialect = DatabaseDialect(_dialect_name(request.alias))
+    source = MigrationStateSchemaSource(
+        alias=request.alias, app_labels=request.app_labels
+    )
+    pipeline = default_generation_pipeline(source)
+    return pipeline.execute(django_configuration(dialect), request.mode)
+
+
+def _dialect_name(alias: str) -> str:
+    vendor = connections[alias].vendor
+    name = _DIALECTS.get(vendor)
+    if name is None:
+        message = (
+            f"PyOQ cannot generate for the Django backend {vendor!r}; "
+            f"supported backends: {', '.join(sorted(_DIALECTS))}"
+        )
+        raise OperationUnavailableError(message)
+    return name
+
+
+__all__ = ("GenerationRequest", "generate")

pyoq/django/management/commands/makemigrations.py

@@ -0,0 +1,53 @@
+"""Django's own `makemigrations`, with generation after it.
+
+A project opts in to this by putting PyOQ ahead of the app that supplies the
+original command, and by setting `codegen_after_makemigrations`. Overriding a
+command a project already runs is not something to do to somebody quietly.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from django.core.management.base import CommandError
+from django.core.management.commands.makemigrations import (
+    Command as MakeMigrations,
+)
+
+from pyoq.django.generation import GenerationRequest, generate
+from pyoq.django.settings import codegen_after_makemigrations
+from pyoq.errors import PyOQError
+
+
+class Command(MakeMigrations):
+    def handle(self, *app_labels: str, **options: Any) -> None:
+        super().handle(*app_labels, **options)
+        if not codegen_after_makemigrations():
+            return
+        if _writes_nothing(options):
+            self.stdout.write(
+                "PyOQ: migrations were not written, so nothing was generated"
+            )
+            return
+        self.stdout.write(f"PyOQ: {_generated(app_labels, options)}")
+
+
+def _writes_nothing(options: dict[str, Any]) -> bool:
+    """Generation reads migration files, so it must not run before they exist.
+
+    A run asked to check or to report leaves the files as they were. Generating
+    then would describe the schema before this change rather than after it, and
+    would look like it had worked.
+    """
+    return bool(options.get("dry_run") or options.get("check_changes"))
+
+
+def _generated(app_labels: tuple[str, ...], options: dict[str, Any]) -> str:
+    request = GenerationRequest(
+        alias=str(options.get("database") or "default"),
+        app_labels=app_labels,
+    )
+    try:
+        return generate(request)
+    except PyOQError as error:
+        raise CommandError(str(error)) from error

pyoq/django/management/commands/pyoq_codegen.py

@@ -0,0 +1,75 @@
+"""Generate PyOQ types from the migrations this project has written."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from django.core.management.base import BaseCommand, CommandError
+from django.db import DEFAULT_DB_ALIAS
+
+from pyoq.django.generation import GenerationRequest, generate
+from pyoq.errors import PyOQError
+from pyoq.generation import GenerationMode
+
+if TYPE_CHECKING:
+    from argparse import ArgumentParser
+
+
+class Command(BaseCommand):
+    help = "Generate typed PyOQ code from this project's migration state."
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument(
+            "app_label",
+            nargs="*",
+            help="Limit generation to these apps. Defaults to all of them.",
+        )
+        parser.add_argument(
+            "--database",
+            default=DEFAULT_DB_ALIAS,
+            help="The alias whose backend decides the column types.",
+        )
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            help="Report what would be written without writing it.",
+        )
+        parser.add_argument(
+            "--check",
+            action="store_true",
+            help="Fail when the generated package is not up to date.",
+        )
+
+    def handle(self, *args: Any, **options: Any) -> None:
+        del args
+        request = GenerationRequest(
+            alias=str(options["database"]),
+            app_labels=tuple(options["app_label"]),
+            mode=_mode(check=options["check"], dry_run=options["dry_run"]),
+        )
+        self.stdout.write(_report(request))
+
+
+def _mode(*, check: bool, dry_run: bool) -> GenerationMode:
+    """Checking and reporting are different questions, so asking both is not.
+
+    One answers whether the package is current and the other describes what a
+    run would produce. A command that quietly picked one would give an answer
+    to a question that was not asked.
+    """
+    if check and dry_run:
+        message = "--check and --dry-run cannot be combined"
+        raise CommandError(message)
+    if check:
+        return GenerationMode.CHECK
+    if dry_run:
+        return GenerationMode.DRY_RUN
+    return GenerationMode.WRITE
+
+
+def _report(request: GenerationRequest) -> str:
+    """A PyOQ failure is the operator's problem, not a traceback."""
+    try:
+        return generate(request)
+    except PyOQError as error:
+        raise CommandError(str(error)) from error

pyoq/django/parameters.py

@@ -0,0 +1,101 @@
+"""Binding values the way Django binds them on the connection it owns.
+
+A dialect adapter writes what its driver accepts. That is not enough here,
+because the rows are shared: Django reads back what PyOQ writes and filters on
+it, so a value has to reach the column in the form Django's own fields put
+there. That form differs per backend and per type, and Django already decides
+it, so each value is handed to the field Django would have used rather than to
+a rule restated here.
+"""
+
+from __future__ import annotations
+
+from datetime import date, datetime, time, timedelta
+from decimal import Decimal
+from enum import Enum
+from typing import TypeAlias, cast
+from uuid import UUID
+
+from django.db import models
+from django.db.backends.base.base import BaseDatabaseWrapper
+from django.db.models import Field
+
+from pyoq.errors import ParameterBindingError
+
+DjangoValue: TypeAlias = (
+    bool
+    | int
+    | float
+    | Decimal
+    | str
+    | bytes
+    | date
+    | datetime
+    | time
+    | timedelta
+    | UUID
+    | None
+)
+"""What a Django backend accepts as a bound value.
+
+Django declares this only in its stubs and under a private name, so PyOQ states
+it rather than reaching for one.
+"""
+
+_FIELDS: tuple[tuple[type[object], Field[object, object]], ...] = (
+    (bool, models.BooleanField()),
+    (UUID, models.UUIDField()),
+    (timedelta, models.DurationField()),
+    (datetime, models.DateTimeField()),
+    (date, models.DateField()),
+    (time, models.TimeField()),
+    (Decimal, models.DecimalField()),
+    (dict, models.JSONField()),
+    (list, models.JSONField()),
+)
+"""The field Django would have used, for each type it stores its own way.
+
+Order matters where one type is another: a datetime is a date, and a boolean
+is an integer, so the narrower entry comes first. Each field prepares a value
+without reading state of its own, so one instance serves every statement.
+"""
+
+
+def adapt_parameters(
+    connection: BaseDatabaseWrapper,
+    values: tuple[object, ...],
+    /,
+) -> tuple[DjangoValue, ...]:
+    """Every bound value, in the form this backend stores it."""
+    return tuple(_adapted(connection, value) for value in values)
+
+
+def _adapted(connection: BaseDatabaseWrapper, value: object) -> DjangoValue:
+    if isinstance(value, Enum):
+        return _adapted(connection, value.value)
+    if isinstance(value, memoryview):
+        return value.tobytes()
+    for value_type, field in _FIELDS:
+        if isinstance(value, value_type):
+            return _prepared(field, connection, value)
+    return _plain(value)
+
+
+def _prepared(
+    field: Field[object, object],
+    connection: BaseDatabaseWrapper,
+    value: object,
+) -> DjangoValue:
+    """Every field above prepares a value into one a backend already binds."""
+    return cast("DjangoValue", field.get_db_prep_save(value, connection))
+
+
+def _plain(value: object) -> DjangoValue:
+    """A value every backend stores alike, which no field has to prepare."""
+    if value is None or isinstance(value, (int, float, str, bytes)):
+        return value
+    message = f"unsupported Django parameter type: {type(value).__name__}"
+    raise ParameterBindingError(message)
+
+
+__all__ = ("DjangoValue", "adapt_parameters")

pyoq/django/schema.py

@@ -0,0 +1,379 @@
+"""The schema a Django project's migrations describe.
+
+Read from migration state rather than from a database, because migrations are
+the authority on what the schema will be and a developer generating types has
+usually not applied them yet. Nothing here opens a connection: Django answers
+what type a field takes on a backend without consulting the server.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, replace
+from typing import TYPE_CHECKING, Protocol
+
+from django.db import DEFAULT_DB_ALIAS, connections
+from django.db.migrations.loader import MigrationLoader
+from django.db.models import CheckConstraint as DjangoCheckConstraint
+from django.db.models import ForeignKey
+
+from pyoq.config import Configuration, DatabaseDialect
+from pyoq.errors import OperationUnavailableError
+from pyoq.schema import (
+    Catalog,
+    CheckConstraint,
+    Column,
+    GeneratedKind,
+    GeneratedValue,
+    Identifier,
+    Index,
+    IndexTerm,
+    Key,
+    KeyKind,
+    MetadataEntry,
+    ObjectReference,
+    Relation,
+    Schema,
+    SchemaSnapshot,
+    SqlExpression,
+    SqlType,
+    Table,
+    TypeKind,
+)
+from pyoq.schema.mysql.types import mysql_type
+from pyoq.schema.postgres.types import postgres_type
+from pyoq.schema.sqlite.types import sqlite_type
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Sequence
+
+    from django.db.backends.base.base import BaseDatabaseWrapper
+    from django.db.models import Field, Model
+    from django.db.models.options import Options
+
+
+class TypeParser(Protocol):
+    def __call__(self, declared_type: str, /) -> SqlType: ...
+
+
+_PARSERS: dict[str, tuple[DatabaseDialect, TypeParser]] = {
+    "sqlite": (DatabaseDialect.SQLITE, sqlite_type),
+    "postgresql": (DatabaseDialect.POSTGRES, postgres_type),
+    "mysql": (DatabaseDialect.MYSQL, mysql_type),
+}
+
+_AUTO_FIELDS = frozenset({"AutoField", "BigAutoField", "SmallAutoField"})
+
+
+@dataclass(frozen=True, slots=True)
+class MigrationStateSchemaSource:
+    """Every table the migrations on disk would create, and nothing else."""
+
+    alias: str = DEFAULT_DB_ALIAS
+    app_labels: tuple[str, ...] = ()
+
+    def load(self, configuration: Configuration) -> SchemaSnapshot:
+        del configuration
+        connection = connections[self.alias]
+        dialect, parse = _parser_for(connection)
+        models = _selected_models(self.app_labels)
+        tables = [_table(model, connection, parse) for model in models]
+        tables.extend(_join_tables(models, connection, parse))
+        schema = Schema(name=None, tables=tuple(_ordered(tables)))
+        return SchemaSnapshot(
+            dialect=dialect,
+            catalogs=(Catalog(name=None, schemas=(schema,)),),
+        )
+
+
+def _parser_for(
+    connection: BaseDatabaseWrapper,
+) -> tuple[DatabaseDialect, TypeParser]:
+    parser = _PARSERS.get(connection.vendor)
+    if parser is None:
+        message = (
+            f"PyOQ cannot read a schema for the Django backend "
+            f"{connection.vendor!r}; supported backends: "
+            f"{', '.join(sorted(_PARSERS))}"
+        )
+        raise OperationUnavailableError(message)
+    return parser
+
+
+def _selected_models(app_labels: tuple[str, ...]) -> tuple[type[Model], ...]:
+    """The models the migrations describe, with no database consulted.
+
+    A loader given no connection reads the migration files alone and never asks
+    which of them have been applied, which is the question that needs a server.
+    """
+    state = MigrationLoader(None).project_state()
+    wanted = frozenset(app_labels)
+    chosen = [
+        state.apps.get_model(label, name)
+        for label, name in sorted(state.models)
+        if not wanted or label in wanted
+    ]
+    _require_known_labels(wanted, frozenset(label for label, _ in state.models))
+    return tuple(model for model in chosen if model._meta.managed)
+
+
+def _require_known_labels(
+    wanted: frozenset[str],
+    available: frozenset[str],
+) -> None:
+    missing = sorted(wanted - available)
+    if not missing:
+        return
+    message = (
+        f"no migrations describe the app {', '.join(missing)!r}; "
+        f"apps with migrations: {', '.join(sorted(available))}"
+    )
+    raise OperationUnavailableError(message)
+
+
+def _table(
+    model: type[Model],
+    connection: BaseDatabaseWrapper,
+    parse: TypeParser,
+) -> Table:
+    meta = model._meta
+    columns = tuple(
+        _column(field, declared, parse)
+        for field, declared in _declared_types(meta, connection)
+    )
+    return Table(
+        name=Identifier(meta.db_table),
+        columns=columns,
+        keys=_keys(meta),
+        relations=_relations(meta),
+        indexes=_indexes(meta),
+        checks=_checks(meta),
+    )
+
+
+def _declared_types(
+    meta: Options[Model],
+    connection: BaseDatabaseWrapper,
+) -> Iterable[tuple[Field[object, object], str]]:
+    """A field without a type of its own has no column to describe."""
+    for field in meta.local_fields:
+        declared = field.db_type(connection)
+        if declared is not None:
+            yield field, declared
+
+
+_MEANT_KINDS: dict[str, TypeKind] = {
+    "JSONField": TypeKind.JSON,
+    "UUIDField": TypeKind.UUID,
+    "DurationField": TypeKind.INTERVAL,
+}
+"""What a field means, where the column it asks for does not say so.
+
+Django declares a JSON column as text, a UUID as a fixed width string, and a
+duration as an integer, so reading the column back would hand a caller the
+stored form rather than the value. The field is the one that knows.
+"""
+
+
+def _data_type(
+    field: Field[object, object],
+    declared: str,
+    parse: TypeParser,
+) -> SqlType:
+    """The declared column, named by what the field puts in it."""
+    stored = parse(declared)
+    meant = _MEANT_KINDS.get(field.get_internal_type())
+    if meant is None or stored.kind is meant:
+        return stored
+    return replace(stored, kind=meant, element_type=None, named_type=None)
+
+
+def _column(
+    field: Field[object, object],
+    declared: str,
+    parse: TypeParser,
+) -> Column:
+    return Column(
+        name=Identifier(_column_of(field)),
+        data_type=_data_type(field, declared, parse),
+        nullable=bool(field.null),
+        default=None,
+        generated=_generated(field),
+        metadata=_column_metadata(field),
+    )
+
+
+def _generated(field: Field[object, object]) -> GeneratedValue | None:
+    """Only a field the database fills in counts as generated here.
+
+    A Django default is applied in Python before the row is sent, so it is not
+    a fact about the column. Recording it as one would tell a caller they may
+    omit a value that the database will then refuse.
+    """
+    if type(field).__name__ not in _AUTO_FIELDS:
+        return None
+    return GeneratedValue(kind=GeneratedKind.IDENTITY_BY_DEFAULT)
+
+
+def _column_metadata(
+    field: Field[object, object],
+) -> tuple[MetadataEntry, ...]:
+    entries = [MetadataEntry(key="django-field", value=type(field).__name__)]
+    if field.has_default():
+        entries.append(MetadataEntry(key="django-default", value="python"))
+    if isinstance(field, ForeignKey):
+        entries.append(MetadataEntry(key="django-on-delete", value=_on_delete(field)))
+    return tuple(entries)
+
+
+def _on_delete(field: ForeignKey[object]) -> str:
+    """What the model asked for, beside what the database was told.
+
+    Django deletes related rows itself, so the constraint carries no action at
+    all. Keeping the model's choice visible costs nothing and stops a reader
+    concluding the database will do it.
+    """
+    return str(getattr(field.remote_field.on_delete, "__name__", "unknown"))
+
+
+def _keys(meta: Options[Model]) -> tuple[Key, ...]:
+    keys: list[Key] = []
+    keys.append(Key(kind=KeyKind.PRIMARY, columns=(Identifier(_column_of(meta.pk)),)))
+    keys.extend(
+        Key(kind=KeyKind.UNIQUE, columns=(Identifier(_column_of(field)),))
+        for field in meta.local_fields
+        if field.unique and not field.primary_key
+    )
+    keys.extend(
+        Key(kind=KeyKind.UNIQUE, columns=_columns(meta, group))
+        for group in meta.unique_together
+    )
+    keys.extend(_constraint_keys(meta))
+    return tuple(keys)
+
+
+def _constraint_keys(meta: Options[Model]) -> Iterable[Key]:
+    """A unique constraint counts only when it holds for every row.
+
+    A constraint carrying a condition is unique among the rows it matches and
+    silently not unique among the rest. Treating one as a key would let a
+    relation be described as reaching a single row when it reaches many.
+    """
+    for constraint in meta.constraints:
+        fields = getattr(constraint, "fields", None)
+        if not fields or _is_partial(constraint):
+            continue
+        yield Key(
+            kind=KeyKind.UNIQUE,
+            columns=_columns(meta, fields),
+            name=Identifier(constraint.name),
+        )
+
+
+def _is_partial(constraint: object) -> bool:
+    return any(
+        getattr(constraint, attribute, None)
+        for attribute in ("condition", "expressions", "nulls_distinct")
+    )
+
+
+def _relations(meta: Options[Model]) -> tuple[Relation, ...]:
+    """Django enforces deletion in Python, so the database is told nothing.
+
+    The referential action recorded here is the one the database will actually
+    take. What the model asked for is kept beside it as a fact about Django.
+    """
+    return tuple(
+        Relation(
+            columns=(Identifier(_column_of(field)),),
+            target=ObjectReference(name=Identifier(_target_table(field))),
+            target_columns=(Identifier(_column_of(field.target_field)),),
+        )
+        for field in _foreign_keys(meta)
+    )
+
+
+def _foreign_keys(meta: Options[Model]) -> Iterable[ForeignKey[object]]:
+    for field in meta.local_fields:
+        if isinstance(field, ForeignKey):
+            yield field
+
+
+def _target_table(field: ForeignKey[object]) -> str:
+    return str(field.related_model._meta.db_table)
+
+
+def _column_of(field: Field[object, object]) -> str:
+    return str(field.column)
+
+
+def _indexes(meta: Options[Model]) -> tuple[Index, ...]:
+    """Only indexes the project named itself.
+
+    Django names the indexes it adds for itself inside a schema editor, which
+    reaches the server. Guessing those names here would put a name in the
+    schema that the database may not agree with.
+    """
+    return tuple(
+        Index(
+            name=Identifier(index.name),
+            terms=tuple(_terms(meta, index.fields)),
+        )
+        for index in meta.indexes
+        if index.fields
+    )
+
+
+def _terms(meta: Options[Model], fields: Sequence[str]) -> Iterable[IndexTerm]:
+    for name in fields:
+        descending = name.startswith("-")
+        column = _column_name(meta, name.lstrip("-"))
+        yield IndexTerm(column=Identifier(column), descending=descending)
+
+
+def _checks(meta: Options[Model]) -> tuple[CheckConstraint, ...]:
+    return tuple(
+        CheckConstraint(
+            expression=SqlExpression(text=str(constraint.condition)),
+            name=Identifier(constraint.name),
+        )
+        for constraint in meta.constraints
+        if isinstance(constraint, DjangoCheckConstraint)
+    )
+
+
+def _columns(
+    meta: Options[Model],
+    names: Iterable[str],
+) -> tuple[Identifier, ...]:
+    return tuple(Identifier(_column_name(meta, name)) for name in names)
+
+
+def _column_name(meta: Options[Model], name: str) -> str:
+    field = meta.get_field(name)
+    column = getattr(field, "column", None)
+    return column if isinstance(column, str) else name
+
+
+def _join_tables(
+    models: tuple[type[Model], ...],
+    connection: BaseDatabaseWrapper,
+    parse: TypeParser,
+) -> Iterable[Table]:
+    """A many-to-many field owns a table that migration state does not list.
+
+    Django creates it from the field rather than from a model of its own, so it
+    has to be described here or the schema would be missing a real table. A
+    through model the project wrote is already a model and is left alone.
+    """
+    for model in models:
+        for field in model._meta.local_many_to_many:
+            through = field.remote_field.through
+            if through is not None and through._meta.auto_created:
+                yield _table(through, connection, parse)
+
+
+def _ordered(tables: Iterable[Table]) -> list[Table]:
+    return sorted(tables, key=lambda table: table.name.value)
+
+
+__all__ = ("MigrationStateSchemaSource",)