pyoq-sql 1.0.0__py3-none-any.whl

pyoq/django/execution.py

@@ -0,0 +1,317 @@
+"""Running PyOQ statements on the connection Django already has open.
+
+Django owns its connections, their aliases, and whatever transaction is in
+progress. PyOQ borrows one rather than opening a pool beside it, because two
+pools against one database is two views of what has been committed.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Generator, Sequence
+from contextlib import AbstractContextManager, contextmanager
+from dataclasses import dataclass
+from typing import Protocol, TypeVar, cast
+
+from django.db import DEFAULT_DB_ALIAS, Error, connections, router, transaction
+from django.db.backends.base.base import BaseDatabaseWrapper
+from django.db.models import Model
+
+from pyoq.django.parameters import DjangoValue, adapt_parameters
+from pyoq.django.timeouts import (
+    MYSQL_TIMEOUT,
+    POSTGRES_TIMEOUT,
+    TimeoutSetting,
+    applied,
+)
+from pyoq.errors import (
+    OperationUnavailableError,
+    QueryCancelledError,
+    QueryExecutionError,
+)
+from pyoq.query.execution import (
+    BulkPlan,
+    CompiledQuery,
+    DatabaseCursor,
+    ExecutionControl,
+    QueryOperations,
+    WriteProvider,
+)
+from pyoq.query.statements import StatementCompiler
+from pyoq.query.write_nodes import WriteNode
+
+OperationResult = TypeVar("OperationResult")
+
+
+class BulkPlanner(Protocol):
+    def plan(self, statement: WriteProvider | WriteNode, /) -> BulkPlan: ...
+
+
+class LastRowCursor(Protocol):
+    @property
+    def lastrowid(self) -> object: ...
+
+
+@dataclass(frozen=True, slots=True)
+class Dialect:
+    """What the backend Django is already speaking needs from PyOQ.
+
+    Values are adapted the same way they are on PyOQ's own connections, because
+    a backend does not care whose connection it is: a decimal or a date still
+    has to reach it in the shape its driver accepts.
+    """
+
+    compiler: StatementCompiler
+    planner: BulkPlanner
+    timeout: TimeoutSetting | None = None
+
+
+def _sqlite() -> Dialect:
+    from pyoq.query.execution import ParameterStyle
+    from pyoq.query.sqlite import (
+        SQLiteBulkPlanner,
+        SQLiteCapabilities,
+        SQLiteCompiler,
+    )
+
+    compiler = SQLiteCompiler(SQLiteCapabilities(parameter_style=ParameterStyle.FORMAT))
+    return Dialect(compiler, SQLiteBulkPlanner(compiler))
+
+
+def _postgres() -> Dialect:
+    from pyoq.query.postgres import PostgresBulkPlanner, PostgresCompiler
+
+    compiler = PostgresCompiler()
+    return Dialect(compiler, PostgresBulkPlanner(compiler), POSTGRES_TIMEOUT)
+
+
+def _mysql() -> Dialect:
+    from pyoq.query.mysql import MySQLBulkPlanner, MySQLCompiler
+
+    compiler = MySQLCompiler()
+    return Dialect(compiler, MySQLBulkPlanner(compiler), MYSQL_TIMEOUT)
+
+
+_DIALECTS: dict[str, Callable[[], Dialect]] = {
+    "sqlite": _sqlite,
+    "postgresql": _postgres,
+    "mysql": _mysql,
+}
+"""How to build each dialect, called once a connection turns out to be one.
+
+A dialect package imports the driver it speaks to, so building these eagerly
+would make the Django extra need every driver PyOQ supports.
+"""
+
+_DRIVER_EXTRAS: dict[str, str] = {"postgresql": "postgres", "mysql": "mysql"}
+"""What a project installs to reach each backend.
+
+SQLite is absent because it needs nothing, and so can never be the one that
+is missing.
+"""
+
+
+def dialect_for(connection: BaseDatabaseWrapper, /) -> Dialect:
+    """Choose the dialect Django is already speaking.
+
+    The connection decides, not the caller. Speaking a different dialect than
+    the one on the other end of the socket is not a choice worth offering.
+    """
+    factory = _DIALECTS.get(connection.vendor)
+    if factory is None:
+        message = (
+            f"PyOQ has no dialect for the Django backend {connection.vendor!r}; "
+            f"supported backends: {', '.join(sorted(_DIALECTS))}"
+        )
+        raise OperationUnavailableError(message)
+    return _built(factory, connection.vendor)
+
+
+def _built(factory: Callable[[], Dialect], vendor: str) -> Dialect:
+    """Say which extra is missing, rather than which module is.
+
+    A project installs the drivers it uses, so reaching a backend it did not
+    install is an ordinary mistake and worth answering with the fix.
+    """
+    try:
+        return factory()
+    except ImportError as error:
+        extra = _DRIVER_EXTRAS[vendor]
+        message = (
+            f"the Django backend {vendor!r} needs a driver PyOQ does not "
+            f"install by default; add it with pip install "
+            f"'pyoq-sql[django,{extra}]'"
+        )
+        raise OperationUnavailableError(message) from error
+
+
+def alias_for(
+    model: type[Model],
+    /,
+    *,
+    write: bool = False,
+    **hints: object,
+) -> str:
+    """Ask Django's routers which database a model lives in.
+
+    Routing is a question about a model, because that is the only thing a
+    router is given to decide on. A caller with a model gets the answer its own
+    routers would give; a caller without one names the alias instead.
+    """
+    if write:
+        return str(router.db_for_write(model, **hints) or DEFAULT_DB_ALIAS)
+    return str(router.db_for_read(model, **hints) or DEFAULT_DB_ALIAS)
+
+
+class DjangoOperations(QueryOperations):
+    """Typed operations against one Django database alias.
+
+    The connection is resolved for each statement rather than held, because
+    Django hands a different connection to each thread and closes them between
+    requests, so holding one would outlive what it belongs to.
+    """
+
+    __slots__ = ("_alias", "_dialect")
+
+    def __init__(self, alias: str = DEFAULT_DB_ALIAS) -> None:
+        self._alias = alias
+        self._dialect = dialect_for(self.connection)
+
+    @classmethod
+    def for_model(
+        cls,
+        model: type[Model],
+        /,
+        *,
+        write: bool = False,
+        **hints: object,
+    ) -> DjangoOperations:
+        """Run against whichever database this project routes a model to."""
+        return cls(alias_for(model, write=write, **hints))
+
+    @property
+    def alias(self) -> str:
+        return self._alias
+
+    @property
+    def connection(self) -> BaseDatabaseWrapper:
+        """Whatever connection Django holds for this alias right now."""
+        return connections[self._alias]
+
+    @property
+    def compiler(self) -> StatementCompiler:
+        return self._dialect.compiler
+
+    @property
+    def in_transaction(self) -> bool:
+        """Whether this database is inside a transaction right now."""
+        return self.connection.in_atomic_block
+
+    def atomic(
+        self,
+        *,
+        savepoint: bool = True,
+        durable: bool = False,
+    ) -> AbstractContextManager[None]:
+        """A transaction on this database, rather than on whichever is default.
+
+        Django's own block covers the default alias unless told otherwise, so a
+        caller running here against another alias would get no transaction at
+        all and no indication of it. Nesting one of these is a savepoint, which
+        is what Django already does.
+
+        PyOQ never commits or rolls back. The block that opened a transaction is
+        the one that ends it, and here that block is Django's.
+        """
+        return transaction.atomic(
+            using=self._alias, savepoint=savepoint, durable=durable
+        )
+
+    def on_commit(
+        self,
+        callback: Callable[[], object],
+        /,
+        *,
+        robust: bool = False,
+    ) -> None:
+        """Run something once this database's transaction has committed.
+
+        Outside a transaction it runs immediately, which is Django's own rule
+        and the right one: there is nothing left to wait for.
+        """
+        transaction.on_commit(callback, using=self._alias, robust=robust)
+
+    def plan_bulk(self, statement: WriteProvider | WriteNode, /) -> BulkPlan:
+        return self._dialect.planner.plan(statement)
+
+    def last_inserted_id(self, cursor: DatabaseCursor) -> int | None:
+        identifier = _last_row_identifier(cursor)
+        if not isinstance(identifier, int) or identifier <= 0:
+            return None
+        return identifier
+
+    def _run(
+        self,
+        statement: CompiledQuery,
+        operation: Callable[[DatabaseCursor], OperationResult],
+        control: ExecutionControl | None,
+    ) -> OperationResult:
+        _checkpoint(control)
+        connection = self.connection
+        adapted = adapt_parameters(connection, statement.parameters)
+        parameters = cast("Sequence[DjangoValue]", adapted)
+        with self._bounded(control), connection.cursor() as cursor:
+            try:
+                # Django's stub omits the duration values its own interval
+                # backends accept, so the call is wider than the stub says.
+                cursor.execute(statement.sql, parameters)  # type: ignore[arg-type]
+                result = operation(cast("DatabaseCursor", cursor))
+            except Error as error:
+                message = f"Django query execution failed: {error}"
+                raise QueryExecutionError(message) from error
+        _checkpoint(control)
+        return result
+
+    @contextmanager
+    def _bounded(self, control: ExecutionControl | None) -> Generator[None]:
+        """Bound the statement where the backend offers a way to.
+
+        SQLite has no session setting for this, and MySQL's covers reads only,
+        so a caller is given what the backend can honour rather than a promise
+        it cannot keep.
+        """
+        setting = self._dialect.timeout
+        seconds = None if control is None else control.timeout
+        if setting is None or seconds is None:
+            yield
+            return
+        with applied(setting, self.connection, seconds):
+            yield
+
+
+def _last_row_identifier(cursor: DatabaseCursor) -> object:
+    """Not every backend offers one, and none of them promise a type."""
+    return getattr(cursor, "lastrowid", None)
+
+
+def _checkpoint(control: ExecutionControl | None) -> None:
+    """Honour a caller's own limits without touching Django's connection.
+
+    A server-side timeout is a change to session state, and the session belongs
+    to Django. What is left is refusing before and after a statement, which is
+    what a caller asking for cancellation can be given honestly.
+    """
+    if control is None:
+        return
+    token = control.cancellation_token
+    if token is not None and token.cancelled:
+        message = "Django query execution was cancelled"
+        raise QueryCancelledError(message)
+
+
+__all__ = (
+    "BulkPlanner",
+    "Dialect",
+    "DjangoOperations",
+    "alias_for",
+    "dialect_for",
+)

pyoq/django/generation.py

@@ -0,0 +1,59 @@
+"""Running the shared generation pipeline for a Django project."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from django.db import DEFAULT_DB_ALIAS, connections
+
+from pyoq.cli.defaults import default_generation_pipeline
+from pyoq.config import DatabaseDialect
+from pyoq.django.schema import MigrationStateSchemaSource
+from pyoq.django.settings import django_configuration
+from pyoq.errors import OperationUnavailableError
+from pyoq.generation import GenerationMode
+
+_DIALECTS = {
+    "sqlite": "sqlite",
+    "postgresql": "postgres",
+    "mysql": "mysql",
+}
+
+
+@dataclass(frozen=True, slots=True)
+class GenerationRequest:
+    """One run of generation, described before anything is written."""
+
+    alias: str = DEFAULT_DB_ALIAS
+    app_labels: tuple[str, ...] = ()
+    mode: GenerationMode = GenerationMode.WRITE
+
+
+def generate(request: GenerationRequest, /) -> str:
+    """The pipeline every other entry point uses, pointed at Django.
+
+    Generation is not reimplemented here. Staging, validation, the manifest,
+    drift, and the atomic replacement all belong to the shared pipeline, and a
+    second copy of them would be a second set of answers.
+    """
+    dialect = DatabaseDialect(_dialect_name(request.alias))
+    source = MigrationStateSchemaSource(
+        alias=request.alias, app_labels=request.app_labels
+    )
+    pipeline = default_generation_pipeline(source)
+    return pipeline.execute(django_configuration(dialect), request.mode)
+
+
+def _dialect_name(alias: str) -> str:
+    vendor = connections[alias].vendor
+    name = _DIALECTS.get(vendor)
+    if name is None:
+        message = (
+            f"PyOQ cannot generate for the Django backend {vendor!r}; "
+            f"supported backends: {', '.join(sorted(_DIALECTS))}"
+        )
+        raise OperationUnavailableError(message)
+    return name
+
+
+__all__ = ("GenerationRequest", "generate")

pyoq/django/management/commands/makemigrations.py

@@ -0,0 +1,53 @@
+"""Django's own `makemigrations`, with generation after it.
+
+A project opts in to this by putting PyOQ ahead of the app that supplies the
+original command, and by setting `codegen_after_makemigrations`. Overriding a
+command a project already runs is not something to do to somebody quietly.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from django.core.management.base import CommandError
+from django.core.management.commands.makemigrations import (
+    Command as MakeMigrations,
+)
+
+from pyoq.django.generation import GenerationRequest, generate
+from pyoq.django.settings import codegen_after_makemigrations
+from pyoq.errors import PyOQError
+
+
+class Command(MakeMigrations):
+    def handle(self, *app_labels: str, **options: Any) -> None:
+        super().handle(*app_labels, **options)
+        if not codegen_after_makemigrations():
+            return
+        if _writes_nothing(options):
+            self.stdout.write(
+                "PyOQ: migrations were not written, so nothing was generated"
+            )
+            return
+        self.stdout.write(f"PyOQ: {_generated(app_labels, options)}")
+
+
+def _writes_nothing(options: dict[str, Any]) -> bool:
+    """Generation reads migration files, so it must not run before they exist.
+
+    A run asked to check or to report leaves the files as they were. Generating
+    then would describe the schema before this change rather than after it, and
+    would look like it had worked.
+    """
+    return bool(options.get("dry_run") or options.get("check_changes"))
+
+
+def _generated(app_labels: tuple[str, ...], options: dict[str, Any]) -> str:
+    request = GenerationRequest(
+        alias=str(options.get("database") or "default"),
+        app_labels=app_labels,
+    )
+    try:
+        return generate(request)
+    except PyOQError as error:
+        raise CommandError(str(error)) from error

pyoq/django/management/commands/pyoq_codegen.py

@@ -0,0 +1,75 @@
+"""Generate PyOQ types from the migrations this project has written."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from django.core.management.base import BaseCommand, CommandError
+from django.db import DEFAULT_DB_ALIAS
+
+from pyoq.django.generation import GenerationRequest, generate
+from pyoq.errors import PyOQError
+from pyoq.generation import GenerationMode
+
+if TYPE_CHECKING:
+    from argparse import ArgumentParser
+
+
+class Command(BaseCommand):
+    help = "Generate typed PyOQ code from this project's migration state."
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument(
+            "app_label",
+            nargs="*",
+            help="Limit generation to these apps. Defaults to all of them.",
+        )
+        parser.add_argument(
+            "--database",
+            default=DEFAULT_DB_ALIAS,
+            help="The alias whose backend decides the column types.",
+        )
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            help="Report what would be written without writing it.",
+        )
+        parser.add_argument(
+            "--check",
+            action="store_true",
+            help="Fail when the generated package is not up to date.",
+        )
+
+    def handle(self, *args: Any, **options: Any) -> None:
+        del args
+        request = GenerationRequest(
+            alias=str(options["database"]),
+            app_labels=tuple(options["app_label"]),
+            mode=_mode(check=options["check"], dry_run=options["dry_run"]),
+        )
+        self.stdout.write(_report(request))
+
+
+def _mode(*, check: bool, dry_run: bool) -> GenerationMode:
+    """Checking and reporting are different questions, so asking both is not.
+
+    One answers whether the package is current and the other describes what a
+    run would produce. A command that quietly picked one would give an answer
+    to a question that was not asked.
+    """
+    if check and dry_run:
+        message = "--check and --dry-run cannot be combined"
+        raise CommandError(message)
+    if check:
+        return GenerationMode.CHECK
+    if dry_run:
+        return GenerationMode.DRY_RUN
+    return GenerationMode.WRITE
+
+
+def _report(request: GenerationRequest) -> str:
+    """A PyOQ failure is the operator's problem, not a traceback."""
+    try:
+        return generate(request)
+    except PyOQError as error:
+        raise CommandError(str(error)) from error

pyoq/django/parameters.py

@@ -0,0 +1,101 @@
+"""Binding values the way Django binds them on the connection it owns.
+
+A dialect adapter writes what its driver accepts. That is not enough here,
+because the rows are shared: Django reads back what PyOQ writes and filters on
+it, so a value has to reach the column in the form Django's own fields put
+there. That form differs per backend and per type, and Django already decides
+it, so each value is handed to the field Django would have used rather than to
+a rule restated here.
+"""
+
+from __future__ import annotations
+
+from datetime import date, datetime, time, timedelta
+from decimal import Decimal
+from enum import Enum
+from typing import TypeAlias, cast
+from uuid import UUID
+
+from django.db import models
+from django.db.backends.base.base import BaseDatabaseWrapper
+from django.db.models import Field
+
+from pyoq.errors import ParameterBindingError
+
+DjangoValue: TypeAlias = (
+    bool
+    | int
+    | float
+    | Decimal
+    | str
+    | bytes
+    | date
+    | datetime
+    | time
+    | timedelta
+    | UUID
+    | None
+)
+"""What a Django backend accepts as a bound value.
+
+Django declares this only in its stubs and under a private name, so PyOQ states
+it rather than reaching for one.
+"""
+
+_FIELDS: tuple[tuple[type[object], Field[object, object]], ...] = (
+    (bool, models.BooleanField()),
+    (UUID, models.UUIDField()),
+    (timedelta, models.DurationField()),
+    (datetime, models.DateTimeField()),
+    (date, models.DateField()),
+    (time, models.TimeField()),
+    (Decimal, models.DecimalField()),
+    (dict, models.JSONField()),
+    (list, models.JSONField()),
+)
+"""The field Django would have used, for each type it stores its own way.
+
+Order matters where one type is another: a datetime is a date, and a boolean
+is an integer, so the narrower entry comes first. Each field prepares a value
+without reading state of its own, so one instance serves every statement.
+"""
+
+
+def adapt_parameters(
+    connection: BaseDatabaseWrapper,
+    values: tuple[object, ...],
+    /,
+) -> tuple[DjangoValue, ...]:
+    """Every bound value, in the form this backend stores it."""
+    return tuple(_adapted(connection, value) for value in values)
+
+
+def _adapted(connection: BaseDatabaseWrapper, value: object) -> DjangoValue:
+    if isinstance(value, Enum):
+        return _adapted(connection, value.value)
+    if isinstance(value, memoryview):
+        return value.tobytes()
+    for value_type, field in _FIELDS:
+        if isinstance(value, value_type):
+            return _prepared(field, connection, value)
+    return _plain(value)
+
+
+def _prepared(
+    field: Field[object, object],
+    connection: BaseDatabaseWrapper,
+    value: object,
+) -> DjangoValue:
+    """Every field above prepares a value into one a backend already binds."""
+    return cast("DjangoValue", field.get_db_prep_save(value, connection))
+
+
+def _plain(value: object) -> DjangoValue:
+    """A value every backend stores alike, which no field has to prepare."""
+    if value is None or isinstance(value, (int, float, str, bytes)):
+        return value
+    message = f"unsupported Django parameter type: {type(value).__name__}"
+    raise ParameterBindingError(message)
+
+
+__all__ = ("DjangoValue", "adapt_parameters")