pyoq-sql 1.0.0__py3-none-any.whl

pyoq/diagnostics/instrumented.py

@@ -0,0 +1,237 @@
+"""Reporting what every statement did, at the point they all pass.
+
+Instrumentation is a database that wraps another one, so a project that wants
+none holds the plain database and pays nothing at all. There is no flag to read
+on the way to the driver because there is nothing in the way.
+"""
+
+from __future__ import annotations
+
+from time import perf_counter
+from typing import TYPE_CHECKING, TypeVar
+
+from pyoq.diagnostics.events import (
+    EventPolicy,
+    StatementFailed,
+    StatementFinished,
+    StatementStarted,
+    reportable_values,
+)
+from pyoq.diagnostics.fingerprint import query_shape
+from pyoq.query.execution import QueryOperations
+from pyoq.query.execution.aio import AsyncQueryOperations
+
+if TYPE_CHECKING:
+    from collections.abc import Awaitable, Callable
+
+    from pyoq.diagnostics.events import EventSink
+    from pyoq.query.execution import (
+        BulkPlan,
+        CompiledQuery,
+        DatabaseCursor,
+        ExecutionControl,
+        WriteProvider,
+    )
+    from pyoq.query.execution.aio import AsyncDatabaseCursor
+    from pyoq.query.statements import StatementCompiler
+    from pyoq.query.write_nodes import WriteNode
+
+OperationResult = TypeVar("OperationResult")
+
+
+class _Rows:
+    """How many rows the cursor reported, which only it can say."""
+
+    __slots__ = ("count",)
+
+    def __init__(self) -> None:
+        self.count: int | None = None
+
+
+class InstrumentedOperations(QueryOperations):
+    """A database that reports every statement it runs."""
+
+    __slots__ = ("_inner", "_policy", "_sink")
+
+    def __init__(
+        self,
+        inner: QueryOperations,
+        sink: EventSink,
+        policy: EventPolicy | None = None,
+    ) -> None:
+        self._inner = inner
+        self._sink = sink
+        self._policy = policy or EventPolicy()
+
+    @property
+    def policy(self) -> EventPolicy:
+        return self._policy
+
+    @property
+    def compiler(self) -> StatementCompiler:
+        return self._inner.compiler
+
+    def plan_bulk(self, statement: WriteProvider | WriteNode, /) -> BulkPlan:
+        return self._inner.plan_bulk(statement)
+
+    def last_inserted_id(self, cursor: DatabaseCursor) -> int | None:
+        return self._inner.last_inserted_id(cursor)
+
+    def _run(
+        self,
+        statement: CompiledQuery,
+        operation: Callable[[DatabaseCursor], OperationResult],
+        control: ExecutionControl | None,
+    ) -> OperationResult:
+        report = _Report(statement, self._sink, self._policy)
+        rows = _Rows()
+        report.started()
+        try:
+            outcome = self._inner._run(statement, _counting(operation, rows), control)
+        except BaseException as error:
+            report.failed(error)
+            raise
+        report.finished(rows.count)
+        return outcome
+
+
+class AsyncInstrumentedOperations(AsyncQueryOperations):
+    """The asynchronous counterpart, reporting at the same point."""
+
+    __slots__ = ("_inner", "_policy", "_sink")
+
+    def __init__(
+        self,
+        inner: AsyncQueryOperations,
+        sink: EventSink,
+        policy: EventPolicy | None = None,
+    ) -> None:
+        self._inner = inner
+        self._sink = sink
+        self._policy = policy or EventPolicy()
+
+    @property
+    def policy(self) -> EventPolicy:
+        return self._policy
+
+    @property
+    def compiler(self) -> StatementCompiler:
+        return self._inner.compiler
+
+    def plan_bulk(self, statement: WriteProvider | WriteNode, /) -> BulkPlan:
+        return self._inner.plan_bulk(statement)
+
+    def last_inserted_id(self, cursor: AsyncDatabaseCursor) -> int | None:
+        return self._inner.last_inserted_id(cursor)
+
+    async def _run(
+        self,
+        statement: CompiledQuery,
+        operation: Callable[[AsyncDatabaseCursor], Awaitable[OperationResult]],
+        control: ExecutionControl | None,
+    ) -> OperationResult:
+        report = _Report(statement, self._sink, self._policy)
+        rows = _Rows()
+        report.started()
+        try:
+            outcome = await self._inner._run(
+                statement, _counting_async(operation, rows), control
+            )
+        except BaseException as error:
+            report.failed(error)
+            raise
+        report.finished(rows.count)
+        return outcome
+
+
+class _Report:
+    """One statement's events, timed from the moment it was described."""
+
+    __slots__ = ("_policy", "_shape", "_sink", "_started", "_statement")
+
+    def __init__(
+        self,
+        statement: CompiledQuery,
+        sink: EventSink,
+        policy: EventPolicy,
+    ) -> None:
+        self._statement = statement
+        self._sink = sink
+        self._policy = policy
+        self._shape = query_shape(statement.sql)
+        self._started = perf_counter()
+
+    def started(self) -> None:
+        self._sink.record(
+            StatementStarted(
+                shape=self._shape,
+                kind=self._statement.statement_kind,
+                parameters=len(self._statement.parameters),
+                sensitive_parameters=len(self._statement.sensitive_parameter_indexes),
+                values=self._values(),
+            )
+        )
+
+    def finished(self, rows: int | None) -> None:
+        elapsed = perf_counter() - self._started
+        threshold = self._policy.slow_after
+        self._sink.record(
+            StatementFinished(
+                shape=self._shape,
+                kind=self._statement.statement_kind,
+                parameters=len(self._statement.parameters),
+                sensitive_parameters=len(self._statement.sensitive_parameter_indexes),
+                duration=elapsed,
+                rows=rows,
+                slow=threshold is not None and elapsed >= threshold,
+                values=self._values(),
+            )
+        )
+
+    def failed(self, error: BaseException) -> None:
+        self._sink.record(
+            StatementFailed(
+                shape=self._shape,
+                kind=self._statement.statement_kind,
+                parameters=len(self._statement.parameters),
+                sensitive_parameters=len(self._statement.sensitive_parameter_indexes),
+                duration=perf_counter() - self._started,
+                failure=type(error).__name__,
+                detail=self._detail(error),
+                values=self._values(),
+            )
+        )
+
+    def _values(self) -> tuple[object, ...]:
+        return reportable_values(self._statement, self._policy)
+
+    def _detail(self, error: BaseException) -> str | None:
+        """A driver names the offending value in its message, so this is asked."""
+        return str(error) if self._policy.include_failure_detail else None
+
+
+def _counting(
+    operation: Callable[[DatabaseCursor], OperationResult],
+    rows: _Rows,
+) -> Callable[[DatabaseCursor], OperationResult]:
+    def counted(cursor: DatabaseCursor) -> OperationResult:
+        outcome = operation(cursor)
+        rows.count = cursor.rowcount
+        return outcome
+
+    return counted
+
+
+def _counting_async(
+    operation: Callable[[AsyncDatabaseCursor], Awaitable[OperationResult]],
+    rows: _Rows,
+) -> Callable[[AsyncDatabaseCursor], Awaitable[OperationResult]]:
+    async def counted(cursor: AsyncDatabaseCursor) -> OperationResult:
+        outcome = await operation(cursor)
+        rows.count = cursor.rowcount
+        return outcome
+
+    return counted
+
+
+__all__ = ("AsyncInstrumentedOperations", "InstrumentedOperations")

pyoq/diagnostics/metrics.py

@@ -0,0 +1,61 @@
+"""What a database is doing right now, as numbers.
+
+A pool already knows its own counts and an observer already knows what has run.
+This puts the two together so one reading describes both, and adds nothing that
+would have to be kept up to date separately.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Protocol
+
+if TYPE_CHECKING:
+    from pyoq.diagnostics.observation import QueryObserver
+    from pyoq.query.execution import PoolStats
+
+
+class ReportsPoolStats(Protocol):
+    """Anything that can say how its connections are being used."""
+
+    @property
+    def stats(self) -> PoolStats: ...
+
+
+@dataclass(frozen=True, slots=True)
+class DatabaseMetrics:
+    """One reading, carrying counts and nothing a query was asked about."""
+
+    open_connections: int
+    idle_connections: int
+    checked_out_connections: int
+    closed: bool
+    statements: int
+    shapes: int
+
+    @property
+    def in_use(self) -> float:
+        """How much of what is open is checked out, between zero and one."""
+        if self.open_connections == 0:
+            return 0.0
+        return self.checked_out_connections / self.open_connections
+
+
+def metrics_of(
+    pool: ReportsPoolStats,
+    /,
+    observer: QueryObserver | None = None,
+) -> DatabaseMetrics:
+    """A reading taken now, from a pool and optionally from what it has run."""
+    stats = pool.stats
+    return DatabaseMetrics(
+        open_connections=stats.open_connections,
+        idle_connections=stats.idle_connections,
+        checked_out_connections=stats.checked_out_connections,
+        closed=stats.closed,
+        statements=0 if observer is None else observer.executions,
+        shapes=0 if observer is None else len(observer.repeated(threshold=1)),
+    )
+
+
+__all__ = ("DatabaseMetrics", "ReportsPoolStats", "metrics_of")

pyoq/diagnostics/observation.py

@@ -0,0 +1,227 @@
+"""Watching what one scope executed, so a repeated query becomes visible.
+
+A query issued once per row of a previous result is the shape of an N+1 access,
+and it is invisible from inside the loop that causes it. Recording shapes for
+the length of a scope makes it visible from outside.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from inspect import currentframe
+from threading import Lock
+from types import FrameType
+
+from pyoq.diagnostics.fingerprint import QueryShape, query_shape
+from pyoq.errors import QueryValidationError
+from pyoq.query.execution import CompiledQuery
+from pyoq.query.execution.results import StatementKind
+
+DEFAULT_SHAPE_LIMIT = 256
+DEFAULT_SITE_LIMIT = 5
+DEFAULT_REPEAT_THRESHOLD = 2
+
+_OWN_PACKAGE = "pyoq."
+
+
+@dataclass(frozen=True, slots=True)
+class CallSite:
+    """Where in the caller's own code a query was issued."""
+
+    file: str
+    line: int
+    function: str
+
+    def __str__(self) -> str:
+        return f"{self.file}:{self.line} in {self.function}"
+
+
+@dataclass(frozen=True, slots=True)
+class RepeatedQuery:
+    """One shape a scope executed more than once, and where from."""
+
+    shape: QueryShape
+    executions: int
+    kinds: frozenset[StatementKind]
+    sites: tuple[CallSite, ...]
+
+    def describe(self) -> str:
+        places = "; ".join(str(site) for site in self.sites)
+        suffix = f" from {places}" if places else ""
+        return f"{self.executions} executions of {self.shape.sql}{suffix}"
+
+
+def _no_kinds() -> set[StatementKind]:
+    return set()
+
+
+def _no_sites() -> dict[CallSite, None]:
+    """An ordered set of places, so a report reads in the order they occurred."""
+    return {}
+
+
+@dataclass(slots=True)
+class _Observed:
+    shape: QueryShape
+    executions: int = 0
+    kinds: set[StatementKind] = field(default_factory=_no_kinds)
+    sites: dict[CallSite, None] = field(default_factory=_no_sites)
+
+
+class QueryObserver:
+    """What one scope executed, kept within a fixed budget of memory.
+
+    Diagnostics must not become the thing that exhausts a process, so the number
+    of distinct shapes and the number of places recorded per shape are both
+    capped. Executions keep being counted after the caps are reached.
+
+    An observer is safe to share. A pool hands connections to whichever thread
+    asks, so an observer watching a whole application is written to from several
+    at once, and reading a report while that happens must not fail.
+    """
+
+    __slots__ = (
+        "_capture_sites",
+        "_executions",
+        "_lock",
+        "_observed",
+        "_shape_limit",
+        "_site_limit",
+        "_unrecorded_shapes",
+    )
+
+    def __init__(
+        self,
+        *,
+        shape_limit: int = DEFAULT_SHAPE_LIMIT,
+        site_limit: int = DEFAULT_SITE_LIMIT,
+        capture_sites: bool = True,
+    ) -> None:
+        _require_positive(shape_limit, "shape limit")
+        _require_positive(site_limit, "site limit")
+        self._shape_limit = shape_limit
+        self._site_limit = site_limit
+        self._capture_sites = capture_sites
+        self._observed: dict[str, _Observed] = {}
+        self._executions = 0
+        self._unrecorded_shapes = 0
+        self._lock = Lock()
+
+    @property
+    def executions(self) -> int:
+        """Every statement this scope executed, including ones not recorded."""
+        return self._executions
+
+    @property
+    def shapes(self) -> int:
+        with self._lock:
+            return len(self._observed)
+
+    @property
+    def unrecorded_shapes(self) -> int:
+        """Distinct shapes seen after the limit, counted but not kept."""
+        return self._unrecorded_shapes
+
+    def record(self, statement: CompiledQuery, /) -> int:
+        """Record a statement and report how often its shape has now run.
+
+        A shape seen after the cap reports nothing, because it is counted but
+        not kept. A query repeated is one shape repeated, so a scope with more
+        distinct shapes than the cap has a different problem.
+        """
+        shape = query_shape(statement.sql)
+        site = self._site()
+        with self._lock:
+            self._executions += 1
+            observed = self._observed.get(shape.digest)
+            if observed is None:
+                if len(self._observed) >= self._shape_limit:
+                    self._unrecorded_shapes += 1
+                    return 0
+                observed = _Observed(shape)
+                self._observed[shape.digest] = observed
+            observed.executions += 1
+            observed.kinds.add(statement.statement_kind)
+            if site is not None and len(observed.sites) < self._site_limit:
+                observed.sites[site] = None
+            return observed.executions
+
+    def report(self, digest: str, /) -> RepeatedQuery | None:
+        """One shape's report, for a caller that already knows which."""
+        with self._lock:
+            observed = self._observed.get(digest)
+            if observed is None:
+                return None
+            return _report(observed)
+
+    def repeated(
+        self,
+        *,
+        threshold: int = DEFAULT_REPEAT_THRESHOLD,
+    ) -> tuple[RepeatedQuery, ...]:
+        """Shapes executed at least ``threshold`` times, busiest first."""
+        _require_positive(threshold, "repeat threshold")
+        with self._lock:
+            matching = [
+                _report(observed)
+                for observed in self._observed.values()
+                if observed.executions >= threshold
+            ]
+        matching.sort(key=lambda entry: (-entry.executions, entry.shape.digest))
+        return tuple(matching)
+
+    def _site(self) -> CallSite | None:
+        """Read the caller's frame outside the lock, where it still is theirs."""
+        return calling_site() if self._capture_sites else None
+
+
+def _report(observed: _Observed) -> RepeatedQuery:
+    return RepeatedQuery(
+        observed.shape,
+        observed.executions,
+        frozenset(observed.kinds),
+        tuple(observed.sites),
+    )
+
+
+def calling_site() -> CallSite | None:
+    """The nearest frame that is not PyOQ's own.
+
+    A query is issued from inside this library, so the frame that matters is
+    the first one above it, which is the caller's own code. An interpreter that
+    does not offer frames reports no place rather than refusing to run.
+    """
+    frame = currentframe()
+    while frame is not None:
+        if not _is_own_frame(frame):
+            return CallSite(
+                frame.f_code.co_filename,
+                frame.f_lineno,
+                frame.f_code.co_qualname,
+            )
+        frame = frame.f_back
+    return None
+
+
+def _is_own_frame(frame: FrameType) -> bool:
+    module = frame.f_globals.get("__name__", "")
+    return isinstance(module, str) and (
+        module == "pyoq" or module.startswith(_OWN_PACKAGE)
+    )
+
+
+def _require_positive(value: int, label: str) -> None:
+    if value < 1:
+        message = f"{label} must be at least one"
+        raise QueryValidationError(message)
+
+
+__all__ = (
+    "DEFAULT_REPEAT_THRESHOLD",
+    "DEFAULT_SHAPE_LIMIT",
+    "DEFAULT_SITE_LIMIT",
+    "CallSite",
+    "QueryObserver",
+    "RepeatedQuery",
+    "calling_site",
+)

pyoq/diagnostics/scoped.py

@@ -0,0 +1,103 @@
+"""Counting what one scope's database work actually does.
+
+A budget is only worth holding if something records against it. Every read and
+write funnels through one place before it reaches a driver, so that is where a
+statement is counted, and nothing about a dialect has to know a budget exists.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, TypeVar
+
+from pyoq.query.execution import QueryOperations
+from pyoq.query.execution.aio import AsyncQueryOperations
+
+if TYPE_CHECKING:
+    from collections.abc import Awaitable, Callable
+
+    from pyoq.diagnostics.budget import QueryScope
+    from pyoq.query.execution import (
+        BulkPlan,
+        CompiledQuery,
+        DatabaseCursor,
+        ExecutionControl,
+        WriteProvider,
+    )
+    from pyoq.query.execution.aio import AsyncDatabaseCursor
+    from pyoq.query.statements import StatementCompiler
+    from pyoq.query.write_nodes import WriteNode
+
+OperationResult = TypeVar("OperationResult")
+
+
+class ScopedOperations(QueryOperations):
+    """A database that reports every statement to a scope before running it.
+
+    Recording happens after compilation and before the driver sees anything, so
+    a statement that a budget refuses never reaches the database at all.
+    """
+
+    __slots__ = ("_inner", "_scope")
+
+    def __init__(self, inner: QueryOperations, scope: QueryScope) -> None:
+        self._inner = inner
+        self._scope = scope
+
+    @property
+    def scope(self) -> QueryScope:
+        return self._scope
+
+    @property
+    def compiler(self) -> StatementCompiler:
+        return self._inner.compiler
+
+    def plan_bulk(self, statement: WriteProvider | WriteNode, /) -> BulkPlan:
+        return self._inner.plan_bulk(statement)
+
+    def last_inserted_id(self, cursor: DatabaseCursor) -> int | None:
+        return self._inner.last_inserted_id(cursor)
+
+    def _run(
+        self,
+        statement: CompiledQuery,
+        operation: Callable[[DatabaseCursor], OperationResult],
+        control: ExecutionControl | None,
+    ) -> OperationResult:
+        self._scope.record(statement)
+        return self._inner._run(statement, operation, control)
+
+
+class AsyncScopedOperations(AsyncQueryOperations):
+    """The asynchronous counterpart, counting at the same point."""
+
+    __slots__ = ("_inner", "_scope")
+
+    def __init__(self, inner: AsyncQueryOperations, scope: QueryScope) -> None:
+        self._inner = inner
+        self._scope = scope
+
+    @property
+    def scope(self) -> QueryScope:
+        return self._scope
+
+    @property
+    def compiler(self) -> StatementCompiler:
+        return self._inner.compiler
+
+    def plan_bulk(self, statement: WriteProvider | WriteNode, /) -> BulkPlan:
+        return self._inner.plan_bulk(statement)
+
+    def last_inserted_id(self, cursor: AsyncDatabaseCursor) -> int | None:
+        return self._inner.last_inserted_id(cursor)
+
+    async def _run(
+        self,
+        statement: CompiledQuery,
+        operation: Callable[[AsyncDatabaseCursor], Awaitable[OperationResult]],
+        control: ExecutionControl | None,
+    ) -> OperationResult:
+        self._scope.record(statement)
+        return await self._inner._run(statement, operation, control)
+
+
+__all__ = ("AsyncScopedOperations", "ScopedOperations")

pyoq/django/__init__.py

@@ -0,0 +1,15 @@
+"""Running PyOQ inside a Django project."""
+
+from pyoq.django.execution import Dialect, DjangoOperations, alias_for, dialect_for
+from pyoq.django.generation import GenerationRequest, generate
+from pyoq.django.schema import MigrationStateSchemaSource
+
+__all__ = (
+    "Dialect",
+    "DjangoOperations",
+    "GenerationRequest",
+    "MigrationStateSchemaSource",
+    "alias_for",
+    "dialect_for",
+    "generate",
+)

pyoq/django/apps.py

@@ -0,0 +1,17 @@
+"""The application entry a project adds to install PyOQ's commands."""
+
+from __future__ import annotations
+
+from django.apps import AppConfig
+
+
+class PyoqConfig(AppConfig):
+    """Named for the toolkit rather than for the package it lives under.
+
+    The label Django would infer from `pyoq.django` is `django`, which reads as
+    the framework itself everywhere a label is shown.
+    """
+
+    name = "pyoq.django"
+    label = "pyoq"
+    verbose_name = "PyOQ"