pyoq-sql 1.0.2__py3-none-any.whl

pyoq/descriptors.py

@@ -0,0 +1,165 @@
+"""Runtime contracts used by generated database types."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import ClassVar, Generic, TypeAlias, TypeVar
+
+from pyoq.query import Expression, FieldNode, ScalarFamily
+from pyoq.relations import RelationCardinality, RelationDirection
+from pyoq.schema import JsonScalar as SchemaJsonScalar
+from pyoq.schema import JsonValue as SchemaJsonValue
+from pyoq.schema import KeyKind, ReferentialAction
+from pyoq.unset import UNSET, UnsetType
+
+Value = TypeVar("Value")
+Row = TypeVar("Row", covariant=True)
+Insert = TypeVar("Insert", covariant=True)
+Update = TypeVar("Update", covariant=True)
+SourceRow = TypeVar("SourceRow", covariant=True)
+TargetRow = TypeVar("TargetRow", covariant=True)
+JsonScalar: TypeAlias = SchemaJsonScalar
+JsonValue: TypeAlias = SchemaJsonValue
+
+
+class Missing:
+    __slots__ = ()
+
+
+class Present:
+    __slots__ = ()
+
+
+@dataclass(frozen=True, slots=True)
+class ColumnDescriptor(Expression[Value], Generic[Value]):
+    database_name: str
+    nullable: bool
+    writable: bool
+    has_default: bool
+    generated: bool
+    scalar_family: ScalarFamily = ScalarFamily.OTHER
+    table_name: str | None = None
+    schema_name: str | None = None
+    catalog_name: str | None = None
+    field_name: str | None = None
+    value_type: type[object] | None = None
+
+    def __post_init__(self) -> None:
+        _require_name(self.database_name, "column")
+        self._initialize_expression(
+            FieldNode(
+                self.database_name,
+                self.table_name,
+                self.schema_name,
+                self.catalog_name,
+                self.scalar_family,
+                self.value_type,
+                self.nullable,
+            ),
+            self.scalar_family,
+        )
+
+
+@dataclass(frozen=True, slots=True)
+class TableDescriptor(Generic[Row, Insert, Update]):
+    COLUMNS: ClassVar[tuple[object, ...]] = ()
+    """Every column this table has, in the order the table declares them.
+
+    Generation fills this in. A table written by hand names no columns, and
+    so cannot be selected from without saying which.
+    """
+
+    database_name: str
+    schema_name: str | None = None
+    catalog_name: str | None = None
+    row_type: type[Row] | None = None
+    """The class one row of this table reads back as.
+
+    Generation fills this in, so a table constant carries the type of what it
+    holds as well as its name. A table written by hand names no row class, and
+    so a row of it can only be read as the values it came back with.
+    """
+
+    def __post_init__(self) -> None:
+        _require_name(self.database_name, "table")
+
+
+@dataclass(frozen=True, slots=True)
+class KeyDescriptor(Generic[Value]):
+    kind: KeyKind
+    column_names: tuple[str, ...]
+    database_name: str | None = None
+
+    def __post_init__(self) -> None:
+        if not self.column_names:
+            message = "key descriptor requires at least one column"
+            raise ValueError(message)
+
+
+@dataclass(frozen=True, slots=True)
+class RelationshipDescriptor(Generic[SourceRow, TargetRow]):
+    source_table: str
+    source_columns: tuple[str, ...]
+    target_table: str
+    target_columns: tuple[str, ...]
+    source_schema: str | None = None
+    source_catalog: str | None = None
+    target_schema: str | None = None
+    target_catalog: str | None = None
+    """Where each side lives, because a table name alone names more than one.
+
+    Two schemas may each hold a `parent` and a `child` joined on the same
+    columns. Without these, a constant generated for one of them describes
+    both, and reading through it answers with whichever was found first.
+    """
+
+    database_name: str | None = None
+    on_update: ReferentialAction = ReferentialAction.NO_ACTION
+    on_delete: ReferentialAction = ReferentialAction.NO_ACTION
+    direction: RelationDirection = RelationDirection.FORWARD
+    cardinality: RelationCardinality = RelationCardinality.TO_ONE
+    optional: bool = False
+    target_row_type: type[TargetRow] | None = None
+    """The class one row of the table this reaches reads back as.
+
+    The same fact as a table constant's own, carried here so that following a
+    relation says what it arrives at without the caller naming it again.
+    """
+
+    def __post_init__(self) -> None:
+        _require_name(self.source_table, "relationship source table")
+        _require_name(self.target_table, "relationship target table")
+        if not self.source_columns or len(self.source_columns) != len(
+            self.target_columns
+        ):
+            message = "relationship descriptor columns must be non-empty and aligned"
+            raise ValueError(message)
+        if self.cardinality is RelationCardinality.TO_MANY and self.optional:
+            message = "a to-many relationship cannot be optional"
+            raise ValueError(message)
+
+    @property
+    def to_one(self) -> bool:
+        return self.cardinality is RelationCardinality.TO_ONE
+
+
+def _require_name(value: str, label: str) -> None:
+    if not value:
+        message = f"{label} name cannot be empty"
+        raise ValueError(message)
+
+
+__all__ = (
+    "UNSET",
+    "ColumnDescriptor",
+    "JsonScalar",
+    "JsonValue",
+    "KeyDescriptor",
+    "Missing",
+    "Present",
+    "RelationCardinality",
+    "RelationDirection",
+    "RelationshipDescriptor",
+    "TableDescriptor",
+    "UnsetType",
+)

pyoq/diagnostics/__init__.py

@@ -0,0 +1,68 @@
+"""Seeing what a scope asked a database to do."""
+
+from pyoq.diagnostics.budget import QueryBudget, QueryScope
+from pyoq.diagnostics.events import (
+    CollectingSink,
+    EventPolicy,
+    EventSink,
+    StatementEvent,
+    StatementFailed,
+    StatementFinished,
+    StatementStarted,
+    reportable_values,
+)
+from pyoq.diagnostics.fingerprint import (
+    CacheMetrics,
+    QueryShape,
+    forget_shapes,
+    normalize_sql,
+    query_shape,
+    shape_cache_metrics,
+)
+from pyoq.diagnostics.instrumented import (
+    AsyncInstrumentedOperations,
+    InstrumentedOperations,
+)
+from pyoq.diagnostics.metrics import DatabaseMetrics, metrics_of
+from pyoq.diagnostics.observation import (
+    DEFAULT_REPEAT_THRESHOLD,
+    DEFAULT_SHAPE_LIMIT,
+    DEFAULT_SITE_LIMIT,
+    CallSite,
+    QueryObserver,
+    RepeatedQuery,
+    calling_site,
+)
+from pyoq.diagnostics.scoped import AsyncScopedOperations, ScopedOperations
+
+__all__ = (
+    "DEFAULT_REPEAT_THRESHOLD",
+    "DEFAULT_SHAPE_LIMIT",
+    "DEFAULT_SITE_LIMIT",
+    "AsyncInstrumentedOperations",
+    "AsyncScopedOperations",
+    "CacheMetrics",
+    "CallSite",
+    "CollectingSink",
+    "DatabaseMetrics",
+    "EventPolicy",
+    "EventSink",
+    "InstrumentedOperations",
+    "QueryBudget",
+    "QueryObserver",
+    "QueryScope",
+    "QueryShape",
+    "RepeatedQuery",
+    "ScopedOperations",
+    "StatementEvent",
+    "StatementFailed",
+    "StatementFinished",
+    "StatementStarted",
+    "calling_site",
+    "forget_shapes",
+    "metrics_of",
+    "normalize_sql",
+    "query_shape",
+    "reportable_values",
+    "shape_cache_metrics",
+)

pyoq/diagnostics/budget.py

@@ -0,0 +1,136 @@
+"""What one scope is allowed to ask a database to do.
+
+A repeated query that is only reported is a repeated query that still ships. A
+budget turns the same observation into a refusal, at the point where the scope
+exceeds what it was allowed rather than after the fact.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import cast
+
+from pyoq.diagnostics.fingerprint import QueryShape, query_shape
+from pyoq.diagnostics.observation import QueryObserver, RepeatedQuery
+from pyoq.errors import QueryBudgetExceededError, QueryValidationError
+from pyoq.query.execution import CompiledQuery
+
+
+@dataclass(frozen=True, slots=True)
+class QueryBudget:
+    """A ceiling on one scope's database work.
+
+    ``maximum_repeats`` is the one that catches an N+1 access, because the shape
+    executed once per row of an earlier result is the shape that repeats.
+    """
+
+    maximum_queries: int | None = None
+    maximum_repeats: int | None = None
+
+    def __post_init__(self) -> None:
+        _require_optional_positive(self.maximum_queries, "maximum queries")
+        _require_optional_positive(self.maximum_repeats, "maximum repeats")
+
+    @property
+    def unlimited(self) -> bool:
+        return self.maximum_queries is None and self.maximum_repeats is None
+
+
+class QueryScope:
+    """One request's worth of database work, watched and bounded.
+
+    The scope holds an observer so that a refusal can say which shape ran too
+    often and where from, which is what makes the refusal actionable rather than
+    merely correct.
+    """
+
+    __slots__ = ("_budget", "_observer")
+
+    def __init__(
+        self,
+        budget: QueryBudget | None = None,
+        *,
+        observer: QueryObserver | None = None,
+    ) -> None:
+        self._budget = budget or QueryBudget()
+        self._observer = observer or QueryObserver()
+
+    @property
+    def budget(self) -> QueryBudget:
+        return self._budget
+
+    @property
+    def observer(self) -> QueryObserver:
+        return self._observer
+
+    @property
+    def executions(self) -> int:
+        return self._observer.executions
+
+    def record(self, statement: CompiledQuery, /) -> None:
+        """Record a statement, refusing the one that exceeds the budget.
+
+        The statement is recorded before it is judged, so a report taken after
+        a refusal includes the execution that caused it.
+        """
+        executions = self._observer.record(statement)
+        if self._budget.unlimited:
+            return
+        self._require_within_total()
+        self._require_within_repeats(statement, executions)
+
+    def repeated(self, *, threshold: int = 2) -> tuple[RepeatedQuery, ...]:
+        return self._observer.repeated(threshold=threshold)
+
+    def _require_within_total(self) -> None:
+        limit = self._budget.maximum_queries
+        if limit is None or self._observer.executions <= limit:
+            return
+        message = (
+            f"this scope executed {self._observer.executions} statements, beyond "
+            f"the {limit} it was allowed"
+        )
+        raise QueryBudgetExceededError(message)
+
+    def _require_within_repeats(
+        self,
+        statement: CompiledQuery,
+        executions: int,
+    ) -> None:
+        """Judge only the shape just recorded, which is the only one that moved.
+
+        Scanning every shape on every statement would make the cost of holding a
+        budget grow with the variety of a scope's queries.
+        """
+        limit = self._budget.maximum_repeats
+        if limit is None or executions <= limit:
+            return
+        digest = query_shape(statement.sql).digest
+        # A count above the limit means the observer kept this shape, and it
+        # never lets one go, so the report is there to be read.
+        repeated = cast("RepeatedQuery", self._observer.report(digest))
+        raise QueryBudgetExceededError(_repeat_message(repeated, limit))
+
+
+def _repeat_message(repeated: RepeatedQuery, limit: int) -> str:
+    places = "; ".join(str(site) for site in repeated.sites)
+    origin = f" from {places}" if places else ""
+    return (
+        f"one statement ran {repeated.executions} times in this scope, beyond "
+        f"the {limit} it was allowed{origin}: {_shape_text(repeated.shape)}"
+    )
+
+
+def _shape_text(shape: QueryShape) -> str:
+    return shape.sql
+
+
+def _require_optional_positive(value: int | None, label: str) -> None:
+    if value is None:
+        return
+    if isinstance(value, bool) or value < 1:
+        message = f"{label} must be a positive integer or None"
+        raise QueryValidationError(message)
+
+
+__all__ = ("QueryBudget", "QueryScope")

pyoq/diagnostics/events.py

@@ -0,0 +1,137 @@
+"""What a statement did, described without what it was asked about.
+
+A shape carries the SQL with its values taken out, which is safe to log. A bound
+value is not, and neither is the message a driver raises: PostgreSQL states the
+offending value inside it, so a failure that says only its type is the one that
+can be reported anywhere.
+
+A project that has decided otherwise says so through a policy, one field at a
+time. A value marked sensitive when the statement was compiled is never included
+whatever the policy says.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Protocol, TypeAlias
+
+if TYPE_CHECKING:
+    from pyoq.diagnostics.fingerprint import QueryShape
+    from pyoq.query.execution import CompiledQuery, StatementKind
+
+
+@dataclass(frozen=True, slots=True)
+class EventPolicy:
+    """What an event may carry beyond the shape of a statement.
+
+    Every field is off, so instrumentation added without a decision reports
+    nothing a query was asked about.
+    """
+
+    include_values: bool = False
+    include_failure_detail: bool = False
+    slow_after: float | None = None
+
+    def __post_init__(self) -> None:
+        if self.slow_after is not None and self.slow_after <= 0:
+            message = "a slow statement threshold must be positive"
+            raise ValueError(message)
+
+
+@dataclass(frozen=True, slots=True)
+class StatementStarted:
+    """A statement about to reach a driver."""
+
+    shape: QueryShape
+    kind: StatementKind
+    parameters: int
+    sensitive_parameters: int
+    values: tuple[object, ...] = ()
+
+
+@dataclass(frozen=True, slots=True)
+class StatementFinished:
+    """A statement the database answered."""
+
+    shape: QueryShape
+    kind: StatementKind
+    parameters: int
+    sensitive_parameters: int
+    duration: float
+    rows: int | None = None
+    slow: bool = False
+    values: tuple[object, ...] = ()
+
+
+@dataclass(frozen=True, slots=True)
+class StatementFailed:
+    """A statement the database refused.
+
+    `failure` names the error's type. Its message is only carried when a policy
+    says so, because a driver states the value that caused the failure in it.
+    """
+
+    shape: QueryShape
+    kind: StatementKind
+    parameters: int
+    sensitive_parameters: int
+    duration: float
+    failure: str
+    detail: str | None = None
+    values: tuple[object, ...] = ()
+
+
+StatementEvent: TypeAlias = StatementStarted | StatementFinished | StatementFailed
+
+
+class EventSink(Protocol):
+    """Somewhere for events to go."""
+
+    def record(self, event: StatementEvent, /) -> None: ...
+
+
+class CollectingSink:
+    """Keeps what it is given, for a caller that wants to look afterwards."""
+
+    __slots__ = ("events",)
+    events: list[StatementEvent]
+
+    def __init__(self) -> None:
+        self.events = []
+
+    def record(self, event: StatementEvent, /) -> None:
+        self.events.append(event)
+
+    def clear(self) -> None:
+        self.events.clear()
+
+
+def reportable_values(
+    statement: CompiledQuery,
+    policy: EventPolicy,
+    /,
+) -> tuple[object, ...]:
+    """The values an event may carry, which is none of them by default.
+
+    A value the compiler marked sensitive is left out even when a policy asks
+    for values, because marking it was the decision that it must not be shown.
+    """
+    if not policy.include_values:
+        return ()
+    return tuple(
+        value
+        for index, value in enumerate(statement.parameters)
+        if index not in statement.sensitive_parameter_indexes
+    )
+
+
+__all__ = (
+    "CollectingSink",
+    "EventPolicy",
+    "EventSink",
+    "StatementEvent",
+    "StatementFailed",
+    "StatementFinished",
+    "StatementStarted",
+    "reportable_values",
+)