pyoq-sql 1.0.2__py3-none-any.whl

pyoq/diagnostics/fingerprint.py

@@ -0,0 +1,267 @@
+"""Recognizing one query shape across the many times it is executed.
+
+A query executed in a loop differs only in its bound values, a query executed
+against a growing list of keys differs only in how many placeholders it carries,
+and a write chunked to fit a parameter limit differs only in how many groups of
+them it carries. All three are one shape, and seeing them as one is what makes a
+repeated query visible.
+
+A literal written into the statement itself is taken out for the same reason,
+and for one more: a shape is reported and logged, so anything left in it is
+reported and logged too. PyOQ binds every value it is given, but raw SQL can
+carry one, and a shape that kept it would put it in a log.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from functools import lru_cache
+from hashlib import blake2b
+
+_ESCAPED_PERCENT = "\x00escaped-percent\x00"
+# PostgreSQL opens a string with a tag it chooses, and closes it with the same.
+_DOLLAR_TAG = re.compile(r"\$[A-Za-z_]\w*\$|\$\$")
+# What quotes a name rather than a value, in the dialects PyOQ speaks.
+# What quotes a name rather than a value. A single quote is absent because a
+# single quoted string is a value, and is read as one before this is asked.
+_NAME_QUOTES = '"`'
+# What begins a comment that runs to the end of its line.
+_LINE_COMMENTS = ("--", "#")
+# A number written in another radix, and one written in the usual way.
+_RADIX_NUMBER = re.compile(r"0[xX][0-9a-fA-F]+|0[bB][01]+")
+_DECIMAL_NUMBER = re.compile(r"\d+(?:\.\d+)?(?:[eE][+-]?\d+)?")
+_PLACEHOLDER = re.compile(r"\$\d+|%s|\?")
+_PLACEHOLDER_RUN = re.compile(r"\?(?:\s*,\s*\?)+")
+_GROUP_RUN = re.compile(r"\(\?\)(?:\s*,\s*\(\?\))+")
+_WHITESPACE = re.compile(r"\s+")
+_DIGEST_BYTES = 16
+# What a shape may carry into a log line or a span attribute.
+_MAXIMUM_SHAPE = 4096
+_TRUNCATED = " ..."
+
+
+@dataclass(frozen=True, slots=True)
+class QueryShape:
+    """One query with its values and their number taken out.
+
+    The shape carries no bound value, so it can be reported and logged without
+    revealing anything a query was asked about.
+    """
+
+    sql: str
+    digest: str
+
+    def __str__(self) -> str:
+        return self.sql
+
+
+# An application runs the same statements over and over, so the shape of one is
+# asked for far more often than it changes. The cache is bounded, because the
+# statements a process runs are not.
+_SHAPE_CACHE_SIZE = 2048
+
+
+@lru_cache(maxsize=_SHAPE_CACHE_SIZE)
+def query_shape(sql: str, /) -> QueryShape:
+    """The shape of a statement, remembered while it keeps being asked for.
+
+    A shape is written to a log and set on a span, so what it carries is
+    bounded. The digest is taken from the whole statement, so two that differ
+    only past the bound are still told apart.
+    """
+    normalized = normalize_sql(sql)
+    return QueryShape(_within_bounds(normalized), _digest(normalized))
+
+
+def _within_bounds(normalized: str) -> str:
+    """A generated statement can be enormous, and a log line cannot."""
+    if len(normalized) <= _MAXIMUM_SHAPE:
+        return normalized
+    return normalized[:_MAXIMUM_SHAPE] + _TRUNCATED
+
+
+def shape_cache_metrics() -> CacheMetrics:
+    """How well the shapes being asked for are already known."""
+    info = query_shape.cache_info()
+    return CacheMetrics(
+        hits=info.hits,
+        misses=info.misses,
+        held=info.currsize,
+        capacity=_SHAPE_CACHE_SIZE,
+    )
+
+
+def forget_shapes() -> None:
+    """Empty the cache, which only a test measuring it should need."""
+    query_shape.cache_clear()
+
+
+@dataclass(frozen=True, slots=True)
+class CacheMetrics:
+    """What a cache has been asked for and how much of it it kept."""
+
+    hits: int
+    misses: int
+    held: int
+    capacity: int
+
+    @property
+    def hit_rate(self) -> float:
+        asked = self.hits + self.misses
+        return 0.0 if asked == 0 else self.hits / asked
+
+
+def normalize_sql(sql: str, /) -> str:
+    """Reduce a statement to the shape it shares with its repetitions.
+
+    The statement is read once, left to right, because what a character means
+    depends on what it is inside. A quote inside a comment starts nothing, and
+    two dashes inside a string are not a comment, so deciding either one without
+    tracking the other gets both wrong.
+    """
+    pieces: list[str] = []
+    position = 0
+    length = len(sql)
+    while position < length:
+        step = (
+            _skip_comment(sql, position)
+            or _take_literal(sql, position)
+            or _take_quoted_name(sql, position)
+        )
+        if step is None:
+            pieces.append(sql[position])
+            position += 1
+            continue
+        text, position = step
+        pieces.append(text)
+    return _collapse("".join(pieces))
+
+
+def _skip_comment(sql: str, position: int) -> tuple[str, int] | None:
+    """A comment is dropped, because it says nothing about the shape."""
+    if sql.startswith(_LINE_COMMENTS, position):
+        ending = sql.find("\n", position)
+        return (" ", len(sql) if ending == -1 else ending)
+    if sql.startswith("/*", position):
+        return (" ", _end_of_block_comment(sql, position))
+    return None
+
+
+def _end_of_block_comment(sql: str, position: int) -> int:
+    """Past the closing marker, counting the nesting PostgreSQL allows.
+
+    A dialect that does not nest would have ended at the first marker, so this
+    can drop more than that dialect needed. Dropping too much loses a little of
+    a shape; dropping too little puts whatever was written there in a log.
+    """
+    depth = 0
+    index = position
+    while index < len(sql):
+        if sql.startswith("/*", index):
+            depth += 1
+            index += 2
+            continue
+        if sql.startswith("*/", index):
+            depth -= 1
+            index += 2
+            if depth == 0:
+                return index
+            continue
+        index += 1
+    return len(sql)
+
+
+def _take_literal(sql: str, position: int) -> tuple[str, int] | None:
+    """Every kind of literal becomes one placeholder."""
+    dollar = _take_dollar_quoted(sql, position)
+    if dollar is not None:
+        return dollar
+    if sql[position] == "'":
+        return ("?", _end_of_quoted(sql, position, "'"))
+    if _starts_number(sql, position):
+        return ("?", _end_of_number(sql, position))
+    return None
+
+
+def _take_dollar_quoted(sql: str, position: int) -> tuple[str, int] | None:
+    """PostgreSQL quotes with a tag of its own choosing, so it is read out."""
+    if sql[position] != "$":
+        return None
+    opening = _DOLLAR_TAG.match(sql, position)
+    if opening is None:
+        return None
+    tag = opening.group(0)
+    ending = sql.find(tag, opening.end())
+    return ("?", len(sql) if ending == -1 else ending + len(tag))
+
+
+def _end_of_quoted(sql: str, position: int, quote: str) -> int:
+    """Past the closing quote, counting a doubled or escaped one as inside."""
+    index = position + 1
+    while index < len(sql):
+        character = sql[index]
+        if character == "\\":
+            index += 2
+            continue
+        if character == quote:
+            if sql.startswith(quote * 2, index):
+                index += 2
+                continue
+            return index + 1
+        index += 1
+    return len(sql)
+
+
+def _take_quoted_name(sql: str, position: int) -> tuple[str, int] | None:
+    """A quoted name is a name, so it is kept as it was written."""
+    quote = sql[position]
+    if quote not in _NAME_QUOTES:
+        return None
+    ending = _end_of_quoted(sql, position, quote)
+    return (sql[position:ending], ending)
+
+
+def _starts_number(sql: str, position: int) -> bool:
+    """A decimal digit, which is narrower than what `isdigit` accepts.
+
+    A superscript is a digit to `isdigit` and not to the pattern that reads the
+    rest of the number, and taking one out a character at a time would mean a
+    shape that changes every time it is taken.
+    """
+    if not sql[position].isdecimal():
+        return False
+    before = sql[position - 1] if position else ""
+    return not (before.isalnum() or before in "_.$")
+
+
+def _end_of_number(sql: str, position: int) -> int:
+    """Past the whole number, in any of the ways one can be written."""
+    radix = _RADIX_NUMBER.match(sql, position)
+    if radix is not None:
+        return radix.end()
+    decimal = _DECIMAL_NUMBER.match(sql, position)
+    return position + 1 if decimal is None else decimal.end()
+
+
+def _collapse(text: str) -> str:
+    text = text.replace("%%", _ESCAPED_PERCENT)
+    text = _PLACEHOLDER.sub("?", text)
+    text = _PLACEHOLDER_RUN.sub("?", text)
+    text = _GROUP_RUN.sub("(?)", text)
+    text = _WHITESPACE.sub(" ", text).strip()
+    return text.replace(_ESCAPED_PERCENT, "%%")
+
+
+def _digest(normalized: str) -> str:
+    return blake2b(normalized.encode(), digest_size=_DIGEST_BYTES).hexdigest()
+
+
+__all__ = (
+    "CacheMetrics",
+    "QueryShape",
+    "forget_shapes",
+    "normalize_sql",
+    "query_shape",
+    "shape_cache_metrics",
+)

pyoq/diagnostics/instrumented.py

@@ -0,0 +1,237 @@
+"""Reporting what every statement did, at the point they all pass.
+
+Instrumentation is a database that wraps another one, so a project that wants
+none holds the plain database and pays nothing at all. There is no flag to read
+on the way to the driver because there is nothing in the way.
+"""
+
+from __future__ import annotations
+
+from time import perf_counter
+from typing import TYPE_CHECKING, TypeVar
+
+from pyoq.diagnostics.events import (
+    EventPolicy,
+    StatementFailed,
+    StatementFinished,
+    StatementStarted,
+    reportable_values,
+)
+from pyoq.diagnostics.fingerprint import query_shape
+from pyoq.query.execution import QueryOperations
+from pyoq.query.execution.aio import AsyncQueryOperations
+
+if TYPE_CHECKING:
+    from collections.abc import Awaitable, Callable
+
+    from pyoq.diagnostics.events import EventSink
+    from pyoq.query.execution import (
+        BulkPlan,
+        CompiledQuery,
+        DatabaseCursor,
+        ExecutionControl,
+        WriteProvider,
+    )
+    from pyoq.query.execution.aio import AsyncDatabaseCursor
+    from pyoq.query.statements import StatementCompiler
+    from pyoq.query.write_nodes import WriteNode
+
+OperationResult = TypeVar("OperationResult")
+
+
+class _Rows:
+    """How many rows the cursor reported, which only it can say."""
+
+    __slots__ = ("count",)
+
+    def __init__(self) -> None:
+        self.count: int | None = None
+
+
+class InstrumentedOperations(QueryOperations):
+    """A database that reports every statement it runs."""
+
+    __slots__ = ("_inner", "_policy", "_sink")
+
+    def __init__(
+        self,
+        inner: QueryOperations,
+        sink: EventSink,
+        policy: EventPolicy | None = None,
+    ) -> None:
+        self._inner = inner
+        self._sink = sink
+        self._policy = policy or EventPolicy()
+
+    @property
+    def policy(self) -> EventPolicy:
+        return self._policy
+
+    @property
+    def compiler(self) -> StatementCompiler:
+        return self._inner.compiler
+
+    def plan_bulk(self, statement: WriteProvider | WriteNode, /) -> BulkPlan:
+        return self._inner.plan_bulk(statement)
+
+    def last_inserted_id(self, cursor: DatabaseCursor) -> int | None:
+        return self._inner.last_inserted_id(cursor)
+
+    def _run(
+        self,
+        statement: CompiledQuery,
+        operation: Callable[[DatabaseCursor], OperationResult],
+        control: ExecutionControl | None,
+    ) -> OperationResult:
+        report = _Report(statement, self._sink, self._policy)
+        rows = _Rows()
+        report.started()
+        try:
+            outcome = self._inner._run(statement, _counting(operation, rows), control)
+        except BaseException as error:
+            report.failed(error)
+            raise
+        report.finished(rows.count)
+        return outcome
+
+
+class AsyncInstrumentedOperations(AsyncQueryOperations):
+    """The asynchronous counterpart, reporting at the same point."""
+
+    __slots__ = ("_inner", "_policy", "_sink")
+
+    def __init__(
+        self,
+        inner: AsyncQueryOperations,
+        sink: EventSink,
+        policy: EventPolicy | None = None,
+    ) -> None:
+        self._inner = inner
+        self._sink = sink
+        self._policy = policy or EventPolicy()
+
+    @property
+    def policy(self) -> EventPolicy:
+        return self._policy
+
+    @property
+    def compiler(self) -> StatementCompiler:
+        return self._inner.compiler
+
+    def plan_bulk(self, statement: WriteProvider | WriteNode, /) -> BulkPlan:
+        return self._inner.plan_bulk(statement)
+
+    def last_inserted_id(self, cursor: AsyncDatabaseCursor) -> int | None:
+        return self._inner.last_inserted_id(cursor)
+
+    async def _run(
+        self,
+        statement: CompiledQuery,
+        operation: Callable[[AsyncDatabaseCursor], Awaitable[OperationResult]],
+        control: ExecutionControl | None,
+    ) -> OperationResult:
+        report = _Report(statement, self._sink, self._policy)
+        rows = _Rows()
+        report.started()
+        try:
+            outcome = await self._inner._run(
+                statement, _counting_async(operation, rows), control
+            )
+        except BaseException as error:
+            report.failed(error)
+            raise
+        report.finished(rows.count)
+        return outcome
+
+
+class _Report:
+    """One statement's events, timed from the moment it was described."""
+
+    __slots__ = ("_policy", "_shape", "_sink", "_started", "_statement")
+
+    def __init__(
+        self,
+        statement: CompiledQuery,
+        sink: EventSink,
+        policy: EventPolicy,
+    ) -> None:
+        self._statement = statement
+        self._sink = sink
+        self._policy = policy
+        self._shape = query_shape(statement.sql)
+        self._started = perf_counter()
+
+    def started(self) -> None:
+        self._sink.record(
+            StatementStarted(
+                shape=self._shape,
+                kind=self._statement.statement_kind,
+                parameters=len(self._statement.parameters),
+                sensitive_parameters=len(self._statement.sensitive_parameter_indexes),
+                values=self._values(),
+            )
+        )
+
+    def finished(self, rows: int | None) -> None:
+        elapsed = perf_counter() - self._started
+        threshold = self._policy.slow_after
+        self._sink.record(
+            StatementFinished(
+                shape=self._shape,
+                kind=self._statement.statement_kind,
+                parameters=len(self._statement.parameters),
+                sensitive_parameters=len(self._statement.sensitive_parameter_indexes),
+                duration=elapsed,
+                rows=rows,
+                slow=threshold is not None and elapsed >= threshold,
+                values=self._values(),
+            )
+        )
+
+    def failed(self, error: BaseException) -> None:
+        self._sink.record(
+            StatementFailed(
+                shape=self._shape,
+                kind=self._statement.statement_kind,
+                parameters=len(self._statement.parameters),
+                sensitive_parameters=len(self._statement.sensitive_parameter_indexes),
+                duration=perf_counter() - self._started,
+                failure=type(error).__name__,
+                detail=self._detail(error),
+                values=self._values(),
+            )
+        )
+
+    def _values(self) -> tuple[object, ...]:
+        return reportable_values(self._statement, self._policy)
+
+    def _detail(self, error: BaseException) -> str | None:
+        """A driver names the offending value in its message, so this is asked."""
+        return str(error) if self._policy.include_failure_detail else None
+
+
+def _counting(
+    operation: Callable[[DatabaseCursor], OperationResult],
+    rows: _Rows,
+) -> Callable[[DatabaseCursor], OperationResult]:
+    def counted(cursor: DatabaseCursor) -> OperationResult:
+        outcome = operation(cursor)
+        rows.count = cursor.rowcount
+        return outcome
+
+    return counted
+
+
+def _counting_async(
+    operation: Callable[[AsyncDatabaseCursor], Awaitable[OperationResult]],
+    rows: _Rows,
+) -> Callable[[AsyncDatabaseCursor], Awaitable[OperationResult]]:
+    async def counted(cursor: AsyncDatabaseCursor) -> OperationResult:
+        outcome = await operation(cursor)
+        rows.count = cursor.rowcount
+        return outcome
+
+    return counted
+
+
+__all__ = ("AsyncInstrumentedOperations", "InstrumentedOperations")

pyoq/diagnostics/metrics.py

@@ -0,0 +1,61 @@
+"""What a database is doing right now, as numbers.
+
+A pool already knows its own counts and an observer already knows what has run.
+This puts the two together so one reading describes both, and adds nothing that
+would have to be kept up to date separately.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Protocol
+
+if TYPE_CHECKING:
+    from pyoq.diagnostics.observation import QueryObserver
+    from pyoq.query.execution import PoolStats
+
+
+class ReportsPoolStats(Protocol):
+    """Anything that can say how its connections are being used."""
+
+    @property
+    def stats(self) -> PoolStats: ...
+
+
+@dataclass(frozen=True, slots=True)
+class DatabaseMetrics:
+    """One reading, carrying counts and nothing a query was asked about."""
+
+    open_connections: int
+    idle_connections: int
+    checked_out_connections: int
+    closed: bool
+    statements: int
+    shapes: int
+
+    @property
+    def in_use(self) -> float:
+        """How much of what is open is checked out, between zero and one."""
+        if self.open_connections == 0:
+            return 0.0
+        return self.checked_out_connections / self.open_connections
+
+
+def metrics_of(
+    pool: ReportsPoolStats,
+    /,
+    observer: QueryObserver | None = None,
+) -> DatabaseMetrics:
+    """A reading taken now, from a pool and optionally from what it has run."""
+    stats = pool.stats
+    return DatabaseMetrics(
+        open_connections=stats.open_connections,
+        idle_connections=stats.idle_connections,
+        checked_out_connections=stats.checked_out_connections,
+        closed=stats.closed,
+        statements=0 if observer is None else observer.executions,
+        shapes=0 if observer is None else len(observer.repeated(threshold=1)),
+    )
+
+
+__all__ = ("DatabaseMetrics", "ReportsPoolStats", "metrics_of")