lorebase-store 0.2.3__py3-none-any.whl

lorebase_store/telemetry.py

@@ -0,0 +1,142 @@
+"""OpenTelemetry instruments of the store. They are no-ops until the process installs
+providers (`lorebase_platform.telemetry.setup_telemetry`, `telemetry.otlp_endpoint` set); the
+process running the store code (Keeper or Scribe) exports them under its own service name."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from threading import Lock
+
+from opentelemetry import metrics
+from opentelemetry.metrics import CallbackOptions, Observation
+
+from lorebase_platform.telemetry import LATENCY_BUCKETS_S
+
+INSTRUMENTATION = "lorebase.store"
+
+meter = metrics.get_meter(INSTRUMENTATION)
+
+COMMIT_DURATION = meter.create_histogram(
+    "lorebase.store.commit.duration",
+    unit="s",
+    description="Duration of a successful commit through the commit pipeline, by origin.",
+    explicit_bucket_boundaries_advisory=LATENCY_BUCKETS_S,
+)
+MERGE_ATTEMPTS = meter.create_counter(
+    "lorebase.store.merge.attempts",
+    unit="{attempt}",
+    description="Merges of a Proposal, by result (merged, fast_forward, up_to_date, conflicts).",
+)
+DEFINER_READS = meter.create_counter(
+    "lorebase.store.reads.definer",
+    unit="{call}",
+    description="Calls of the security definer read functions, by function.",
+)
+JOBS_CLAIMED = meter.create_counter(
+    "lorebase.jobs.claimed", unit="{job}", description="Jobs claimed from the queue, by type."
+)
+JOBS_RECLAIMED = meter.create_counter(
+    "lorebase.jobs.reclaimed",
+    unit="{job}",
+    description="Running jobs whose worker stopped sending heartbeats, taken back from it, by "
+    "type and outcome (requeued, failed).",
+)
+JOBS_RELEASED = meter.create_counter(
+    "lorebase.jobs.released",
+    unit="{job}",
+    description="Running jobs a stopping worker put back in the queue, by type.",
+)
+WEBHOOK_OUTCOMES = meter.create_counter(
+    "lorebase.webhooks.deliveries",
+    unit="{delivery}",
+    description="Webhook delivery attempts, by outcome (delivered, retrying, dead).",
+)
+WEBHOOK_TIMEOUTS = meter.create_counter(
+    "lorebase.webhooks.timeouts",
+    unit="{attempt}",
+    description="Webhook delivery attempts cut off by webhooks.timeout_s.",
+)
+OUTBOX_HANDOFFS = meter.create_counter(
+    "lorebase.outbox.handoffs",
+    unit="{event}",
+    description="Outbox events handed off by a consumer, by consumer.",
+)
+OUTBOX_PRUNED = meter.create_counter(
+    "lorebase.outbox.pruned",
+    unit="{event}",
+    description="Outbox events deleted, by reason (retention: handed off by every consumer; "
+    "expired: older than store.outbox.max_age_s while still pending).",
+)
+
+
+class PruneBacklog:
+    """Outbox events still eligible for pruning after the latest pruning pass of this process
+    (capped, see `OutboxRepository.prunable`); nothing is reported before the first pass."""
+
+    def __init__(self) -> None:
+        self._lock = Lock()
+        self._value: int | None = None
+
+    def set(self, value: int) -> None:
+        with self._lock:
+            self._value = value
+
+    def observe(self, _options: CallbackOptions | None = None) -> Iterable[Observation]:
+        with self._lock:
+            value = self._value
+        return [] if value is None else [Observation(value)]
+
+
+OUTBOX_PRUNE_BACKLOG = PruneBacklog()
+meter.create_observable_gauge(
+    "lorebase.outbox.prune.backlog",
+    callbacks=[OUTBOX_PRUNE_BACKLOG.observe],
+    unit="{event}",
+    description="Outbox events eligible for pruning left after the latest pruning pass "
+    "(counted up to a cap).",
+)
+OPERATION_EVENTS = meter.create_counter(
+    "lorebase.operations.events",
+    unit="{event}",
+    description="Operation side effect events written to the outbox with a committed execution.",
+)
+GC_RUNS = meter.create_counter(
+    "lorebase.gc.runs", unit="{run}", description="Completed garbage collections of a Domain."
+)
+GC_FREED_BYTES = meter.create_counter(
+    "lorebase.gc.freed_bytes",
+    unit="By",
+    description="Bytes of tree nodes and objects deleted by garbage collection.",
+)
+
+BRANCH_JOB_ROWS = meter.create_counter(
+    "lorebase.store.branch_jobs.rows",
+    unit="{row}",
+    description="Rows read by background branch jobs, by job (materialize: base rows copied "
+    "to a branch, delete: rows of a deleted branch removed, sensitivity_refresh: Records whose "
+    "title and search vector were checked).",
+)
+BRANCH_JOB_RUNS = meter.create_counter(
+    "lorebase.store.branch_jobs.runs",
+    unit="{run}",
+    description="Runs of background branch jobs, by job and outcome (done, deferred, waiting, "
+    "skipped, cancelled, failed; for copies also retried: failed for good, queued again "
+    "with backoff; fallback: the overlay was detached and is rebuilt from its tree; "
+    "given_up: no attempt left).",
+)
+BRANCH_JOB_BATCH_ROWS = meter.create_histogram(
+    "lorebase.store.branch_jobs.batch.rows",
+    unit="{row}",
+    description="Rows a batch of a background branch job was sized for, by job: batches "
+    "shrink while commits wait for the branch lock and grow back when none does.",
+)
+BRANCH_JOB_BATCH_DURATION = meter.create_histogram(
+    "lorebase.store.branch_jobs.batch.duration",
+    unit="s",
+    description="Duration of one batch transaction of a background branch job, by job.",
+    explicit_bucket_boundaries_advisory=LATENCY_BUCKETS_S,
+)
+
+
+def record_definer_call(function: str) -> None:
+    DEFINER_READS.add(1, {"function": function})

lorebase_store/testing.py

@@ -0,0 +1,108 @@
+"""Postgres for tests: an existing server or a disposable container.
+
+`postgres_server()` yields an asyncpg URL of a server where the role may create databases and
+roles. It uses `LOREBASE_TEST_DATABASE_URL` when set; otherwise it starts a Postgres 16 +
+pgvector container with testcontainers (image overridable with `LOREBASE_TEST_POSTGRES_IMAGE`).
+`scratch_database()` creates an empty database on that server and drops it afterwards.
+`nested_plans()` collects the plans of statements run inside functions (`auto_explain`).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import uuid
+from collections.abc import AsyncIterator, Iterator
+from contextlib import asynccontextmanager, contextmanager
+from typing import Any
+
+from sqlalchemy import URL, text
+from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
+from sqlalchemy.pool import NullPool
+
+from lorebase_store.db import async_url
+
+DATABASE_URL_ENV = "LOREBASE_TEST_DATABASE_URL"
+IMAGE_ENV = "LOREBASE_TEST_POSTGRES_IMAGE"
+DEFAULT_IMAGE = "pgvector/pgvector:pg16"
+SCRATCH_PREFIX = "lorebase_test_"
+
+
+@contextmanager
+def postgres_server() -> Iterator[str]:
+    """URL of a Postgres server for integration tests."""
+    configured = os.environ.get(DATABASE_URL_ENV)
+    if configured:
+        yield async_url(configured).render_as_string(hide_password=False)
+        return
+    try:
+        from testcontainers.community.postgres import PostgresContainer
+    except ImportError as exc:
+        raise RuntimeError(
+            f"set {DATABASE_URL_ENV} or install the store dev dependencies "
+            "(uv sync --all-packages) to start a Postgres container"
+        ) from exc
+    with PostgresContainer(os.environ.get(IMAGE_ENV, DEFAULT_IMAGE), driver="asyncpg") as pg:
+        yield pg.get_connection_url()
+
+
+async def _execute_autocommit(url: URL, statement: str) -> None:
+    engine = create_async_engine(url, poolclass=NullPool, isolation_level="AUTOCOMMIT")
+    try:
+        async with engine.connect() as connection:
+            await connection.execute(text(statement))
+    finally:
+        await engine.dispose()
+
+
+_AUTO_EXPLAIN = (
+    "LOAD 'auto_explain'",
+    "SET LOCAL auto_explain.log_min_duration = 0",
+    "SET LOCAL auto_explain.log_nested_statements = on",
+    "SET LOCAL auto_explain.log_format = json",
+    "SET LOCAL client_min_messages = log",
+)
+
+
+@asynccontextmanager
+async def nested_plans(session: AsyncSession) -> AsyncIterator[list[dict[str, Any]]]:
+    """Plans (EXPLAIN JSON `Plan` nodes) of the statements the session's transaction runs
+    meanwhile, including those run inside functions, as `auto_explain` reports them.
+
+    Loading the module needs a superuser: enter before `SET LOCAL ROLE`. Raises
+    `sqlalchemy.exc.DBAPIError` where the module cannot be loaded.
+    """
+    connection = await session.connection()
+    for statement in _AUTO_EXPLAIN:
+        await connection.exec_driver_sql(statement)
+    driver = (await connection.get_raw_connection()).driver_connection
+    assert driver is not None
+    plans: list[dict[str, Any]] = []
+
+    def listen(_connection: Any, message: Any) -> None:
+        _, marker, body = str(message.message).partition("plan:\n")
+        if marker:
+            plans.append(json.loads(body)["Plan"])
+
+    driver.add_log_listener(listen)
+    try:
+        yield plans
+    finally:
+        # A top-level statement's plan is logged when its portal closes, at the next statement.
+        await connection.exec_driver_sql("SELECT 1")
+        await asyncio.sleep(0)  # log messages are dispatched through the event loop
+        driver.remove_log_listener(listen)
+        await connection.exec_driver_sql("SET LOCAL client_min_messages = notice")
+
+
+@contextmanager
+def scratch_database(server_url: str) -> Iterator[str]:
+    """Create a uniquely named empty database on `server_url`, yield its URL, then drop it."""
+    server = async_url(server_url)
+    name = f"{SCRATCH_PREFIX}{uuid.uuid4().hex[:12]}"
+    asyncio.run(_execute_autocommit(server, f'CREATE DATABASE "{name}"'))
+    try:
+        yield server.set(database=name).render_as_string(hide_password=False)
+    finally:
+        asyncio.run(_execute_autocommit(server, f'DROP DATABASE IF EXISTS "{name}" WITH (FORCE)'))