matrx-orm 3.1.71__tar.gz → 3.1.72__tar.gz

{matrx_orm-3.1.71 → matrx_orm-3.1.72}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: matrx-orm
-Version: 3.1.71
+Version: 3.1.72
 Summary: Async-first PostgreSQL ORM with bidirectional migrations, schema introspection, many-to-many relationships, and built-in state caching
 Project-URL: Homepage, https://github.com/AI-Matrix-Engine/aidream-current
 Project-URL: Repository, https://github.com/AI-Matrix-Engine/aidream-current
@@ -25,7 +25,7 @@ Requires-Python: >=3.13
 Requires-Dist: asyncpg>=0.31.0
 Requires-Dist: cryptography>=43.0
 Requires-Dist: gitpython>=3.1.40
-Requires-Dist: matrx-utils>=2.0.14
+Requires-Dist: matrx-utils>=2.0.15
 Requires-Dist: psycopg-pool>=3.2.5
 Requires-Dist: psycopg[binary]>=3.2.5
 Requires-Dist: python-dotenv>=1.0

{matrx_orm-3.1.71 → matrx_orm-3.1.72}/matrx_orm/__init__.py

@@ -197,6 +197,12 @@ from .core.pydantic_bridge import (
     clear_schema_cache,
     model_json_schema,
 )
+from .core.pool_watch import (
+    PoolHolderSnapshot,
+    PoolSaturationEvent,
+    connection_holder_census,
+    register_pool_saturation_sink,
+)
 from .core.pydantic_bridge import (
     validate_input as validate_model_input,
 )
@@ -562,6 +568,11 @@ __all__ = [
     # ── Pagination ─────────────────────────────────────────────────────────────
     "Paginator",
     "Page",
+    # ── Pool saturation observability ───────────────────────────────────────────
+    "PoolHolderSnapshot",
+    "PoolSaturationEvent",
+    "connection_holder_census",
+    "register_pool_saturation_sink",
     # ── Signals ────────────────────────────────────────────────────────────────
     "Signal",
     "pre_create",

{matrx_orm-3.1.71 → matrx_orm-3.1.72}/matrx_orm/core/FEATURE.md

@@ -53,7 +53,25 @@ connection and can strand old holders in release/reset.
 
 Pinned by `tests/level1/test_pool_maintenance.py`.
 
-Verified against code: 2026-08-13.
+**Every checkout is attributable.** `AsyncDatabaseManager.get_connection` and
+the sanctioned caller-supplied-pool helper register the holder's asyncio task,
+checkout call site/stack, current suspended call site/stack, and age. On the
+first acquire timeout in an incident, `pool_watch` emits a critical red census
+and invokes host sinks; subsequent retry timeouts do not spam another incident.
+Once a release returns an idle slot, the alarm re-arms. The final
+`PoolAcquireTimeoutError.details` carries both the first snapshot and the current
+census, so retry/persistence casualties preserve the diagnosis.
+
+**Release is bounded and cancellation-safe.** Return/reset runs in a dedicated,
+shielded task with a 10-second asyncpg release timeout. A holder remains visible
+as `releasing` until reset really finishes. If reset exceeds the bound, asyncpg
+discards the physical connection and the failure is logged loudly rather than
+leaving an invisible checkout forever.
+
+Pinned by `tests/level1/test_pool_holder_census.py` and
+`tests/level1/test_pool_maintenance.py`.
+
+Verified against code: 2026-08-15.
 
 ## Transaction-pooled role scope
 
@@ -73,6 +91,7 @@ Verified against code: 2026-08-15.
 
 - 2026-08-15: Pinned transaction-local RLS role cleanup across success,
   exception, and early-return paths.
+- 2026-08-15: Added first-timeout pool holder census, host alarm seam, tracked raw-pool helper, and bounded cancellation-safe release.
 - 2026-08-14: Public models now render `public.table` SQL targets; bare public
   relation names can no longer disable application logs or write-failure capture.
 - 2026-08-13: Registry, admin, recovery, catalog, and diagnostic identities are now exact `schema.table`; bare names are accepted only as uncached discovery terms and ambiguity fails loudly.

{matrx_orm-3.1.71 → matrx_orm-3.1.72}/matrx_orm/core/async_db_manager.py

@@ -91,6 +91,7 @@ def _integrity_constraint_kind(exc: asyncpg.exceptions.IntegrityConstraintViolat
 # read path survives the same burst.
 POOL_ACQUIRE_RETRIES = 2  # total acquire attempts = 1 + POOL_ACQUIRE_RETRIES
 POOL_ACQUIRE_RETRY_BASE_BACKOFF = 0.25  # seconds; doubled per attempt
+POOL_RELEASE_TIMEOUT_SECONDS = 10.0
 
 # ── read-statement retry (command-timeout self-heal) ──────────────────────────
 # A ``command_timeout`` on a READ (SELECT) is provably side-effect-free to retry
@@ -576,8 +577,7 @@ class AsyncDatabaseManager:
         functions, so `call_function` cannot reach them; this is the sanctioned
         accessor.
         """
-        pool = await cls.get_pool(config_name)
-        async with pool.acquire() as conn:
+        async with cls.get_connection(config_name) as conn:
             row = await conn.fetchrow(
                 "SELECT current_user AS current_user, session_user AS session_user, "
                 "current_database() AS database, "
@@ -643,7 +643,9 @@ class AsyncDatabaseManager:
 
                 growth_lock = cls._growth_locks.setdefault(config_name, asyncio.Lock())
                 async with growth_lock:
-                    await maintain_one_connection(pool, target_size=pool_min)
+                    await maintain_one_connection(
+                        pool, target_size=pool_min, config_name=config_name
+                    )
                 failures = 0
                 sweep_count = max(pool.get_size(), pool_min, 1)
                 interval = max(
@@ -718,6 +720,7 @@ class AsyncDatabaseManager:
         config_name = resolve_database_name(config_name)
         pool = await cls.get_pool(config_name)
         conn = None
+        first_timeout_event = None
         try:
             # Bounded acquire-retry: a timeout here means NO SQL was sent (the
             # connection was never handed out), so retrying is provably
@@ -731,13 +734,42 @@ class AsyncDatabaseManager:
                         conn = await cls._acquire_bounded_growth(config_name, pool)
                     break
                 except TimeoutError:
-                    if _acquire_attempt < POOL_ACQUIRE_RETRIES:
-                        backoff = POOL_ACQUIRE_RETRY_BASE_BACKOFF * (2**_acquire_attempt)
+                    from matrx_orm.core.pool_watch import emit_first_pool_timeout
+
+                    try:
+                        _snap_size = pool.get_size()
+                    except Exception:
+                        _snap_size = -1
+                    try:
+                        _snap_idle = pool.get_idle_size()
+                    except Exception:
+                        _snap_idle = -1
+                    try:
+                        _snap_max = pool.get_max_size()
+                    except Exception:
+                        _snap_max = -1
+                    try:
+                        _snap_min = get_database_config(config_name).get(
+                            "pool_min", pool.get_min_size()
+                        )
+                    except Exception:
                         try:
-                            _snap_size = pool.get_size()
-                            _snap_idle = pool.get_idle_size()
+                            _snap_min = pool.get_min_size()
                         except Exception:
-                            _snap_size = _snap_idle = -1
+                            _snap_min = -1
+                    timeout_event = emit_first_pool_timeout(
+                        config_name=config_name,
+                        timeout_seconds=timeout,
+                        acquire_attempt=_acquire_attempt + 1,
+                        pool_size=_snap_size,
+                        pool_max=_snap_max,
+                        pool_idle=_snap_idle,
+                        pool_min=_snap_min,
+                    )
+                    if first_timeout_event is None:
+                        first_timeout_event = timeout_event
+                    if _acquire_attempt < POOL_ACQUIRE_RETRIES:
+                        backoff = POOL_ACQUIRE_RETRY_BASE_BACKOFF * (2**_acquire_attempt)
                         logger.warning(
                             "[matrx_orm] Acquire timeout for '%s' (attempt %d/%d, "
                             "size=%s idle=%s) — retrying in %.2fs (no SQL sent, safe).",
@@ -782,8 +814,7 @@ class AsyncDatabaseManager:
 
             if conn is not None:
                 logger.warning(
-                    "[matrx_orm] Acquire healed for '%s' after replacing a "
-                    "zero-connection pool.",
+                    "[matrx_orm] Acquire healed for '%s' after replacing a zero-connection pool.",
                     config_name,
                 )
             else:
@@ -791,13 +822,28 @@ class AsyncDatabaseManager:
 
                 try:
                     _pool_size = pool.get_size()
+                except Exception:
+                    _pool_size = -1
+                try:
                     _pool_idle = pool.get_idle_size()
+                except Exception:
+                    _pool_idle = -1
+                try:
                     _pool_max = pool.get_max_size()
+                except Exception:
+                    _pool_max = -1
+                try:
                     _pool_min = get_database_config(config_name).get(
                         "pool_min", pool.get_min_size()
                     )
                 except Exception:
-                    _pool_size = _pool_idle = _pool_max = _pool_min = -1
+                    try:
+                        _pool_min = pool.get_min_size()
+                    except Exception:
+                        _pool_min = -1
+                from matrx_orm.core.pool_watch import connection_holder_census
+
+                holders = connection_holder_census(config_name)
                 _timeout_err = PoolAcquireTimeoutError(
                     model=None,
                     config_name=config_name,
@@ -808,10 +854,14 @@ class AsyncDatabaseManager:
                         "pool_idle": _pool_idle,
                         "pool_min": _pool_min,
                         "acquire_attempts": (
-                            2 + POOL_ACQUIRE_RETRIES
-                            if rebuilt
-                            else 1 + POOL_ACQUIRE_RETRIES
+                            2 + POOL_ACQUIRE_RETRIES if rebuilt else 1 + POOL_ACQUIRE_RETRIES
+                        ),
+                        "first_timeout": (
+                            first_timeout_event.model_dump(mode="json")
+                            if first_timeout_event is not None
+                            else None
                         ),
+                        "holders": [holder.model_dump(mode="json") for holder in holders],
                     },
                 )
                 maybe_activate_queue(config_name, _timeout_err)
@@ -852,6 +902,17 @@ class AsyncDatabaseManager:
             if classified is None:
                 raise
             raise classified from None
+        from matrx_orm.core.pool_watch import (
+            mark_connection_releasing,
+            note_pool_recovered,
+            track_connection_holder,
+            untrack_connection_holder,
+        )
+
+        # Attribute the checkout immediately. Poison cleanup below is database
+        # work too; if it stalls, the holder census must show who owns the slot.
+        track_connection_holder(config_name, conn)
+
         # Acquired successfully — yield outside the timeout scope so the
         # user's work isn't bounded by it, and release in finally so leaks
         # are impossible regardless of how the consumer exits.
@@ -888,7 +949,11 @@ class AsyncDatabaseManager:
                 conn.terminate()
             except Exception:
                 pass
-            await pool.release(conn)
+            mark_connection_releasing(config_name, conn)
+            try:
+                await pool.release(conn, timeout=POOL_RELEASE_TIMEOUT_SECONDS)
+            finally:
+                untrack_connection_holder(config_name, conn)
             raise StateError(
                 model=None,
                 operation="acquire_connection",
@@ -900,7 +965,43 @@ class AsyncDatabaseManager:
         try:
             yield conn
         finally:
-            await pool.release(conn)
+            mark_connection_releasing(config_name, conn)
+
+            async def _release() -> None:
+                try:
+                    await pool.release(conn, timeout=POOL_RELEASE_TIMEOUT_SECONDS)
+                finally:
+                    untrack_connection_holder(config_name, conn)
+
+            # A caller cancellation must not strand a checkout while asyncpg
+            # resets it. The release has its own hard bound; asyncpg terminates
+            # the physical connection when reset exceeds that bound.
+            release_task = asyncio.create_task(_release(), name=f"matrx-orm-release:{config_name}")
+
+            def _release_done(task: asyncio.Task[None]) -> None:
+                if task.cancelled():
+                    return
+                release_exc = task.exception()
+                if release_exc is not None:
+                    logger.error(
+                        "[matrx_orm] Connection release failed for '%s': %s",
+                        config_name,
+                        release_exc,
+                    )
+                    return
+                try:
+                    if pool.get_idle_size() > 0:
+                        note_pool_recovered(config_name)
+                except Exception:
+                    pass
+
+            release_task.add_done_callback(_release_done)
+            try:
+                await asyncio.shield(release_task)
+            except asyncio.CancelledError:
+                # The dedicated release task keeps running to completion and its
+                # body/callback own census cleanup and recovery-latch reset.
+                raise
 
     @classmethod
     async def execute_query(

{matrx_orm-3.1.71 → matrx_orm-3.1.72}/matrx_orm/core/pool_maintenance.py

@@ -59,6 +59,7 @@ async def maintain_one_connection(
     *,
     target_size: int,
     timeout_seconds: float = 5.0,
+    config_name: str = "<pool-maintenance>",
 ) -> bool:
     """Probe one idle connection or grow one disconnected holder, never more."""
     queue, raw_queue = _pool_queue(pool)
@@ -83,10 +84,19 @@ async def maintain_one_connection(
     try:
         async with asyncio.timeout(timeout_seconds):
             proxy = await holder.acquire()
+            from matrx_orm.core.pool_watch import track_connection_holder
+
+            track_connection_holder(config_name, proxy)
             await proxy.fetchval("SELECT 1")
         return True
     finally:
         if proxy is not None:
+            from matrx_orm.core.pool_watch import (
+                mark_connection_releasing,
+                untrack_connection_holder,
+            )
+
+            mark_connection_releasing(config_name, proxy)
             try:
                 await pool.release(proxy, timeout=timeout_seconds)
             except Exception:
@@ -95,6 +105,8 @@ async def maintain_one_connection(
                 if holder not in raw_queue and getattr(holder, "_in_use", None) is None:
                     queue.put_nowait(holder)
                 raise
+            finally:
+                untrack_connection_holder(config_name, proxy)
         elif holder not in raw_queue:
             # connect/setup failed before a proxy was established.
             queue.put_nowait(holder)

matrx_orm-3.1.72/matrx_orm/core/pool_watch.py

@@ -0,0 +1,318 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+import traceback
+from collections.abc import AsyncIterator, Callable
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict
+
+from matrx_utils import vcprint
+
+logger = logging.getLogger("matrx_orm.pool_watch")
+
+_STACK_LIMIT = 24
+_DISPLAY_STACK_LIMIT = 8
+
+
+class PoolHolderSnapshot(BaseModel):
+    """One connection currently checked out from a watched ORM pool."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    task_name: str
+    task_id: int
+    state: Literal["held", "releasing"]
+    age_seconds: float
+    acquired_at: datetime
+    checkout_site: str
+    checkout_stack: tuple[str, ...]
+    current_site: str | None
+    current_stack: tuple[str, ...]
+    task_done: bool
+    task_cancelled: bool
+
+
+class PoolSaturationEvent(BaseModel):
+    """Diagnostic snapshot captured on a pool's first acquire timeout."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    config_name: str
+    timeout_seconds: float
+    acquire_attempt: int
+    occurred_at: datetime
+    pool_size: int
+    pool_max: int
+    pool_idle: int
+    pool_min: int
+    waiter_task_name: str
+    waiter_site: str
+    holders: tuple[PoolHolderSnapshot, ...]
+
+
+PoolSaturationSink = Callable[[PoolSaturationEvent], None]
+
+
+@dataclass(slots=True)
+class _TrackedHolder:
+    config_name: str
+    connection: Any
+    task: asyncio.Task[Any]
+    acquired_monotonic: float
+    acquired_at: datetime
+    checkout_site: str
+    checkout_stack: tuple[str, ...]
+    state: Literal["held", "releasing"] = "held"
+
+
+_holders: dict[str, dict[int, _TrackedHolder]] = {}
+_sinks: list[PoolSaturationSink] = []
+_open_saturation_incidents: set[str] = set()
+
+
+def _format_frame(frame: traceback.FrameSummary) -> str:
+    return f"{frame.filename}:{frame.lineno} in {frame.name}"
+
+
+def _is_orm_frame(frame: traceback.FrameSummary) -> bool:
+    normalized = frame.filename.replace("\\", "/")
+    return "/matrx_orm/" in normalized
+
+
+def _is_context_wrapper_frame(frame: traceback.FrameSummary) -> bool:
+    normalized = frame.filename.replace("\\", "/")
+    return normalized.endswith("/contextlib.py") or "/asyncio/" in normalized
+
+
+def _capture_checkout_stack() -> tuple[str, tuple[str, ...]]:
+    frames = traceback.extract_stack(limit=_STACK_LIMIT)[:-1]
+    formatted = tuple(_format_frame(frame) for frame in frames[-_DISPLAY_STACK_LIMIT:])
+    external = next(
+        (
+            frame
+            for frame in reversed(frames)
+            if not _is_orm_frame(frame) and not _is_context_wrapper_frame(frame)
+        ),
+        None,
+    )
+    site = _format_frame(external or frames[-1]) if frames else "<stack unavailable>"
+    return site, formatted
+
+
+def _task_stack(task: asyncio.Task[Any]) -> tuple[str | None, tuple[str, ...]]:
+    try:
+        frames = task.get_stack(limit=_DISPLAY_STACK_LIMIT)
+    except Exception:
+        return None, ()
+    formatted = tuple(
+        f"{frame.f_code.co_filename}:{frame.f_lineno} in {frame.f_code.co_name}" for frame in frames
+    )
+    return (formatted[-1] if formatted else None), formatted
+
+
+def track_connection_holder(config_name: str, connection: Any) -> None:
+    """Register the current task as the holder of ``connection``."""
+    task = asyncio.current_task()
+    if task is None:
+        return
+    checkout_site, checkout_stack = _capture_checkout_stack()
+    tracked = _TrackedHolder(
+        config_name=config_name,
+        connection=connection,
+        task=task,
+        acquired_monotonic=time.monotonic(),
+        acquired_at=datetime.now(UTC),
+        checkout_site=checkout_site,
+        checkout_stack=checkout_stack,
+    )
+    _holders.setdefault(config_name, {})[id(connection)] = tracked
+
+
+def mark_connection_releasing(config_name: str, connection: Any) -> None:
+    """Mark a checkout as resetting/returning rather than caller-held."""
+    tracked = _holders.get(config_name, {}).get(id(connection))
+    if tracked is not None:
+        tracked.state = "releasing"
+
+
+def untrack_connection_holder(config_name: str, connection: Any) -> None:
+    """Remove a connection after its pool release actually completes."""
+    config_holders = _holders.get(config_name)
+    if config_holders is None:
+        return
+    config_holders.pop(id(connection), None)
+    if not config_holders:
+        _holders.pop(config_name, None)
+
+
+def connection_holder_census(config_name: str) -> tuple[PoolHolderSnapshot, ...]:
+    """Return oldest-first snapshots of every tracked checkout for a pool."""
+    now = time.monotonic()
+    snapshots: list[PoolHolderSnapshot] = []
+    for tracked in list(_holders.get(config_name, {}).values()):
+        current_site, current_stack = _task_stack(tracked.task)
+        snapshots.append(
+            PoolHolderSnapshot(
+                task_name=tracked.task.get_name(),
+                task_id=id(tracked.task),
+                state=tracked.state,
+                age_seconds=round(max(0.0, now - tracked.acquired_monotonic), 3),
+                acquired_at=tracked.acquired_at,
+                checkout_site=tracked.checkout_site,
+                checkout_stack=tracked.checkout_stack,
+                current_site=current_site,
+                current_stack=current_stack,
+                task_done=tracked.task.done(),
+                task_cancelled=tracked.task.cancelled(),
+            )
+        )
+    snapshots.sort(key=lambda holder: holder.age_seconds, reverse=True)
+    return tuple(snapshots)
+
+
+def register_pool_saturation_sink(sink: PoolSaturationSink) -> None:
+    """Register a synchronous first-timeout observer (normally a task scheduler)."""
+    if sink not in _sinks:
+        _sinks.append(sink)
+
+
+def reset_pool_watch_state() -> None:
+    """Clear process state for isolated tests."""
+    _holders.clear()
+    _sinks.clear()
+    _open_saturation_incidents.clear()
+
+
+def note_pool_recovered(config_name: str) -> None:
+    """Re-arm the first-timeout alarm after the pool has an idle connection."""
+    _open_saturation_incidents.discard(config_name)
+
+
+@asynccontextmanager
+async def watched_pool_acquire(
+    pool: Any,
+    *,
+    config_name: str = "<external-pool>",
+    timeout_seconds: float = 10.0,
+) -> AsyncIterator[Any]:
+    """Track a caller-supplied pool checkout and alarm if its wait times out."""
+    acquired = False
+    connection: Any = None
+    try:
+        async with pool.acquire(timeout=timeout_seconds) as connection:
+            acquired = True
+            track_connection_holder(config_name, connection)
+            try:
+                yield connection
+            finally:
+                mark_connection_releasing(config_name, connection)
+    except TimeoutError:
+        # A TimeoutError from the caller's body is not an acquire timeout.
+        if acquired:
+            raise
+
+        def _stat(method: str) -> int:
+            try:
+                return int(getattr(pool, method)())
+            except Exception:
+                return -1
+
+        emit_first_pool_timeout(
+            config_name=config_name,
+            timeout_seconds=timeout_seconds,
+            acquire_attempt=1,
+            pool_size=_stat("get_size"),
+            pool_max=_stat("get_max_size"),
+            pool_idle=_stat("get_idle_size"),
+            pool_min=_stat("get_min_size"),
+        )
+        raise
+    finally:
+        if acquired:
+            untrack_connection_holder(config_name, connection)
+            try:
+                if pool.get_idle_size() > 0:
+                    note_pool_recovered(config_name)
+            except Exception:
+                pass
+
+
+def emit_first_pool_timeout(
+    *,
+    config_name: str,
+    timeout_seconds: float,
+    acquire_attempt: int,
+    pool_size: int,
+    pool_max: int,
+    pool_idle: int,
+    pool_min: int,
+) -> PoolSaturationEvent:
+    """Scream once per saturation incident and return its holder census."""
+    waiter = asyncio.current_task()
+    waiter_site, _ = _capture_checkout_stack()
+    event = PoolSaturationEvent(
+        config_name=config_name,
+        timeout_seconds=timeout_seconds,
+        acquire_attempt=acquire_attempt,
+        occurred_at=datetime.now(UTC),
+        pool_size=pool_size,
+        pool_max=pool_max,
+        pool_idle=pool_idle,
+        pool_min=pool_min,
+        waiter_task_name=waiter.get_name() if waiter is not None else "<no task>",
+        waiter_site=waiter_site,
+        holders=connection_holder_census(config_name),
+    )
+    if config_name in _open_saturation_incidents:
+        return event
+    _open_saturation_incidents.add(config_name)
+
+    holder_lines = [
+        (
+            f"  - task={holder.task_name!r} state={holder.state} "
+            f"age={holder.age_seconds:.3f}s checkout={holder.checkout_site} "
+            f"current={holder.current_site or '<done>'}"
+        )
+        for holder in event.holders
+    ]
+    if not holder_lines:
+        holder_lines.append("  - <no tracked holders; inspect raw pool bypasses or pool growth>")
+    message = (
+        "\n🚨 MATRX ORM POOL SATURATION — FIRST ACQUIRE TIMEOUT\n"
+        f"config={config_name} attempt={acquire_attempt} timeout={timeout_seconds:.3f}s "
+        f"pool={pool_idle} idle / {pool_size} open / {pool_max} max (min={pool_min})\n"
+        f"waiter={event.waiter_task_name!r} at {event.waiter_site}\n"
+        f"tracked_holders={len(event.holders)}\n" + "\n".join(holder_lines)
+    )
+    vcprint(message, color="red")
+    logger.critical(
+        "pool_saturation_first_timeout",
+        extra={"pool_saturation": event.model_dump(mode="json")},
+    )
+    for sink in list(_sinks):
+        try:
+            sink(event)
+        except Exception as exc:  # noqa: BLE001 - observability never masks the timeout
+            logger.exception("pool saturation sink %r failed: %s", sink, exc)
+    return event
+
+
+__all__ = [
+    "PoolHolderSnapshot",
+    "PoolSaturationEvent",
+    "connection_holder_census",
+    "emit_first_pool_timeout",
+    "mark_connection_releasing",
+    "note_pool_recovered",
+    "register_pool_saturation_sink",
+    "reset_pool_watch_state",
+    "track_connection_holder",
+    "untrack_connection_holder",
+    "watched_pool_acquire",
+]