pz-cli 5.0.0__py3-none-any.whl

pz_foundation/db/connection_manager.py

@@ -0,0 +1,689 @@
+from __future__ import annotations
+
+import asyncio
+import functools
+import inspect
+import time
+import traceback
+from functools import lru_cache
+from typing import Any, Awaitable, Callable, TypeVar, cast
+
+from fastapi import HTTPException
+from ..observability import get_logger
+
+try:
+    from tortoise.exceptions import OperationalError
+except Exception:  # pragma: no cover
+    OperationalError = Exception  # type: ignore[assignment]
+
+from .tortoise_manager import tortoise_manager
+from .lifecycle_status import connection_lifecycle_status
+from .connection_probe import DBConnectionReadinessProbeResult, describe_connection_pool, probe_connection_readiness
+from .production_pool_fallback import maybe_apply_production_pool_replenishment_fallback
+
+
+logger = get_logger(__name__)
+
+T = TypeVar("T")
+AsyncFunc = Callable[..., Any]
+
+try:  # pragma: no cover
+    from asyncpg.exceptions import ConnectionDoesNotExistError as _AsyncpgConnDoesNotExistError  # type: ignore
+    from asyncpg.exceptions import InterfaceError as _AsyncpgInterfaceError  # type: ignore
+
+    _ASYNC_PG_CONN_ERRORS: tuple[type[BaseException], ...] = (
+        _AsyncpgConnDoesNotExistError,
+        _AsyncpgInterfaceError,
+    )
+except Exception:  # pragma: no cover
+    _ASYNC_PG_CONN_ERRORS = ()
+
+try:  # pragma: no cover
+    from psycopg_pool import PoolTimeout as _PsycopgPoolTimeout  # type: ignore
+except Exception:  # pragma: no cover
+    _PsycopgPoolTimeout = None  # type: ignore[assignment]
+
+try:  # pragma: no cover
+    from ..observability.metrics import metrics  # type: ignore
+except Exception:  # pragma: no cover
+    class _M:
+        db_reconnects_total = type("N", (), {"inc": staticmethod(lambda *a, **k: None)})()
+        db_retries_total = type("N", (), {"inc": staticmethod(lambda *a, **k: None)})()
+
+    metrics = _M()  # type: ignore
+
+
+def _is_transient_db_error(exc: BaseException) -> bool:
+    if _is_capacity_or_query_timeout_error(exc):
+        return False
+    if _is_writer_role_changed_error(exc):
+        return True
+    try:
+        if _ASYNC_PG_CONN_ERRORS and isinstance(exc, _ASYNC_PG_CONN_ERRORS):
+            return True
+    except Exception:
+        pass
+    message = str(exc or "").lower()
+    transient_markers = (
+        "lost connection",
+        "connection reset",
+        "connection was closed",
+        "connection does not exist",
+        "server closed the connection",
+        "terminating connection",
+        "broken pipe",
+        "connection refused",
+        "timed out",
+        "timeout",
+        "pool is closing",
+    )
+    return any(marker in message for marker in transient_markers) or isinstance(exc, TimeoutError)
+
+
+def _is_writer_role_changed_error(exc: BaseException) -> bool:
+    message = str(exc or "").lower()
+    return (
+        "read-only transaction" in message
+        or "cannot execute insert in a read-only transaction" in message
+        or "cannot execute update in a read-only transaction" in message
+        or "cannot execute delete in a read-only transaction" in message
+        or "default_transaction_read_only" in message
+    )
+
+
+def _is_pool_exhausted_error(exc: BaseException) -> bool:
+    if _PsycopgPoolTimeout is not None and isinstance(exc, _PsycopgPoolTimeout):
+        return True
+    message = str(exc or "").lower()
+    return (
+        "couldn't get a connection" in message
+        or "could not get a connection" in message
+        or "pool timeout" in message
+        or "connection pool exhausted" in message
+    )
+
+
+def _is_capacity_or_query_timeout_error(exc: BaseException) -> bool:
+    if _is_pool_exhausted_error(exc):
+        return True
+    message = str(exc or "").lower()
+    return (
+        "statement timeout" in message
+        or "canceling statement due to statement timeout" in message
+        or "lock timeout" in message
+        or "idle-in-transaction timeout" in message
+        or "idle in transaction session timeout" in message
+    )
+
+
+def _describe_alias_pool(alias: str) -> dict[str, Any]:
+    try:
+        has_connection = getattr(tortoise_manager, "has_connection", None)
+        if callable(has_connection) and not bool(has_connection(alias)):
+            return {"present": False, "status": "not_configured"}
+        connection = tortoise_manager.get_connection(alias)
+    except Exception as exc:  # noqa: BLE001
+        return {"present": False, "status": "lookup_failed", "error_type": type(exc).__name__}
+    return describe_connection_pool(connection)
+
+
+def _describe_known_pools() -> dict[str, dict[str, Any]]:
+    return {
+        alias: status
+        for alias in ("default", "read_only")
+        if (status := _describe_alias_pool(alias)).get("present") is not False
+        or status.get("status") != "not_configured"
+    }
+
+
+def _safe_error_message(exc: BaseException) -> str:
+    text = str(exc or "").replace("\n", " ").strip()
+    if len(text) > 300:
+        return f"{text[:300]}..."
+    return text
+
+
+def _diagnostic_timeout_from_config(config: dict[str, Any] | None, alias: str) -> float:
+    credentials = _connection_credentials(config, alias)
+    values: list[float] = []
+    for key in ("timeout", "connect_timeout"):
+        try:
+            value = float(credentials.get(key, 0) or 0)
+        except Exception:
+            value = 0
+        if value > 0:
+            values.append(value)
+    if not values:
+        return 2.0
+    return max(0.2, min(2.0, *values))
+
+
+def _effective_config(config: dict[str, Any] | None) -> dict[str, Any] | None:
+    if isinstance(config, dict):
+        return config
+    runtime_config = getattr(tortoise_manager, "_last_config", None)
+    return runtime_config if isinstance(runtime_config, dict) else None
+
+
+def _connection_credentials(config: dict[str, Any] | None, alias: str) -> dict[str, Any]:
+    try:
+        payload = dict(((_effective_config(config) or {}).get("connections", {}) or {}).get(alias, {}) or {})
+    except Exception:
+        return {}
+    credentials = payload.get("credentials")
+    return dict(credentials) if isinstance(credentials, dict) else {}
+
+
+def _scalar_float(value: Any) -> float | None:
+    try:
+        number = float(value)
+    except Exception:
+        return None
+    return number if number >= 0 else None
+
+
+def _pool_config_snapshot(config: dict[str, Any] | None, alias: str) -> dict[str, Any]:
+    credentials = _connection_credentials(config, alias)
+    payload: dict[str, Any] = {}
+    for key in (
+        "minsize",
+        "maxsize",
+        "timeout",
+        "connect_timeout",
+        "max_lifetime",
+        "max_idle",
+        "reconnect_timeout",
+    ):
+        value = credentials.get(key)
+        if isinstance(value, (bool, int, float, str)) and value not in ("", None):
+            payload[key] = value
+    return payload
+
+
+def _numeric_pool_value(pool_status: dict[str, Any], *keys: str) -> float | None:
+    stats = pool_status.get("stats")
+    for key in keys:
+        value = pool_status.get(key)
+        if value is None and isinstance(stats, dict):
+            value = stats.get(key)
+        number = _scalar_float(value)
+        if number is not None:
+            return number
+    return None
+
+
+def _diagnose_capacity_failure(
+    *,
+    config: dict[str, Any] | None,
+    alias: str,
+    error: BaseException,
+    pool_status: dict[str, Any],
+    independent_probe: dict[str, Any],
+    manager_uptime_seconds: float,
+) -> dict[str, Any]:
+    pool_config = _pool_config_snapshot(config, alias)
+    category = "query_timeout_or_lock_wait"
+    if _is_pool_exhausted_error(error):
+        if independent_probe.get("attempted") and independent_probe.get("reachable") is False:
+            category = "database_unreachable_or_auth"
+        else:
+            available = _numeric_pool_value(pool_status, "available", "pool_available")
+            waiting = _numeric_pool_value(pool_status, "waiting", "requests_waiting")
+            connections = _numeric_pool_value(pool_status, "connections", "pool_size")
+            max_size = _numeric_pool_value(pool_status, "max_size") or _scalar_float(pool_config.get("maxsize"))
+            growing = bool(pool_status.get("growing"))
+            if connections is not None and max_size is not None and connections < max_size and (growing or available == 0):
+                category = "pool_replenishment_stalled"
+            elif available == 0 and waiting and waiting > 0:
+                category = "application_pool_depleted"
+            else:
+                category = "application_pool_capacity"
+
+    max_lifetime = _scalar_float(pool_config.get("max_lifetime"))
+    lifetime_window = False
+    if max_lifetime and manager_uptime_seconds >= max(0.0, max_lifetime - 120.0):
+        lifetime_window = True
+
+    return {
+        "category": category,
+        "alias": alias,
+        "database_reachable": independent_probe.get("reachable"),
+        "independent_probe_status": independent_probe.get("status"),
+        "pool_config": pool_config,
+        "manager_uptime_seconds": round(manager_uptime_seconds, 3),
+        "pool_lifetime_rotation_window": lifetime_window,
+    }
+
+
+async def _probe_independent_database(config: dict[str, Any] | None, alias: str) -> dict[str, Any]:
+    config = _effective_config(config)
+    try:
+        connection_payload = dict(((config or {}).get("connections", {}) or {}).get(alias, {}) or {})
+    except Exception as exc:  # noqa: BLE001
+        return {"attempted": False, "status": "config_unavailable", "error_type": type(exc).__name__}
+
+    engine = str(connection_payload.get("engine") or "").strip()
+    if "postgres" not in engine and "psycopg" not in engine:
+        return {"attempted": False, "status": "unsupported_engine", "engine": engine}
+
+    credentials = _connection_credentials(config, alias)
+    required = ("host", "port", "user", "database")
+    if any(not str(credentials.get(key, "") or "").strip() for key in required):
+        return {"attempted": False, "status": "incomplete_credentials"}
+
+    try:
+        import psycopg  # type: ignore
+    except Exception as exc:  # pragma: no cover
+        return {"attempted": False, "status": "psycopg_unavailable", "error_type": type(exc).__name__}
+
+    timeout = _diagnostic_timeout_from_config(config, alias)
+    connect_timeout = int(max(1, timeout))
+    conninfo_kwargs: dict[str, Any] = {
+        "host": credentials.get("host"),
+        "port": credentials.get("port"),
+        "user": credentials.get("user"),
+        "password": credentials.get("password"),
+        "dbname": credentials.get("database"),
+    }
+    server_options = str(credentials.get("server_options") or "").strip()
+    if server_options:
+        conninfo_kwargs["options"] = server_options
+    for key in ("sslmode", "sslrootcert", "sslcert", "sslkey", "sslpassword"):
+        value = credentials.get(key)
+        if value not in (None, ""):
+            conninfo_kwargs[key] = value
+
+    async def _run_probe() -> None:
+        conn = await psycopg.AsyncConnection.connect(
+            psycopg.conninfo.make_conninfo(**conninfo_kwargs),
+            autocommit=True,
+            connect_timeout=connect_timeout,
+        )
+        try:
+            await conn.execute("SELECT 1")
+        finally:
+            await conn.close()
+
+    try:
+        await asyncio.wait_for(_run_probe(), timeout=timeout)
+    except Exception as exc:  # noqa: BLE001
+        return {
+            "attempted": True,
+            "reachable": False,
+            "status": "failed",
+            "error_type": type(exc).__name__,
+            "error": _safe_error_message(exc),
+            "timeout": timeout,
+        }
+    return {"attempted": True, "reachable": True, "status": "ok", "timeout": timeout}
+
+
+class DBConnectionManager:
+    def __init__(self) -> None:
+        self._settings: Any = None
+        self._last_check_time = 0.0
+        self._check_interval = 60.0
+        self._connection_healthy = True
+        self._last_reconnect_attempt_time = 0.0
+        self._reconnect_cooldown = 0.1
+        self._last_reconnect_exception: BaseException | None = None
+        self._last_independent_diagnostic_time = 0.0
+        self._independent_diagnostic_cooldown = 60.0
+        self._manager_started_at = time.monotonic()
+        self._lock = asyncio.Lock()
+        self._config_builder: Callable[[], dict[str, Any]] | None = None
+        self._cluster_factory: Callable[[], Any] | None = None
+
+    def configure(
+        self,
+        settings: Any = None,
+        *,
+        config_builder: Callable[[], dict[str, Any]] | None = None,
+        cluster_factory: Callable[[], Any] | None = None,
+    ) -> None:
+        self._settings = settings
+        self._config_builder = config_builder
+        self._cluster_factory = cluster_factory
+
+    async def execute_with_retry(self, func: AsyncFunc, *args: Any, **kwargs: Any) -> Any:
+        max_retries = 3
+        retry_delay = 0.5
+        for attempt in range(max_retries):
+            try:
+                if self._config_builder is not None or self._cluster_factory is not None:
+                    await tortoise_manager.ensure_initialized(
+                        config_builder=self._config_builder,
+                        cluster_factory=self._cluster_factory,
+                    )
+                else:
+                    await tortoise_manager.ensure_initialized()
+                result = await func(*args, **kwargs)
+                self._connection_healthy = True
+                self._last_reconnect_attempt_time = 0.0
+                self._last_reconnect_exception = None
+                return result
+            except OperationalError as exc:  # type: ignore[misc]
+                if _is_capacity_or_query_timeout_error(exc):
+                    logger.error("数据库容量或查询超时，不重建连接池: %s; pools=%s", exc, _describe_known_pools())
+                    raise
+                if not _is_transient_db_error(exc):
+                    logger.error("数据库操作错误: %s", exc)
+                    raise
+                self._connection_healthy = False
+                try:
+                    metrics.db_retries_total.inc()
+                except Exception:
+                    pass
+                if attempt == max_retries - 1:
+                    logger.error("数据库连接失败，已重试 %d 次: %s", max_retries, exc)
+                    raise
+                logger.warning("数据库连接失败，正在重试 (%d/%d): %s", attempt + 1, max_retries, exc)
+                try:
+                    await self.check_and_reconnect(force=True, rebuild_first=True)
+                except Exception as reconnect_exc:  # noqa: BLE001
+                    logger.warning("数据库重连尝试失败，将继续按重试策略执行: %s", reconnect_exc)
+                await asyncio.sleep(retry_delay * (2**attempt))
+            except HTTPException as exc:
+                status = getattr(exc, "status_code", None)
+                if isinstance(status, int) and 400 <= status < 500:
+                    detail = getattr(exc, "detail", "")
+                    try:
+                        msg = detail if isinstance(detail, str) else str(detail)
+                    except Exception:
+                        msg = ""
+                    logger.warning("HTTP %s: %s", status, msg)
+                else:
+                    logger.error("执行数据库操作时发生错误: %s", traceback.format_exc())
+                raise
+            except Exception as exc:  # noqa: BLE001
+                if _is_capacity_or_query_timeout_error(exc):
+                    logger.error("数据库容量或查询超时，不重建连接池: %s; pools=%s", exc, _describe_known_pools())
+                    raise
+                if not _is_transient_db_error(exc):
+                    logger.error("执行数据库操作时发生错误: %s", traceback.format_exc())
+                    raise
+                self._connection_healthy = False
+                if attempt == max_retries - 1:
+                    logger.error("数据库连接失败，已重试 %d 次: %s", max_retries, exc)
+                    raise
+                logger.warning("数据库连接失败，正在重试 (%d/%d): %s", attempt + 1, max_retries, exc)
+                try:
+                    await self.check_and_reconnect(force=True, rebuild_first=True)
+                except Exception as reconnect_exc:  # noqa: BLE001
+                    logger.warning("数据库重连尝试失败，将继续按重试策略执行: %s", reconnect_exc)
+                await asyncio.sleep(retry_delay * (2**attempt))
+
+    async def retry(self, func: Callable[..., Awaitable[Any]], *args: Any, **kwargs: Any) -> Any:
+        return await self.execute_with_retry(func, *args, **kwargs)
+
+    async def run(self, factory: Callable[[], Awaitable[Any]]) -> Any:
+        async def _call():
+            return await factory()
+
+        return await self.execute_with_retry(_call)
+
+    def with_retry(self, func: AsyncFunc) -> AsyncFunc:
+        @functools.wraps(func)
+        async def wrapper(*args: Any, **kwargs: Any) -> Any:
+            if not self._connection_healthy:
+                await self.check_and_reconnect(force=True)
+            try:
+                return await self.execute_with_retry(func, *args, **kwargs)
+            except OperationalError as exc:  # type: ignore[misc]
+                if _is_capacity_or_query_timeout_error(exc) or not _is_transient_db_error(exc):
+                    raise
+                await self.check_and_reconnect(force=True, rebuild_first=True)
+                return await self.execute_with_retry(func, *args, **kwargs)
+
+        try:
+            wrapper.__signature__ = inspect.signature(func)  # type: ignore[attr-defined]
+        except Exception:
+            pass
+        try:
+            import typing as _typing
+
+            try:
+                resolved = _typing.get_type_hints(func)
+            except Exception:
+                resolved = getattr(func, "__annotations__", {}) or {}
+            wrapper.__annotations__ = resolved  # type: ignore[attr-defined]
+        except Exception:
+            pass
+        return cast(AsyncFunc, wrapper)
+
+    async def check_and_reconnect(self, *, force: bool = False, rebuild_first: bool = False) -> None:
+        async with self._lock:
+            current_time = asyncio.get_event_loop().time()
+            if not force and (current_time - self._last_check_time < self._check_interval and self._connection_healthy):
+                return
+            self._last_check_time = current_time
+            if not rebuild_first:
+                try:
+                    if self._config_builder is not None or self._cluster_factory is not None:
+                        await tortoise_manager.ensure_initialized(
+                            config_builder=self._config_builder,
+                            cluster_factory=self._cluster_factory,
+                        )
+                    else:
+                        await tortoise_manager.ensure_initialized()
+                    status = connection_lifecycle_status(tortoise_manager)
+                    if status.default_ready:
+                        self._connection_healthy = True
+                        self._last_reconnect_attempt_time = 0.0
+                        self._last_reconnect_exception = None
+                        return
+                    self._connection_healthy = False
+                    logger.warning("数据库连接生命周期不可用，准备执行安全重连: %s", status)
+                except Exception as exc:  # noqa: BLE001
+                    if not _is_transient_db_error(exc):
+                        raise
+                    self._connection_healthy = False
+                    logger.warning("数据库连接检查失败，准备执行安全重连: %s", exc)
+            else:
+                self._connection_healthy = False
+                logger.warning("数据库连接已在业务路径失败，跳过旧连接池探活并准备安全重连")
+
+            reconnect_started_at = asyncio.get_event_loop().time()
+            if (
+                self._last_reconnect_attempt_time
+                and reconnect_started_at - self._last_reconnect_attempt_time < self._reconnect_cooldown
+                and self._last_reconnect_exception is not None
+            ):
+                logger.debug("跳过重复数据库重连，仍在冷却窗口内")
+                raise RuntimeError("database reconnect throttled during cooldown") from self._last_reconnect_exception
+
+            self._last_reconnect_attempt_time = reconnect_started_at
+            try:
+                await tortoise_manager.reconnect()
+                status = connection_lifecycle_status(tortoise_manager)
+                if not status.default_ready:
+                    raise RuntimeError(f"database lifecycle not ready after reconnect: {status}")
+                self._connection_healthy = True
+                self._last_reconnect_attempt_time = 0.0
+                self._last_reconnect_exception = None
+                try:
+                    metrics.db_reconnects_total.inc()
+                except Exception:
+                    pass
+                logger.info("数据库连接已通过 alias 级重建恢复")
+            except Exception as reconnect_exc:  # noqa: BLE001
+                self._connection_healthy = False
+                self._last_reconnect_exception = reconnect_exc
+                logger.warning("数据库连接重建失败: %s", reconnect_exc)
+                raise
+
+    async def probe_and_reconnect(self, *, force: bool = False, alias: str = "default") -> None:
+        capacity_failure: tuple[BaseException, DBConnectionReadinessProbeResult, dict[str, Any] | None] | None = None
+        non_transient_failure: tuple[BaseException, DBConnectionReadinessProbeResult] | None = None
+        async with self._lock:
+            current_time = asyncio.get_event_loop().time()
+            if not force and (current_time - self._last_check_time < self._check_interval and self._connection_healthy):
+                return
+            self._last_check_time = current_time
+
+            config: dict[str, Any] | None = None
+            if self._config_builder is not None or self._cluster_factory is not None:
+                config = self._config_builder() if self._config_builder is not None else None
+                await tortoise_manager.ensure_initialized(
+                    config_builder=(lambda: config) if config is not None else self._config_builder,
+                    cluster_factory=self._cluster_factory,
+                )
+            else:
+                await tortoise_manager.ensure_initialized()
+
+            result = await probe_connection_readiness(tortoise_manager, alias=alias)
+            if result.ready:
+                self._connection_healthy = True
+                self._last_reconnect_attempt_time = 0.0
+                self._last_reconnect_exception = None
+                return
+
+            self._connection_healthy = False
+            if result.error is not None and _is_capacity_or_query_timeout_error(result.error):
+                capacity_failure = (result.error, result, config)
+
+            elif result.error is not None and not _is_transient_db_error(result.error):
+                non_transient_failure = (result.error, result)
+
+        if capacity_failure is not None:
+            error, result, config = capacity_failure
+            diagnostic = await self._run_independent_database_diagnostic(config, alias)
+            pool_status = result.pool_status or _describe_alias_pool(alias)
+            diagnosis = _diagnose_capacity_failure(
+                config=config,
+                alias=alias,
+                error=error,
+                pool_status=pool_status,
+                independent_probe=diagnostic,
+                manager_uptime_seconds=time.monotonic() - self._manager_started_at,
+            )
+            fallback_result = await maybe_apply_production_pool_replenishment_fallback(
+                alias=alias,
+                error=error,
+                lifecycle=result.lifecycle,
+                pool_status=pool_status,
+                independent_probe=diagnostic,
+                diagnosis=diagnosis,
+                reconnect=tortoise_manager.reconnect,
+                lifecycle_reader=lambda: connection_lifecycle_status(tortoise_manager),
+            )
+            if fallback_result.recovered:
+                self._connection_healthy = True
+                self._last_reconnect_attempt_time = 0.0
+                self._last_reconnect_exception = None
+                try:
+                    metrics.db_reconnects_total.inc()
+                except Exception:
+                    pass
+                return
+            if fallback_result.attempted and fallback_result.error is not None:
+                self._last_reconnect_exception = fallback_result.error
+            logger.error(
+                "数据库容量或查询超时，不重建连接池: %s; alias=%s lifecycle=%s "
+                "probe_status=%s pool_status=%s independent_probe=%s diagnosis=%s fallback=%s",
+                error,
+                alias,
+                result.lifecycle,
+                result.status,
+                pool_status,
+                diagnostic,
+                diagnosis,
+                fallback_result,
+            )
+            raise error
+
+        if non_transient_failure is not None:
+            error, result = non_transient_failure
+            logger.warning(
+                "数据库探活失败但非断连信号，不执行连接池重建: %s; alias=%s lifecycle=%s probe_status=%s pool_status=%s",
+                error,
+                alias,
+                result.lifecycle,
+                result.status,
+                result.pool_status or _describe_alias_pool(alias),
+            )
+            raise error
+
+        await self.check_and_reconnect(force=True, rebuild_first=True)
+
+    async def _run_independent_database_diagnostic(
+        self,
+        config: dict[str, Any] | None,
+        alias: str,
+    ) -> dict[str, Any]:
+        now = asyncio.get_event_loop().time()
+        if (
+            self._last_independent_diagnostic_time
+            and now - self._last_independent_diagnostic_time < self._independent_diagnostic_cooldown
+        ):
+            return {
+                "attempted": False,
+                "status": "cooldown",
+                "cooldown": self._independent_diagnostic_cooldown,
+            }
+        self._last_independent_diagnostic_time = now
+        return await _probe_independent_database(config, alias)
+
+    async def ensure_ready(self) -> None:
+        if not self._connection_healthy:
+            await self.check_and_reconnect(force=True)
+        else:
+            try:
+                if self._config_builder is not None or self._cluster_factory is not None:
+                    await tortoise_manager.ensure_initialized(
+                        config_builder=self._config_builder,
+                        cluster_factory=self._cluster_factory,
+                    )
+                else:
+                    await tortoise_manager.ensure_initialized()
+                self._connection_healthy = True
+                self._last_reconnect_attempt_time = 0.0
+                self._last_reconnect_exception = None
+            except Exception:
+                self._connection_healthy = False
+                await self.check_and_reconnect(force=True)
+
+    def wrap(self, target: Any) -> Any:
+        manager = self
+
+        class _DBRetryProxy:
+            def __getattr__(self, name: str):
+                attr = getattr(target, name)
+                if inspect.iscoroutinefunction(attr):
+                    async def _afn(*args, **kwargs):
+                        return await manager.execute_with_retry(attr, *args, **kwargs)
+
+                    return _afn
+                return attr
+
+            def __call__(self, *args, **kwargs):
+                if inspect.iscoroutinefunction(target):
+                    async def _afn():
+                        return await target(*args, **kwargs)
+
+                    return manager.execute_with_retry(_afn)
+                return target(*args, **kwargs)
+
+        return _DBRetryProxy()
+
+
+@lru_cache(maxsize=1)
+def get_db_manager() -> DBConnectionManager:
+    return DBConnectionManager()
+
+
+class _DBManagerProxy:
+    def __getattr__(self, name: str):
+        return getattr(get_db_manager(), name)
+
+    def __setattr__(self, name: str, value: Any) -> None:
+        setattr(get_db_manager(), name, value)
+
+    def __delattr__(self, name: str) -> None:
+        delattr(get_db_manager(), name)
+
+
+db_manager = _DBManagerProxy()
+
+
+__all__ = ["DBConnectionManager", "db_manager", "get_db_manager"]