ddutils 0.3.0__tar.gz → 0.3.1__tar.gz

{ddutils-0.3.0 → ddutils-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddutils
-Version: 0.3.0
+Version: 0.3.1
 Summary: Domain Driven Utils Library
 Keywords: python,ddutils
 Author: davyddd
@@ -519,6 +519,58 @@ that planted a key would be inherited by every request task, making all requests
 same reason the key is not reset when the block ends, so `scope()` must be entered in a task that does not
 outlive the block (a request handler, an actor run).
 
+## Connections
+
+`ddutils.connections` sits between a `ScopedRegistry` and the code that uses a connection: a query, a cache
+lookup, a Kafka send. A registry holds one connection per scope: a pooled client per process, a session
+per task. The connections module adds two things on top:
+
+- `ConnectionManagerFactory(registries, default, connection_manager_class=ConnectionManager)` resolves a
+  connection alias to its registry, like Django's `using`. Calling it opens a connection manager on the
+  chosen registry; `registry(alias)` returns the registry itself; `close()` clears every registry for the
+  current scope, calling the connection's destructor, which is the shutdown hook of process-scoped clients.
+- `ConnectionManager(registry)` is the default connection manager: an async context manager that hands out
+  the connection of the registry's current scope and releases nothing on exit. Enough for clients that are
+  pools themselves, such as ClickHouse, Redis or a Kafka producer.
+
+Databases with transactions pass their own `connection_manager_class`: a unit of work that begins, commits
+or rolls back and closes the session on exit. It depends on the driver, so it lives in the project.
+
+```python
+import os
+from enum import Enum
+
+from ddutils.connections import ConnectionManagerFactory
+from ddutils.scoped_registry import ScopedRegistry
+
+
+class ClickhouseDB(str, Enum):
+    PRIMARY = 'primary'
+
+
+client_registries = {
+    db: ScopedRegistry(create_func=make_client_for(db), scope_func=os.getpid, destructor_method_name='close')
+    for db in ClickhouseDB
+}
+clickhouse = ConnectionManagerFactory(client_registries, default=ClickhouseDB.PRIMARY)
+
+
+async with clickhouse() as client:
+    result = await client.query(sql)
+
+
+# a transactional database: `Atomic` is your unit of work over `registry.scope()`
+atomic = ConnectionManagerFactory(session_registries, default=PostgresDB.PRIMARY, connection_manager_class=Atomic)
+
+
+async with atomic(alias=PostgresDB.REPLICA) as session:
+    ...
+
+
+# on shutdown: closes the clients of every alias that was used in this process
+await clickhouse.close()
+```
+
 ## Function Exception Extraction
 
 AST-based introspection for extracting exceptions from function source code.

{ddutils-0.3.0 → ddutils-0.3.1}/README.md

@@ -501,6 +501,58 @@ that planted a key would be inherited by every request task, making all requests
 same reason the key is not reset when the block ends, so `scope()` must be entered in a task that does not
 outlive the block (a request handler, an actor run).
 
+## Connections
+
+`ddutils.connections` sits between a `ScopedRegistry` and the code that uses a connection: a query, a cache
+lookup, a Kafka send. A registry holds one connection per scope: a pooled client per process, a session
+per task. The connections module adds two things on top:
+
+- `ConnectionManagerFactory(registries, default, connection_manager_class=ConnectionManager)` resolves a
+  connection alias to its registry, like Django's `using`. Calling it opens a connection manager on the
+  chosen registry; `registry(alias)` returns the registry itself; `close()` clears every registry for the
+  current scope, calling the connection's destructor, which is the shutdown hook of process-scoped clients.
+- `ConnectionManager(registry)` is the default connection manager: an async context manager that hands out
+  the connection of the registry's current scope and releases nothing on exit. Enough for clients that are
+  pools themselves, such as ClickHouse, Redis or a Kafka producer.
+
+Databases with transactions pass their own `connection_manager_class`: a unit of work that begins, commits
+or rolls back and closes the session on exit. It depends on the driver, so it lives in the project.
+
+```python
+import os
+from enum import Enum
+
+from ddutils.connections import ConnectionManagerFactory
+from ddutils.scoped_registry import ScopedRegistry
+
+
+class ClickhouseDB(str, Enum):
+    PRIMARY = 'primary'
+
+
+client_registries = {
+    db: ScopedRegistry(create_func=make_client_for(db), scope_func=os.getpid, destructor_method_name='close')
+    for db in ClickhouseDB
+}
+clickhouse = ConnectionManagerFactory(client_registries, default=ClickhouseDB.PRIMARY)
+
+
+async with clickhouse() as client:
+    result = await client.query(sql)
+
+
+# a transactional database: `Atomic` is your unit of work over `registry.scope()`
+atomic = ConnectionManagerFactory(session_registries, default=PostgresDB.PRIMARY, connection_manager_class=Atomic)
+
+
+async with atomic(alias=PostgresDB.REPLICA) as session:
+    ...
+
+
+# on shutdown: closes the clients of every alias that was used in this process
+await clickhouse.close()
+```
+
 ## Function Exception Extraction
 
 AST-based introspection for extracting exceptions from function source code.

ddutils-0.3.1/ddutils/connections.py

@@ -0,0 +1,81 @@
+from collections.abc import Callable, Hashable, Mapping
+from typing import Any, Generic, TypeVar
+
+from ddutils.scoped_registry import ScopedRegistry
+
+# The name a connection is registered under and selected by, like Django's `using` alias
+ConnectionAliasT = TypeVar('ConnectionAliasT', bound=Hashable)
+# The object a registry holds for a connection: a pooled client, a session
+ConnectionT = TypeVar('ConnectionT')
+# The context manager the factory opens on the chosen registry
+ConnectionManagerT = TypeVar('ConnectionManagerT')
+
+
+class ConnectionManager(Generic[ConnectionT]):
+    """
+    Default connection manager: hands out the connection of the registry's current scope and releases nothing on exit.
+
+    The connection is created on first use via `registry.set()` and kept for the life of its scope,
+    typically a process, since a client is itself a connection pool (ClickHouse, Redis, a Kafka producer). Databases
+    with transactions need their own connection manager on top of the registry, e.g. a unit of work that begins,
+    commits and closes a session on exit.
+    """
+
+    def __init__(self, registry: ScopedRegistry[ConnectionT]) -> None:
+        self._registry = registry
+
+    async def __aenter__(self) -> ConnectionT:
+        return await self._registry.set()
+
+    async def __aexit__(self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: Any) -> None:
+        return None
+
+
+class ConnectionManagerFactory(Generic[ConnectionAliasT, ConnectionT, ConnectionManagerT]):
+    """
+    Resolves a connection alias to its registry and opens a connection manager on it, like Django's `using`.
+
+    `connection_manager_class` is the context manager opened on the chosen registry: `ConnectionManager` by
+    default, a project class for anything that needs setup and teardown around the connection (transactions):
+
+        clickhouse = ConnectionManagerFactory(client_registries, default=ClickhouseAlias.PRIMARY)
+        atomic = ConnectionManagerFactory(session_registries, default=PostgresAlias.PRIMARY, connection_manager_class=Atomic)
+
+        async with clickhouse() as client: ...
+        async with atomic(alias=PostgresAlias.REPLICA) as session: ...
+        await clickhouse.close()  # on shutdown
+    """
+
+    def __init__(
+        self,
+        registries: Mapping[ConnectionAliasT, ScopedRegistry[ConnectionT]],
+        default: ConnectionAliasT,
+        connection_manager_class: Callable[[ScopedRegistry[ConnectionT]], ConnectionManagerT] = ConnectionManager,  # type: ignore[assignment]
+    ) -> None:
+        if default not in registries:
+            raise ValueError(f'Default connection {default!r} is not among the registries')
+        self.registries = registries
+        self.default = default
+        self.connection_manager_class = connection_manager_class
+
+    def registry(self, alias: Hashable | None = None) -> ScopedRegistry[ConnectionT]:
+        """The registry of `alias`, or of the default connection; `Hashable` because `Adapter.alias` is untyped."""
+        alias = self.default if alias is None else alias
+        if alias not in self.registries:
+            raise KeyError(f'Unknown connection {alias!r}; expected one of {list(self.registries)!r}')
+        return self.registries[alias]  # type: ignore[index]
+
+    def __call__(self, alias: Hashable | None = None) -> ConnectionManagerT:
+        return self.connection_manager_class(self.registry(alias))
+
+    async def close(self) -> None:
+        """
+        Clears every registry for the current scope, calling the connection's destructor: the shutdown hook of
+        a process-scoped client (a pool, a producer with a buffer to flush). A registry without a connection in
+        the current scope is left alone, so the hook is safe to call for aliases that were never used.
+        """
+        for registry in self.registries.values():
+            await registry.clear()
+
+
+__all__ = ('ConnectionManager', 'ConnectionManagerFactory')

{ddutils-0.3.0 → ddutils-0.3.1}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "ddutils"
 description = "Domain Driven Utils Library"
-version = "0.3.0"
+version = "0.3.1"
 license = "MIT"
 license-files = ["LICENSE"]
 readme = "README.md"

{ddutils-0.3.0 → ddutils-0.3.1}/pyproject.toml.orig

@@ -1,7 +1,7 @@
 [project]
 name = "ddutils"
 description = "Domain Driven Utils Library"
-version = "0.3.0"
+version = "0.3.1"
 authors = [{ name = "davyddd" }]
 license = "MIT"
 license-files = ["LICENSE"]