altcodepro-polydb-python 2.5.24__py3-none-any.whl → 2.6.0__py3-none-any.whl

{altcodepro_polydb_python-2.5.24.dist-info → altcodepro_polydb_python-2.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: altcodepro-polydb-python
-Version: 2.5.24
+Version: 2.6.0
 Summary: Production-ready multi-cloud database abstraction layer with connection pooling, retry logic, and thread safety
 Author: AltCodePro
 Project-URL: Homepage, https://github.com/altcodepro/polydb-python
@@ -23,7 +23,7 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: psycopg2-binary>=2.9.11
+Requires-Dist: psycopg[binary,pool]>=3.2
 Requires-Dist: tenacity>=9.1.4
 Requires-Dist: redis>=6.4.0
 Requires-Dist: python-dotenv>=1.1.1

{altcodepro_polydb_python-2.5.24.dist-info → altcodepro_polydb_python-2.6.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-altcodepro_polydb_python-2.5.24.dist-info/licenses/LICENSE,sha256=9X8GLocsBwy-5aR5JGOt2SAMDDPs9Qv-YnqmHBHOXrw,1067
+altcodepro_polydb_python-2.6.0.dist-info/licenses/LICENSE,sha256=9X8GLocsBwy-5aR5JGOt2SAMDDPs9Qv-YnqmHBHOXrw,1067
 polydb/PolyDB.py,sha256=Lp0QtG7D89Z6P6nt9fVoT8rVHAy8xlE2kFRSS70r-mk,23672
 polydb/__init__.py,sha256=XuPMkOUcluxI8TIMNZCPyI1MZg77DC-10FKANEVobME,1876
 polydb/advanced_query.py,sha256=M9sKmdF2PsLZWSc-zKpXRcwaDumbYYTsibmL_IAjKP4,6931
@@ -6,7 +6,7 @@ polydb/aio.py,sha256=0EQ6hXrf6N4If6pcht6L9B7qz0PU2yGahyGzEss2eQg,8340
 polydb/batch.py,sha256=ulvSdtVeofsOiVNVAQwrE9tXIIaP8fjiW1Sbpz3dcv0,4093
 polydb/cache.py,sha256=ArI0zRbuhAYd6kn4TPVMBHBptdRmuZyy05_hwIATUsQ,11784
 polydb/cloudDatabaseFactory.py,sha256=eJiNKycdaC9nO2BKWPjkig3GaNGkNjoJci6yVJC7A_c,25301
-polydb/databaseFactory.py,sha256=l0ODaR3HbTN4KNpC_oEPqN1ussorXGfKnRRq2r5xt6Q,60202
+polydb/databaseFactory.py,sha256=MWQhfsumX0_7G3N6XSotCHRqqtzBikT1Ogy4lqJI_hw,61404
 polydb/errors.py,sha256=AtXCLdzkHE2ncUaE1P0wH5HmIyW3NXx9KHL-fCejjA4,1832
 polydb/json_safe.py,sha256=jj3ZA6gImGwgTO0dCZVcckjeUHtewa2HP6OA1-u6VZ0,137
 polydb/models.py,sha256=Pq_63FXu1IiTxl-RshZ2vmoe-5m5k6qft-tXpHLn-70,12600
@@ -15,6 +15,7 @@ polydb/multitenancy.py,sha256=pmbrfdtycQiETnBvOqpvXzkGy2tw_ZVDLo7JOaJ9Z6M,7813
 polydb/overflow_gc.py,sha256=hgKx6ZZPVNGX6ejQ5hB7SjxI1XT9_r60kMERQ8tl9KA,6499
 polydb/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 polydb/query.py,sha256=17V-e70df9NaiaUYPN5_MQz7O0vRdBaCR4xlZU3F2YE,8511
+polydb/read_batch.py,sha256=kDol5XnIsXKfac5xy6R3FJQbcK2zHj7ce1ndtQJvlXA,6004
 polydb/registry.py,sha256=RD_elvFXcmhTdCyZDm2f3ej0elxqhArnSJ2aO9k5VCU,2352
 polydb/retry.py,sha256=0rILN3-auuzomOXPveIAvgnhBbtXXGNKvxg6qcRgN4g,4926
 polydb/schema.py,sha256=v3lnD9jUoBippMj2eLNwvmykPygO0nFx806X7cmeb8Q,9840
@@ -22,6 +23,7 @@ polydb/security.py,sha256=IF8Oh6Y6YU9xSjz_pi6lotfJuhIUJgP7cCmpGc8bz6w,20158
 polydb/types.py,sha256=UbO5tQ2oPfviszWAvZvhuYv6PMp_WhgPFgUa5tTVgKI,3026
 polydb/utils.py,sha256=YS-BhBpQfQsOf-z_i1zw_kvMYwJ05Zn1S2Xt6K4_2ak,3171
 polydb/validation.py,sha256=DK9-AFndTr0cLodpc3KX0wnABDpw2vPB4H6hp67BJKE,4570
+polydb/write_behind.py,sha256=I8WEtAHZvuswX-ItJAVWhsz1oDcPpJmfp4E8YzWGGmk,5548
 polydb/adapters/AWSSecretsManagerAdapter.py,sha256=W4B3xhQFcBAlNlwQ4uTgcDmRrOmZr6v-suC4qihTpJw,2747
 polydb/adapters/AzureBlobStorageAdapter.py,sha256=ZTovSC_CVduGnIoIQUUeNxDQlf-IbFDBRzjGDk8U9Io,7084
 polydb/adapters/AzureFileStorageAdapter.py,sha256=VZNprqlBXCuWUgtqClNT-NrQmRf-XFYEiRA2BLbf-Sc,7046
@@ -41,7 +43,7 @@ polydb/adapters/GCPSecretManagerAdapter.py,sha256=ZFL2FGh7uTaBfpUo-HKXZAxS9s-w0w
 polydb/adapters/GCPStorageAdapter.py,sha256=ce7vCOW46XbdjiupAgMgkgopy3IV7nM5YvIwEhLIKvQ,7133
 polydb/adapters/KafkaQueueAdapter.py,sha256=_FrcI6JLIoJHPOtSIL2q1uXHeao9ZkGMyUnT68p36HQ,14329
 polydb/adapters/MongoDBAdapter.py,sha256=Rs1L9gbGj2hHiJuz62Ms7uRK6sppHc1aIO4MNrtZ3Zo,11750
-polydb/adapters/PostgreSQLAdapter.py,sha256=n73HfBIXCF8Em4bHPvRVvipOJKdt9qZuZWdzRNCxS3w,48166
+polydb/adapters/PostgreSQLAdapter.py,sha256=8Q-O-6o37p0inwv_EgWMU7M8KthY0CDgbF9XV-lF2SQ,61367
 polydb/adapters/RabbitMQAdapter.py,sha256=rMeqWXQmnfRX_pakaZX6q1bHkZ8E1boQ_Pkj3byl4E4,19903
 polydb/adapters/S3Adapter.py,sha256=ZkMR91u1e4SnrXSUbV3siKFYdw0ykYHJfyP2cUJUVso,7564
 polydb/adapters/S3CompatibleAdapter.py,sha256=Qj9SUtCnOFNL9c05wNVjICxZacFCnee-Wn_-hh0lwwE,7006
@@ -69,7 +71,7 @@ polydb/observability/logging.py,sha256=FWYky-Mg1IhLpwuaKuLIsTROBZc9bI_-AO-y53nvE
 polydb/services/__init__.py,sha256=Zs3PZbHutgPdPjJatygvtYvKPZMBt1qKSqESbFRPHJs,241
 polydb/services/compliance_service.py,sha256=UMjCbQ4z13-w6pT4RLQpMi_u1zbShEpiz8kawA2WsSU,5075
 polydb/services/security_service.py,sha256=gGo_j9Ca_rgvxS7s6C5JUktugQBj1PV1DXpMnOYAGbI,4992
-altcodepro_polydb_python-2.5.24.dist-info/METADATA,sha256=6Osh8k039JDAzMEhKD7RIpHstGqXvw7rBgMVis9o8UM,15599
-altcodepro_polydb_python-2.5.24.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-altcodepro_polydb_python-2.5.24.dist-info/top_level.txt,sha256=WgLFWJoYjUhwvyPxJFl6jYLrVFuBJDX3OABf4ocwk_E,7
-altcodepro_polydb_python-2.5.24.dist-info/RECORD,,
+altcodepro_polydb_python-2.6.0.dist-info/METADATA,sha256=Pvi9KDJMCNteoahMUpIH5bM2Igc-MS47jXbmVZ6Pk3I,15600
+altcodepro_polydb_python-2.6.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+altcodepro_polydb_python-2.6.0.dist-info/top_level.txt,sha256=WgLFWJoYjUhwvyPxJFl6jYLrVFuBJDX3OABf4ocwk_E,7
+altcodepro_polydb_python-2.6.0.dist-info/RECORD,,

polydb/adapters/PostgreSQLAdapter.py

@@ -5,17 +5,21 @@ import json
 import os
 import threading
 import time
+import weakref
 from contextlib import contextmanager
 from datetime import date, datetime
 from decimal import Decimal
 from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
 
-import psycopg2.extensions
-from psycopg2 import sql as pg_sql
-from psycopg2.extras import Json
+import psycopg
+import psycopg.pq
+import psycopg_pool
+from psycopg import sql as pg_sql
+from psycopg.types.json import Jsonb as Json
 
 from ..errors import ConnectionError, DatabaseError, InsufficientBalanceError
 from ..query import Operator, QueryBuilder
+from ..read_batch import SelectSpec, current_batch
 from ..retry import retry
 from ..types import JsonDict, Lookup
 from ..utils import validate_column_name, validate_table_name
@@ -35,6 +39,80 @@ from ..utils import validate_column_name, validate_table_name
 ALLOWED_SESSION_VARS = frozenset({"app.tenant_id"})
 
 
+
+class PoolError(Exception):
+    """Raised when no connection frees up within the acquire timeout."""
+
+
+def _close_pool_quietly(pool) -> None:
+    # A pool that is dropped without closeall() (a test, a process shutting down) must not raise from __del__ -- psycopg_pool's own
+    # finalizer does, because its worker threads outlive the object.
+    try:
+        pool.close(timeout=1.0)
+    except Exception:
+        pass
+
+
+class LazyPool:
+    """A small, lazy, bounded connection pool on psycopg 3 (`psycopg_pool.ConnectionPool`).
+
+    Why not the old eager pool: every process used to open `POSTGRES_MIN_CONNECTIONS` connections the moment it started and keep
+    them for its whole life. With a handful of processes (api plus one worker per lane) and a database that only allows a few dozen
+    clients, the idle reservations alone filled the server ("remaining connection slots are reserved ...") and a new revision or
+    worker could not even start.
+
+    This pool opens connections ONLY when a request needs one, keeps at most `keep` (default 1) idle for reuse, and closes every
+    other idle connection after `max_idle` seconds. `max_size` is a hard ceiling per process. A request that finds every connection
+    in use waits up to `acquire_timeout` seconds instead of failing at once."""
+
+    def __init__(self, dsn: str, *, max_size: int, keep: int = 1, acquire_timeout: float = 10.0,
+                 max_idle: float = 30.0, max_lifetime: float = 1800.0):
+        self.maxconn = max_size
+        self._timeout = float(acquire_timeout)
+        keep = max(0, min(keep, max_size))
+        self._pool = psycopg_pool.ConnectionPool(
+            conninfo=dsn,
+            min_size=keep,
+            max_size=max_size,
+            timeout=self._timeout,
+            max_idle=max_idle,
+            max_lifetime=max_lifetime,
+            reconnect_timeout=30.0,
+            # prepare_threshold=None: no server-side prepared statements (they pin per-connection state and break behind a
+            # transaction-mode pooler); autocommit off keeps the engine's explicit commit/rollback contract.
+            kwargs={"autocommit": False, "prepare_threshold": None},
+            open=False,
+            name="polydb-pg",
+        )
+        self._pool.open()
+        weakref.finalize(self, _close_pool_quietly, self._pool)
+
+    def getconn(self, key=None):
+        try:
+            return self._pool.getconn(timeout=self._timeout)
+        except psycopg_pool.PoolTimeout as e:
+            raise PoolError(
+                f"connection pool exhausted: waited {self._timeout:g}s for one of {self.maxconn} connections"
+            ) from e
+
+    def putconn(self, conn=None, key=None, close=False):
+        if conn is None:
+            return
+        if close:
+            try:
+                conn.close()
+            except Exception:
+                pass
+        self._pool.putconn(conn)
+
+    def closeall(self):
+        self._pool.close()
+
+    def used_count(self) -> int:
+        stats = self._pool.get_stats()
+        return max(0, int(stats.get("pool_size", 0)) - int(stats.get("pool_available", 0)))
+
+
 class PostgreSQLAdapter:
     """PostgreSQL with full LINQ support, connection pooling, JSON/JSONB support"""
 
@@ -59,7 +137,7 @@ class PostgreSQLAdapter:
 
     def _is_idle(self, conn) -> bool:
         try:
-            return conn.info.transaction_status == psycopg2.extensions.TRANSACTION_STATUS_IDLE
+            return conn.info.transaction_status == psycopg.pq.TransactionStatus.IDLE
         except Exception:
             return False
 
@@ -91,8 +169,6 @@ class PostgreSQLAdapter:
         try:
             from urllib.parse import parse_qs, quote, urlencode, urlparse, urlunparse
 
-            import psycopg2.pool
-
             dsn = self.connection_string
             if "postgresql://" in dsn:
                 parsed = urlparse(dsn)
@@ -132,17 +208,23 @@ class PostgreSQLAdapter:
                 parsed = parsed._replace(query=new_query)
                 dsn = urlunparse(parsed)
 
-            _maxconn = int(os.getenv("POSTGRES_MAX_CONNECTIONS", "100"))
+            _maxconn = int(os.getenv("POSTGRES_MAX_CONNECTIONS", "10"))
             with self._lock:
                 if not self._pool:
-                    self._pool = psycopg2.pool.ThreadedConnectionPool(
-                        minconn=int(os.getenv("POSTGRES_MIN_CONNECTIONS", "5")),
-                        maxconn=_maxconn,
-                        dsn=dsn,
+                    # One probe connection up front so a bad DSN / full server fails here, loudly, exactly as the old eager pool
+                    # did -- and is closed again at once (the pool itself opens connections only on demand).
+                    with psycopg.connect(dsn, connect_timeout=int(os.getenv("POSTGRES_CONNECT_TIMEOUT_S", "30"))) as probe:
+                        probe.execute("SELECT 1")
+                    self._pool = LazyPool(
+                        dsn,
+                        max_size=_maxconn,
+                        keep=int(os.getenv("POSTGRES_POOL_IDLE_KEEP", "1")),
+                        acquire_timeout=float(os.getenv("POSTGRES_POOL_ACQUIRE_TIMEOUT_S", "10")),
+                        max_idle=float(os.getenv("POSTGRES_POOL_MAX_IDLE_S", "30")),
                     )
                     self.logger.info(
-                        "PostgreSQL pool initialized: min=%s max=%s",
-                        os.getenv("POSTGRES_MIN_CONNECTIONS", "5"),
+                        "PostgreSQL pool initialized (lazy): idle-keep=%s max=%s",
+                        os.getenv("POSTGRES_POOL_IDLE_KEEP", "1"),
                         _maxconn,
                     )
         except Exception as e:
@@ -152,8 +234,8 @@ class PostgreSQLAdapter:
         if not self._pool:
             return
         try:
-            maxconn = int(os.getenv("POSTGRES_MAX_CONNECTIONS", "100"))
-            used_count = len(getattr(self._pool, "_used", {}))
+            maxconn = self._pool.maxconn
+            used_count = self._pool.used_count()
             pct = (used_count / maxconn * 100) if maxconn else 0
             if pct > 80:
                 self.logger.warning(
@@ -204,7 +286,9 @@ class PostgreSQLAdapter:
         table: str = "",
     ) -> float:
         t0 = time.perf_counter()
-        cursor.execute(sql, params)
+        # psycopg 3 parses `%` as a placeholder whenever a params sequence is given, even an empty one (psycopg2 did not):
+        # no parameters means None, so raw SQL / DDL containing a literal `%` or several statements still runs.
+        cursor.execute(sql, params if params else None)
         duration_ms = (time.perf_counter() - t0) * 1000.0
 
         self.logger.debug(
@@ -323,7 +407,7 @@ class PostgreSQLAdapter:
             return Json(self._json_safe(v))
         if isinstance(v, (list, tuple)):
             # A bare list/tuple of scalars is a real Postgres array column
-            # (e.g. TEXT[]) -- psycopg2 adapts a plain Python list to that
+            # (e.g. TEXT[]) -- psycopg adapts a plain Python list to that
             # natively, so it must pass through untouched here, exactly
             # like _serialize_param already does for query parameters
             # (below). Only a list/tuple actually containing dicts is
@@ -394,7 +478,7 @@ class PostgreSQLAdapter:
         both are simply "a string that starts with `{` and ends with
         `}`." This adapter deliberately does NOT try to solve that
         ambiguity generically (declaring the column JSONB instead sidesteps
-        it entirely -- psycopg2 registers global JSON/JSONB typecasters
+        it entirely -- psycopg registers JSON/JSONB loaders
         for OIDs 114/3802 automatically at import time, so a genuine
         `json`/`jsonb` column never even reaches this heuristic, `isinstance
         (v, str)` already `False` by the time this method sees it); a
@@ -473,10 +557,91 @@ class PostgreSQLAdapter:
             if own_conn and conn:
                 self._return_connection(conn)
 
+    # ---------------------------------------------------------------------
+    # INSERT MANY (pipelined)
+    # ---------------------------------------------------------------------
+
+    def insert_many(
+        self,
+        table: str,
+        rows: List[JsonDict],
+        *,
+        session_vars: Optional[Dict[str, str]] = None,
+    ) -> int:
+        """Insert many rows in ONE transaction and ONE network round trip (psycopg 3 `executemany` runs in pipeline mode).
+
+        All-or-nothing: if any row fails the whole batch is rolled back and a DatabaseError is raised (the caller -- see
+        `write_behind.WriteBehind` -- then retries the rows one by one). Rows may have different column sets; they are grouped by
+        column set, and every group is sent inside the same transaction. No RETURNING: this is for rows nobody reads back."""
+        table = validate_table_name(table)
+        if not rows:
+            return 0
+        groups: Dict[Tuple[str, ...], List[JsonDict]] = {}
+        for row in rows:
+            cols = tuple(row.keys())
+            for c in cols:
+                validate_column_name(c)
+            groups.setdefault(cols, []).append(row)
+
+        conn = self._get_connection()
+        try:
+            self._apply_session_vars(conn, session_vars)
+            t0 = time.perf_counter()
+            with conn.cursor() as cursor:
+                for cols, group in groups.items():
+                    placeholders = ", ".join(["%s"] * len(cols))
+                    sql = f"INSERT INTO {table} ({', '.join(cols)}) VALUES ({placeholders})"
+                    cursor.executemany(sql, [[self._serialize_value(r[c]) for c in cols] for r in group])
+            conn.commit()
+            self._log_batch(len(rows), (time.perf_counter() - t0) * 1000.0)
+            return len(rows)
+        except Exception as e:
+            try:
+                conn.rollback()
+            except Exception:
+                pass
+            raise DatabaseError(f"Insert many failed: {str(e)}")
+        finally:
+            self._return_connection(conn)
+
     # ---------------------------------------------------------------------
     # SELECT
     # ---------------------------------------------------------------------
 
+    @staticmethod
+    def _build_select(
+        table: str,
+        query: Optional[Lookup],
+        limit: Optional[int],
+        offset: Optional[int],
+        fields: Optional[List[str]],
+    ) -> Tuple[str, List[Any]]:
+        """The SQL text and bind parameters of a plain `select()` -- a pure function, shared by the single and the pipelined path."""
+        sql = f"SELECT {', '.join(fields)} FROM {table}" if fields else f"SELECT * FROM {table}"
+        params: List[Any] = []
+        if query:
+            where_parts: List[str] = []
+            for k, v in query.items():
+                validate_column_name(k)
+                if v is None:
+                    where_parts.append(f"{k} IS NULL")
+                elif isinstance(v, (list, tuple)):
+                    placeholders = ",".join(["%s"] * len(v))
+                    where_parts.append(f"{k} IN ({placeholders})")
+                    params.extend(list(v))
+                else:
+                    where_parts.append(f"{k} = %s")
+                    params.append(v)
+            if where_parts:
+                sql += " WHERE " + " AND ".join(where_parts)
+        if limit:
+            sql += " LIMIT %s"
+            params.append(limit)
+        if offset:
+            sql += " OFFSET %s"
+            params.append(offset)
+        return sql, params
+
     @retry(max_attempts=3, delay=1.0, exceptions=(DatabaseError,))
     def select(
         self,
@@ -493,6 +658,16 @@ class PostgreSQLAdapter:
         if fields:
             for f in fields:
                 validate_column_name(f)
+
+        # Inside a ReadBatch (the engine's coll.parallel) this query is queued and sent together with its siblings in one
+        # pipelined round trip on one connection -- see polydb/read_batch.py. A caller-supplied transaction is never batched.
+        batch = current_batch() if tx is None else None
+        if batch is not None:
+            return batch.submit(
+                self,
+                SelectSpec(table=table, query=query, limit=limit, offset=offset, fields=fields, session_vars=session_vars),
+            )
+
         conn = tx
         own_conn = False
         if not conn:
@@ -502,31 +677,7 @@ class PostgreSQLAdapter:
         try:
             self._apply_session_vars(conn, session_vars)
             cursor = conn.cursor()
-            sql = f"SELECT {', '.join(fields)} FROM {table}" if fields else f"SELECT * FROM {table}"
-            params: List[Any] = []
-
-            if query:
-                where_parts: List[str] = []
-                for k, v in query.items():
-                    validate_column_name(k)
-                    if v is None:
-                        where_parts.append(f"{k} IS NULL")
-                    elif isinstance(v, (list, tuple)):
-                        placeholders = ",".join(["%s"] * len(v))
-                        where_parts.append(f"{k} IN ({placeholders})")
-                        params.extend(list(v))
-                    else:
-                        where_parts.append(f"{k} = %s")
-                        params.append(v)
-                if where_parts:
-                    sql += " WHERE " + " AND ".join(where_parts)
-
-            if limit:
-                sql += " LIMIT %s"
-                params.append(limit)
-            if offset:
-                sql += " OFFSET %s"
-                params.append(offset)
+            sql, params = self._build_select(table, query, limit, offset, fields)
 
             self._timed_execute(
                 cursor, sql, self._serialize_params(params), operation="select", table=table
@@ -548,6 +699,119 @@ class PostgreSQLAdapter:
             if own_conn and conn:
                 self._return_connection(conn)
 
+    def select_many(self, specs: List[SelectSpec]) -> List[Any]:
+        """Run several SELECTs in ONE round trip on ONE connection (psycopg 3 pipeline mode).
+
+        Returns one entry per spec, in order: the rows (list of dicts), or a `DatabaseError` for a query that failed. A failing
+        query never poisons the others: Postgres skips every statement after an error until the next sync point, so the specs
+        that were skipped are simply sent again in a second pipeline (one extra round trip per failure, nothing in the common
+        case).
+
+        Every query keeps the guarantees a connection of its own would give it: it runs in its own READ ONLY transaction, with its
+        own `session_vars` applied via `set_config(name, value, true)` (transaction-local, so row-level security sees exactly the
+        tenant that spec asked for and nothing leaks to the next spec or the next borrower of the connection)."""
+        results: List[Any] = [None] * len(specs)
+        if not specs:
+            return results
+
+        built: List[Optional[Tuple[str, List[Any]]]] = []
+        for i, sp in enumerate(specs):
+            try:
+                table = validate_table_name(sp.table)
+                if sp.fields:
+                    for f in sp.fields:
+                        validate_column_name(f)
+                if sp.builder is not None:
+                    sql, params = self._build_linq(table, sp.builder)
+                else:
+                    sql, params = self._build_select(table, sp.query, sp.limit, sp.offset, sp.fields)
+                for k in (sp.session_vars or {}):
+                    if k not in ALLOWED_SESSION_VARS:
+                        raise DatabaseError(
+                            f"session_vars: {k!r} is not in the allow-list of Postgres "
+                            f"session variables this adapter will set"
+                        )
+                built.append((sql, self._serialize_params(params)))
+            except Exception as e:  # a malformed spec fails alone
+                built.append(None)
+                results[i] = e if isinstance(e, DatabaseError) else DatabaseError(f"Select failed: {e}")
+
+        pending = [i for i in range(len(specs)) if built[i] is not None]
+        if not pending:
+            return results
+
+        conn = self._get_connection()
+        t0 = time.perf_counter()
+        broken = False
+        try:
+            conn.autocommit = True
+            while pending:
+                cursors: Dict[int, Any] = {}
+                pipeline_error: Optional[BaseException] = None
+                try:
+                    with conn.pipeline():
+                        for i in pending:
+                            sql, params = built[i]  # type: ignore[misc]
+                            conn.execute("ROLLBACK")  # no-op when idle; clears a transaction the previous failure left aborted
+                            conn.execute("BEGIN READ ONLY")
+                            for k, v in (specs[i].session_vars or {}).items():
+                                conn.execute("SELECT set_config(%s, %s, true)", [k, str(v)])
+                            cur = conn.cursor()
+                            cur.execute(sql, params)
+                            conn.execute("COMMIT")
+                            cursors[i] = cur
+                except Exception as e:  # the first failing statement; everything after it was skipped by the server
+                    pipeline_error = e
+
+                skipped: List[int] = []
+                failed_seen = False
+                for i in pending:
+                    if failed_seen:
+                        skipped.append(i)
+                        continue
+                    cur = cursors.get(i)
+                    try:
+                        if specs[i].builder is not None and specs[i].builder.count_only:
+                            results[i] = cur.fetchone()[0]  # type: ignore[union-attr]
+                        else:
+                            columns = [d[0] for d in cur.description]  # type: ignore[union-attr]
+                            results[i] = [self._deserialize_row(dict(zip(columns, r))) for r in cur.fetchall()]  # type: ignore[union-attr]
+                    except Exception as e:
+                        failed_seen = True
+                        cause = pipeline_error if pipeline_error is not None else e
+                        results[i] = DatabaseError(f"Select failed: {str(cause)}")
+                if not failed_seen and pipeline_error is not None:
+                    # the pipeline failed after every result was already in (e.g. COMMIT of the last spec): nothing is lost
+                    pass
+                pending = skipped
+        except Exception:
+            broken = True
+            raise
+        finally:
+            try:
+                if not conn.closed:
+                    try:
+                        conn.rollback()
+                    except Exception:
+                        pass
+                    conn.autocommit = False
+            except Exception:
+                broken = True
+            if broken:
+                self._pool.putconn(conn, close=True)  # type: ignore[union-attr]
+            else:
+                self._return_connection(conn)
+            self._log_batch(len(specs), (time.perf_counter() - t0) * 1000.0)
+        return results
+
+    def _log_batch(self, n: int, duration_ms: float) -> None:
+        self.logger.debug("SQL pipeline executed: statements=%d duration_ms=%.1f", n, duration_ms)
+        if duration_ms > self._slow_query_ms:
+            self.logger.warning(
+                "Slow query detected: operation=select_many statements=%d duration_ms=%.1f threshold=%.1f",
+                n, duration_ms, self._slow_query_ms,
+            )
+
     # ---------------------------------------------------------------------
     # SELECT PAGE
     # ---------------------------------------------------------------------
@@ -767,11 +1031,63 @@ class PostgreSQLAdapter:
     # LINQ QUERY
     # ---------------------------------------------------------------------
 
+    @staticmethod
+    def _build_linq(table: str, builder: QueryBuilder) -> Tuple[str, List[Any]]:
+        """The SQL text and bind parameters of a `query_linq()` -- a pure function, shared by the single and the pipelined path."""
+        if builder.count_only:
+            sql = f"SELECT COUNT(*) FROM {table}"
+        elif builder.selected_fields:
+            for f in builder.selected_fields:
+                validate_column_name(f)
+            fields = ", ".join(builder.selected_fields)
+            if builder.distinct_flag:
+                sql = f"SELECT DISTINCT {fields} FROM {table}"
+            else:
+                sql = f"SELECT {fields} FROM {table}"
+        else:
+            sql = f"SELECT * FROM {table}"
+
+        params: List[Any] = []
+        where_clause, where_params = builder.to_sql_where()
+        if where_clause:
+            sql += f" WHERE {where_clause}"
+            params.extend(where_params)
+
+        if builder.group_by_fields:
+            for f in builder.group_by_fields:
+                validate_column_name(f)
+            sql += f" GROUP BY {', '.join(builder.group_by_fields)}"
+
+        if builder.order_by_fields:
+            order_parts = []
+            for field, desc in builder.order_by_fields:
+                validate_column_name(field)
+                direction = "DESC" if desc else "ASC"
+                order_parts.append(f"{field} {direction}")
+            sql += f" ORDER BY {', '.join(order_parts)}"
+
+        if builder.take_count is not None:
+            sql += " LIMIT %s"
+            params.append(builder.take_count)
+
+        if builder.skip_count:
+            sql += " OFFSET %s"
+            params.append(builder.skip_count)
+        return sql, params
+
     @retry(max_attempts=3, delay=1.0, exceptions=(DatabaseError,))
     def query_linq(
-        self, table: str, builder: QueryBuilder, tx: Optional[Any] = None
+        self,
+        table: str,
+        builder: QueryBuilder,
+        tx: Optional[Any] = None,
+        *,
+        session_vars: Optional[Dict[str, str]] = None,
     ) -> Union[List[JsonDict], int]:
         table = validate_table_name(table)
+        batch = current_batch() if tx is None else None
+        if batch is not None:
+            return batch.submit(self, SelectSpec(table=table, builder=builder, session_vars=session_vars))
         conn = tx
         own_conn = False
         if not conn:
@@ -779,46 +1095,10 @@ class PostgreSQLAdapter:
             own_conn = True
 
         try:
+            if own_conn:
+                self._apply_session_vars(conn, session_vars)
             cursor = conn.cursor()
-            if builder.count_only:
-                sql = f"SELECT COUNT(*) FROM {table}"
-            elif builder.selected_fields:
-                for f in builder.selected_fields:
-                    validate_column_name(f)
-                fields = ", ".join(builder.selected_fields)
-                if builder.distinct_flag:
-                    sql = f"SELECT DISTINCT {fields} FROM {table}"
-                else:
-                    sql = f"SELECT {fields} FROM {table}"
-            else:
-                sql = f"SELECT * FROM {table}"
-
-            params: List[Any] = []
-            where_clause, where_params = builder.to_sql_where()
-            if where_clause:
-                sql += f" WHERE {where_clause}"
-                params.extend(where_params)
-
-            if builder.group_by_fields:
-                for f in builder.group_by_fields:
-                    validate_column_name(f)
-                sql += f" GROUP BY {', '.join(builder.group_by_fields)}"
-
-            if builder.order_by_fields:
-                order_parts = []
-                for field, desc in builder.order_by_fields:
-                    validate_column_name(field)
-                    direction = "DESC" if desc else "ASC"
-                    order_parts.append(f"{field} {direction}")
-                sql += f" ORDER BY {', '.join(order_parts)}"
-
-            if builder.take_count is not None:
-                sql += " LIMIT %s"
-                params.append(builder.take_count)
-
-            if builder.skip_count:
-                sql += " OFFSET %s"
-                params.append(builder.skip_count)
+            sql, params = self._build_linq(table, builder)
 
             self._timed_execute(
                 cursor, sql, self._serialize_params(params), operation="query_linq", table=table

polydb/databaseFactory.py

@@ -425,6 +425,20 @@ class DatabaseFactory:
     # CREATE
     # ──────────────────────────────────────────────────────────
 
+    def _write_behind_queue(self):
+        wb = getattr(self, "_wb", None)
+        if wb is None:
+            from .write_behind import WriteBehind
+
+            wb = self._wb = WriteBehind()
+        return wb
+
+    def flush_background_writes(self, timeout: float = 10.0) -> None:
+        """Block until every `create(..., background=True)` row has been written. A no-op when none were queued."""
+        wb = getattr(self, "_wb", None)
+        if wb is not None:
+            wb.flush(timeout)
+
     def create(
         self,
         model: Union[type, str],
@@ -432,10 +446,19 @@ class DatabaseFactory:
         *,
         engine_override: Optional[EngineOverride] = None,
         session_vars: Optional[Dict[str, str]] = None,
+        background: bool = False,
     ) -> JsonDict:
+        """`background=True` is for telemetry-style rows nobody reads back in the same request (audit / event / query log): the row is
+        queued and written by one background thread in batches -- see write_behind.py. It returns the row as it will be stored (not a
+        database round trip), and applies only to a SQL model; anything else is written inline as usual."""
         meta = _extract_meta(model)
         name = _model_name(model)
         data = self._inject_audit_fields(data, is_create=True)
+        if background and self._is_sql(meta, engine_override) and not getattr(meta, "encrypted_fields", None):
+            self._write_behind_queue().submit(
+                self._adapters_for(model, meta, engine_override).sql, meta.table, data, session_vars
+            )
+            return data
 
         encrypted_fields = getattr(meta, "encrypted_fields", [])
         if self.encryption and encrypted_fields:

polydb/read_batch.py

@@ -0,0 +1,134 @@
+# src/polydb/read_batch.py
+"""Coalesce the SQL reads several threads make at the same moment into ONE pipelined round trip on ONE connection.
+
+The problem this solves: a request that needs N independent reads used to pay N network round trips in a row (each query waits
+for the previous answer), or -- when run on N threads -- held N Postgres connections at once and could drain the connection pool
+or the server's connection slots. Postgres runs one statement at a time per connection, but with psycopg 3 *pipeline mode* a
+client can send many statements without waiting for each answer, so N reads cost about one round trip and one connection.
+
+How it plugs in: a caller (the engine's `coll.parallel`) creates a `ReadBatch` for the tasks it is about to run and makes it the
+current batch for those threads. `PostgreSQLAdapter.select()` -- the single place every SQL read goes through -- checks for a
+current batch and, if there is one, hands its query to the batch instead of executing it, then waits for its own rows. Everything
+above `select()` (permission checks, tenant scoping, audit, caches, decryption) runs untouched on each task's own thread; only the
+final "send this SELECT to the database" step is shared.
+
+A batch flushes as soon as every still-running task is waiting on it, or after `window_s` (so a task busy with non-database work
+cannot hold the others up). Each query still runs in its own read-only transaction with its own `session_vars`
+(`set_config(..., true)`), so row-level security sees exactly the tenant it would have seen on a connection of its own.
+"""
+from __future__ import annotations
+
+import contextvars
+import os
+import threading
+from contextlib import contextmanager
+from dataclasses import dataclass
+from typing import Any, Dict, Iterator, List, Optional
+
+
+@dataclass
+class SelectSpec:
+    """One SELECT to run, already validated by `PostgreSQLAdapter.select()`."""
+
+    table: str
+    query: Optional[Dict[str, Any]] = None
+    limit: Optional[int] = None
+    offset: Optional[int] = None
+    fields: Optional[List[str]] = None
+    session_vars: Optional[Dict[str, str]] = None
+    # A `QueryBuilder` (LINQ-style query: projection, ORDER BY, operators, count). When set, `query`/`limit`/`offset`/`fields` are
+    # not used; the result is the row list, or the integer for a count-only builder.
+    builder: Optional[Any] = None
+
+
+class _Pending:
+    __slots__ = ("adapter", "spec", "event", "rows", "error")
+
+    def __init__(self, adapter: Any, spec: SelectSpec):
+        self.adapter = adapter
+        self.spec = spec
+        self.event = threading.Event()
+        self.rows: Any = None
+        self.error: Optional[BaseException] = None
+
+
+_current: contextvars.ContextVar[Optional["ReadBatch"]] = contextvars.ContextVar("polydb_read_batch", default=None)
+
+
+def current_batch() -> Optional["ReadBatch"]:
+    return _current.get()
+
+
+@contextmanager
+def use_read_batch(batch: Optional["ReadBatch"]) -> Iterator[None]:
+    """Make `batch` the current batch for this context (and for contexts copied from it afterwards)."""
+    token = _current.set(batch)
+    try:
+        yield
+    finally:
+        _current.reset(token)
+
+
+class ReadBatch:
+    """Collects SELECTs submitted by up to `parties` threads and runs them together.
+
+    Each party must call `task_done()` exactly once when it has finished (success or failure), so the batch knows nobody else is
+    coming and can flush what is waiting."""
+
+    def __init__(self, parties: int, window_s: Optional[float] = None):
+        self._active = max(0, int(parties))
+        self._window = (
+            float(window_s) if window_s is not None else float(os.getenv("POLYDB_READ_BATCH_WINDOW_MS", "5")) / 1000.0
+        )
+        self._lock = threading.Lock()
+        self._queue: List[_Pending] = []
+        self.flushes = 0          # how many pipelined round trips were made (for tests / diagnostics)
+        self.statements = 0       # how many SELECTs went through the batch
+
+    # -- called from a task's thread, inside PostgreSQLAdapter.select() ------------------------------------------------------
+    def submit(self, adapter: Any, spec: SelectSpec) -> Any:
+        item = _Pending(adapter, spec)
+        with self._lock:
+            self._queue.append(item)
+            everyone_is_waiting = len(self._queue) >= self._active
+        if everyone_is_waiting:
+            self._flush()
+        elif not item.event.wait(self._window):
+            self._flush()  # the window elapsed: send whatever has queued up
+        item.event.wait()  # another thread may be executing our query right now
+        if item.error is not None:
+            raise item.error
+        return item.rows if item.rows is not None else []
+
+    def task_done(self) -> None:
+        with self._lock:
+            self._active -= 1
+            waiting = bool(self._queue) and len(self._queue) >= self._active
+        if waiting:
+            self._flush()
+
+    # -- internals ------------------------------------------------------------------------------------------------------------
+    def _flush(self) -> None:
+        with self._lock:
+            batch, self._queue = self._queue, []
+        if not batch:
+            return
+        groups: Dict[int, List[_Pending]] = {}
+        for it in batch:
+            groups.setdefault(id(it.adapter), []).append(it)
+        try:
+            for items in groups.values():
+                try:
+                    outcomes = items[0].adapter.select_many([i.spec for i in items])
+                except BaseException as exc:  # noqa: BLE001 -- every waiter must be released
+                    outcomes = [exc] * len(items)
+                self.flushes += 1
+                self.statements += len(items)
+                for it, out in zip(items, outcomes):
+                    if isinstance(out, BaseException):
+                        it.error = out
+                    else:
+                        it.rows = out
+        finally:
+            for it in batch:
+                it.event.set()

polydb/write_behind.py

@@ -0,0 +1,110 @@
+# src/polydb/write_behind.py
+"""Write-behind queue: telemetry-style inserts (audit log, event log, query log) written by ONE background thread, in batches.
+
+Why: the engine writes an audit row, an event row and a query-log row for every action it runs. Written inline, each of those is its
+own transaction on a pooled connection -- two network round trips apiece, taken on whatever thread is running the action. A request
+that runs 40 steps paid ~120 extra transactions before it could answer, and a `coll.parallel` group did it from several threads at
+once, each holding its own connection. Here the caller only appends the row to an in-memory queue and moves on; one writer thread
+groups what has queued up per table and sends each group as ONE pipelined transaction on ONE connection
+(`PostgreSQLAdapter.insert_many`).
+
+Trade-off, stated plainly: a row still queued when the process is killed hard (SIGKILL / OOM) is lost; a normal exit flushes
+(`atexit`). A full queue falls back to writing inline, so nothing is dropped under load. Only for rows nobody reads back in the
+same request -- never for data a caller depends on.
+"""
+from __future__ import annotations
+
+import atexit
+import logging
+import os
+import queue
+import threading
+import time
+from typing import Any, Dict, List, Optional, Tuple
+
+logger = logging.getLogger(__name__)
+
+_QUEUE_MAX = int(os.getenv("POLYDB_WRITE_BEHIND_QUEUE_MAX", "20000"))
+_BATCH_MAX = int(os.getenv("POLYDB_WRITE_BEHIND_BATCH_MAX", "200"))
+_LINGER_S = float(os.getenv("POLYDB_WRITE_BEHIND_LINGER_MS", "25")) / 1000.0
+
+_Item = Tuple[Any, str, Dict[str, Any], Optional[Dict[str, str]]]  # (sql adapter, table, row, session_vars)
+
+
+class WriteBehind:
+    def __init__(self) -> None:
+        self._queue: "queue.Queue[_Item]" = queue.Queue(maxsize=_QUEUE_MAX)
+        self._thread: Optional[threading.Thread] = None
+        self._start_lock = threading.Lock()
+        self.batches = 0     # pipelined transactions written (diagnostics / tests)
+        self.rows = 0        # rows written through the queue
+        atexit.register(self.flush)
+
+    # ------------------------------------------------------------------------------------------------------------------
+    def submit(self, adapter: Any, table: str, row: Dict[str, Any], session_vars: Optional[Dict[str, str]] = None) -> None:
+        self._ensure_thread()
+        try:
+            self._queue.put_nowait((adapter, table, row, session_vars))
+        except queue.Full:
+            # Never drop a row: when the queue is full the caller writes it inline, exactly as it would without write-behind.
+            adapter.insert(table, row, session_vars=session_vars)
+
+    def flush(self, timeout: float = 10.0) -> None:
+        """Block until every queued row has been written (or `timeout` seconds pass)."""
+        if self._thread is None:
+            return
+        deadline = time.monotonic() + timeout
+        while (not self._queue.empty() or self._queue.unfinished_tasks) and time.monotonic() < deadline:
+            time.sleep(0.005)
+
+    # ------------------------------------------------------------------------------------------------------------------
+    def _ensure_thread(self) -> None:
+        if self._thread is not None and self._thread.is_alive():
+            return
+        with self._start_lock:
+            if self._thread is None or not self._thread.is_alive():
+                self._thread = threading.Thread(target=self._run, name="polydb-write-behind", daemon=True)
+                self._thread.start()
+
+    def _run(self) -> None:
+        while True:
+            first = self._queue.get()
+            batch: List[_Item] = [first]
+            deadline = time.monotonic() + _LINGER_S
+            while len(batch) < _BATCH_MAX:
+                remaining = deadline - time.monotonic()
+                if remaining <= 0:
+                    break
+                try:
+                    batch.append(self._queue.get(timeout=remaining))
+                except queue.Empty:
+                    break
+            try:
+                self._write(batch)
+            except Exception:  # noqa: BLE001 -- best-effort, the writer thread must never die
+                logger.warning("write-behind: batch write failed", exc_info=True)
+            finally:
+                for _ in batch:
+                    self._queue.task_done()
+
+    def _write(self, batch: List[_Item]) -> None:
+        groups: Dict[Tuple[int, str, Tuple[Tuple[str, str], ...]], List[_Item]] = {}
+        for item in batch:
+            adapter, table, _row, session_vars = item
+            key = (id(adapter), table, tuple(sorted((session_vars or {}).items())))
+            groups.setdefault(key, []).append(item)
+        for items in groups.values():
+            adapter, table, _row, session_vars = items[0]
+            rows = [i[2] for i in items]
+            try:
+                adapter.insert_many(table, rows, session_vars=session_vars)
+                self.batches += 1
+                self.rows += len(rows)
+            except Exception:  # noqa: BLE001 -- one bad row must not cost the rest: retry them one by one
+                logger.warning("write-behind: pipelined insert into %s failed; retrying row by row", table, exc_info=True)
+                for r in rows:
+                    try:
+                        adapter.insert(table, r, session_vars=session_vars)
+                        self.rows += 1
+                    except Exception:  # noqa: BLE001
+                        logger.warning("write-behind: dropped one row for %s", table, exc_info=True)