eventsourcing 9.2.22__py3-none-any.whl → 9.3.0__py3-none-any.whl

eventsourcing/postgres.py

@@ -1,35 +1,19 @@
 from __future__ import annotations
 
+import logging
 from contextlib import contextmanager
-from itertools import chain
-from threading import Lock
-from types import TracebackType
-from typing import (
-    Any,
-    Dict,
-    Iterator,
-    List,
-    Optional,
-    Sequence,
-    Set,
-    Tuple,
-    Type,
-    Union,
-)
-from uuid import NAMESPACE_URL, UUID, uuid5
+from typing import TYPE_CHECKING, Any, Callable, Iterator, List, Sequence
 
-import psycopg2
-import psycopg2.errors
-import psycopg2.extras
-from psycopg2.errorcodes import DUPLICATE_PREPARED_STATEMENT
-from psycopg2.extensions import connection, cursor
+import psycopg
+import psycopg.errors
+import psycopg_pool
+from psycopg import Connection, Cursor
+from psycopg.rows import DictRow, dict_row
+from typing_extensions import Self
 
 from eventsourcing.persistence import (
     AggregateRecorder,
     ApplicationRecorder,
-    Connection,
-    ConnectionPool,
-    Cursor,
     DatabaseError,
     DataError,
     InfrastructureFactory,
@@ -45,74 +29,32 @@ from eventsourcing.persistence import (
     StoredEvent,
     Tracking,
 )
-from eventsourcing.utils import Environment, retry, strtobool
-
-psycopg2.extras.register_uuid()
-
-
-class PostgresCursor(Cursor):
-    def __init__(self, pg_cursor: cursor):
-        self.pg_cursor = pg_cursor
-
-    def __enter__(self, *args: Any, **kwargs: Any) -> "PostgresCursor":
-        self.pg_cursor.__enter__(*args, **kwargs)
-        return self
-
-    def __exit__(self, *args: Any, **kwargs: Any) -> None:
-        return self.pg_cursor.__exit__(*args, **kwargs)
-
-    def mogrify(self, statement: str, params: Any = None) -> bytes:
-        return self.pg_cursor.mogrify(statement, vars=params)
-
-    def execute(self, statement: Union[str, bytes], params: Any = None) -> None:
-        self.pg_cursor.execute(query=statement, vars=params)
-
-    def fetchall(self) -> Any:
-        return self.pg_cursor.fetchall()
-
-    def fetchone(self) -> Any:
-        return self.pg_cursor.fetchone()
+from eventsourcing.utils import Environment, resolve_topic, retry, strtobool
 
-    @property
-    def closed(self) -> bool:
-        return self.pg_cursor.closed
+if TYPE_CHECKING:  # pragma: nocover
+    from uuid import UUID
 
+logging.getLogger("psycopg.pool").setLevel(logging.CRITICAL)
+logging.getLogger("psycopg").setLevel(logging.CRITICAL)
 
-class PostgresConnection(Connection[PostgresCursor]):
-    def __init__(self, pg_conn: connection, max_age: Optional[float]):
-        super().__init__(max_age=max_age)
-        self._pg_conn = pg_conn
-        self.is_prepared: Set[str] = set()
 
-    @contextmanager
-    def transaction(self, commit: bool) -> Iterator[PostgresCursor]:
-        # Context managed transaction.
-        with PostgresTransaction(self, commit) as curs:
-            # Context managed cursor.
-            with curs:
-                yield curs
-
-    def cursor(self) -> PostgresCursor:
-        return PostgresCursor(
-            self._pg_conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
-        )
-
-    def rollback(self) -> None:
-        self._pg_conn.rollback()
-
-    def commit(self) -> None:
-        self._pg_conn.commit()
-
-    def _close(self) -> None:
-        self._pg_conn.close()
-        super()._close()
+class ConnectionPool(psycopg_pool.ConnectionPool[Any]):
+    def __init__(
+        self,
+        *args: Any,
+        get_password_func: Callable[[], str] | None = None,
+        **kwargs: Any,
+    ) -> None:
+        self.get_password_func = get_password_func
+        super().__init__(*args, **kwargs)
 
-    @property
-    def closed(self) -> bool:
-        return bool(self._pg_conn.closed)
+    def _connect(self, timeout: float | None = None) -> Connection[Any]:
+        if self.get_password_func:
+            self.kwargs["password"] = self.get_password_func()
+        return super()._connect(timeout=timeout)
 
 
-class PostgresConnectionPool(ConnectionPool[PostgresConnection]):
+class PostgresDatastore:
     def __init__(
         self,
         dbname: str,
@@ -120,152 +62,93 @@ class PostgresConnectionPool(ConnectionPool[PostgresConnection]):
         port: str,
         user: str,
         password: str,
-        connect_timeout: int = 5,
+        *,
+        connect_timeout: int = 30,
         idle_in_transaction_session_timeout: int = 0,
-        pool_size: int = 1,
-        max_overflow: int = 0,
-        pool_timeout: float = 5.0,
-        max_age: Optional[float] = None,
+        pool_size: int = 2,
+        max_overflow: int = 2,
+        max_waiting: int = 0,
+        conn_max_age: float = 60 * 60.0,
         pre_ping: bool = False,
+        lock_timeout: int = 0,
+        schema: str = "",
+        pool_open_timeout: int | None = None,
+        get_password_func: Callable[[], str] | None = None,
     ):
-        self.dbname = dbname
-        self.host = host
-        self.port = port
-        self.user = user
-        self.password = password
-        self.connect_timeout = connect_timeout
         self.idle_in_transaction_session_timeout = idle_in_transaction_session_timeout
-        super().__init__(
-            pool_size=pool_size,
-            max_overflow=max_overflow,
-            pool_timeout=pool_timeout,
-            max_age=max_age,
-            pre_ping=pre_ping,
-            mutually_exclusive_read_write=False,
+        self.pre_ping = pre_ping
+        self.pool_open_timeout = pool_open_timeout
+
+        check = ConnectionPool.check_connection if pre_ping else None
+        self.pool = ConnectionPool(
+            get_password_func=get_password_func,
+            connection_class=Connection[DictRow],
+            kwargs={
+                "dbname": dbname,
+                "host": host,
+                "port": port,
+                "user": user,
+                "password": password,
+                "row_factory": dict_row,
+            },
+            min_size=pool_size,
+            max_size=pool_size + max_overflow,
+            open=False,
+            configure=self.after_connect,
+            timeout=connect_timeout,
+            max_waiting=max_waiting,
+            max_lifetime=conn_max_age,
+            check=check,
         )
+        self.lock_timeout = lock_timeout
+        self.schema = schema.strip()
 
-    def _create_connection(self) -> PostgresConnection:
-        # Make a connection to a database.
-        try:
-            pg_conn = psycopg2.connect(
-                dbname=self.dbname,
-                host=self.host,
-                port=self.port,
-                user=self.user,
-                password=self.password,
-                connect_timeout=self.connect_timeout,
-            )
-        except psycopg2.OperationalError as e:
-            raise OperationalError(e) from e
-        pg_conn.cursor().execute(
-            f"SET idle_in_transaction_session_timeout = "
+    def after_connect(self, conn: Connection[DictRow]) -> None:
+        conn.autocommit = True
+        conn.cursor().execute(
+            "SET idle_in_transaction_session_timeout = "
             f"'{self.idle_in_transaction_session_timeout}s'"
         )
-        return PostgresConnection(pg_conn, max_age=self.max_age)
-
-
-class PostgresTransaction:
-    def __init__(self, conn: PostgresConnection, commit: bool):
-        self.conn = conn
-        self.commit = commit
-        self.has_entered = False
 
-    def __enter__(self) -> PostgresCursor:
-        self.has_entered = True
-        return self.conn.cursor()
-
-    def __exit__(
-        self,
-        exc_type: Type[BaseException],
-        exc_val: BaseException,
-        exc_tb: TracebackType,
-    ) -> None:
+    @contextmanager
+    def get_connection(self) -> Iterator[Connection[DictRow]]:
         try:
-            if exc_val:
-                self.conn.rollback()
-                raise exc_val
-            elif not self.commit:
-                self.conn.rollback()
-            else:
-                self.conn.commit()
-        except psycopg2.InterfaceError as e:
-            self.conn.close()
+            wait = self.pool_open_timeout is not None
+            timeout = self.pool_open_timeout or 30.0
+            self.pool.open(wait, timeout)
+
+            with self.pool.connection() as conn:
+                yield conn
+        except psycopg.InterfaceError as e:
+            # conn.close()
             raise InterfaceError(str(e)) from e
-        except psycopg2.DataError as e:
-            raise DataError(str(e)) from e
-        except psycopg2.OperationalError as e:
-            self.conn.close()
+        except psycopg.OperationalError as e:
+            # conn.close()
             raise OperationalError(str(e)) from e
-        except psycopg2.IntegrityError as e:
+        except psycopg.DataError as e:
+            raise DataError(str(e)) from e
+        except psycopg.IntegrityError as e:
             raise IntegrityError(str(e)) from e
-        except psycopg2.InternalError as e:
+        except psycopg.InternalError as e:
             raise InternalError(str(e)) from e
-        except psycopg2.ProgrammingError as e:
+        except psycopg.ProgrammingError as e:
             raise ProgrammingError(str(e)) from e
-        except psycopg2.NotSupportedError as e:
+        except psycopg.NotSupportedError as e:
             raise NotSupportedError(str(e)) from e
-        except psycopg2.DatabaseError as e:
+        except psycopg.DatabaseError as e:
             raise DatabaseError(str(e)) from e
-        except psycopg2.Error as e:
+        except psycopg.Error as e:
+            # conn.close()
             raise PersistenceError(str(e)) from e
-
-
-class PostgresDatastore:
-    def __init__(
-        self,
-        dbname: str,
-        host: str,
-        port: str,
-        user: str,
-        password: str,
-        connect_timeout: int = 5,
-        idle_in_transaction_session_timeout: int = 0,
-        pool_size: int = 2,
-        max_overflow: int = 2,
-        pool_timeout: float = 5.0,
-        conn_max_age: Optional[float] = None,
-        pre_ping: bool = False,
-        lock_timeout: int = 0,
-        schema: str = "",
-    ):
-        self.pool = PostgresConnectionPool(
-            dbname=dbname,
-            host=host,
-            port=port,
-            user=user,
-            password=password,
-            connect_timeout=connect_timeout,
-            idle_in_transaction_session_timeout=idle_in_transaction_session_timeout,
-            pool_size=pool_size,
-            max_overflow=max_overflow,
-            pool_timeout=pool_timeout,
-            max_age=conn_max_age,
-            pre_ping=pre_ping,
-        )
-        self.lock_timeout = lock_timeout
-        self.schema = schema.strip()
-
-    @contextmanager
-    def transaction(self, commit: bool) -> Iterator[PostgresCursor]:
-        with self.get_connection() as conn:
-            with conn.transaction(commit) as curs:
-                yield curs
+        except Exception:
+            # conn.close()
+            raise
 
     @contextmanager
-    def get_connection(self) -> Iterator[PostgresConnection]:
-        conn = self.pool.get_connection()
-        try:
-            yield conn
-        finally:
-            self.pool.put_connection(conn)
-
-    def report_on_prepared_statements(
-        self,
-    ) -> Tuple[List[List[Union[bool, str]]], List[str]]:
-        with self.get_connection() as conn:
-            with conn.cursor() as curs:
-                curs.execute("SELECT * from pg_prepared_statements")
-                return sorted(curs.fetchall()), sorted(conn.is_prepared)
+    def transaction(self, *, commit: bool = False) -> Iterator[Cursor[DictRow]]:
+        conn: Connection[DictRow]
+        with self.get_connection() as conn, conn.transaction(force_rollback=not commit):
+            yield conn.cursor()
 
     def close(self) -> None:
         self.pool.close()
@@ -273,8 +156,11 @@ class PostgresDatastore:
     def __del__(self) -> None:
         self.close()
 
+    def __enter__(self) -> Self:
+        return self
 
-PG_IDENTIFIER_MAX_LEN = 63
+    def __exit__(self, *args: object, **kwargs: Any) -> None:
+        self.close()
 
 
 class PostgresAggregateRecorder(AggregateRecorder):
@@ -283,8 +169,6 @@ class PostgresAggregateRecorder(AggregateRecorder):
         datastore: PostgresDatastore,
         events_table_name: str,
     ):
-        self.statement_name_aliases: Dict[str, str] = {}
-        self.statement_name_aliases_lock = Lock()
         self.check_table_name_length(events_table_name, datastore.schema)
         self.datastore = datastore
         self.events_table_name = events_table_name
@@ -300,15 +184,12 @@ class PostgresAggregateRecorder(AggregateRecorder):
 
         self.create_table_statements = self.construct_create_table_statements()
         self.insert_events_statement = (
-            f"INSERT INTO {self.events_table_name} VALUES ($1, $2, $3, $4)"
-        )
-        self.insert_events_statement_name = f"insert_{events_table_name}".replace(
-            ".", "_"
+            f"INSERT INTO {self.events_table_name} VALUES (%s, %s, %s, %s)"
         )
         self.select_events_statement = (
-            f"SELECT * FROM {self.events_table_name} WHERE originator_id = $1"
+            f"SELECT * FROM {self.events_table_name} WHERE originator_id = %s"
         )
-        self.lock_statements: List[str] = []
+        self.lock_table_statements: List[str] = []
 
     @staticmethod
     def check_table_name_length(table_name: str, schema_name: str) -> None:
@@ -318,47 +199,8 @@ class PostgresAggregateRecorder(AggregateRecorder):
         else:
             unqualified_table_name = table_name
         if len(unqualified_table_name) > 63:
-            raise ProgrammingError(f"Table name too long: {unqualified_table_name}")
-
-    def get_statement_alias(self, statement_name: str) -> str:
-        try:
-            alias = self.statement_name_aliases[statement_name]
-        except KeyError:
-            with self.statement_name_aliases_lock:
-                try:
-                    alias = self.statement_name_aliases[statement_name]
-                except KeyError:
-                    existing_aliases = self.statement_name_aliases.values()
-                    if (
-                        len(statement_name) <= PG_IDENTIFIER_MAX_LEN
-                        and statement_name not in existing_aliases
-                    ):
-                        alias = statement_name
-                        self.statement_name_aliases[statement_name] = alias
-                    else:
-                        uid = uuid5(
-                            NAMESPACE_URL, f"/statement_names/{statement_name}"
-                        ).hex
-                        alias = uid
-                        for i in range(len(uid)):  # pragma: no cover
-                            preserve_end = 21
-                            preserve_start = (
-                                PG_IDENTIFIER_MAX_LEN - preserve_end - i - 2
-                            )
-                            uuid5_tail = i
-                            candidate = (
-                                statement_name[:preserve_start]
-                                + "_"
-                                + (uid[-uuid5_tail:] if i else "")
-                                + "_"
-                                + statement_name[-preserve_end:]
-                            )
-                            assert len(alias) <= PG_IDENTIFIER_MAX_LEN
-                            if candidate not in existing_aliases:
-                                alias = candidate
-                                break
-                        self.statement_name_aliases[statement_name] = alias
-        return alias
+            msg = f"Table name too long: {unqualified_table_name}"
+            raise ProgrammingError(msg)
 
     def construct_create_table_statements(self) -> List[str]:
         statement = (
@@ -377,155 +219,121 @@ class PostgresAggregateRecorder(AggregateRecorder):
     def create_table(self) -> None:
         with self.datastore.transaction(commit=True) as curs:
             for statement in self.create_table_statements:
-                curs.execute(statement)
-            pass  # for Coverage 5.5 bug with CPython 3.10.0rc1
+                curs.execute(statement, prepare=False)
 
     @retry((InterfaceError, OperationalError), max_attempts=10, wait=0.2)
     def insert_events(
         self, stored_events: List[StoredEvent], **kwargs: Any
-    ) -> Optional[Sequence[int]]:
+    ) -> Sequence[int] | None:
+        conn: Connection[DictRow]
+        exc: Exception | None = None
+        notification_ids: Sequence[int] | None = None
         with self.datastore.get_connection() as conn:
-            self._prepare_insert_events(conn)
-            with conn.transaction(commit=True) as curs:
-                return self._insert_events(curs, stored_events, **kwargs)
-
-    def _prepare_insert_events(self, conn: PostgresConnection) -> None:
-        self._prepare(
-            conn,
-            self.insert_events_statement_name,
-            self.insert_events_statement,
-        )
-
-    def _prepare(
-        self, conn: PostgresConnection, statement_name: str, statement: str
-    ) -> str:
-        statement_name_alias = self.get_statement_alias(statement_name)
-        if statement_name not in conn.is_prepared:
-            curs: PostgresCursor
-            with conn.transaction(commit=True) as curs:
-                try:
-                    lock_timeout = self.datastore.lock_timeout
-                    curs.execute(f"SET LOCAL lock_timeout = '{lock_timeout}s'")
-                    curs.execute(f"PREPARE {statement_name_alias} AS " + statement)
-                except psycopg2.errors.lookup(DUPLICATE_PREPARED_STATEMENT):  # noqa
-                    pass
-                conn.is_prepared.add(statement_name)
-        return statement_name_alias
+            with conn.pipeline() as pipeline, conn.transaction():
+                # Do other things first, so they can be pipelined too.
+                with conn.cursor() as curs:
+                    self._insert_events(curs, stored_events, **kwargs)
+                # Then use a different cursor for the executemany() call.
+                with conn.cursor() as curs:
+                    try:
+                        self._insert_stored_events(curs, stored_events, **kwargs)
+                        # Sync now, so any uniqueness constraint violation causes an
+                        # IntegrityError to be raised here, rather an InternalError
+                        # being raised sometime later e.g. when commit() is called.
+                        pipeline.sync()
+                        notification_ids = self._fetch_ids_after_insert_events(
+                            curs, stored_events, **kwargs
+                        )
+                    except Exception as e:
+                        # Avoid psycopg emitting a pipeline warning.
+                        exc = e
+            if exc:
+                # Reraise exception after pipeline context manager has exited.
+                raise exc
+        return notification_ids
 
     def _insert_events(
         self,
-        c: PostgresCursor,
+        c: Cursor[DictRow],
         stored_events: List[StoredEvent],
         **kwargs: Any,
-    ) -> Optional[Sequence[int]]:
-        # Acquire "EXCLUSIVE" table lock, to serialize inserts so that
-        # insertion of notification IDs is monotonic for notification log
-        # readers. We want concurrent transactions to commit inserted
-        # notification_id values in order, and by locking the table for writes,
-        # it can be guaranteed. The EXCLUSIVE lock mode does not block
-        # the ACCESS SHARE lock which is acquired during SELECT statements,
-        # so the table can be read concurrently. However, INSERT normally
-        # just acquires ROW EXCLUSIVE locks, which risks interleaving of
-        # many inserts in one transaction with many insert in another
-        # transaction. Since one transaction will commit before another,
-        # the possibility arises for readers that are tailing a notification
-        # log to miss items inserted later but with lower notification IDs.
-        # https://www.postgresql.org/docs/current/explicit-locking.html#LOCKING-TABLES
-        # https://www.postgresql.org/docs/9.1/sql-lock.html
-        # https://stackoverflow.com/questions/45866187/guarantee-monotonicity-of
-        # -postgresql-serial-column-values-by-commit-order
-
-        len_stored_events = len(stored_events)
+    ) -> None:
+        pass
 
+    def _insert_stored_events(
+        self,
+        c: Cursor[DictRow],
+        stored_events: List[StoredEvent],
+        **_: Any,
+    ) -> None:
         # Only do something if there is something to do.
-        if len_stored_events > 0:
-            # Mogrify the table lock statements.
-            lock_sqls = (c.mogrify(s) for s in self.lock_statements)
-
-            # Prepare the commands before getting the table lock.
-            alias = self.statement_name_aliases[self.insert_events_statement_name]
-            page_size = 500
-            pages = [
-                (
-                    c.mogrify(
-                        f"EXECUTE {alias}(%s, %s, %s, %s)",
-                        (
-                            stored_event.originator_id,
-                            stored_event.originator_version,
-                            stored_event.topic,
-                            stored_event.state,
-                        ),
+        if len(stored_events) > 0:
+            self._lock_table(c)
+
+            # Insert events.
+            c.executemany(
+                query=self.insert_events_statement,
+                params_seq=[
+                    (
+                        stored_event.originator_id,
+                        stored_event.originator_version,
+                        stored_event.topic,
+                        stored_event.state,
                     )
-                    for stored_event in page
-                )
-                for page in (
-                    stored_events[ndx : min(ndx + page_size, len_stored_events)]
-                    for ndx in range(0, len_stored_events, page_size)
-                )
-            ]
-            commands = [
-                b"; ".join(page)
-                for page in chain([chain(lock_sqls, pages[0])], pages[1:])
-            ]
+                    for stored_event in stored_events
+                ],
+                returning="RETURNING" in self.insert_events_statement,
+            )
 
-            # Execute the commands.
-            for command in commands:
-                c.execute(command)
+    def _lock_table(self, c: Cursor[DictRow]) -> None:
+        pass
+
+    def _fetch_ids_after_insert_events(
+        self,
+        c: Cursor[DictRow],
+        stored_events: List[StoredEvent],
+        **kwargs: Any,
+    ) -> Sequence[int] | None:
         return None
 
     @retry((InterfaceError, OperationalError), max_attempts=10, wait=0.2)
     def select_events(
         self,
         originator_id: UUID,
-        gt: Optional[int] = None,
-        lte: Optional[int] = None,
+        *,
+        gt: int | None = None,
+        lte: int | None = None,
         desc: bool = False,
-        limit: Optional[int] = None,
+        limit: int | None = None,
     ) -> List[StoredEvent]:
-        parts = [self.select_events_statement]
+        statement = self.select_events_statement
         params: List[Any] = [originator_id]
-        statement_name = f"select_{self.events_table_name}".replace(".", "_")
         if gt is not None:
             params.append(gt)
-            parts.append(f"AND originator_version > ${len(params)}")
-            statement_name += "_gt"
+            statement += " AND originator_version > %s"
         if lte is not None:
             params.append(lte)
-            parts.append(f"AND originator_version <= ${len(params)}")
-            statement_name += "_lte"
-        parts.append("ORDER BY originator_version")
+            statement += " AND originator_version <= %s"
+        statement += " ORDER BY originator_version"
         if desc is False:
-            parts.append("ASC")
+            statement += " ASC"
         else:
-            parts.append("DESC")
-            statement_name += "_desc"
+            statement += " DESC"
         if limit is not None:
             params.append(limit)
-            parts.append(f"LIMIT ${len(params)}")
-            statement_name += "_limit"
-        statement = " ".join(parts)
-
-        stored_events = []
-
-        with self.datastore.get_connection() as conn:
-            alias = self._prepare(conn, statement_name, statement)
-
-            with conn.transaction(commit=False) as curs:
-                curs.execute(
-                    f"EXECUTE {alias}({', '.join(['%s' for _ in params])})",
-                    params,
+            statement += " LIMIT %s"
+
+        with self.datastore.get_connection() as conn, conn.cursor() as curs:
+            curs.execute(statement, params, prepare=True)
+            return [
+                StoredEvent(
+                    originator_id=row["originator_id"],
+                    originator_version=row["originator_version"],
+                    topic=row["topic"],
+                    state=bytes(row["state"]),
                 )
-                for row in curs.fetchall():
-                    stored_events.append(
-                        StoredEvent(
-                            originator_id=row["originator_id"],
-                            originator_version=row["originator_version"],
-                            topic=row["topic"],
-                            state=bytes(row["state"]),
-                        )
-                    )
-                pass  # for Coverage 5.5 bug with CPython 3.10.0rc1
-            return stored_events
+                for row in curs.fetchall()
+            ]
 
 
 class PostgresApplicationRecorder(PostgresAggregateRecorder, ApplicationRecorder):
@@ -535,45 +343,42 @@ class PostgresApplicationRecorder(PostgresAggregateRecorder, ApplicationRecorder
         events_table_name: str = "stored_events",
     ):
         super().__init__(datastore, events_table_name)
-        self.insert_events_statement = (
-            f"INSERT INTO {self.events_table_name} VALUES ($1, $2, $3, $4) "
-            f"RETURNING notification_id"
-        )
+        self.insert_events_statement += " RETURNING notification_id"
         self.max_notification_id_statement = (
             f"SELECT MAX(notification_id) FROM {self.events_table_name}"
         )
-        self.max_notification_id_statement_name = (
-            f"max_notification_id_{events_table_name}".replace(".", "_")
-        )
-        self.lock_statements = [
+        self.lock_table_statements = [
             f"SET LOCAL lock_timeout = '{self.datastore.lock_timeout}s'",
             f"LOCK TABLE {self.events_table_name} IN EXCLUSIVE MODE",
         ]
 
     def construct_create_table_statements(self) -> List[str]:
-        statements = [
-            "CREATE TABLE IF NOT EXISTS "
-            f"{self.events_table_name} ("
-            "originator_id uuid NOT NULL, "
-            "originator_version bigint NOT NULL, "
-            "topic text, "
-            "state bytea, "
-            "notification_id bigserial, "
-            "PRIMARY KEY "
-            "(originator_id, originator_version)) "
-            "WITH (autovacuum_enabled=false)",
-            f"CREATE UNIQUE INDEX IF NOT EXISTS "
-            f"{self.notification_id_index_name}"
-            f"ON {self.events_table_name} (notification_id ASC);",
+        return [
+            (
+                "CREATE TABLE IF NOT EXISTS "
+                f"{self.events_table_name} ("
+                "originator_id uuid NOT NULL, "
+                "originator_version bigint NOT NULL, "
+                "topic text, "
+                "state bytea, "
+                "notification_id bigserial, "
+                "PRIMARY KEY "
+                "(originator_id, originator_version)) "
+                "WITH (autovacuum_enabled=false)"
+            ),
+            (
+                "CREATE UNIQUE INDEX IF NOT EXISTS "
+                f"{self.notification_id_index_name}"
+                f"ON {self.events_table_name} (notification_id ASC);"
+            ),
         ]
-        return statements
 
     @retry((InterfaceError, OperationalError), max_attempts=10, wait=0.2)
     def select_notifications(
         self,
         start: int,
         limit: int,
-        stop: Optional[int] = None,
+        stop: int | None = None,
         topics: Sequence[str] = (),
     ) -> List[Notification]:
         """
@@ -581,86 +386,91 @@ class PostgresApplicationRecorder(PostgresAggregateRecorder, ApplicationRecorder
         from 'start', limited by 'limit'.
         """
 
-        params: List[Union[int, str, Sequence[str]]] = [start]
-        statement = (
-            "SELECT * " f"FROM {self.events_table_name} " "WHERE notification_id>=$1 "
-        )
-        statement_name = f"select_notifications_{self.events_table_name}".replace(
-            ".", "_"
-        )
+        params: List[int | str | Sequence[str]] = [start]
+        statement = f"SELECT * FROM {self.events_table_name} WHERE notification_id>=%s"
 
         if stop is not None:
             params.append(stop)
-            statement += f"AND notification_id <= ${len(params)} "
-            statement_name += "_stop"
+            statement += " AND notification_id <= %s"
 
         if topics:
             params.append(topics)
-            statement += f"AND topic = ANY(${len(params)}) "
-            statement_name += "_topics"
+            statement += " AND topic = ANY(%s)"
 
         params.append(limit)
-        statement += "ORDER BY notification_id " f"LIMIT ${len(params)}"
-
-        notifications = []
-        with self.datastore.get_connection() as conn:
-            alias = self._prepare(
-                conn,
-                statement_name,
-                statement,
-            )
-            with conn.transaction(commit=False) as curs:
-                curs.execute(
-                    f"EXECUTE {alias}({', '.join(['%s' for _ in params])})",
-                    params,
+        statement += " ORDER BY notification_id LIMIT %s"
+
+        connection = self.datastore.get_connection()
+        with connection as conn, conn.cursor() as curs:
+            curs.execute(statement, params, prepare=True)
+            return [
+                Notification(
+                    id=row["notification_id"],
+                    originator_id=row["originator_id"],
+                    originator_version=row["originator_version"],
+                    topic=row["topic"],
+                    state=bytes(row["state"]),
                 )
-                for row in curs.fetchall():
-                    notifications.append(
-                        Notification(
-                            id=row["notification_id"],
-                            originator_id=row["originator_id"],
-                            originator_version=row["originator_version"],
-                            topic=row["topic"],
-                            state=bytes(row["state"]),
-                        )
-                    )
-                pass  # for Coverage 5.5 bug with CPython 3.10.0rc1
-        return notifications
+                for row in curs.fetchall()
+            ]
 
     @retry((InterfaceError, OperationalError), max_attempts=10, wait=0.2)
     def max_notification_id(self) -> int:
         """
         Returns the maximum notification ID.
         """
-        statement_name = self.max_notification_id_statement_name
-        with self.datastore.get_connection() as conn:
-            statement_alias = self._prepare(
-                conn, statement_name, self.max_notification_id_statement
-            )
-            with conn.transaction(commit=False) as curs:
-                curs.execute(
-                    f"EXECUTE {statement_alias}",
-                )
-                max_id = curs.fetchone()[0] or 0
-        return max_id
+        conn: Connection[DictRow]
+        with self.datastore.get_connection() as conn, conn.cursor() as curs:
+            curs.execute(self.max_notification_id_statement)
+            fetchone = curs.fetchone()
+            assert fetchone is not None
+            return fetchone["max"] or 0
+
+    def _lock_table(self, c: Cursor[DictRow]) -> None:
+        # Acquire "EXCLUSIVE" table lock, to serialize transactions that insert
+        # stored events, so that readers don't pass over gaps that are filled in
+        # later. We want each transaction that will be issued with notifications
+        # IDs by the notification ID sequence to receive all its notification IDs
+        # and then commit, before another transaction is issued with any notification
+        # IDs. In other words, we want the insert order to be the same as the commit
+        # order. We can accomplish this by locking the table for writes. The
+        # EXCLUSIVE lock mode does not block SELECT statements, which acquire an
+        # ACCESS SHARE lock, so the stored events table can be read concurrently
+        # with writes and other reads. However, INSERT statements normally just
+        # acquires ROW EXCLUSIVE locks, which risks the interleaving (within the
+        # recorded sequence of notification IDs) of stored events from one transaction
+        # with those of another transaction. And since one transaction will always
+        # commit before another, the possibility arises when using ROW EXCLUSIVE locks
+        # for readers that are tailing a notification log to miss items inserted later
+        # but issued with lower notification IDs.
+        # https://www.postgresql.org/docs/current/explicit-locking.html#LOCKING-TABLES
+        # https://www.postgresql.org/docs/9.1/sql-lock.html
+        # https://stackoverflow.com/questions/45866187/guarantee-monotonicity-of
+        # -postgresql-serial-column-values-by-commit-order
+        for lock_statement in self.lock_table_statements:
+            c.execute(lock_statement, prepare=True)
 
-    def _insert_events(
+    def _fetch_ids_after_insert_events(
         self,
-        c: PostgresCursor,
+        c: Cursor[DictRow],
         stored_events: List[StoredEvent],
         **kwargs: Any,
-    ) -> Optional[Sequence[int]]:
-        super()._insert_events(c, stored_events, **kwargs)
-        if stored_events:
-            last_notification_id = c.fetchone()[0]
-            notification_ids = list(
-                range(
-                    last_notification_id - len(stored_events) + 1,
-                    last_notification_id + 1,
-                )
-            )
-        else:
-            notification_ids = []
+    ) -> Sequence[int] | None:
+        notification_ids: List[int] = []
+        len_events = len(stored_events)
+        if len_events:
+            if (
+                (c.statusmessage == "SET")
+                and c.nextset()
+                and (c.statusmessage == "LOCK TABLE")
+            ):
+                while c.nextset() and len(notification_ids) != len_events:
+                    row = c.fetchone()
+                    assert row is not None
+                    notification_ids.append(row["notification_id"])
+            if len(notification_ids) != len(stored_events):
+                msg = "Couldn't get all notification IDs"
+                raise ProgrammingError(msg)
         return notification_ids
 
 
@@ -675,26 +485,17 @@ class PostgresProcessRecorder(PostgresApplicationRecorder, ProcessRecorder):
         self.tracking_table_name = tracking_table_name
         super().__init__(datastore, events_table_name)
         self.insert_tracking_statement = (
-            f"INSERT INTO {self.tracking_table_name} VALUES ($1, $2)"
-        )
-        self.insert_tracking_statement_name = f"insert_{tracking_table_name}".replace(
-            ".", "_"
+            f"INSERT INTO {self.tracking_table_name} VALUES (%s, %s)"
         )
         self.max_tracking_id_statement = (
             "SELECT MAX(notification_id) "
             f"FROM {self.tracking_table_name} "
-            "WHERE application_name=$1"
+            "WHERE application_name=%s"
         )
         self.count_tracking_id_statement = (
             "SELECT COUNT(*) "
             f"FROM {self.tracking_table_name} "
-            "WHERE application_name=$1 AND notification_id=$2"
-        )
-        self.max_tracking_id_statement_name = (
-            f"max_tracking_id_{tracking_table_name}".replace(".", "_")
-        )
-        self.count_tracking_id_statement_name = (
-            f"count_tracking_id_{tracking_table_name}".replace(".", "_")
+            "WHERE application_name=%s AND notification_id=%s"
         )
 
     def construct_create_table_statements(self) -> List[str]:
@@ -711,61 +512,46 @@ class PostgresProcessRecorder(PostgresApplicationRecorder, ProcessRecorder):
 
     @retry((InterfaceError, OperationalError), max_attempts=10, wait=0.2)
     def max_tracking_id(self, application_name: str) -> int:
-        statement_name = self.max_tracking_id_statement_name
-        with self.datastore.get_connection() as conn:
-            statement_alias = self._prepare(
-                conn, statement_name, self.max_tracking_id_statement
+        with self.datastore.get_connection() as conn, conn.cursor() as curs:
+            curs.execute(
+                query=self.max_tracking_id_statement,
+                params=(application_name,),
+                prepare=True,
             )
-
-            with conn.transaction(commit=False) as curs:
-                curs.execute(
-                    f"EXECUTE {statement_alias}(%s)",
-                    (application_name,),
-                )
-                max_id = curs.fetchone()[0] or 0
-        return max_id
+            fetchone = curs.fetchone()
+            assert fetchone is not None
+            return fetchone["max"] or 0
 
     @retry((InterfaceError, OperationalError), max_attempts=10, wait=0.2)
     def has_tracking_id(self, application_name: str, notification_id: int) -> bool:
-        statement_name = self.count_tracking_id_statement_name
-        with self.datastore.get_connection() as conn:
-            statement_alias = self._prepare(
-                conn, statement_name, self.count_tracking_id_statement
+        conn: Connection[DictRow]
+        with self.datastore.get_connection() as conn, conn.cursor() as curs:
+            curs.execute(
+                query=self.count_tracking_id_statement,
+                params=(application_name, notification_id),
+                prepare=True,
             )
-
-            with conn.transaction(commit=False) as curs:
-                curs.execute(
-                    f"EXECUTE {statement_alias}(%s, %s)",
-                    (application_name, notification_id),
-                )
-                return bool(curs.fetchone()[0])
-
-    def _prepare_insert_events(self, conn: PostgresConnection) -> None:
-        super()._prepare_insert_events(conn)
-        self._prepare(
-            conn, self.insert_tracking_statement_name, self.insert_tracking_statement
-        )
+            fetchone = curs.fetchone()
+            assert fetchone is not None
+            return bool(fetchone["count"])
 
     def _insert_events(
         self,
-        c: PostgresCursor,
+        c: Cursor[DictRow],
         stored_events: List[StoredEvent],
         **kwargs: Any,
-    ) -> Optional[Sequence[int]]:
-        notification_ids = super()._insert_events(c, stored_events, **kwargs)
-        tracking: Optional[Tracking] = kwargs.get("tracking", None)
+    ) -> None:
+        tracking: Tracking | None = kwargs.get("tracking", None)
         if tracking is not None:
-            statement_alias = self.statement_name_aliases[
-                self.insert_tracking_statement_name
-            ]
             c.execute(
-                f"EXECUTE {statement_alias}(%s, %s)",
-                (
+                query=self.insert_tracking_statement,
+                params=(
                     tracking.application_name,
                     tracking.notification_id,
                 ),
+                prepare=True,
             )
-        return notification_ids
+        super()._insert_events(c, stored_events, **kwargs)
 
 
 class Factory(InfrastructureFactory):
@@ -773,14 +559,15 @@ class Factory(InfrastructureFactory):
     POSTGRES_HOST = "POSTGRES_HOST"
     POSTGRES_PORT = "POSTGRES_PORT"
     POSTGRES_USER = "POSTGRES_USER"
-    POSTGRES_PASSWORD = "POSTGRES_PASSWORD"
+    POSTGRES_PASSWORD = "POSTGRES_PASSWORD"  # noqa: S105
+    POSTGRES_GET_PASSWORD_TOPIC = "POSTGRES_GET_PASSWORD_TOPIC"  # noqa: S105
     POSTGRES_CONNECT_TIMEOUT = "POSTGRES_CONNECT_TIMEOUT"
     POSTGRES_CONN_MAX_AGE = "POSTGRES_CONN_MAX_AGE"
     POSTGRES_PRE_PING = "POSTGRES_PRE_PING"
-    POSTGRES_POOL_TIMEOUT = "POSTGRES_POOL_TIMEOUT"
+    POSTGRES_MAX_WAITING = "POSTGRES_MAX_WAITING"
     POSTGRES_LOCK_TIMEOUT = "POSTGRES_LOCK_TIMEOUT"
     POSTGRES_POOL_SIZE = "POSTGRES_POOL_SIZE"
-    POSTGRES_POOL_MAX_OVERFLOW = "POSTGRES_POOL_MAX_OVERFLOW"
+    POSTGRES_MAX_OVERFLOW = "POSTGRES_MAX_OVERFLOW"
     POSTGRES_IDLE_IN_TRANSACTION_SESSION_TIMEOUT = (
         "POSTGRES_IDLE_IN_TRANSACTION_SESSION_TIMEOUT"
     )
@@ -795,54 +582,61 @@ class Factory(InfrastructureFactory):
         super().__init__(env)
         dbname = self.env.get(self.POSTGRES_DBNAME)
         if dbname is None:
-            raise EnvironmentError(
+            msg = (
                 "Postgres database name not found "
                 "in environment with key "
                 f"'{self.POSTGRES_DBNAME}'"
             )
+            raise OSError(msg)
 
         host = self.env.get(self.POSTGRES_HOST)
         if host is None:
-            raise EnvironmentError(
+            msg = (
                 "Postgres host not found "
                 "in environment with key "
                 f"'{self.POSTGRES_HOST}'"
             )
+            raise OSError(msg)
 
         port = self.env.get(self.POSTGRES_PORT) or "5432"
 
         user = self.env.get(self.POSTGRES_USER)
         if user is None:
-            raise EnvironmentError(
+            msg = (
                 "Postgres user not found "
                 "in environment with key "
                 f"'{self.POSTGRES_USER}'"
             )
+            raise OSError(msg)
+
+        get_password_func = None
+        get_password_topic = self.env.get(self.POSTGRES_GET_PASSWORD_TOPIC)
+        if not get_password_topic:
+            password = self.env.get(self.POSTGRES_PASSWORD)
+            if password is None:
+                msg = (
+                    "Postgres password not found "
+                    "in environment with key "
+                    f"'{self.POSTGRES_PASSWORD}'"
+                )
+                raise OSError(msg)
+        else:
+            get_password_func = resolve_topic(get_password_topic)
+            password = ""
 
-        password = self.env.get(self.POSTGRES_PASSWORD)
-        if password is None:
-            raise EnvironmentError(
-                "Postgres password not found "
-                "in environment with key "
-                f"'{self.POSTGRES_PASSWORD}'"
-            )
-
-        connect_timeout: Optional[int]
+        connect_timeout = 30
         connect_timeout_str = self.env.get(self.POSTGRES_CONNECT_TIMEOUT)
-        if connect_timeout_str is None:
-            connect_timeout = 5
-        elif connect_timeout_str == "":
-            connect_timeout = 5
-        else:
+        if connect_timeout_str:
             try:
                 connect_timeout = int(connect_timeout_str)
             except ValueError:
-                raise EnvironmentError(
-                    f"Postgres environment value for key "
+                msg = (
+                    "Postgres environment value for key "
                     f"'{self.POSTGRES_CONNECT_TIMEOUT}' is invalid. "
-                    f"If set, an integer or empty string is expected: "
+                    "If set, an integer or empty string is expected: "
                     f"'{connect_timeout_str}'"
                 )
+                raise OSError(msg) from None
 
         idle_in_transaction_session_timeout_str = (
             self.env.get(self.POSTGRES_IDLE_IN_TRANSACTION_SESSION_TIMEOUT) or "5"
@@ -853,80 +647,69 @@ class Factory(InfrastructureFactory):
                 idle_in_transaction_session_timeout_str
             )
         except ValueError:
-            raise EnvironmentError(
-                f"Postgres environment value for key "
+            msg = (
+                "Postgres environment value for key "
                 f"'{self.POSTGRES_IDLE_IN_TRANSACTION_SESSION_TIMEOUT}' is invalid. "
-                f"If set, an integer or empty string is expected: "
+                "If set, an integer or empty string is expected: "
                 f"'{idle_in_transaction_session_timeout_str}'"
             )
+            raise OSError(msg) from None
 
-        pool_size: Optional[int]
+        pool_size = 5
         pool_size_str = self.env.get(self.POSTGRES_POOL_SIZE)
-        if pool_size_str is None:
-            pool_size = 5
-        elif pool_size_str == "":
-            pool_size = 5
-        else:
+        if pool_size_str:
             try:
                 pool_size = int(pool_size_str)
             except ValueError:
-                raise EnvironmentError(
-                    f"Postgres environment value for key "
+                msg = (
+                    "Postgres environment value for key "
                     f"'{self.POSTGRES_POOL_SIZE}' is invalid. "
-                    f"If set, an integer or empty string is expected: "
+                    "If set, an integer or empty string is expected: "
                     f"'{pool_size_str}'"
                 )
+                raise OSError(msg) from None
 
-        pool_max_overflow: Optional[int]
-        pool_max_overflow_str = self.env.get(self.POSTGRES_POOL_MAX_OVERFLOW)
-        if pool_max_overflow_str is None:
-            pool_max_overflow = 10
-        elif pool_max_overflow_str == "":
-            pool_max_overflow = 10
-        else:
+        pool_max_overflow = 10
+        pool_max_overflow_str = self.env.get(self.POSTGRES_MAX_OVERFLOW)
+        if pool_max_overflow_str:
             try:
                 pool_max_overflow = int(pool_max_overflow_str)
             except ValueError:
-                raise EnvironmentError(
-                    f"Postgres environment value for key "
-                    f"'{self.POSTGRES_POOL_MAX_OVERFLOW}' is invalid. "
-                    f"If set, an integer or empty string is expected: "
+                msg = (
+                    "Postgres environment value for key "
+                    f"'{self.POSTGRES_MAX_OVERFLOW}' is invalid. "
+                    "If set, an integer or empty string is expected: "
                     f"'{pool_max_overflow_str}'"
                 )
+                raise OSError(msg) from None
 
-        pool_timeout: Optional[float]
-        pool_timeout_str = self.env.get(self.POSTGRES_POOL_TIMEOUT)
-        if pool_timeout_str is None:
-            pool_timeout = 30
-        elif pool_timeout_str == "":
-            pool_timeout = 30
-        else:
+        max_waiting = 0
+        max_waiting_str = self.env.get(self.POSTGRES_MAX_WAITING)
+        if max_waiting_str:
             try:
-                pool_timeout = float(pool_timeout_str)
+                max_waiting = int(max_waiting_str)
             except ValueError:
-                raise EnvironmentError(
-                    f"Postgres environment value for key "
-                    f"'{self.POSTGRES_POOL_TIMEOUT}' is invalid. "
-                    f"If set, a float or empty string is expected: "
-                    f"'{pool_timeout_str}'"
+                msg = (
+                    "Postgres environment value for key "
+                    f"'{self.POSTGRES_MAX_WAITING}' is invalid. "
+                    "If set, an integer or empty string is expected: "
+                    f"'{max_waiting_str}'"
                 )
+                raise OSError(msg) from None
 
-        conn_max_age: Optional[float]
+        conn_max_age = 60 * 60.0
         conn_max_age_str = self.env.get(self.POSTGRES_CONN_MAX_AGE)
-        if conn_max_age_str is None:
-            conn_max_age = None
-        elif conn_max_age_str == "":
-            conn_max_age = None
-        else:
+        if conn_max_age_str:
             try:
                 conn_max_age = float(conn_max_age_str)
             except ValueError:
-                raise EnvironmentError(
-                    f"Postgres environment value for key "
+                msg = (
+                    "Postgres environment value for key "
                     f"'{self.POSTGRES_CONN_MAX_AGE}' is invalid. "
-                    f"If set, a float or empty string is expected: "
+                    "If set, a float or empty string is expected: "
                     f"'{conn_max_age_str}'"
                 )
+                raise OSError(msg) from None
 
         pre_ping = strtobool(self.env.get(self.POSTGRES_PRE_PING) or "no")
 
@@ -935,12 +718,13 @@ class Factory(InfrastructureFactory):
         try:
             lock_timeout = int(lock_timeout_str)
         except ValueError:
-            raise EnvironmentError(
-                f"Postgres environment value for key "
+            msg = (
+                "Postgres environment value for key "
                 f"'{self.POSTGRES_LOCK_TIMEOUT}' is invalid. "
-                f"If set, an integer or empty string is expected: "
+                "If set, an integer or empty string is expected: "
                 f"'{lock_timeout_str}'"
             )
+            raise OSError(msg) from None
 
         schema = self.env.get(self.POSTGRES_SCHEMA) or ""
 
@@ -950,17 +734,21 @@ class Factory(InfrastructureFactory):
             port=port,
             user=user,
             password=password,
+            get_password_func=get_password_func,
             connect_timeout=connect_timeout,
             idle_in_transaction_session_timeout=idle_in_transaction_session_timeout,
             pool_size=pool_size,
             max_overflow=pool_max_overflow,
-            pool_timeout=pool_timeout,
+            max_waiting=max_waiting,
             conn_max_age=conn_max_age,
             pre_ping=pre_ping,
             lock_timeout=lock_timeout,
             schema=schema,
         )
 
+    def env_create_table(self) -> bool:
+        return strtobool(self.env.get(self.CREATE_TABLE) or "yes")
+
     def aggregate_recorder(self, purpose: str = "events") -> AggregateRecorder:
         prefix = self.env.name.lower() or "stored"
         events_table_name = prefix + "_" + purpose
@@ -1004,8 +792,9 @@ class Factory(InfrastructureFactory):
             recorder.create_table()
         return recorder
 
-    def env_create_table(self) -> bool:
-        return strtobool(self.env.get(self.CREATE_TABLE) or "yes")
-
     def close(self) -> None:
-        self.datastore.close()
+        if hasattr(self, "datastore"):
+            self.datastore.close()
+
+    def __del__(self) -> None:
+        self.close()