continuo-postgres-adapter 0.3.0__tar.gz → 0.3.2__tar.gz

{continuo_postgres_adapter-0.3.0 → continuo_postgres_adapter-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: continuo-postgres-adapter
-Version: 0.3.0
+Version: 0.3.2
 Summary: Postgres warehouse adapter for Continuo: validation and python-node runtime.
 Author: Simone Carolini
 Maintainer: Simone Carolini
@@ -8,9 +8,9 @@ Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python :: 3.14
 Requires-Python: >=3.14
-Requires-Dist: continuo-engine-contract==0.8.0
-Requires-Dist: psycopg2-binary==2.9.12
-Requires-Dist: pyarrow==25.0.1
+Requires-Dist: continuo-engine-contract==0.9.0
+Requires-Dist: psycopg2-binary==2.9.13
+Requires-Dist: pyarrow==26.0.0
 Description-Content-Type: text/markdown
 
 # continuo-postgres-adapter
@@ -27,6 +27,13 @@ registers itself under entry-point group `continuo_engine.adapters` as
 Connection env: `POSTGRES_HOST`, `POSTGRES_DB`, `POSTGRES_USER` (required);
 `POSTGRES_PORT` (default 5432), `POSTGRES_PASSWORD` (default empty).
 
+`from_env` tries a connection that no server answered (every address of the
+host refused, had no route or sat on an unreachable network) up to 5 times,
+2 s apart, logging each retry: a Job pod's first connection can be refused
+for a few seconds after the pod starts, until the cluster's NetworkPolicy
+engine has admitted the pod. Every other failure, an authentication failure or
+a connect timeout on any address among them, is raised at once.
+
 The connection runs with `autocommit = True`; every method owns its
 transaction explicitly. `load()` and `ensure_table()` open an explicit
 `BEGIN` around their statements and `COMMIT`/`ROLLBACK` themselves, so a

{continuo_postgres_adapter-0.3.0 → continuo_postgres_adapter-0.3.2}/README.md

@@ -12,6 +12,13 @@ registers itself under entry-point group `continuo_engine.adapters` as
 Connection env: `POSTGRES_HOST`, `POSTGRES_DB`, `POSTGRES_USER` (required);
 `POSTGRES_PORT` (default 5432), `POSTGRES_PASSWORD` (default empty).
 
+`from_env` tries a connection that no server answered (every address of the
+host refused, had no route or sat on an unreachable network) up to 5 times,
+2 s apart, logging each retry: a Job pod's first connection can be refused
+for a few seconds after the pod starts, until the cluster's NetworkPolicy
+engine has admitted the pod. Every other failure, an authentication failure or
+a connect timeout on any address among them, is raised at once.
+
 The connection runs with `autocommit = True`; every method owns its
 transaction explicitly. `load()` and `ensure_table()` open an explicit
 `BEGIN` around their statements and `COMMIT`/`ROLLBACK` themselves, so a

{continuo_postgres_adapter-0.3.0 → continuo_postgres_adapter-0.3.2}/continuo_postgres_adapter/adapter.py

@@ -28,8 +28,10 @@ import contextlib
 import hashlib
 import logging
 import os
+import re
+import time
 
-from collections.abc import Iterator, Sequence
+from collections.abc import Callable, Iterator, Sequence
 from typing import Any
 
 import psycopg2  # type: ignore[import-untyped]
@@ -79,6 +81,42 @@ _DEFAULT_INDEX_TYPE = "btree"
 # postgres actually stores.
 _MAX_IDENTIFIER_BYTES = 63
 
+# A Job pod's first connection can be refused for a few seconds after the pod
+# starts, until the cluster's NetworkPolicy engine has admitted the pod: an
+# engine that rejects (k3s's kube-router) refuses at once instead of dropping
+# the packets. So a connection the server never answered, refused or
+# unroutable, is tried again. A connect timeout is not: an engine that drops
+# packets only delays the first connection, which TCP's own retransmissions
+# carry through, so a timeout means the server stayed silent for the whole
+# timeout and every retry would wait that long again.
+_CONNECT_ATTEMPTS = 5
+_CONNECT_RETRY_SECONDS = 2.0
+# libpq writes one line per address it tried, the reason after "failed: "
+# (libpq 14 and later) or after "could not connect to server: " (libpq 13 and
+# earlier); pgcode is None for every connection failure, so the text is all
+# there is.
+_ADDRESS_FAILURE = re.compile(r"(?:failed|could not connect to server): ([^\n]*)")
+_UNREACHABLE_REASONS = frozenset({"Connection refused", "No route to host", "Network is unreachable"})
+
+
+def _is_unreachable(exc: "psycopg2.OperationalError") -> bool:
+    """Whether *exc* says that every address of the host refused or could not be routed to.
+
+    A host name can have several addresses, and libpq reports each one it
+    tried. One that timed out means a retry would wait out its timeout again;
+    a FATAL line came from a server that answered (a failed authentication, an
+    unknown database). libpq 13 and earlier print a timeout as a bare
+    "timeout expired" line, with no prefix.
+    """
+    message = str(exc)
+    reasons = [reason.strip() for reason in _ADDRESS_FAILURE.findall(message)]
+    return (
+        bool(reasons)
+        and all(reason in _UNREACHABLE_REASONS for reason in reasons)
+        and "FATAL:" not in message
+        and "timeout expired" not in message
+    )
+
 
 def _truncate_explicit_identifier(name: str) -> str:
     """Truncate an explicit, author-given identifier to at most 63 bytes.
@@ -418,16 +456,35 @@ class PostgresAdapter(WarehouseAdapter, AppendCapable, UpsertCapable):
         return ["POSTGRES_HOST", "POSTGRES_DB", "POSTGRES_USER"]
 
     @classmethod
-    def from_env(cls) -> "PostgresAdapter":
-        """Connect from POSTGRES_* env (port defaults 5432, password empty)."""
-        conn = psycopg2.connect(
-            host=os.environ["POSTGRES_HOST"],
-            port=os.environ.get("POSTGRES_PORT", "5432"),
-            dbname=os.environ["POSTGRES_DB"],
-            user=os.environ["POSTGRES_USER"],
-            password=os.environ.get("POSTGRES_PASSWORD", ""),
-        )
-        return cls(conn)
+    def from_env(cls, *, sleep: Callable[[float], None] = time.sleep) -> "PostgresAdapter":
+        """Connect from POSTGRES_* env (port defaults 5432, password empty).
+
+        A connection no server answered (refused, no route, network
+        unreachable) is tried again, up to ``_CONNECT_ATTEMPTS`` times in all,
+        ``_CONNECT_RETRY_SECONDS`` apart, each retry logged; the last failure
+        is raised. Every other failure is raised at once.
+        """
+        params = {
+            "host": os.environ["POSTGRES_HOST"],
+            "port": os.environ.get("POSTGRES_PORT", "5432"),
+            "dbname": os.environ["POSTGRES_DB"],
+            "user": os.environ["POSTGRES_USER"],
+            "password": os.environ.get("POSTGRES_PASSWORD", ""),
+        }
+        for attempt in range(1, _CONNECT_ATTEMPTS + 1):
+            try:
+                conn = psycopg2.connect(**params)
+            except psycopg2.OperationalError as exc:
+                if attempt == _CONNECT_ATTEMPTS or not _is_unreachable(exc):
+                    raise
+                logger.warning(
+                    "postgres not reachable yet (attempt %d of %d), retrying in %.0f s: %s",
+                    attempt, _CONNECT_ATTEMPTS, _CONNECT_RETRY_SECONDS, str(exc).strip(),
+                )
+                sleep(_CONNECT_RETRY_SECONDS)
+                continue
+            return cls(conn)
+        raise AssertionError("unreachable")  # pragma: no cover
 
     # --- Schema lifecycle ---------------------------------------------------
 
@@ -706,7 +763,7 @@ class PostgresAdapter(WarehouseAdapter, AppendCapable, UpsertCapable):
         this engine's physical-layout vocabulary — ``indexes`` — validated by
         :func:`_validated_indexes` before any DDL runs, so a malformed config
         never leaves a half-built table behind (fail closed). Required and
-        keyword-only, matching the contract 0.6.0 port signature. The create
+        keyword-only, matching the contract 0.8.0 port signature. The create
         and every index either all land or none do, in the same explicit
         BEGIN/COMMIT/ROLLBACK shape build_empty_from_columns uses.
 

{continuo_postgres_adapter-0.3.0 → continuo_postgres_adapter-0.3.2}/pyproject.toml

@@ -1,5 +1,5 @@
 [project]
-version = "0.3.0"
+version = "0.3.2"
 name = "continuo-postgres-adapter"
 description = "Postgres warehouse adapter for Continuo: validation and python-node runtime."
 authors = [{ name = "Simone Carolini" }]
@@ -12,9 +12,9 @@ dependencies = [
     # of this adapter would otherwise accept whatever contract release is
     # newest. This adapter subclasses the contract's WarehouseAdapter port
     # directly, so the pair must stay the tested one.
-    "continuo-engine-contract==0.8.0",
-    "psycopg2-binary==2.9.12",
-    "pyarrow==25.0.1",
+    "continuo-engine-contract==0.9.0",
+    "psycopg2-binary==2.9.13",
+    "pyarrow==26.0.0",
 ]
 classifiers = [
     "Development Status :: 4 - Beta",
@@ -27,8 +27,8 @@ postgres = "continuo_postgres_adapter.adapter:PostgresAdapter"
 
 [dependency-groups]
 dev = [
-    "ruff==0.16.5",
-    "mypy==2.3.1",
+    "ruff==0.17.1",
+    "mypy==2.4.0",
 ]
 test = [
     "pytest==9.1.1",

continuo_postgres_adapter-0.3.2/tests/test_adapter_postgres_connect.py

@@ -0,0 +1,209 @@
+"""Unit tests for ``PostgresAdapter.from_env``'s first connection.
+
+A Job pod's first connection can be refused for a few seconds after the pod
+starts, while the cluster's NetworkPolicy engine has not yet admitted the
+pod; an engine that rejects (k3s's kube-router) refuses at once rather than
+dropping packets. ``from_env`` retries only a connection the server never
+answered, refused or unreachable, and raises every other failure at once.
+
+The scripted cases raise ``psycopg2.OperationalError`` with libpq's own
+message text (``pgcode`` is ``None`` for every connection failure, so the text
+is all there is). One case connects to a local port nothing listens on, so it
+checks the classification against what the installed libpq really says.
+"""
+import logging
+import socket
+
+import psycopg2  # type: ignore[import-untyped]
+import pytest
+
+import continuo_postgres_adapter.adapter as adapter_module
+from continuo_postgres_adapter.adapter import PostgresAdapter
+
+REFUSED = (
+    'connection to server at "continuo-pg-rw" (10.43.101.94), port 5432 failed: '
+    "Connection refused\n\tIs the server running on that host and accepting TCP/IP connections?\n"
+)
+# libpq 13 and older word a refused connection this way.
+REFUSED_LIBPQ_13 = (
+    "could not connect to server: Connection refused\n\tIs the server running on host "
+    '"continuo-pg-rw" (10.43.101.94) and accepting\n\tTCP/IP connections on port 5432?\n'
+)
+NO_ROUTE = (
+    'connection to server at "continuo-pg-rw" (10.43.101.94), port 5432 failed: '
+    "No route to host\n\tIs the server running on that host and accepting TCP/IP connections?\n"
+)
+NET_UNREACHABLE = (
+    'connection to server at "continuo-pg-rw" (10.43.101.94), port 5432 failed: '
+    "Network is unreachable\n\tIs the server running on that host and accepting TCP/IP connections?\n"
+)
+AUTH_FAILED = (
+    'connection to server at "continuo-pg-rw" (10.43.101.94), port 5432 failed: '
+    'FATAL:  password authentication failed for user "continuo"\n'
+)
+NO_DATABASE = (
+    'connection to server at "continuo-pg-rw" (10.43.101.94), port 5432 failed: '
+    'FATAL:  database "warehouse" does not exist\n'
+)
+TIMED_OUT = 'connection to server at "continuo-pg-rw" (10.43.101.94), port 5432 failed: timeout expired\n'
+UNKNOWN_HOST = (
+    'could not translate host name "continuo-pg-rw" to address: Name or service not known\n'
+)
+# A host name with two addresses: the server answered on the second, so the
+# failure is the server's verdict, not an unreachable host.
+REFUSED_THEN_AUTH_FAILED = (
+    'connection to server at "localhost" (::1), port 5432 failed: Connection refused\n'
+    "\tIs the server running on that host and accepting TCP/IP connections?\n"
+    'connection to server at "localhost" (127.0.0.1), port 5432 failed: '
+    'FATAL:  password authentication failed for user "continuo"\n'
+)
+# libpq reports every address it tried in one message. One address that stayed
+# silent means the host is not just unadmitted yet, and a retry would wait out
+# its timeout again (libpq 17's text, a real dual-stack localhost).
+REFUSED_THEN_TIMED_OUT = (
+    'connection to server at "localhost" (::1), port 5432 failed: Connection refused\n'
+    "\tIs the server running on that host and accepting TCP/IP connections?\n"
+    'connection to server at "localhost" (127.0.0.1), port 5432 failed: timeout expired\n'
+)
+TIMED_OUT_THEN_REFUSED = (
+    'connection to server at "localhost" (127.0.0.1), port 5432 failed: timeout expired\n'
+    'connection to server at "localhost" (::1), port 5432 failed: Connection refused\n'
+    "\tIs the server running on that host and accepting TCP/IP connections?\n"
+)
+# libpq 13 and earlier print a timeout as a bare line.
+REFUSED_THEN_TIMED_OUT_LIBPQ_13 = (
+    "could not connect to server: Connection refused\n\tIs the server running on host "
+    '"localhost" (::1) and accepting\n\tTCP/IP connections on port 5432?\n'
+    "timeout expired\n"
+)
+REFUSED_TWICE = (
+    'connection to server at "localhost" (::1), port 5432 failed: Connection refused\n'
+    "\tIs the server running on that host and accepting TCP/IP connections?\n"
+    'connection to server at "localhost" (127.0.0.1), port 5432 failed: Connection refused\n'
+    "\tIs the server running on that host and accepting TCP/IP connections?\n"
+)
+
+
+class _Connection:
+    """What PostgresAdapter.__init__ touches of a psycopg2 connection."""
+
+    autocommit = False
+
+    def close(self):
+        pass
+
+
+class _ConnectScript:
+    """psycopg2.connect replacement: raises the next scripted error (None = connect)."""
+
+    def __init__(self, errors):
+        self.errors = list(errors)
+        self.calls: list[dict] = []
+
+    def __call__(self, **kwargs):
+        self.calls.append(kwargs)
+        error = self.errors.pop(0) if self.errors else None
+        if error is not None:
+            raise psycopg2.OperationalError(error)
+        return _Connection()
+
+
+@pytest.fixture
+def postgres_env(monkeypatch):
+    monkeypatch.setenv("POSTGRES_HOST", "continuo-pg-rw")
+    monkeypatch.setenv("POSTGRES_PORT", "5432")
+    monkeypatch.setenv("POSTGRES_DB", "warehouse")
+    monkeypatch.setenv("POSTGRES_USER", "continuo")
+    monkeypatch.setenv("POSTGRES_PASSWORD", "s3cret")
+
+
+@pytest.fixture
+def connect_with(monkeypatch, postgres_env):
+    def run(errors):
+        script = _ConnectScript(errors)
+        monkeypatch.setattr(adapter_module.psycopg2, "connect", script)
+        return script
+
+    return run
+
+
+@pytest.mark.parametrize("message", [REFUSED, REFUSED_LIBPQ_13, NO_ROUTE, NET_UNREACHABLE, REFUSED_TWICE],
+                         ids=["refused", "refused-libpq-13", "no-route", "net-unreachable", "refused-twice"])
+def test_an_unreachable_server_is_retried_until_it_answers(connect_with, message):
+    script = connect_with([message, message, None])
+    sleeps: list[float] = []
+
+    adapter = PostgresAdapter.from_env(sleep=sleeps.append)
+
+    assert isinstance(adapter, PostgresAdapter)
+    assert len(script.calls) == 3
+    assert sleeps == [2.0, 2.0]
+
+
+def test_every_retry_connects_with_the_same_parameters(connect_with):
+    script = connect_with([REFUSED, None])
+
+    PostgresAdapter.from_env(sleep=lambda _seconds: None)
+
+    expected = dict(host="continuo-pg-rw", port="5432", dbname="warehouse", user="continuo", password="s3cret")
+    assert script.calls == [expected, expected]
+
+
+def test_a_server_that_stays_unreachable_fails_after_five_attempts(connect_with):
+    script = connect_with([REFUSED] * 50)
+    sleeps: list[float] = []
+
+    with pytest.raises(psycopg2.OperationalError, match="Connection refused"):
+        PostgresAdapter.from_env(sleep=sleeps.append)
+
+    assert len(script.calls) == 5
+    assert sleeps == [2.0, 2.0, 2.0, 2.0]
+
+
+def test_each_retry_is_logged_with_the_drivers_message(connect_with, caplog):
+    connect_with([REFUSED, REFUSED, None])
+    caplog.set_level(logging.INFO, logger="continuo_postgres_adapter")
+
+    PostgresAdapter.from_env(sleep=lambda _seconds: None)
+
+    warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
+    assert len(warnings) == 2
+    assert all("port 5432 failed: Connection refused" in r.getMessage() for r in warnings)
+
+
+@pytest.mark.parametrize("message", [
+    AUTH_FAILED, NO_DATABASE, TIMED_OUT, UNKNOWN_HOST, REFUSED_THEN_AUTH_FAILED,
+    REFUSED_THEN_TIMED_OUT, TIMED_OUT_THEN_REFUSED, REFUSED_THEN_TIMED_OUT_LIBPQ_13,
+], ids=["auth-failed", "no-database", "timed-out", "unknown-host", "refused-then-auth-failed",
+        "refused-then-timed-out", "timed-out-then-refused", "refused-then-timed-out-libpq-13"])
+def test_every_other_connection_failure_is_raised_at_once(connect_with, message):
+    script = connect_with([message, None])
+    sleeps: list[float] = []
+
+    with pytest.raises(psycopg2.OperationalError) as caught:
+        PostgresAdapter.from_env(sleep=sleeps.append)
+
+    assert str(caught.value) == message
+    assert len(script.calls) == 1 and sleeps == []
+
+
+def _free_port() -> int:
+    """A loopback port no socket holds, so a connection to it is refused.
+
+    A port merely bound, without listen(), is not enough: macOS drops the
+    connection's SYN instead of refusing it, and the connect times out.
+    """
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
+        probe.bind(("127.0.0.1", 0))
+        return probe.getsockname()[1]
+
+
+def test_a_real_refused_connection_is_retried(monkeypatch, postgres_env):
+    monkeypatch.setenv("POSTGRES_HOST", "127.0.0.1")
+    monkeypatch.setenv("POSTGRES_PORT", str(_free_port()))
+    sleeps: list[float] = []
+
+    with pytest.raises(psycopg2.OperationalError, match="Connection refused"):
+        PostgresAdapter.from_env(sleep=sleeps.append)
+
+    assert sleeps == [2.0, 2.0, 2.0, 2.0]

{continuo_postgres_adapter-0.3.0 → continuo_postgres_adapter-0.3.2}/tests/test_integration_postgres_conformance.py

@@ -18,6 +18,7 @@ from continuo_engine_contract.testing.conformance import (
     AppendConformance,
     ConformanceBase,
     IntrospectionConformance,
+    StartupConformance,
     UpsertConformance,
     table_of,
 )
@@ -48,6 +49,28 @@ def schema(adapter):
     adapter.drop_schema(name)
 
 
+@pytest.fixture
+def warehouse_address():
+    return "127.0.0.1", int(PG["port"])
+
+
+@pytest.fixture
+def adapter_from_env(monkeypatch):
+    def build(host, port):
+        monkeypatch.setenv("POSTGRES_HOST", host)
+        monkeypatch.setenv("POSTGRES_PORT", str(port))
+        monkeypatch.setenv("POSTGRES_DB", PG["dbname"])
+        monkeypatch.setenv("POSTGRES_USER", PG["user"])
+        monkeypatch.setenv("POSTGRES_PASSWORD", PG["password"])
+        return PostgresAdapter.from_env()
+
+    return build
+
+
+class TestPostgresStartup(StartupConformance):
+    pass
+
+
 class TestPostgresIntrospection(IntrospectionConformance):
     pass
 

{continuo_postgres_adapter-0.3.0 → continuo_postgres_adapter-0.3.2}/tests/test_integration_runtime_postgres.py

@@ -85,6 +85,35 @@ def _seed(schema: str, table: str, ddl: str, rows_sql: str) -> None:
     conn.close()
 
 
+@pytest.fixture()
+def pg_env(monkeypatch):
+    monkeypatch.setenv("POSTGRES_HOST", PG["host"])
+    monkeypatch.setenv("POSTGRES_PORT", str(PG["port"]))
+    monkeypatch.setenv("POSTGRES_DB", PG["dbname"])
+    monkeypatch.setenv("POSTGRES_USER", PG["user"])
+    monkeypatch.setenv("POSTGRES_PASSWORD", PG["password"])
+
+
+@pytest.mark.integration
+def test_from_env_connects_on_the_first_try(pg_env):
+    sleeps: list[float] = []
+    a = PostgresAdapter.from_env(sleep=sleeps.append)
+    try:
+        assert a.fetch("SELECT 1 AS one").to_pylist() == [{"one": 1}]
+    finally:
+        a.close()
+    assert sleeps == []
+
+
+@pytest.mark.integration
+def test_from_env_raises_a_failed_authentication_at_once(pg_env, monkeypatch):
+    monkeypatch.setenv("POSTGRES_PASSWORD", "not-the-password")
+    sleeps: list[float] = []
+    with pytest.raises(psycopg2.OperationalError, match="password authentication failed"):
+        PostgresAdapter.from_env(sleep=sleeps.append)
+    assert sleeps == []
+
+
 @pytest.mark.integration
 def test_ensure_table_creates_typed_table_with_not_null(clean_schema):
     """ensure_table creates a typed table; NOT NULL columns show up in the DDL."""