pgsync 4.0.0__tar.gz → 4.1.0__tar.gz

{pgsync-4.0.0 → pgsync-4.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: pgsync
-Version: 4.0.0
+Version: 4.1.0
 Summary: Postgres to Elasticsearch/OpenSearch sync
 Home-page: https://github.com/toluaina/pgsync
 Author: Tolu Aina
@@ -31,34 +31,34 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 License-File: AUTHORS.rst
 Requires-Dist: async-timeout==5.0.1
-Requires-Dist: boto3==1.37.11
-Requires-Dist: botocore==1.37.11
-Requires-Dist: certifi==2025.1.31
-Requires-Dist: charset-normalizer==3.4.1
+Requires-Dist: backports-datetime-fromisoformat==2.0.3
+Requires-Dist: boto3==1.38.44
+Requires-Dist: botocore==1.38.44
+Requires-Dist: certifi==2025.6.15
+Requires-Dist: charset-normalizer==3.4.2
 Requires-Dist: click==8.1.8
-Requires-Dist: elastic-transport==8.17.0
-Requires-Dist: elasticsearch==8.17.2
-Requires-Dist: elasticsearch-dsl==8.17.1
-Requires-Dist: environs==14.1.1
+Requires-Dist: elastic-transport==8.17.1
+Requires-Dist: elasticsearch==8.18.1
+Requires-Dist: elasticsearch-dsl==8.18.0
+Requires-Dist: environs==14.2.0
 Requires-Dist: events==0.5
-Requires-Dist: greenlet==3.1.1
+Requires-Dist: greenlet==3.2.3
 Requires-Dist: idna==3.10
 Requires-Dist: jmespath==1.0.1
-Requires-Dist: marshmallow==3.26.1
+Requires-Dist: marshmallow==4.0.0
 Requires-Dist: opensearch-dsl==2.1.0
-Requires-Dist: opensearch-py==2.8.0
-Requires-Dist: packaging==24.2
+Requires-Dist: opensearch-py==3.0.0
 Requires-Dist: psycopg2-binary==2.9.10
 Requires-Dist: python-dateutil==2.9.0.post0
-Requires-Dist: python-dotenv==1.0.1
-Requires-Dist: redis==5.2.1
-Requires-Dist: requests==2.32.3
+Requires-Dist: python-dotenv==1.1.1
+Requires-Dist: redis==6.2.0
+Requires-Dist: requests==2.32.4
 Requires-Dist: requests-aws4auth==1.3.1
-Requires-Dist: s3transfer==0.11.4
+Requires-Dist: s3transfer==0.13.0
 Requires-Dist: six==1.17.0
-Requires-Dist: sqlalchemy==2.0.39
+Requires-Dist: sqlalchemy==2.0.41
 Requires-Dist: sqlparse==0.5.3
-Requires-Dist: typing-extensions==4.12.2
+Requires-Dist: typing-extensions==4.14.0
 Requires-Dist: urllib3==1.26.20
 Dynamic: author
 Dynamic: author-email
@@ -68,6 +68,7 @@ Dynamic: description-content-type
 Dynamic: home-page
 Dynamic: keywords
 Dynamic: license
+Dynamic: license-file
 Dynamic: maintainer
 Dynamic: maintainer-email
 Dynamic: project-url

{pgsync-4.0.0 → pgsync-4.1.0}/bin/bootstrap

@@ -18,16 +18,26 @@ logger = logging.getLogger(__name__)
     help="Schema config",
     type=click.Path(exists=True),
 )
-@click.option("--host", "-h", help="PG_HOST overide")
+@click.option("--host", "-h", help="PG_HOST override")
 @click.option("--password", is_flag=True, help="Prompt for database password")
-@click.option("--port", "-p", help="PG_PORT overide", type=int)
+@click.option("--port", "-p", help="PG_PORT override", type=int)
 @click.option(
     "--teardown",
     "-t",
     is_flag=True,
     help="Teardown database triggers and replication slots",
 )
-@click.option("--user", "-u", help="PG_USER overide")
+@click.option(
+    "--no-create",
+    "-nc",
+    is_flag=True,
+    help=(
+        "Skip DDL statement for objects "
+        "(Functions, Views & Replication slots) creation"
+    ),
+    default=False,
+)
+@click.option("--user", "-u", help="PG_USER override")
 @click.option(
     "--verbose",
     "-v",
@@ -43,6 +53,7 @@ def main(
     host: str,
     port: int,
     verbose: bool,
+    no_create: bool = False,
 ) -> None:
     """Application onetime Bootstrap."""
     kwargs: dict = {
@@ -75,7 +86,7 @@ def main(
         if teardown:
             sync.teardown()
             continue
-        sync.setup()
+        sync.setup(no_create=no_create)
         logger.info(f"Bootstrap: {sync.database}")
 
 

{pgsync-4.0.0 → pgsync-4.1.0}/bin/parallel_sync

@@ -39,7 +39,6 @@ filtered based on the page number and row numbers.
 This systematic and parallel approach optimizes the synchronization process,
 especially in environments challenged by network latency.
 """
-
 import asyncio
 import multiprocessing
 import os
@@ -48,6 +47,7 @@ import sys
 import typing as t
 from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
 from dataclasses import dataclass
+from pathlib import Path
 from queue import Queue
 from threading import Thread
 
@@ -415,10 +415,10 @@ def main(config: str, nprocs: int, mode: str, verbose: bool) -> None:
     - Track progress across cpus/threads
     - Handle KeyboardInterrupt Exception
     """
-
-    show_settings()
     config: str = get_config(config)
 
+    show_settings(config)
+
     for doc in config_loader(config):
         tasks: t.Generator = fetch_tasks(doc)
         if mode == "synchronous":

{pgsync-4.0.0 → pgsync-4.1.0}/pgsync/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = "Tolu Aina"
 __email__ = "tolu@pgsync.com"
-__version__ = "4.0.0"
+__version__ = "4.1.0"

{pgsync-4.0.0 → pgsync-4.1.0}/pgsync/base.py

@@ -2,7 +2,9 @@
 
 import logging
 import os
+import time
 import typing as t
+from contextlib import contextmanager
 
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql  # noqa
@@ -77,20 +79,20 @@ class Payload(object):
 
     def __init__(
         self,
-        tg_op: str = t.Optional[None],
-        table: str = t.Optional[None],
-        schema: str = t.Optional[None],
-        old: dict = t.Optional[None],
-        new: dict = t.Optional[None],
-        xmin: int = t.Optional[None],
-        indices: t.List[str] = t.Optional[None],
+        tg_op: t.Optional[str] = None,
+        table: t.Optional[str] = None,
+        schema: t.Optional[str] = None,
+        old: t.Optional[t.Dict[str, t.Any]] = None,
+        new: t.Optional[t.Dict[str, t.Any]] = None,
+        xmin: t.Optional[int] = None,
+        indices: t.Optional[t.List[str]] = None,
     ):
-        self.tg_op: str = tg_op
-        self.table: str = table
-        self.schema: str = schema
-        self.old: dict = old or {}
-        self.new: dict = new or {}
-        self.xmin: str = xmin
+        self.tg_op: t.Optional[str] = tg_op
+        self.table: t.Optional[str] = table
+        self.schema: t.Optional[str] = schema
+        self.old: t.Dict[str, t.Any] = old or {}
+        self.new: t.Dict[str, t.Any] = new or {}
+        self.xmin: t.Optional[int] = xmin
         self.indices: t.List[str] = indices
 
     @property
@@ -224,22 +226,27 @@ class Base(object):
 
     def _can_create_replication_slot(self, slot_name: str) -> None:
         """Check if the given user can create and destroy replication slots."""
-        if self.replication_slots(slot_name):
-            logger.exception(f"Replication slot {slot_name} already exists")
-            self.drop_replication_slot(slot_name)
+        with self.advisory_lock(
+            slot_name, max_retries=None, retry_interval=0.1
+        ):
+            if self.replication_slots(slot_name):
+                logger.exception(
+                    f"Replication slot {slot_name} already exists"
+                )
+                self.drop_replication_slot(slot_name)
 
-        try:
-            self.create_replication_slot(slot_name)
+            try:
+                self.create_replication_slot(slot_name)
 
-        except Exception as e:
-            logger.exception(f"{e}")
-            raise ReplicationSlotError(
-                f'PG_USER "{self.engine.url.username}" needs to be '
-                f"superuser or have permission to read, create and destroy "
-                f"replication slots to perform this action.\n{e}"
-            )
-        else:
-            self.drop_replication_slot(slot_name)
+            except Exception as e:
+                logger.exception(f"{e}")
+                raise ReplicationSlotError(
+                    f'PG_USER "{self.engine.url.username}" needs to be '
+                    f"superuser or have permission to read, create and destroy "
+                    f"replication slots to perform this action.\n{e}"
+                )
+            else:
+                self.drop_replication_slot(slot_name)
 
     # Tables...
     def models(self, table: str, schema: str) -> sa.sql.Alias:
@@ -413,20 +420,23 @@ class Base(object):
 
         SELECT * FROM PG_REPLICATION_SLOTS
         """
-        return self.fetchall(
-            sa.select("*")
-            .select_from(sa.text("PG_REPLICATION_SLOTS"))
-            .where(
-                sa.and_(
-                    *[
-                        sa.column("slot_name") == slot_name,
-                        sa.column("slot_type") == slot_type,
-                        sa.column("plugin") == plugin,
-                    ]
-                )
-            ),
-            label="replication_slots",
-        )
+        with self.advisory_lock(
+            slot_name, max_retries=None, retry_interval=0.1
+        ):
+            return self.fetchall(
+                sa.select("*")
+                .select_from(sa.text("PG_REPLICATION_SLOTS"))
+                .where(
+                    sa.and_(
+                        *[
+                            sa.column("slot_name") == slot_name,
+                            sa.column("slot_type") == slot_type,
+                            sa.column("plugin") == plugin,
+                        ]
+                    )
+                ),
+                label="replication_slots",
+            )
 
     def create_replication_slot(self, slot_name: str) -> None:
         """Create a replication slot.
@@ -439,14 +449,17 @@ class Base(object):
         """
         logger.debug(f"Creating replication slot: {slot_name}")
         try:
-            self.execute(
-                sa.select("*").select_from(
-                    sa.func.PG_CREATE_LOGICAL_REPLICATION_SLOT(
-                        slot_name,
-                        PLUGIN,
+            with self.advisory_lock(
+                slot_name, max_retries=None, retry_interval=0.1
+            ):
+                self.execute(
+                    sa.select("*").select_from(
+                        sa.func.PG_CREATE_LOGICAL_REPLICATION_SLOT(
+                            slot_name,
+                            PLUGIN,
+                        )
                     )
                 )
-            )
         except Exception as e:
             logger.exception(f"{e}")
             raise
@@ -457,16 +470,84 @@ class Base(object):
         logger.debug(f"Dropping replication slot: {slot_name}")
         if self.replication_slots(slot_name):
             try:
-                self.execute(
-                    sa.select("*").select_from(
-                        sa.func.PG_DROP_REPLICATION_SLOT(slot_name),
+                with self.advisory_lock(
+                    slot_name, max_retries=None, retry_interval=0.1
+                ):
+                    self.execute(
+                        sa.select("*").select_from(
+                            sa.func.PG_DROP_REPLICATION_SLOT(slot_name),
+                        )
                     )
-                )
             except Exception as e:
                 logger.exception(f"{e}")
                 raise
         logger.debug(f"Dropped replication slot: {slot_name}")
 
+    def advisory_key(self, slot_name: str) -> int:
+        """Compute a stable bigint advisory key from slot name."""
+        return self.fetchone(
+            sa.text("SELECT HASHTEXT(:slot)::BIGINT").bindparams(
+                slot=slot_name
+            )
+        )[0]
+
+    def pg_try_advisory_lock(self, key: int) -> bool:
+        """
+        Attempts to acquire an advisory lock based on a hashed slot name.
+
+        Returns:
+            bool: True if the lock was acquired, False otherwise.
+        """
+        result = self.fetchone(
+            sa.text("SELECT PG_TRY_ADVISORY_LOCK(:key)").bindparams(key=key)
+        )
+        return result[0] if result else False
+
+    def pg_advisory_unlock(self, key: int) -> bool:
+        """
+        Releases an advisory lock associated with the hashed slot name.
+
+        Returns:
+            bool: True if the lock was released, False if it was not held.
+        """
+        result = self.fetchone(
+            sa.text("SELECT PG_ADVISORY_UNLOCK(:key)").bindparams(key=key)
+        )
+        return result[0] if result else False
+
+    @contextmanager
+    def advisory_lock(
+        self,
+        slot_name: str,
+        max_retries: int = 5,
+        retry_interval: float = 1.0,
+        backoff_type: str = "fixed",  # or "exponential"
+        backoff_factor: float = 2.0,
+    ):
+        """Context manager to acquire a PostgreSQL advisory lock with optional retries."""
+        key: int = self.advisory_key(slot_name)
+        attempt: int = 0
+        delay: int = retry_interval
+        while True:
+            if self.pg_try_advisory_lock(key):
+                break
+
+            if max_retries is not None and attempt >= max_retries:
+                raise RuntimeError(
+                    f"Failed to acquire advisory lock for '{slot_name}' after {max_retries} retries."
+                )
+
+            time.sleep(delay)
+            if backoff_type == "exponential":
+                delay *= backoff_factor
+
+            attempt += 1
+
+        try:
+            yield
+        finally:
+            self.pg_advisory_unlock(key)
+
     def _logical_slot_changes(
         self,
         slot_name: str,
@@ -555,17 +636,22 @@ class Base(object):
         To get ALL changes and data in existing replication slot:
         SELECT * FROM PG_LOGICAL_SLOT_GET_CHANGES('testdb', NULL, NULL)
         """
-        statement: sa.sql.Select = self._logical_slot_changes(
-            slot_name,
-            sa.func.PG_LOGICAL_SLOT_GET_CHANGES,
-            txmin=txmin,
-            txmax=txmax,
-            upto_lsn=upto_lsn,
-            upto_nchanges=upto_nchanges,
-            limit=limit,
-            offset=offset,
-        )
-        self.execute(statement, options=dict(stream_results=STREAM_RESULTS))
+        with self.advisory_lock(
+            slot_name, max_retries=None, retry_interval=0.1
+        ):
+            statement: sa.sql.Select = self._logical_slot_changes(
+                slot_name,
+                sa.func.PG_LOGICAL_SLOT_GET_CHANGES,
+                txmin=txmin,
+                txmax=txmax,
+                upto_lsn=upto_lsn,
+                upto_nchanges=upto_nchanges,
+                limit=limit,
+                offset=offset,
+            )
+            self.execute(
+                statement, options=dict(stream_results=STREAM_RESULTS)
+            )
 
     def logical_slot_peek_changes(
         self,
@@ -581,17 +667,20 @@ class Base(object):
 
         SELECT * FROM PG_LOGICAL_SLOT_PEEK_CHANGES('testdb', NULL, 1)
         """
-        statement: sa.sql.Select = self._logical_slot_changes(
-            slot_name,
-            sa.func.PG_LOGICAL_SLOT_PEEK_CHANGES,
-            txmin=txmin,
-            txmax=txmax,
-            upto_lsn=upto_lsn,
-            upto_nchanges=upto_nchanges,
-            limit=limit,
-            offset=offset,
-        )
-        return self.fetchall(statement)
+        with self.advisory_lock(
+            slot_name, max_retries=None, retry_interval=0.1
+        ):
+            statement: sa.sql.Select = self._logical_slot_changes(
+                slot_name,
+                sa.func.PG_LOGICAL_SLOT_PEEK_CHANGES,
+                txmin=txmin,
+                txmax=txmax,
+                upto_lsn=upto_lsn,
+                upto_nchanges=upto_nchanges,
+                limit=limit,
+                offset=offset,
+            )
+            return self.fetchall(statement)
 
     def logical_slot_count_changes(
         self,
@@ -615,6 +704,10 @@ class Base(object):
             ).scalar()
 
     # Views...
+
+    def view_exists(self, name: str, schema: str) -> bool:
+        return name in self.views(schema)
+
     def create_view(
         self,
         index: str,
@@ -636,7 +729,9 @@ class Base(object):
     def drop_view(self, schema: str) -> None:
         """Drop a view."""
         logger.debug(f"Dropping view: {schema}.{MATERIALIZED_VIEW}")
-        with self.engine.connect() as conn:
+        with self.engine.connect().execution_options(
+            isolation_level="AUTOCOMMIT"
+        ) as conn:
             conn.execute(DropView(schema, MATERIALIZED_VIEW))
         logger.debug(f"Dropped view: {schema}.{MATERIALIZED_VIEW}")
 
@@ -645,16 +740,44 @@ class Base(object):
     ) -> None:
         """Refresh a materialized view."""
         logger.debug(f"Refreshing view: {schema}.{name}")
-        with self.engine.connect() as conn:
+        with self.engine.connect().execution_options(
+            isolation_level="AUTOCOMMIT"
+        ) as conn:
             conn.execute(RefreshView(schema, name, concurrently=concurrently))
         logger.debug(f"Refreshed view: {schema}.{name}")
 
     # Triggers...
+
+    def trigger_exists(self, trigger: str, table: str, schema: str) -> bool:
+        """
+        Return True if the user-defined trigger is already present on table in schema.
+        """
+        sql: str = """
+            SELECT EXISTS (
+                SELECT 1
+                FROM   pg_trigger     AS t
+                JOIN   pg_class       AS c  ON c.oid = t.tgrelid
+                JOIN   pg_namespace   AS n  ON n.oid = c.relnamespace
+                WHERE  NOT t.tgisinternal           -- exclude system triggers
+                AND  t.tgname   = :trigge
+                AND  c.relname  = :table
+                AND  n.nspname  = :schema
+            )
+        """
+        params: dict = dict(
+            trigger=trigger,
+            table=table,
+            schema=schema,
+        )
+        with self.engine.connect() as conn:
+            return bool(conn.execute(sa.text(sql), params).scalar())
+
     def create_triggers(
         self,
         schema: str,
         tables: t.Optional[t.List[str]] = None,
         join_queries: bool = False,
+        if_not_exists: bool = False,
     ) -> None:
         """Create a database triggers."""
         queries: t.List[str] = []
@@ -668,13 +791,18 @@ class Base(object):
                 ("notify", "ROW", ["INSERT", "UPDATE", "DELETE"]),
                 ("truncate", "STATEMENT", ["TRUNCATE"]),
             ]:
-                self.drop_triggers(schema, [table])
-                queries.append(
-                    f'CREATE TRIGGER "{schema}_{table}_{name}" '
-                    f'AFTER {" OR ".join(tg_op)} ON "{schema}"."{table}" '
-                    f"FOR EACH {level} EXECUTE PROCEDURE "
-                    f"{schema}.{TRIGGER_FUNC}()",
-                )
+
+                if if_not_exists or not self.view_exists(
+                    MATERIALIZED_VIEW, schema
+                ):
+
+                    self.drop_triggers(schema, [table])
+                    queries.append(
+                        f'CREATE TRIGGER "{schema}_{table}_{name}" '
+                        f'AFTER {" OR ".join(tg_op)} ON "{schema}"."{table}" '
+                        f"FOR EACH {level} EXECUTE PROCEDURE "
+                        f"{schema}.{TRIGGER_FUNC}()",
+                    )
         if join_queries:
             if queries:
                 self.execute(sa.text("; ".join(queries)))
@@ -706,6 +834,16 @@ class Base(object):
             for query in queries:
                 self.execute(sa.text(query))
 
+    def function_exists(self, schema: str) -> bool:
+        """Check if the trigger function exists."""
+        return self.exists(
+            sa.text(
+                f"SELECT 1 FROM pg_proc WHERE proname = :name "
+                f"AND pronamespace = (SELECT oid FROM pg_namespace "
+                f"WHERE nspname = :schema)"
+            ).bindparams(name=TRIGGER_FUNC, schema=schema),
+        )
+
     def create_function(self, schema: str) -> None:
         self.execute(
             sa.text(
@@ -895,6 +1033,20 @@ class Base(object):
         with self.engine.connect() as conn:
             return conn.execute(statement).fetchall()
 
+    def exists(
+        self,
+        statement: sa.sql.Select,
+        label: t.Optional[str] = None,
+        literal_binds: bool = False,
+    ) -> t.List[sa.engine.Row]:
+        if self.verbose:
+            compiled_query(statement, label=label, literal_binds=literal_binds)
+        with self.engine.connect() as conn:
+            result = conn.execute(statement).fetchone()
+            if result is None:
+                return False
+            return result[0] > 0
+
     def fetchmany(
         self,
         statement: sa.sql.Select,

{pgsync-4.0.0 → pgsync-4.1.0}/pgsync/redisqueue.py

@@ -7,7 +7,11 @@ import typing as t
 from redis import Redis
 from redis.exceptions import ConnectionError
 
-from .settings import REDIS_READ_CHUNK_SIZE, REDIS_SOCKET_TIMEOUT
+from .settings import (
+    REDIS_READ_CHUNK_SIZE,
+    REDIS_RETRY_ON_TIMEOUT,
+    REDIS_SOCKET_TIMEOUT,
+)
 from .urls import get_redis_url
 
 logger = logging.getLogger(__name__)
@@ -20,10 +24,12 @@ class RedisQueue(object):
         """Init Simple Queue with Redis Backend."""
         url: str = get_redis_url(**kwargs)
         self.key: str = f"{namespace}:{name}"
+        self._meta_key: str = f"{self.key}:meta"
         try:
             self.__db: Redis = Redis.from_url(
                 url,
                 socket_timeout=REDIS_SOCKET_TIMEOUT,
+                retry_on_timeout=REDIS_RETRY_ON_TIMEOUT,
             )
             self.__db.ping()
         except ConnectionError as e:
@@ -54,3 +60,12 @@ class RedisQueue(object):
         """Delete all items from the named queue."""
         logger.info(f"Deleting redis key: {self.key}")
         self.__db.delete(self.key)
+
+    def set_meta(self, value: t.Any) -> None:
+        """Store an arbitrary JSON-serialisable value in a dedicated key."""
+        self.__db.set(self._meta_key, json.dumps(value))
+
+    def get_meta(self, default: t.Any = None) -> t.Any:
+        """Retrieve the stored value (or *default* if nothing is set)."""
+        raw = self.__db.get(self._meta_key)
+        return json.loads(raw) if raw is not None else default