pgsync 6.1.1__tar.gz → 7.0.0__tar.gz

{pgsync-6.1.1 → pgsync-7.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pgsync
-Version: 6.1.1
+Version: 7.0.0
 Summary: Postgres/MySQL/MariaDB to Elasticsearch/OpenSearch sync
 Home-page: https://github.com/toluaina/pgsync
 Author: Tolu Aina
@@ -33,8 +33,8 @@ License-File: LICENSE
 License-File: AUTHORS.rst
 Requires-Dist: async-timeout==5.0.1
 Requires-Dist: backports-datetime-fromisoformat==2.0.3
-Requires-Dist: boto3==1.41.5
-Requires-Dist: botocore==1.41.5
+Requires-Dist: boto3==1.42.2
+Requires-Dist: botocore==1.42.2
 Requires-Dist: certifi==2025.11.12
 Requires-Dist: charset-normalizer==3.4.4
 Requires-Dist: click==8.1.8
@@ -58,7 +58,7 @@ Requires-Dist: python-dotenv==1.2.1
 Requires-Dist: redis==7.0.1
 Requires-Dist: requests==2.32.5
 Requires-Dist: requests-aws4auth==1.3.1
-Requires-Dist: s3transfer==0.15.0
+Requires-Dist: s3transfer==0.16.0
 Requires-Dist: six==1.17.0
 Requires-Dist: sqlalchemy==2.0.44
 Requires-Dist: sqlparse==0.5.4

{pgsync-6.1.1 → pgsync-7.0.0}/pgsync/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = "Tolu Aina"
 __email__ = "tolu@pgsync.com"
-__version__ = "6.1.1"
+__version__ = "7.0.0"

{pgsync-6.1.1 → pgsync-7.0.0}/pgsync/base.py

@@ -1404,14 +1404,24 @@ def _pg_engine(
 def pg_execute(
     engine: sa.engine.Engine,
     statement: sa.sql.Select,
-    values: t.Optional[list] = None,
+    values: t.Optional[t.Mapping] = None,
     options: t.Optional[dict] = None,
-) -> None:
+) -> sa.engine.Result:
+    """Execute a query statement."""
     with engine.connect() as conn:
         if options:
             conn = conn.execution_options(**options)
-        conn.execute(statement, values)
-        conn.commit()
+
+        # Don't pass `None` as values
+        if values is not None:
+            result = conn.execute(statement, values)
+        else:
+            result = conn.execute(statement)
+
+        # If caller did NOT request AUTOCOMMIT, commit the transaction
+        if not (options and options.get("isolation_level") == "AUTOCOMMIT"):
+            conn.commit()
+        return result
 
 
 def create_schema(database: str, schema: str, echo: bool = False) -> None:

{pgsync-6.1.1 → pgsync-7.0.0}/pgsync/redisqueue.py

@@ -25,15 +25,30 @@ class RedisQueue(object):
         url: str = get_redis_url(**kwargs)
         self.key: str = f"{namespace}:{name}"
         self._meta_key: str = f"{self.key}:meta"
+        self.__db: Redis = Redis.from_url(
+            url,
+            socket_timeout=REDIS_SOCKET_TIMEOUT,
+            retry_on_timeout=REDIS_RETRY_ON_TIMEOUT,
+        )
+        try:
+            self.ping()
+        except ConnectionError:
+            raise
+
+    def ping(self) -> bool:
+        """
+        Ping the Redis server to check connectivity.
+
+        Returns:
+            True if the server responded to PING.
+
+        Raises:
+            RedisConnectionError: If the server cannot be reached.
+        """
         try:
-            self.__db: Redis = Redis.from_url(
-                url,
-                socket_timeout=REDIS_SOCKET_TIMEOUT,
-                retry_on_timeout=REDIS_RETRY_ON_TIMEOUT,
-            )
-            self.__db.ping()
-        except ConnectionError as e:
-            logger.exception(f"Redis server is not running: {e}")
+            return bool(self.__db.ping())
+        except ConnectionError as exc:
+            logger.exception("Redis server is not reachable when pinging.")
             raise
 
     @property

{pgsync-6.1.1 → pgsync-7.0.0}/pgsync/sync.py

@@ -28,6 +28,7 @@ from pymysqlreplication.row_event import (
     UpdateRowsEvent,
     WriteRowsEvent,
 )
+from redis.exceptions import ConnectionError
 
 from pgsync.settings import IS_MYSQL_COMPAT
 
@@ -126,7 +127,7 @@ class Sync(Base, metaclass=Singleton):
             self.models, nodes=self.nodes, database=doc["database"]
         )
         if bootstrap:
-            self.setup(wal=wal, polling=polling)
+            self.setup(polling=polling)
 
         if validate:
             self.validate(repl_slots=repl_slots, polling=polling)
@@ -217,8 +218,9 @@ class Sync(Base, metaclass=Singleton):
             # ensure Redis is reachable
             try:
                 self.redis.ping()
-            except Exception as e:
-                raise RuntimeError(f"Cannot reach Redis: {e}")
+            except ConnectionError:
+                raise
+
         else:
             # ensure the checkpoint dirpath is valid
             if not Path(settings.CHECKPOINT_PATH).exists():
@@ -320,9 +322,7 @@ class Sync(Base, metaclass=Singleton):
             routing=self.routing,
         )
 
-    def setup(
-        self, no_create: bool = False, wal: bool = False, polling: bool = False
-    ) -> None:
+    def setup(self, no_create: bool = False, polling: bool = False) -> None:
         """Create the database triggers and replication slot.
         Generally bootstrap should not require Redis as it is optional in certain cases.
         """
@@ -344,6 +344,12 @@ class Sync(Base, metaclass=Singleton):
 
             if not polling:
                 for schema in self.schemas:
+                    if schema not in self.tree.schemas:
+                        logger.warning(
+                            f"Schema '{schema}' not found in node definitions. Skipping..."
+                        )
+                        continue
+
                     # TODO: move if_not_exists to the function
                     if if_not_exists or not self.function_exists(schema):
 
@@ -419,7 +425,9 @@ class Sync(Base, metaclass=Singleton):
                     self.create_replication_slot(self.__name)
 
     def teardown(
-        self, drop_view: bool = True, polling: bool = False, wal: bool = False
+        self,
+        drop_view: bool = True,
+        polling: bool = False,
     ) -> None:
         """Drop the database triggers and replication slot."""
         if self.is_mysql_compat:
@@ -440,7 +448,7 @@ class Sync(Base, metaclass=Singleton):
                 )
 
             try:
-                if self._redis is None:
+                if self.redis is None:
                     raise RuntimeError("Redis is not configured.")
                 self.redis.delete()
             except Exception as e:
@@ -448,6 +456,12 @@ class Sync(Base, metaclass=Singleton):
 
             if not polling:
                 for schema in self.schemas:
+                    if schema not in self.tree.schemas:
+                        logger.warning(
+                            f"Schema '{schema}' not found in node definitions. Skipping..."
+                        )
+                        continue
+
                     tables: t.Set = set()
                     for node in self.tree.traverse_breadth_first():
                         tables |= set(
@@ -466,7 +480,7 @@ class Sync(Base, metaclass=Singleton):
                         self.drop_view(schema)
                         self.drop_function(schema)
 
-            if not wal:
+            if not polling:
                 self.drop_replication_slot(self.__name)
 
     def get_doc_id(self, primary_keys: t.List[str], table: str) -> str:
@@ -1049,6 +1063,11 @@ class Sync(Base, metaclass=Singleton):
             if _filters:
                 filters[self.tree.root.table].extend(_filters)
 
+            # also check through table with a direct references to root
+            _filters = self._through_node_resolver(node, payloads, _filters)
+            if _filters:
+                filters[self.tree.root.table].extend(_filters)
+
         elif node.table in self.tree.tables:
             if node.is_root:
                 for payload in payloads:

{pgsync-6.1.1 → pgsync-7.0.0}/pgsync/trigger.py

@@ -27,7 +27,7 @@ BEGIN
 
     IF TG_OP = 'DELETE' THEN
 
-        SELECT primary_keys INTO _primary_keys
+        SELECT primary_keys, indices INTO _primary_keys, _indices
         FROM {MATERIALIZED_VIEW}
         WHERE table_name = TG_TABLE_NAME;
 

{pgsync-6.1.1 → pgsync-7.0.0}/pgsync.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pgsync
-Version: 6.1.1
+Version: 7.0.0
 Summary: Postgres/MySQL/MariaDB to Elasticsearch/OpenSearch sync
 Home-page: https://github.com/toluaina/pgsync
 Author: Tolu Aina
@@ -33,8 +33,8 @@ License-File: LICENSE
 License-File: AUTHORS.rst
 Requires-Dist: async-timeout==5.0.1
 Requires-Dist: backports-datetime-fromisoformat==2.0.3
-Requires-Dist: boto3==1.41.5
-Requires-Dist: botocore==1.41.5
+Requires-Dist: boto3==1.42.2
+Requires-Dist: botocore==1.42.2
 Requires-Dist: certifi==2025.11.12
 Requires-Dist: charset-normalizer==3.4.4
 Requires-Dist: click==8.1.8
@@ -58,7 +58,7 @@ Requires-Dist: python-dotenv==1.2.1
 Requires-Dist: redis==7.0.1
 Requires-Dist: requests==2.32.5
 Requires-Dist: requests-aws4auth==1.3.1
-Requires-Dist: s3transfer==0.15.0
+Requires-Dist: s3transfer==0.16.0
 Requires-Dist: six==1.17.0
 Requires-Dist: sqlalchemy==2.0.44
 Requires-Dist: sqlparse==0.5.4

{pgsync-6.1.1 → pgsync-7.0.0}/pgsync.egg-info/requires.txt

@@ -1,7 +1,7 @@
 async-timeout==5.0.1
 backports-datetime-fromisoformat==2.0.3
-boto3==1.41.5
-botocore==1.41.5
+boto3==1.42.2
+botocore==1.42.2
 certifi==2025.11.12
 charset-normalizer==3.4.4
 click==8.1.8
@@ -25,7 +25,7 @@ python-dotenv==1.2.1
 redis==7.0.1
 requests==2.32.5
 requests-aws4auth==1.3.1
-s3transfer==0.15.0
+s3transfer==0.16.0
 six==1.17.0
 sqlalchemy==2.0.44
 sqlparse==0.5.4

{pgsync-6.1.1 → pgsync-7.0.0}/tests/test_redisqueue.py

@@ -41,7 +41,7 @@ class TestRedisQueue(object):
         mock_get_redis_url.assert_called_once()
         mock_ping.assert_called_once()
         mock_logger.exception.assert_called_once_with(
-            "Redis server is not running: pong"
+            "Redis server is not reachable when pinging."
         )
 
     def test_qsize(self, mocker):

{pgsync-6.1.1 → pgsync-7.0.0}/tests/test_sync.py

@@ -4,12 +4,15 @@ import importlib
 import os
 import typing as t
 from collections import namedtuple
+from types import SimpleNamespace
+from unittest.mock import Mock
 
 import pytest
 from mock import ANY, call, patch
 
 from pgsync.base import Base, Payload
 from pgsync.exc import (
+    ForeignKeyError,
     InvalidTGOPError,
     PrimaryKeyNotFoundError,
     RDSError,
@@ -960,7 +963,7 @@ class TestSync(object):
                         )
                     mock_drop_view.assert_called_once_with("public")
                 mock_drop_function.assert_called_once_with("public")
-            mock_redis_delete.assert_not_called()
+            mock_redis_delete.assert_called_once()
             assert os.path.exists(sync.checkpoint_file) is False
 
         with patch("pgsync.sync.logger") as mock_logger:
@@ -969,9 +972,6 @@ class TestSync(object):
                 sync.teardown()
                 assert mock_logger.warning.call_args_list == [
                     call("Checkpoint file not found: ./.testdb_testdb"),
-                    call(
-                        "Could not clear Redis checkpoint queue: Redis is not configured."
-                    ),
                 ]
 
     def test_root(self, sync):
@@ -1082,3 +1082,371 @@ class TestSync(object):
         mock_logger.debug.assert_called_once_with(f"_poll_redis: {items}")
         mock_time.sleep.assert_called_once_with(settings.REDIS_POLL_INTERVAL)
         assert sync.count["redis"] == 2
+
+    def test_insert_op_non_root_uses_foreign_keys_and_resolvers(self, sync):
+        """
+        Non-root, non-through node:
+        - uses foreign_keys to populate parent filters
+        - uses _root_foreign_key_resolver + _through_node_resolver to populate root filters
+        """
+
+        # Stub parent and child nodes (duck-typed; don't need the real Node class)
+        parent_node = SimpleNamespace(
+            name="parent",
+            table="parent",
+            parent=None,
+            is_root=False,
+            is_through=False,
+        )
+        node = SimpleNamespace(
+            name="child",
+            table="child",
+            parent=parent_node,
+            is_root=False,
+            is_through=False,
+        )
+
+        # Stub the tree structure
+        sync.tree = SimpleNamespace(
+            tables={"child", "parent", "root"},
+            root=SimpleNamespace(
+                table="root",
+                model=SimpleNamespace(primary_keys=["id"]),
+                parent=None,
+            ),
+        )
+
+        # foreign_keys[node.name] and foreign_keys[node.parent.name] share a key
+        # so that the inner equality condition is hit.
+        sync.query_builder = Mock()
+        sync.query_builder.get_foreign_keys.return_value = {
+            "child": ["id"],
+            "parent": ["id"],
+        }
+
+        # Make the root resolvers predictable
+        sync._root_foreign_key_resolver = Mock(return_value=[{"root_id": 1}])
+
+        def through_node_resolver(node_arg, payloads_arg, filters_arg):
+            # emulate "extend" behaviour
+            filters_arg.append({"root_id": 2})
+            return filters_arg
+
+        sync._through_node_resolver = through_node_resolver
+
+        filters: dict[str, t.List[dict]] = {
+            "parent": [],
+            "root": [],
+        }
+
+        payloads: t.List[Payload] = [
+            Payload(
+                tg_op="INSERT",
+                table="child",
+                new={"id": 99},  # matches foreign_keys["child"] and ["parent"]
+            )
+        ]
+
+        result = sync._insert_op(node, filters, payloads)
+
+        # 1) Parent should get a filter derived from the foreign key
+        assert {"id": 99} in result["parent"]
+
+        # 2) Root should get filters coming back from both resolvers
+        assert {"root_id": 1} in result["root"]
+        assert {"root_id": 2} in result["root"]
+
+        # Sanity: we didn't create any unexpected tables in the filters
+        assert set(result.keys()) == {"parent", "root"}
+
+    def test_insert_op_non_root_falls_back_to__get_foreign_keys(self, sync):
+        """
+        When get_foreign_keys raises ForeignKeyError, _get_foreign_keys is used.
+        """
+
+        parent_node = SimpleNamespace(
+            name="parent",
+            table="parent",
+            parent=None,
+            is_root=False,
+            is_through=False,
+        )
+        node = SimpleNamespace(
+            name="child",
+            table="child",
+            parent=parent_node,
+            is_root=False,
+            is_through=False,
+        )
+
+        sync.tree = SimpleNamespace(
+            tables={"child", "parent", "root"},
+            root=SimpleNamespace(
+                table="root",
+                model=SimpleNamespace(primary_keys=["id"]),
+                parent=None,
+            ),
+        )
+
+        sync.query_builder = Mock()
+        # Primary method fails...
+        sync.query_builder.get_foreign_keys.side_effect = ForeignKeyError(
+            "no fk"
+        )
+        # ...fallback provides the mapping actually used
+        sync.query_builder._get_foreign_keys.return_value = {
+            "child": ["id"],
+            "parent": ["id"],
+        }
+
+        sync._root_foreign_key_resolver = Mock(return_value=[])
+        sync._through_node_resolver = Mock(return_value=[])
+
+        filters: dict[str, t.List[dict]] = {
+            "parent": [],
+            "root": [],
+        }
+
+        payloads: t.List[Payload] = [
+            Payload(
+                tg_op="INSERT",
+                table="child",
+                new={"id": 123},
+            )
+        ]
+
+        result = sync._insert_op(node, filters, payloads)
+
+        # Parent filter must come from fallback mapping
+        assert {"id": 123} in result["parent"]
+
+        sync.query_builder.get_foreign_keys.assert_called_once()
+        sync.query_builder._get_foreign_keys.assert_called_once()
+
+    def test_insert_op_through_node_populates_parent_and_root(self, sync):
+        """
+        Through node:
+        - Uses FKs to populate parent filters
+        - Uses _root_primary_key_resolver for parent and grandparent
+        - Uses _through_node_resolver for additional root filters
+        """
+        grandparent = SimpleNamespace(
+            name="grand",
+            table="grand",
+            parent=None,
+            is_root=False,
+            is_through=False,
+        )
+        parent = SimpleNamespace(
+            name="parent",
+            table="parent",
+            parent=grandparent,
+            is_root=False,
+            is_through=False,
+        )
+        node = SimpleNamespace(
+            name="through",
+            table="through_table",
+            parent=parent,
+            is_root=False,
+            is_through=True,
+        )
+
+        sync.tree = SimpleNamespace(
+            tables={"grand", "parent", "through_table", "root"},
+            root=SimpleNamespace(
+                table="root",
+                model=SimpleNamespace(primary_keys=["id"]),
+                parent=None,
+            ),
+        )
+
+        # FKs used for the "if column in payload.data" part
+        sync.query_builder = SimpleNamespace()
+        sync.query_builder.get_foreign_keys = lambda parent_node, child_node: {
+            "through": ["parent_id"],
+        }
+
+        # Track calls so we know we hit parent and grandparent
+        resolver_calls: list[str] = []
+
+        def root_primary_key_resolver(node_arg, payloads_arg, filters_arg):
+            """
+            Emulate real behaviour: mutate filters_arg and return it.
+            """
+            resolver_calls.append(node_arg.name)
+            if node_arg is parent:
+                filters_arg.append({"id": 10})
+            elif node_arg is grandparent:
+                filters_arg.append({"id": 20})
+            return filters_arg
+
+        def through_node_resolver(node_arg, payloads_arg, filters_arg):
+            """
+            Also mutates filters_arg and returns it.
+            """
+            filters_arg.append({"id": 30})
+            return filters_arg
+
+        sync._root_primary_key_resolver = root_primary_key_resolver
+        sync._through_node_resolver = through_node_resolver
+
+        filters: dict[str, t.List[dict]] = {
+            "parent": [],
+            "root": [],
+        }
+
+        payloads: t.List[Payload] = [
+            Payload(
+                tg_op="INSERT",
+                table="through_table",
+                new={"parent_id": 5},
+            )
+        ]
+
+        result = sync._insert_op(node, filters, payloads)
+
+        # Parent got populated from FK
+        assert {"parent_id": 5} in result["parent"]
+
+        root_filters = result["root"]
+
+        # Root got filters from both root resolvers and through resolver
+        assert {"id": 10} in root_filters
+        assert {"id": 20} in root_filters
+        assert {"id": 30} in root_filters
+
+        # We called the resolver for both parent and grandparent
+        assert resolver_calls == ["parent", "grand"]
+
+    def test_insert_op_ignores_node_not_in_tree_tables(self, sync):
+        """
+        When node.table is not in self.tree.tables and node.is_through is False,
+        the function should return filters unchanged.
+        """
+
+        node = SimpleNamespace(
+            name="other",
+            table="other_table",
+            parent=None,
+            is_root=False,
+            is_through=False,
+        )
+
+        sync.tree = SimpleNamespace(
+            tables={"book", "publisher"},  # "other_table" not listed
+            root=SimpleNamespace(
+                table="book",
+                model=SimpleNamespace(primary_keys=["isbn"]),
+                parent=None,
+            ),
+        )
+
+        original_filters = {"book": [{"isbn": "001"}]}
+        filters = {"book": [{"isbn": "001"}]}
+
+        payloads = []  # no payloads; nothing should happen
+
+        result = sync._insert_op(node, filters, payloads)
+
+        # Same content, no mutation beyond what was already there
+        assert result == original_filters
+
+    def test_insert_op_root_node_with_composite_primary_key(self, sync):
+        """
+        Root node: ensure we correctly build filters for composite PKs.
+        """
+
+        # Root node
+        node = SimpleNamespace(
+            name="order",
+            table="order",
+            parent=None,
+            is_root=True,
+            is_through=False,
+        )
+
+        sync.tree = SimpleNamespace(
+            tables={"order"},
+            root=SimpleNamespace(
+                table="order",
+                model=SimpleNamespace(primary_keys=["id", "version"]),
+                parent=None,
+            ),
+        )
+
+        filters: dict[str, t.List[dict]] = {"order": []}
+
+        payloads: t.List[Payload] = [
+            Payload(
+                tg_op="INSERT",
+                table="order",
+                new={"id": 1, "version": 2, "other": "ignored"},
+            )
+        ]
+
+        result = sync._insert_op(node, filters, payloads)
+
+        # Should only contain PK fields, not extra ones
+        assert result == {"order": [{"id": 1, "version": 2}]}
+
+    def test_insert_op_non_root_with_mismatched_foreign_keys(self, sync):
+        """
+        Non-root, non-through node where FK names don't match between child/parent.
+        - We should not add anything to parent filters.
+        """
+
+        parent_node = SimpleNamespace(
+            name="parent",
+            table="parent",
+            parent=None,
+            is_root=False,
+            is_through=False,
+        )
+        node = SimpleNamespace(
+            name="child",
+            table="child",
+            parent=parent_node,
+            is_root=False,
+            is_through=False,
+        )
+
+        sync.tree = SimpleNamespace(
+            tables={"parent", "child", "root"},
+            root=SimpleNamespace(
+                table="root",
+                model=SimpleNamespace(primary_keys=["id"]),
+                parent=None,
+            ),
+        )
+
+        sync.query_builder = Mock()
+        # Note: no matching key names between child and parent
+        sync.query_builder.get_foreign_keys.return_value = {
+            "child": ["child_id"],
+            "parent": ["parent_id"],
+        }
+
+        # No root/through filters for this test
+        sync._root_foreign_key_resolver = Mock(return_value=[])
+        sync._through_node_resolver = Mock(return_value=[])
+
+        filters: dict[str, t.List[dict]] = {
+            "parent": [],
+            "root": [],
+        }
+
+        payloads: t.List[Payload] = [
+            Payload(
+                tg_op="INSERT",
+                table="child",
+                new={"child_id": 42},
+            )
+        ]
+
+        result = sync._insert_op(node, filters, payloads)
+
+        # Parent remains empty because no FK name match
+        assert result["parent"] == []
+        # Root remains unchanged because resolvers returned empty
+        assert result["root"] == []