pgsync 3.1.0__tar.gz → 3.2.1__tar.gz

{pgsync-3.1.0 → pgsync-3.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pgsync
-Version: 3.1.0
+Version: 3.2.1
 Summary: Postgres to Elasticsearch/OpenSearch sync
 Home-page: https://github.com/toluaina/pgsync
 Author: Tolu Aina
@@ -13,7 +13,7 @@ Project-URL: Funding, https://github.com/sponsors/toluaina
 Project-URL: Source, https://github.com/toluaina/pgsync
 Project-URL: Web, https://pgsync.com
 Project-URL: Documentation, https://pgsync.com
-Keywords: pgsync,elasticsearch,opensearch,postgres,change data capture
+Keywords: change data capture,elasticsearch,opensearch,pgsync,postgres
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Natural Language :: English
@@ -31,34 +31,35 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 License-File: AUTHORS.rst
 Requires-Dist: async-timeout==4.0.3
-Requires-Dist: boto3==1.34.11
-Requires-Dist: botocore==1.34.11
-Requires-Dist: certifi==2023.11.17
+Requires-Dist: boto3==1.35.5
+Requires-Dist: botocore==1.35.5
+Requires-Dist: certifi==2024.7.4
 Requires-Dist: charset-normalizer==3.3.2
 Requires-Dist: click==8.1.7
-Requires-Dist: elastic-transport==8.11.0
-Requires-Dist: elasticsearch==8.11.1
-Requires-Dist: elasticsearch-dsl==8.11.0
-Requires-Dist: environs==10.0.0
+Requires-Dist: elastic-transport==8.15.0
+Requires-Dist: elasticsearch==8.15.0
+Requires-Dist: elasticsearch-dsl==8.15.1
+Requires-Dist: environs==11.0.0
+Requires-Dist: events==0.5
 Requires-Dist: greenlet==3.0.3
-Requires-Dist: idna==3.6
+Requires-Dist: idna==3.8
 Requires-Dist: jmespath==1.0.1
-Requires-Dist: marshmallow==3.20.1
+Requires-Dist: marshmallow==3.22.0
 Requires-Dist: opensearch-dsl==2.1.0
-Requires-Dist: opensearch-py==2.4.2
-Requires-Dist: packaging==23.2
+Requires-Dist: opensearch-py==2.7.1
+Requires-Dist: packaging==24.1
 Requires-Dist: psycopg2-binary==2.9.9
-Requires-Dist: python-dateutil==2.8.2
-Requires-Dist: python-dotenv==1.0.0
-Requires-Dist: redis==5.0.1
-Requires-Dist: requests==2.31.0
-Requires-Dist: requests-aws4auth==1.2.3
-Requires-Dist: s3transfer==0.10.0
+Requires-Dist: python-dateutil==2.9.0.post0
+Requires-Dist: python-dotenv==1.0.1
+Requires-Dist: redis==5.0.8
+Requires-Dist: requests==2.32.3
+Requires-Dist: requests-aws4auth==1.3.1
+Requires-Dist: s3transfer==0.10.2
 Requires-Dist: six==1.16.0
-Requires-Dist: sqlalchemy==2.0.25
-Requires-Dist: sqlparse==0.4.4
-Requires-Dist: typing-extensions==4.9.0
-Requires-Dist: urllib3==1.26.18
+Requires-Dist: sqlalchemy==2.0.32
+Requires-Dist: sqlparse==0.5.1
+Requires-Dist: typing-extensions==4.12.2
+Requires-Dist: urllib3==1.26.19
 
 # PostgreSQL to Elasticsearch/OpenSearch sync
 

{pgsync-3.1.0 → pgsync-3.2.1}/bin/bootstrap

@@ -35,7 +35,15 @@ logger = logging.getLogger(__name__)
     default=False,
     help="Turn on verbosity",
 )
-def main(teardown, config, user, password, host, port, verbose):
+def main(
+    teardown: bool,
+    config: str,
+    user: str,
+    password: bool,
+    host: str,
+    port: int,
+    verbose: bool,
+) -> None:
     """Application onetime Bootstrap."""
     kwargs: dict = {
         "user": user,

{pgsync-3.1.0 → pgsync-3.2.1}/bin/parallel_sync

@@ -409,7 +409,7 @@ def run_task(
     ),
     default="multiprocess_async",
 )
-def main(config, nprocs, mode, verbose):
+def main(config: str, nprocs: int, mode: str, verbose: bool) -> None:
     """
     TODO:
     - Track progress across cpus/threads

{pgsync-3.1.0 → pgsync-3.2.1}/pgsync/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = "Tolu Aina"
 __email__ = "tolu@pgsync.com"
-__version__ = "3.1.0"
+__version__ = "3.2.1"

{pgsync-3.1.0 → pgsync-3.2.1}/pgsync/base.py

@@ -1,4 +1,5 @@
 """PGSync Base."""
+
 import logging
 import os
 import typing as t
@@ -48,6 +49,15 @@ except ImportError:
 
 logger = logging.getLogger(__name__)
 
+SSL_MODES = (
+    "allow",
+    "disable",
+    "prefer",
+    "require",
+    "verify-ca",
+    "verify-full",
+)
+
 
 class Payload(object):
     """
@@ -141,6 +151,36 @@ class TupleIdentifierType(sa.types.UserDefinedType):
 
 
 class Base(object):
+    INT_TYPES = (
+        "bigint",
+        "bigserial",
+        "int",
+        "int2",
+        "int4",
+        "int8",
+        "integer",
+        "serial",
+        "serial2",
+        "serial4",
+        "serial8",
+        "smallint",
+        "smallserial",
+    )
+    FLOAT_TYPES = (
+        "double precision",
+        "float4",
+        "float8",
+        "real",
+    )
+    CHAR_TYPES = (
+        "char",
+        "character",
+        "character varying",
+        "text",
+        "uuid",
+        "varchar",
+    )
+
     def __init__(
         self, database: str, verbose: bool = False, *args, **kwargs
     ) -> None:
@@ -433,7 +473,7 @@ class Base(object):
         func: sa.sql.functions._FunctionGenerator,
         txmin: t.Optional[int] = None,
         txmax: t.Optional[int] = None,
-        upto_lsn: t.Optional[int] = None,
+        upto_lsn: t.Optional[str] = None,
         upto_nchanges: t.Optional[int] = None,
         limit: t.Optional[int] = None,
         offset: t.Optional[int] = None,
@@ -446,7 +486,7 @@ class Base(object):
             func (sa.sql.functions._FunctionGenerator): The function to use to read from the slot.
             txmin (Optional[int], optional): The minimum transaction ID to read from. Defaults to None.
             txmax (Optional[int], optional): The maximum transaction ID to read from. Defaults to None.
-            upto_lsn (Optional[int], optional): The maximum LSN to read up to. Defaults to None.
+            upto_lsn (Optional[str], optional): The maximum LSN to read up to. Defaults to None.
             upto_nchanges (Optional[int], optional): The maximum number of changes to read. Defaults to None.
             limit (Optional[int], optional): The maximum number of rows to return. Defaults to None.
             offset (Optional[int], optional): The number of rows to skip before returning. Defaults to None.
@@ -489,12 +529,20 @@ class Base(object):
             statement = statement.offset(offset)
         return statement
 
+    @property
+    def current_wal_lsn(self) -> str:
+        return self.fetchone(
+            sa.select(sa.func.MAX(sa.text("pg_current_wal_lsn"))).select_from(
+                sa.func.PG_CURRENT_WAL_LSN()
+            )
+        )[0]
+
     def logical_slot_get_changes(
         self,
         slot_name: str,
         txmin: t.Optional[int] = None,
         txmax: t.Optional[int] = None,
-        upto_lsn: t.Optional[int] = None,
+        upto_lsn: t.Optional[str] = None,
         upto_nchanges: t.Optional[int] = None,
         limit: t.Optional[int] = None,
         offset: t.Optional[int] = None,
@@ -524,7 +572,7 @@ class Base(object):
         slot_name: str,
         txmin: t.Optional[int] = None,
         txmax: t.Optional[int] = None,
-        upto_lsn: t.Optional[int] = None,
+        upto_lsn: t.Optional[str] = None,
         upto_nchanges: t.Optional[int] = None,
         limit: t.Optional[int] = None,
         offset: t.Optional[int] = None,
@@ -550,7 +598,7 @@ class Base(object):
         slot_name: str,
         txmin: t.Optional[int] = None,
         txmax: t.Optional[int] = None,
-        upto_lsn: t.Optional[int] = None,
+        upto_lsn: t.Optional[str] = None,
         upto_nchanges: t.Optional[int] = None,
     ) -> int:
         statement: sa.sql.Select = self._logical_slot_changes(
@@ -730,45 +778,20 @@ class Base(object):
 
         NB: All integers are long in python3 and call to convert is just int
         """
+        if self.verbose:
+            logger.debug(f"type: {type_} value: {value}")
         if value.lower() == "null":
             return None
-
-        if type_.lower() in (
-            "bigint",
-            "bigserial",
-            "int",
-            "int2",
-            "int4",
-            "int8",
-            "integer",
-            "serial",
-            "serial2",
-            "serial4",
-            "serial8",
-            "smallint",
-            "smallserial",
-        ):
+        if type_.lower() in self.INT_TYPES:
             try:
                 value = int(value)
             except ValueError:
                 raise
-        if type_.lower() in (
-            "char",
-            "character",
-            "character varying",
-            "text",
-            "uuid",
-            "varchar",
-        ):
+        if type_.lower() in self.CHAR_TYPES:
             value = value.lstrip("'").rstrip("'")
         if type_.lower() == "boolean":
             value = bool(value)
-        if type_.lower() in (
-            "double precision",
-            "float4",
-            "float8",
-            "real",
-        ):
+        if type_.lower() in self.FLOAT_TYPES:
             try:
                 value = float(value)
             except ValueError:
@@ -999,14 +1022,7 @@ def _pg_engine(
     sslrootcert = sslrootcert or PG_SSLROOTCERT
 
     if sslmode:
-        if sslmode not in (
-            "allow",
-            "disable",
-            "prefer",
-            "require",
-            "verify-ca",
-            "verify-full",
-        ):
+        if sslmode not in SSL_MODES:
             raise ValueError(f'Invalid sslmode: "{sslmode}"')
         connect_args["sslmode"] = sslmode
 

{pgsync-3.1.0 → pgsync-3.2.1}/pgsync/constants.py

@@ -89,6 +89,7 @@ ELASTICSEARCH_TYPES = [
     "constant_keyword",
     "date",
     "date_range",
+    "dense_vector",
     "double",
     "double_range",
     "flattened",
@@ -207,5 +208,5 @@ LOGICAL_SLOT_PREFIX = re.compile(
     r"table\s\"?(?P<schema>[\w-]+)\"?.\"?(?P<table>[\w-]+)\"?:\s(?P<tg_op>[A-Z]+):"  # noqa E501
 )
 LOGICAL_SLOT_SUFFIX = re.compile(
-    '\s(?P<key>"?\w+"?)\[(?P<type>[\w\s]+)\]:(?P<value>[\w\'"\-]+)'
+    r'\s(?P<key>"?\w+"?)\[(?P<type>[\w\s]+)\]:(?P<value>(?:"[^"]*"|\'[^\']*\'|null|\d+e[+-]?\d+|\w+))'
 )

{pgsync-3.1.0 → pgsync-3.2.1}/pgsync/helper.py

@@ -1,4 +1,5 @@
 """PGSync helpers."""
+
 import logging
 import os
 import typing as t

{pgsync-3.1.0 → pgsync-3.2.1}/pgsync/node.py

@@ -1,4 +1,5 @@
 """PGSync Node class representation."""
+
 from __future__ import annotations
 
 import re
@@ -68,6 +69,9 @@ class ForeignKey:
     def __str__(self):
         return f"foreign_key: {self.parent}:{self.child}"
 
+    def __repr__(self):
+        return self.__str__()
+
 
 @dataclass
 class Relationship:
@@ -113,6 +117,9 @@ class Relationship:
     def __str__(self):
         return f"relationship: {self.variant}.{self.type}:{self.tables}"
 
+    def __repr__(self):
+        return self.__str__()
+
 
 @dataclass
 class Node(object):
@@ -277,6 +284,7 @@ class Tree(threading.local):
     def __post_init__(self):
         self.tables: t.Set[str] = set()
         self.__nodes: t.Dict[Node] = {}
+        self.__schemas: t.Set[str] = set()
         self.root: t.Optional[Node] = None
         self.build(self.nodes)
 
@@ -327,6 +335,7 @@ class Tree(threading.local):
             node.add_child(self.build(child))
 
         self.__nodes[key] = node
+        self.__schemas.add(schema)
         return node
 
     def get_node(self, table: str, schema: str) -> Node:
@@ -345,3 +354,7 @@ class Tree(threading.local):
             else:
                 raise RuntimeError(f"Node for {schema}.{table} not found")
         return self.__nodes[key]
+
+    @property
+    def schemas(self) -> t.Set[str]:
+        return self.__schemas

{pgsync-3.1.0 → pgsync-3.2.1}/pgsync/plugin.py

@@ -1,6 +1,8 @@
 """PGSync Plugin."""
+
 import logging
 import os
+import sys
 import typing as t
 from abc import ABC, abstractmethod
 from importlib import import_module
@@ -42,7 +44,14 @@ class Plugins(object):
         self.plugins: list = []
         self._paths: list = []
         logger.debug(f"Reloading plugins from package: {self.package}")
-        self.walk(self.package)
+        # skip in test
+        if "test" not in sys.argv[0]:
+            self.walk(self.package)
+
+        # main plugin ordering
+        self.plugins = sorted(
+            self.plugins, key=lambda x: self.names.index(x.name)
+        )
 
     def walk(self, package: str) -> None:
         """Recursively walk the supplied package and fetch all plugins."""

{pgsync-3.1.0 → pgsync-3.2.1}/pgsync/querybuilder.py

@@ -1,4 +1,5 @@
 """PGSync QueryBuilder."""
+
 import threading
 import typing as t
 from collections import defaultdict
@@ -20,6 +21,23 @@ class QueryBuilder(threading.local):
         self.isouter: bool = True
         self._cache: dict = {}
 
+    def _eval_expression(
+        self, expression: sa.sql.elements.BinaryExpression
+    ) -> sa.sql.elements.BinaryExpression:
+        if isinstance(
+            expression.left.type, sa.dialects.postgresql.UUID
+        ) or isinstance(expression.right.type, sa.dialects.postgresql.UUID):
+            if not isinstance(
+                expression.left.type, sa.dialects.postgresql.UUID
+            ) or not isinstance(
+                expression.right.type, sa.dialects.postgresql.UUID
+            ):
+                # handle UUID typed expressions:
+                # psycopg2.errors.UndefinedFunction: operator does not exist: uuid = integer
+                return expression.left is None
+
+        return expression
+
     def _build_filters(
         self, filters: t.Dict[str, t.List[dict]], node: Node
     ) -> t.Optional[sa.sql.elements.BooleanClauseList]:
@@ -45,7 +63,11 @@ class QueryBuilder(threading.local):
                 for values in filters.get(node.table):
                     where: t.List = []
                     for column, value in values.items():
-                        where.append(node.model.c[column] == value)
+                        where.append(
+                            self._eval_expression(
+                                node.model.c[column] == value
+                            )
+                        )
                     # and clause is applied for composite primary keys
                     clause.append(sa.and_(*where))
                 return sa.or_(*clause)

{pgsync-3.1.0 → pgsync-3.2.1}/pgsync/redisqueue.py

@@ -1,4 +1,5 @@
 """PGSync RedisQueue."""
+
 import json
 import logging
 import typing as t

{pgsync-3.1.0 → pgsync-3.2.1}/pgsync/search_client.py

@@ -1,4 +1,5 @@
 """PGSync SearchClient helper."""
+
 import logging
 import typing as t
 from collections import defaultdict
@@ -173,7 +174,7 @@ class SearchClient(object):
     ):
         """Bulk index, update, delete docs to Elasticsearch/OpenSearch."""
         if settings.ELASTICSEARCH_STREAMING_BULK:
-            for ok, _ in self.streaming_bulk(
+            for ok, info in self.streaming_bulk(
                 self.__client,
                 actions,
                 index=index,
@@ -188,10 +189,12 @@ class SearchClient(object):
             ):
                 if ok:
                     self.doc_count += 1
+                else:
+                    logger.error(f"Document failed to index: {info}")
         else:
             # parallel bulk consumes more memory and is also more likely
             # to result in 429 errors.
-            for ok, _ in self.parallel_bulk(
+            for ok, info in self.parallel_bulk(
                 self.__client,
                 actions,
                 thread_count=thread_count,
@@ -205,6 +208,8 @@ class SearchClient(object):
             ):
                 if ok:
                     self.doc_count += 1
+                else:
+                    logger.error(f"Document failed to index: {info}")
 
     def refresh(self, indices: t.List[str]) -> None:
         """Refresh the Elasticsearch/OpenSearch index."""
@@ -245,7 +250,7 @@ class SearchClient(object):
             if "is out of range for a long" not in str(e):
                 raise
 
-    def search(self, index: str, body: dict):
+    def search(self, index: str, body: dict) -> t.Any:
         """
         Search in Elasticsearch/OpenSearch.
 
@@ -259,6 +264,7 @@ class SearchClient(object):
         tree: Tree,
         setting: t.Optional[dict] = None,
         mapping: t.Optional[dict] = None,
+        mappings: t.Optional[dict] = None,
         routing: t.Optional[str] = None,
     ) -> None:
         """Create Elasticsearch/OpenSearch setting and mapping if required."""
@@ -267,7 +273,8 @@ class SearchClient(object):
         if not self.__client.indices.exists(index=index):
             if setting:
                 body.update(**{"settings": {"index": setting}})
-
+            if mappings:
+                body.update(**{"mappings": {"index": mappings}})
             if mapping:
                 if "dynamic_templates" in mapping:
                     body.update(**{"mappings": mapping})
@@ -381,9 +388,9 @@ def get_search_client(
                     service,
                     session_token=credentials.token,
                 ),
-                use_ssl=True,
                 verify_certs=True,
                 connection_class=connection_class,
+                timeout=settings.ELASTICSEARCH_TIMEOUT,
             )
         elif settings.ELASTICSEARCH:
             return client(
@@ -395,18 +402,18 @@ def get_search_client(
                     service,
                     session_token=credentials.token,
                 ),
-                use_ssl=True,
                 verify_certs=True,
                 node_class=node_class,
+                timeout=settings.ELASTICSEARCH_TIMEOUT,
             )
     else:
         hosts: t.List[str] = [url]
         # API
         cloud_id: t.Optional[str] = settings.ELASTICSEARCH_CLOUD_ID
         api_key: t.Optional[t.Union[str, t.Tuple[str, str]]] = None
-        http_auth: t.Optional[
-            t.Union[str, t.Tuple[str, str]]
-        ] = settings.ELASTICSEARCH_HTTP_AUTH
+        http_auth: t.Optional[t.Union[str, t.Tuple[str, str]]] = (
+            settings.ELASTICSEARCH_HTTP_AUTH
+        )
         if (
             settings.ELASTICSEARCH_API_KEY_ID
             and settings.ELASTICSEARCH_API_KEY
@@ -424,17 +431,15 @@ def get_search_client(
         ca_certs: t.Optional[str] = settings.ELASTICSEARCH_CA_CERTS
         client_cert: t.Optional[str] = settings.ELASTICSEARCH_CLIENT_CERT
         client_key: t.Optional[str] = settings.ELASTICSEARCH_CLIENT_KEY
-        ssl_assert_hostname: t.Optional[
-            str
-        ] = settings.ELASTICSEARCH_SSL_ASSERT_HOSTNAME
-        ssl_assert_fingerprint: t.Optional[
-            str
-        ] = settings.ELASTICSEARCH_SSL_ASSERT_FINGERPRINT
+        ssl_assert_hostname: t.Optional[str] = (
+            settings.ELASTICSEARCH_SSL_ASSERT_HOSTNAME
+        )
+        ssl_assert_fingerprint: t.Optional[str] = (
+            settings.ELASTICSEARCH_SSL_ASSERT_FINGERPRINT
+        )
         ssl_version: t.Optional[int] = settings.ELASTICSEARCH_SSL_VERSION
         ssl_context: t.Optional[t.Any] = settings.ELASTICSEARCH_SSL_CONTEXT
         ssl_show_warn: bool = settings.ELASTICSEARCH_SSL_SHOW_WARN
-        # Transport
-        timeout: float = settings.ELASTICSEARCH_TIMEOUT
         return client(
             hosts=hosts,
             http_auth=http_auth,
@@ -453,6 +458,5 @@ def get_search_client(
             ssl_version=ssl_version,
             ssl_context=ssl_context,
             ssl_show_warn=ssl_show_warn,
-            # use_ssl=use_ssl,
-            timeout=timeout,
+            timeout=settings.ELASTICSEARCH_TIMEOUT,
         )

{pgsync-3.1.0 → pgsync-3.2.1}/pgsync/settings.py

@@ -4,6 +4,7 @@ This module contains the settings for PGSync.
 It reads environment variables from a .env file and sets default values for each variable.
 The variables are used to configure various parameters such as block size, checkpoint path, polling interval, etc.
 """
+
 import logging
 import logging.config
 import os
@@ -148,7 +149,7 @@ elif ELASTICSEARCH:
 OPENSEARCH_AWS_HOSTED = env.bool("OPENSEARCH_AWS_HOSTED", default=False)
 OPENSEARCH_AWS_SERVERLESS = env.bool(
     "OPENSEARCH_AWS_SERVERLESS", default=False
-)  # noqa E501
+)
 
 # Postgres:
 PG_HOST = env.str("PG_HOST", default="localhost")
@@ -160,6 +161,7 @@ PG_USER = env.str("PG_USER")
 
 # Redis:
 REDIS_AUTH = env.str("REDIS_AUTH", default=None)
+REDIS_USER = env.str("REDIS_USER", default=None)
 REDIS_DB = env.int("REDIS_DB", default=0)
 REDIS_HOST = env.str("REDIS_HOST", default="localhost")
 # redis poll interval (in secs)