pgsync 3.1.0__tar.gz → 3.2.0__tar.gz

{pgsync-3.1.0 → pgsync-3.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pgsync
-Version: 3.1.0
+Version: 3.2.0
 Summary: Postgres to Elasticsearch/OpenSearch sync
 Home-page: https://github.com/toluaina/pgsync
 Author: Tolu Aina
@@ -13,7 +13,7 @@ Project-URL: Funding, https://github.com/sponsors/toluaina
 Project-URL: Source, https://github.com/toluaina/pgsync
 Project-URL: Web, https://pgsync.com
 Project-URL: Documentation, https://pgsync.com
-Keywords: pgsync,elasticsearch,opensearch,postgres,change data capture
+Keywords: change data capture,elasticsearch,opensearch,pgsync,postgres
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Natural Language :: English
@@ -31,34 +31,35 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 License-File: AUTHORS.rst
 Requires-Dist: async-timeout==4.0.3
-Requires-Dist: boto3==1.34.11
-Requires-Dist: botocore==1.34.11
-Requires-Dist: certifi==2023.11.17
+Requires-Dist: boto3==1.34.142
+Requires-Dist: botocore==1.34.142
+Requires-Dist: certifi==2024.7.4
 Requires-Dist: charset-normalizer==3.3.2
 Requires-Dist: click==8.1.7
-Requires-Dist: elastic-transport==8.11.0
-Requires-Dist: elasticsearch==8.11.1
-Requires-Dist: elasticsearch-dsl==8.11.0
-Requires-Dist: environs==10.0.0
+Requires-Dist: elastic-transport==8.13.1
+Requires-Dist: elasticsearch==8.14.0
+Requires-Dist: elasticsearch-dsl==8.14.0
+Requires-Dist: environs==11.0.0
+Requires-Dist: events==0.5
 Requires-Dist: greenlet==3.0.3
-Requires-Dist: idna==3.6
+Requires-Dist: idna==3.7
 Requires-Dist: jmespath==1.0.1
-Requires-Dist: marshmallow==3.20.1
+Requires-Dist: marshmallow==3.21.3
 Requires-Dist: opensearch-dsl==2.1.0
-Requires-Dist: opensearch-py==2.4.2
-Requires-Dist: packaging==23.2
+Requires-Dist: opensearch-py==2.6.0
+Requires-Dist: packaging==24.1
 Requires-Dist: psycopg2-binary==2.9.9
-Requires-Dist: python-dateutil==2.8.2
-Requires-Dist: python-dotenv==1.0.0
-Requires-Dist: redis==5.0.1
-Requires-Dist: requests==2.31.0
+Requires-Dist: python-dateutil==2.9.0.post0
+Requires-Dist: python-dotenv==1.0.1
+Requires-Dist: redis==5.0.7
+Requires-Dist: requests==2.32.3
 Requires-Dist: requests-aws4auth==1.2.3
-Requires-Dist: s3transfer==0.10.0
+Requires-Dist: s3transfer==0.10.2
 Requires-Dist: six==1.16.0
-Requires-Dist: sqlalchemy==2.0.25
-Requires-Dist: sqlparse==0.4.4
-Requires-Dist: typing-extensions==4.9.0
-Requires-Dist: urllib3==1.26.18
+Requires-Dist: sqlalchemy==2.0.31
+Requires-Dist: sqlparse==0.5.0
+Requires-Dist: typing-extensions==4.12.2
+Requires-Dist: urllib3==1.26.19
 
 # PostgreSQL to Elasticsearch/OpenSearch sync
 

{pgsync-3.1.0 → pgsync-3.2.0}/pgsync/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = "Tolu Aina"
 __email__ = "tolu@pgsync.com"
-__version__ = "3.1.0"
+__version__ = "3.2.0"

{pgsync-3.1.0 → pgsync-3.2.0}/pgsync/base.py

@@ -1,4 +1,5 @@
 """PGSync Base."""
+
 import logging
 import os
 import typing as t
@@ -48,6 +49,15 @@ except ImportError:
 
 logger = logging.getLogger(__name__)
 
+SSL_MODES = (
+    "allow",
+    "disable",
+    "prefer",
+    "require",
+    "verify-ca",
+    "verify-full",
+)
+
 
 class Payload(object):
     """
@@ -141,6 +151,36 @@ class TupleIdentifierType(sa.types.UserDefinedType):
 
 
 class Base(object):
+    INT_TYPES = (
+        "bigint",
+        "bigserial",
+        "int",
+        "int2",
+        "int4",
+        "int8",
+        "integer",
+        "serial",
+        "serial2",
+        "serial4",
+        "serial8",
+        "smallint",
+        "smallserial",
+    )
+    FLOAT_TYPES = (
+        "double precision",
+        "float4",
+        "float8",
+        "real",
+    )
+    CHAR_TYPES = (
+        "char",
+        "character",
+        "character varying",
+        "text",
+        "uuid",
+        "varchar",
+    )
+
     def __init__(
         self, database: str, verbose: bool = False, *args, **kwargs
     ) -> None:
@@ -433,7 +473,7 @@ class Base(object):
         func: sa.sql.functions._FunctionGenerator,
         txmin: t.Optional[int] = None,
         txmax: t.Optional[int] = None,
-        upto_lsn: t.Optional[int] = None,
+        upto_lsn: t.Optional[str] = None,
         upto_nchanges: t.Optional[int] = None,
         limit: t.Optional[int] = None,
         offset: t.Optional[int] = None,
@@ -446,7 +486,7 @@ class Base(object):
             func (sa.sql.functions._FunctionGenerator): The function to use to read from the slot.
             txmin (Optional[int], optional): The minimum transaction ID to read from. Defaults to None.
             txmax (Optional[int], optional): The maximum transaction ID to read from. Defaults to None.
-            upto_lsn (Optional[int], optional): The maximum LSN to read up to. Defaults to None.
+            upto_lsn (Optional[str], optional): The maximum LSN to read up to. Defaults to None.
             upto_nchanges (Optional[int], optional): The maximum number of changes to read. Defaults to None.
             limit (Optional[int], optional): The maximum number of rows to return. Defaults to None.
             offset (Optional[int], optional): The number of rows to skip before returning. Defaults to None.
@@ -489,12 +529,20 @@ class Base(object):
             statement = statement.offset(offset)
         return statement
 
+    @property
+    def current_wal_lsn(self) -> str:
+        return self.fetchone(
+            sa.select(sa.func.MAX(sa.text("pg_current_wal_lsn"))).select_from(
+                sa.func.PG_CURRENT_WAL_LSN()
+            )
+        )[0]
+
     def logical_slot_get_changes(
         self,
         slot_name: str,
         txmin: t.Optional[int] = None,
         txmax: t.Optional[int] = None,
-        upto_lsn: t.Optional[int] = None,
+        upto_lsn: t.Optional[str] = None,
         upto_nchanges: t.Optional[int] = None,
         limit: t.Optional[int] = None,
         offset: t.Optional[int] = None,
@@ -524,7 +572,7 @@ class Base(object):
         slot_name: str,
         txmin: t.Optional[int] = None,
         txmax: t.Optional[int] = None,
-        upto_lsn: t.Optional[int] = None,
+        upto_lsn: t.Optional[str] = None,
         upto_nchanges: t.Optional[int] = None,
         limit: t.Optional[int] = None,
         offset: t.Optional[int] = None,
@@ -550,7 +598,7 @@ class Base(object):
         slot_name: str,
         txmin: t.Optional[int] = None,
         txmax: t.Optional[int] = None,
-        upto_lsn: t.Optional[int] = None,
+        upto_lsn: t.Optional[str] = None,
         upto_nchanges: t.Optional[int] = None,
     ) -> int:
         statement: sa.sql.Select = self._logical_slot_changes(
@@ -732,43 +780,16 @@ class Base(object):
         """
         if value.lower() == "null":
             return None
-
-        if type_.lower() in (
-            "bigint",
-            "bigserial",
-            "int",
-            "int2",
-            "int4",
-            "int8",
-            "integer",
-            "serial",
-            "serial2",
-            "serial4",
-            "serial8",
-            "smallint",
-            "smallserial",
-        ):
+        if type_.lower() in self.INT_TYPES:
             try:
                 value = int(value)
             except ValueError:
                 raise
-        if type_.lower() in (
-            "char",
-            "character",
-            "character varying",
-            "text",
-            "uuid",
-            "varchar",
-        ):
+        if type_.lower() in self.CHAR_TYPES:
             value = value.lstrip("'").rstrip("'")
         if type_.lower() == "boolean":
             value = bool(value)
-        if type_.lower() in (
-            "double precision",
-            "float4",
-            "float8",
-            "real",
-        ):
+        if type_.lower() in self.FLOAT_TYPES:
             try:
                 value = float(value)
             except ValueError:
@@ -999,14 +1020,7 @@ def _pg_engine(
     sslrootcert = sslrootcert or PG_SSLROOTCERT
 
     if sslmode:
-        if sslmode not in (
-            "allow",
-            "disable",
-            "prefer",
-            "require",
-            "verify-ca",
-            "verify-full",
-        ):
+        if sslmode not in SSL_MODES:
             raise ValueError(f'Invalid sslmode: "{sslmode}"')
         connect_args["sslmode"] = sslmode
 

{pgsync-3.1.0 → pgsync-3.2.0}/pgsync/constants.py

@@ -89,6 +89,7 @@ ELASTICSEARCH_TYPES = [
     "constant_keyword",
     "date",
     "date_range",
+    "dense_vector",
     "double",
     "double_range",
     "flattened",
@@ -207,5 +208,5 @@ LOGICAL_SLOT_PREFIX = re.compile(
     r"table\s\"?(?P<schema>[\w-]+)\"?.\"?(?P<table>[\w-]+)\"?:\s(?P<tg_op>[A-Z]+):"  # noqa E501
 )
 LOGICAL_SLOT_SUFFIX = re.compile(
-    '\s(?P<key>"?\w+"?)\[(?P<type>[\w\s]+)\]:(?P<value>[\w\'"\-]+)'
+    '\s(?P<key>"?\w+"?)\[(?P<type>[\w\s]+)\]:(?P<value>[\w\'"\-\\.\\+]+)'
 )

{pgsync-3.1.0 → pgsync-3.2.0}/pgsync/helper.py

@@ -1,4 +1,5 @@
 """PGSync helpers."""
+
 import logging
 import os
 import typing as t

{pgsync-3.1.0 → pgsync-3.2.0}/pgsync/node.py

@@ -1,4 +1,5 @@
 """PGSync Node class representation."""
+
 from __future__ import annotations
 
 import re
@@ -68,6 +69,9 @@ class ForeignKey:
     def __str__(self):
         return f"foreign_key: {self.parent}:{self.child}"
 
+    def __repr__(self):
+        return self.__str__()
+
 
 @dataclass
 class Relationship:
@@ -113,6 +117,9 @@ class Relationship:
     def __str__(self):
         return f"relationship: {self.variant}.{self.type}:{self.tables}"
 
+    def __repr__(self):
+        return self.__str__()
+
 
 @dataclass
 class Node(object):

{pgsync-3.1.0 → pgsync-3.2.0}/pgsync/plugin.py

@@ -1,6 +1,8 @@
 """PGSync Plugin."""
+
 import logging
 import os
+import sys
 import typing as t
 from abc import ABC, abstractmethod
 from importlib import import_module
@@ -42,7 +44,9 @@ class Plugins(object):
         self.plugins: list = []
         self._paths: list = []
         logger.debug(f"Reloading plugins from package: {self.package}")
-        self.walk(self.package)
+        # skip in test
+        if "test" not in sys.argv[0]:
+            self.walk(self.package)
 
     def walk(self, package: str) -> None:
         """Recursively walk the supplied package and fetch all plugins."""

{pgsync-3.1.0 → pgsync-3.2.0}/pgsync/querybuilder.py

@@ -1,4 +1,5 @@
 """PGSync QueryBuilder."""
+
 import threading
 import typing as t
 from collections import defaultdict
@@ -20,6 +21,23 @@ class QueryBuilder(threading.local):
         self.isouter: bool = True
         self._cache: dict = {}
 
+    def _eval_expression(
+        self, expression: sa.sql.elements.BinaryExpression
+    ) -> sa.sql.elements.BinaryExpression:
+        if isinstance(
+            expression.left.type, sa.dialects.postgresql.UUID
+        ) or isinstance(expression.right.type, sa.dialects.postgresql.UUID):
+            if not isinstance(
+                expression.left.type, sa.dialects.postgresql.UUID
+            ) or not isinstance(
+                expression.right.type, sa.dialects.postgresql.UUID
+            ):
+                # handle UUID typed expressions:
+                # psycopg2.errors.UndefinedFunction: operator does not exist: uuid = integer
+                return expression.left is None
+
+        return expression
+
     def _build_filters(
         self, filters: t.Dict[str, t.List[dict]], node: Node
     ) -> t.Optional[sa.sql.elements.BooleanClauseList]:
@@ -45,7 +63,11 @@ class QueryBuilder(threading.local):
                 for values in filters.get(node.table):
                     where: t.List = []
                     for column, value in values.items():
-                        where.append(node.model.c[column] == value)
+                        where.append(
+                            self._eval_expression(
+                                node.model.c[column] == value
+                            )
+                        )
                     # and clause is applied for composite primary keys
                     clause.append(sa.and_(*where))
                 return sa.or_(*clause)

{pgsync-3.1.0 → pgsync-3.2.0}/pgsync/redisqueue.py

@@ -1,4 +1,5 @@
 """PGSync RedisQueue."""
+
 import json
 import logging
 import typing as t

{pgsync-3.1.0 → pgsync-3.2.0}/pgsync/search_client.py

@@ -1,4 +1,5 @@
 """PGSync SearchClient helper."""
+
 import logging
 import typing as t
 from collections import defaultdict
@@ -245,7 +246,7 @@ class SearchClient(object):
             if "is out of range for a long" not in str(e):
                 raise
 
-    def search(self, index: str, body: dict):
+    def search(self, index: str, body: dict) -> t.Any:
         """
         Search in Elasticsearch/OpenSearch.
 
@@ -259,6 +260,7 @@ class SearchClient(object):
         tree: Tree,
         setting: t.Optional[dict] = None,
         mapping: t.Optional[dict] = None,
+        mappings: t.Optional[dict] = None,
         routing: t.Optional[str] = None,
     ) -> None:
         """Create Elasticsearch/OpenSearch setting and mapping if required."""
@@ -267,7 +269,8 @@ class SearchClient(object):
         if not self.__client.indices.exists(index=index):
             if setting:
                 body.update(**{"settings": {"index": setting}})
-
+            if mappings:
+                body.update(**{"mappings": {"index": mappings}})
             if mapping:
                 if "dynamic_templates" in mapping:
                     body.update(**{"mappings": mapping})
@@ -404,9 +407,9 @@ def get_search_client(
         # API
         cloud_id: t.Optional[str] = settings.ELASTICSEARCH_CLOUD_ID
         api_key: t.Optional[t.Union[str, t.Tuple[str, str]]] = None
-        http_auth: t.Optional[
-            t.Union[str, t.Tuple[str, str]]
-        ] = settings.ELASTICSEARCH_HTTP_AUTH
+        http_auth: t.Optional[t.Union[str, t.Tuple[str, str]]] = (
+            settings.ELASTICSEARCH_HTTP_AUTH
+        )
         if (
             settings.ELASTICSEARCH_API_KEY_ID
             and settings.ELASTICSEARCH_API_KEY
@@ -424,12 +427,12 @@ def get_search_client(
         ca_certs: t.Optional[str] = settings.ELASTICSEARCH_CA_CERTS
         client_cert: t.Optional[str] = settings.ELASTICSEARCH_CLIENT_CERT
         client_key: t.Optional[str] = settings.ELASTICSEARCH_CLIENT_KEY
-        ssl_assert_hostname: t.Optional[
-            str
-        ] = settings.ELASTICSEARCH_SSL_ASSERT_HOSTNAME
-        ssl_assert_fingerprint: t.Optional[
-            str
-        ] = settings.ELASTICSEARCH_SSL_ASSERT_FINGERPRINT
+        ssl_assert_hostname: t.Optional[str] = (
+            settings.ELASTICSEARCH_SSL_ASSERT_HOSTNAME
+        )
+        ssl_assert_fingerprint: t.Optional[str] = (
+            settings.ELASTICSEARCH_SSL_ASSERT_FINGERPRINT
+        )
         ssl_version: t.Optional[int] = settings.ELASTICSEARCH_SSL_VERSION
         ssl_context: t.Optional[t.Any] = settings.ELASTICSEARCH_SSL_CONTEXT
         ssl_show_warn: bool = settings.ELASTICSEARCH_SSL_SHOW_WARN

{pgsync-3.1.0 → pgsync-3.2.0}/pgsync/settings.py

@@ -4,6 +4,7 @@ This module contains the settings for PGSync.
 It reads environment variables from a .env file and sets default values for each variable.
 The variables are used to configure various parameters such as block size, checkpoint path, polling interval, etc.
 """
+
 import logging
 import logging.config
 import os
@@ -148,7 +149,7 @@ elif ELASTICSEARCH:
 OPENSEARCH_AWS_HOSTED = env.bool("OPENSEARCH_AWS_HOSTED", default=False)
 OPENSEARCH_AWS_SERVERLESS = env.bool(
     "OPENSEARCH_AWS_SERVERLESS", default=False
-)  # noqa E501
+)
 
 # Postgres:
 PG_HOST = env.str("PG_HOST", default="localhost")

{pgsync-3.1.0 → pgsync-3.2.0}/pgsync/sync.py

@@ -1,4 +1,5 @@
 """Sync module."""
+
 import asyncio
 import json
 import logging
@@ -81,6 +82,7 @@ class Sync(Base, metaclass=Singleton):
         self.nodes: dict = doc.get("nodes", {})
         self.setting: dict = doc.get("setting")
         self.mapping: dict = doc.get("mapping")
+        self.mappings: dict = doc.get("mappings")
         self.routing: str = doc.get("routing")
         super().__init__(
             doc.get("database", self.index), verbose=verbose, **kwargs
@@ -254,6 +256,7 @@ class Sync(Base, metaclass=Singleton):
             self.tree,
             setting=self.setting,
             mapping=self.mapping,
+            mappings=self.mappings,
             routing=self.routing,
         )
 
@@ -347,6 +350,7 @@ class Sync(Base, metaclass=Singleton):
         txmin: t.Optional[int] = None,
         txmax: t.Optional[int] = None,
         upto_nchanges: t.Optional[int] = None,
+        upto_lsn: t.Optional[str] = None,
     ) -> None:
         """
         Process changes from the db logical replication logs.
@@ -374,25 +378,15 @@ class Sync(Base, metaclass=Singleton):
         # minimize the tmp file disk usage when calling
         # PG_LOGICAL_SLOT_PEEK_CHANGES and PG_LOGICAL_SLOT_GET_CHANGES
         # by limiting to a smaller batch size.
-        offset: int = 0
-        total: int = 0
-        limit: int = settings.LOGICAL_SLOT_CHUNK_SIZE
-        count: int = self.logical_slot_count_changes(
-            self.__name,
-            txmin=txmin,
-            txmax=txmax,
-            upto_nchanges=upto_nchanges,
-        )
         while True:
             changes: int = self.logical_slot_peek_changes(
                 self.__name,
                 txmin=txmin,
                 txmax=txmax,
                 upto_nchanges=upto_nchanges,
-                limit=limit,
-                offset=offset,
+                upto_lsn=upto_lsn,
             )
-            if not changes or total > count:
+            if not changes:
                 break
 
             rows: list = []
@@ -447,11 +441,8 @@ class Sync(Base, metaclass=Singleton):
                 txmin=txmin,
                 txmax=txmax,
                 upto_nchanges=upto_nchanges,
-                limit=limit,
-                offset=offset,
+                upto_lsn=upto_lsn,
             )
-            offset += limit
-            total += len(changes)
             self.count["xlog"] += len(rows)
 
     def _root_primary_key_resolver(
@@ -1120,7 +1111,7 @@ class Sync(Base, metaclass=Singleton):
                 notification: t.AnyStr = conn.notifies.pop(0)
                 if notification.channel == self.database:
                     payload = json.loads(notification.payload)
-                    if self.index in payload["indices"]:
+                    if payload["indices"] and self.index in payload["indices"]:
                         payloads.append(payload)
                         logger.debug(f"poll_db: {payload}")
                         self.count["db"] += 1
@@ -1142,7 +1133,7 @@ class Sync(Base, metaclass=Singleton):
             notification: t.AnyStr = self.conn.notifies.pop(0)
             if notification.channel == self.database:
                 payload = json.loads(notification.payload)
-                if self.index in payload["indices"]:
+                if payload["indices"] and self.index in payload["indices"]:
                     self.redis.push([payload])
                     logger.debug(f"async_poll: {payload}")
                     self.count["db"] += 1
@@ -1224,13 +1215,22 @@ class Sync(Base, metaclass=Singleton):
         """Pull data from db."""
         txmin: int = self.checkpoint
         txmax: int = self.txid_current
+        # this is the max lsn we should go upto
+        upto_lsn: str = self.current_wal_lsn
+        upto_nchanges: int = settings.LOGICAL_SLOT_CHUNK_SIZE
+
         logger.debug(f"pull txmin: {txmin} - txmax: {txmax}")
         # forward pass sync
         self.search_client.bulk(
             self.index, self.sync(txmin=txmin, txmax=txmax)
         )
         # now sync up to txmax to capture everything we may have missed
-        self.logical_slot_changes(txmin=txmin, txmax=txmax, upto_nchanges=None)
+        self.logical_slot_changes(
+            txmin=txmin,
+            txmax=txmax,
+            upto_nchanges=upto_nchanges,
+            upto_lsn=upto_lsn,
+        )
         self.checkpoint: int = txmax or self.txid_current
         self._truncate = True
 
@@ -1267,6 +1267,7 @@ class Sync(Base, metaclass=Singleton):
             await asyncio.sleep(settings.LOG_INTERVAL)
 
     def _status(self, label: str) -> None:
+        # TODO: indicate if we are processing logical logs or not
         sys.stdout.write(
             f"{label} {self.database}:{self.index} "
             f"Xlog: [{self.count['xlog']:,}] => "

{pgsync-3.1.0 → pgsync-3.2.0}/pgsync/transform.py

@@ -1,4 +1,5 @@
 """PGSync Transform."""
+
 import logging
 import typing as t
 

{pgsync-3.1.0 → pgsync-3.2.0}/pgsync/utils.py

@@ -1,4 +1,5 @@
 """PGSync utils."""
+
 import json
 import logging
 import os

{pgsync-3.1.0 → pgsync-3.2.0}/pgsync/view.py

@@ -1,4 +1,5 @@
 """PGSync views."""
+
 import logging
 import typing as t
 import warnings
@@ -457,15 +458,21 @@ def create_view(
             [
                 (
                     table_name,
-                    array(fields["primary_keys"])
-                    if fields.get("primary_keys")
-                    else None,
-                    array(fields.get("foreign_keys"))
-                    if fields.get("foreign_keys")
-                    else None,
-                    array(fields.get("indices"))
-                    if fields.get("indices")
-                    else None,
+                    (
+                        array(fields["primary_keys"])
+                        if fields.get("primary_keys")
+                        else None
+                    ),
+                    (
+                        array(fields.get("foreign_keys"))
+                        if fields.get("foreign_keys")
+                        else None
+                    ),
+                    (
+                        array(fields.get("indices"))
+                        if fields.get("indices")
+                        else None
+                    ),
                 )
                 for table_name, fields in rows.items()
             ]

{pgsync-3.1.0 → pgsync-3.2.0}/pgsync.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pgsync
-Version: 3.1.0
+Version: 3.2.0
 Summary: Postgres to Elasticsearch/OpenSearch sync
 Home-page: https://github.com/toluaina/pgsync
 Author: Tolu Aina
@@ -13,7 +13,7 @@ Project-URL: Funding, https://github.com/sponsors/toluaina
 Project-URL: Source, https://github.com/toluaina/pgsync
 Project-URL: Web, https://pgsync.com
 Project-URL: Documentation, https://pgsync.com
-Keywords: pgsync,elasticsearch,opensearch,postgres,change data capture
+Keywords: change data capture,elasticsearch,opensearch,pgsync,postgres
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Natural Language :: English
@@ -31,34 +31,35 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 License-File: AUTHORS.rst
 Requires-Dist: async-timeout==4.0.3
-Requires-Dist: boto3==1.34.11
-Requires-Dist: botocore==1.34.11
-Requires-Dist: certifi==2023.11.17
+Requires-Dist: boto3==1.34.142
+Requires-Dist: botocore==1.34.142
+Requires-Dist: certifi==2024.7.4
 Requires-Dist: charset-normalizer==3.3.2
 Requires-Dist: click==8.1.7
-Requires-Dist: elastic-transport==8.11.0
-Requires-Dist: elasticsearch==8.11.1
-Requires-Dist: elasticsearch-dsl==8.11.0
-Requires-Dist: environs==10.0.0
+Requires-Dist: elastic-transport==8.13.1
+Requires-Dist: elasticsearch==8.14.0
+Requires-Dist: elasticsearch-dsl==8.14.0
+Requires-Dist: environs==11.0.0
+Requires-Dist: events==0.5
 Requires-Dist: greenlet==3.0.3
-Requires-Dist: idna==3.6
+Requires-Dist: idna==3.7
 Requires-Dist: jmespath==1.0.1
-Requires-Dist: marshmallow==3.20.1
+Requires-Dist: marshmallow==3.21.3
 Requires-Dist: opensearch-dsl==2.1.0
-Requires-Dist: opensearch-py==2.4.2
-Requires-Dist: packaging==23.2
+Requires-Dist: opensearch-py==2.6.0
+Requires-Dist: packaging==24.1
 Requires-Dist: psycopg2-binary==2.9.9
-Requires-Dist: python-dateutil==2.8.2
-Requires-Dist: python-dotenv==1.0.0
-Requires-Dist: redis==5.0.1
-Requires-Dist: requests==2.31.0
+Requires-Dist: python-dateutil==2.9.0.post0
+Requires-Dist: python-dotenv==1.0.1
+Requires-Dist: redis==5.0.7
+Requires-Dist: requests==2.32.3
 Requires-Dist: requests-aws4auth==1.2.3
-Requires-Dist: s3transfer==0.10.0
+Requires-Dist: s3transfer==0.10.2
 Requires-Dist: six==1.16.0
-Requires-Dist: sqlalchemy==2.0.25
-Requires-Dist: sqlparse==0.4.4
-Requires-Dist: typing-extensions==4.9.0
-Requires-Dist: urllib3==1.26.18
+Requires-Dist: sqlalchemy==2.0.31
+Requires-Dist: sqlparse==0.5.0
+Requires-Dist: typing-extensions==4.12.2
+Requires-Dist: urllib3==1.26.19
 
 # PostgreSQL to Elasticsearch/OpenSearch sync
 

pgsync-3.2.0/pgsync.egg-info/requires.txt

@@ -0,0 +1,30 @@
+async-timeout==4.0.3
+boto3==1.34.142
+botocore==1.34.142
+certifi==2024.7.4
+charset-normalizer==3.3.2
+click==8.1.7
+elastic-transport==8.13.1
+elasticsearch==8.14.0
+elasticsearch-dsl==8.14.0
+environs==11.0.0
+events==0.5
+greenlet==3.0.3
+idna==3.7
+jmespath==1.0.1
+marshmallow==3.21.3
+opensearch-dsl==2.1.0
+opensearch-py==2.6.0
+packaging==24.1
+psycopg2-binary==2.9.9
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+redis==5.0.7
+requests==2.32.3
+requests-aws4auth==1.2.3
+s3transfer==0.10.2
+six==1.16.0
+sqlalchemy==2.0.31
+sqlparse==0.5.0
+typing-extensions==4.12.2
+urllib3==1.26.19

{pgsync-3.1.0 → pgsync-3.2.0}/setup.py

@@ -28,11 +28,11 @@ PYTHON_REQUIRES = ">=3.8.0"
 VERSION = get_version()
 INSTALL_REQUIRES = []
 KEYWORDS = [
-    "pgsync",
+    "change data capture",
     "elasticsearch",
     "opensearch",
+    "pgsync",
     "postgres",
-    "change data capture",
 ]
 CLASSIFIERS = [
     "Development Status :: 5 - Production/Stable",
@@ -48,11 +48,7 @@ CLASSIFIERS = [
     "License :: OSI Approved :: MIT License",
     "Operating System :: OS Independent",
 ]
-SCRIPTS = [
-    "bin/pgsync",
-    "bin/bootstrap",
-    "bin/parallel_sync",
-]
+SCRIPTS = ["bin/bootstrap", "bin/parallel_sync", "bin/pgsync"]
 SETUP_REQUIRES = ["pytest-runner"]
 TESTS_REQUIRE = ["pytest"]
 

{pgsync-3.1.0 → pgsync-3.2.0}/tests/conftest.py

@@ -1,4 +1,5 @@
 """Generic fixtures for PGSync tests."""
+
 import logging
 import os
 

{pgsync-3.1.0 → pgsync-3.2.0}/tests/test_base.py

@@ -462,6 +462,32 @@ class TestBase(object):
             pg_base.parse_logical_slot(row)
             assert '"Unknown UNKNOWN operation for row:' in str(excinfo.value)
 
+    def test_parse_logical_slot_with_double_precision(
+        self,
+        connection,
+    ):
+        pg_base = Base(connection.engine.url.database)
+        row = """
+        table public.book: UPDATE: id[integer]:1 isbn[character varying]:'001' title[character varying]:'It' description[character varying]:'Stephens Kings It' copyright[character varying]:null tags[jsonb]:'["a", "b", "c"]' doc[jsonb]:'{"a": {"b": {"c": [0, 1, 2, 3, 4]}}, "i": 73, "x": [{"y": 0, "z": 5}, {"y": 1, "z": 6}], "bool": true, "lastname": "Judye", "firstname": "Glenda", "generation": {"name": "X"}, "nick_names": ["Beatriz", "Jean", "Carilyn", "Carol-Jean", "Sara-Ann"], "coordinates": {"lat": 21.1, "lon": 32.9}}' publisher_id[integer]:1 publish_date[timestamp without time zone]:'1980-01-01 00:00:00' quad[double precision]:2e+58
+        """  # noqa E501
+        payload = pg_base.parse_logical_slot(row)
+        assert payload.data == {
+            "id": 1,
+            "isbn": "001",
+            "title": "It",
+            "description": "Stephens",
+            "copyright": None,
+            "tags": "'",
+            "doc": "'",
+            "publisher_id": 1,
+            "publish_date": "'1980-01-01",
+            "quad": 2e58,
+        }
+        assert payload.old == {}
+        assert payload.schema == "public"
+        assert payload.table == "book"
+        assert payload.tg_op == "UPDATE"
+
     def test_fetchone(self, connection):
         pg_base = Base(connection.engine.url.database, verbose=True)
         with patch("pgsync.base.compiled_query") as mock_compiled_query:

{pgsync-3.1.0 → pgsync-3.2.0}/tests/test_constants.py

@@ -1,4 +1,5 @@
 """Constants tests."""
+
 from pgsync.constants import (
     ELASTICSEARCH_MAPPING_PARAMETERS,
     ELASTICSEARCH_TYPES,
@@ -98,6 +99,7 @@ class TestConstants(object):
                 "constant_keyword",
                 "date",
                 "double",
+                "dense_vector",
                 "float",
                 "geo_point",
                 "geo_shape",

{pgsync-3.1.0 → pgsync-3.2.0}/tests/test_env_vars.py

@@ -1,4 +1,5 @@
 """Env vars tests."""
+
 import pytest
 
 

{pgsync-3.1.0 → pgsync-3.2.0}/tests/test_helper.py

@@ -1,4 +1,5 @@
 """Helper tests."""
+
 import pytest
 import sqlalchemy as sa
 from mock import ANY, call, patch

{pgsync-3.1.0 → pgsync-3.2.0}/tests/test_log_handlers.py

@@ -1,4 +1,5 @@
 """Loghandler tests."""
+
 import pytest
 
 

{pgsync-3.1.0 → pgsync-3.2.0}/tests/test_node.py

@@ -1,4 +1,5 @@
 """Node tests."""
+
 import pytest
 
 from pgsync.base import Base

{pgsync-3.1.0 → pgsync-3.2.0}/tests/test_query_builder.py

@@ -1,4 +1,5 @@
 """QueryBuilder tests."""
+
 import pytest
 
 from pgsync.base import Base

{pgsync-3.1.0 → pgsync-3.2.0}/tests/test_search_client.py

@@ -1,4 +1,5 @@
 """SearchClient tests."""
+
 import importlib
 
 import elastic_transport

{pgsync-3.1.0 → pgsync-3.2.0}/tests/test_settings.py

@@ -1,4 +1,5 @@
 """Tests for the settings module."""
+
 import importlib
 
 from pgsync import settings

{pgsync-3.1.0 → pgsync-3.2.0}/tests/test_sync.py

@@ -1,4 +1,5 @@
 """Sync tests."""
+
 import importlib
 import os
 import typing as t
@@ -81,8 +82,7 @@ class TestSync(object):
                     txmin=None,
                     txmax=None,
                     upto_nchanges=None,
-                    limit=settings.LOGICAL_SLOT_CHUNK_SIZE,
-                    offset=0,
+                    upto_lsn=None,
                 )
                 mock_sync.assert_not_called()
 
@@ -98,8 +98,7 @@ class TestSync(object):
                     txmin=None,
                     txmax=None,
                     upto_nchanges=None,
-                    limit=settings.LOGICAL_SLOT_CHUNK_SIZE,
-                    offset=0,
+                    upto_lsn=None,
                 )
                 mock_sync.assert_not_called()
 
@@ -127,8 +126,7 @@ class TestSync(object):
                         txmin=None,
                         txmax=None,
                         upto_nchanges=None,
-                        limit=settings.LOGICAL_SLOT_CHUNK_SIZE,
-                        offset=0,
+                        upto_lsn=None,
                     )
                     mock_get.assert_called_once()
                     mock_sync.assert_called_once()
@@ -355,7 +353,10 @@ class TestSync(object):
             txmin = None
             txmax = sync.txid_current - 1
             mock_get.assert_called_once_with(
-                txmin=txmin, txmax=txmax, upto_nchanges=None
+                txmin=txmin,
+                txmax=txmax,
+                upto_nchanges=settings.LOGICAL_SLOT_CHUNK_SIZE,
+                upto_lsn=ANY,
             )
             mock_logger.debug.assert_called_once_with(
                 f"pull txmin: {txmin} - txmax: {txmax}"
@@ -762,7 +763,12 @@ class TestSync(object):
     def test_create_setting(self, mock_es, sync):
         sync.create_setting()
         mock_es.assert_called_once_with(
-            "testdb", ANY, setting=None, mapping=None, routing=None
+            "testdb",
+            ANY,
+            setting=None,
+            mapping=None,
+            mappings=None,
+            routing=None,
         )
 
     @patch("pgsync.sync.Sync.teardown")

{pgsync-3.1.0 → pgsync-3.2.0}/tests/test_sync_root.py

@@ -1,4 +1,5 @@
 """Tests for `pgsync` package."""
+
 import mock
 import pytest
 

{pgsync-3.1.0 → pgsync-3.2.0}/tests/test_transform.py

@@ -1,4 +1,5 @@
 """Transform tests."""
+
 import pytest
 
 from pgsync.constants import CONCAT_TRANSFORM, RENAME_TRANSFORM

{pgsync-3.1.0 → pgsync-3.2.0}/tests/test_trigger.py

@@ -1,4 +1,5 @@
 """Trigger tests."""
+
 import pytest
 import sqlalchemy as sa
 

{pgsync-3.1.0 → pgsync-3.2.0}/tests/test_urls.py

@@ -1,4 +1,5 @@
 """URLS tests."""
+
 import pytest
 from mock import MagicMock, patch
 

{pgsync-3.1.0 → pgsync-3.2.0}/tests/test_utils.py

@@ -1,4 +1,5 @@
 """Utils tests."""
+
 import os
 from urllib.parse import ParseResult, urlparse
 

pgsync-3.1.0/pgsync.egg-info/requires.txt

@@ -1,29 +0,0 @@
-async-timeout==4.0.3
-boto3==1.34.11
-botocore==1.34.11
-certifi==2023.11.17
-charset-normalizer==3.3.2
-click==8.1.7
-elastic-transport==8.11.0
-elasticsearch==8.11.1
-elasticsearch-dsl==8.11.0
-environs==10.0.0
-greenlet==3.0.3
-idna==3.6
-jmespath==1.0.1
-marshmallow==3.20.1
-opensearch-dsl==2.1.0
-opensearch-py==2.4.2
-packaging==23.2
-psycopg2-binary==2.9.9
-python-dateutil==2.8.2
-python-dotenv==1.0.0
-redis==5.0.1
-requests==2.31.0
-requests-aws4auth==1.2.3
-s3transfer==0.10.0
-six==1.16.0
-sqlalchemy==2.0.25
-sqlparse==0.4.4
-typing-extensions==4.9.0
-urllib3==1.26.18