pgsync 7.0.1__tar.gz → 7.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pgsync-7.0.1 → pgsync-7.0.2}/PKG-INFO +4 -5
- {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/__init__.py +1 -1
- {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/base.py +71 -19
- {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/constants.py +15 -2
- {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/querybuilder.py +1 -1
- {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/sync.py +2 -1
- {pgsync-7.0.1 → pgsync-7.0.2}/pgsync.egg-info/PKG-INFO +4 -5
- {pgsync-7.0.1 → pgsync-7.0.2}/pgsync.egg-info/requires.txt +3 -4
- {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_base.py +2 -2
- {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_constants.py +228 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_sync.py +2 -6
- {pgsync-7.0.1 → pgsync-7.0.2}/AUTHORS.rst +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/CONTRIBUTING.rst +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/HISTORY.rst +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/LICENSE +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/MANIFEST.in +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/README.md +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/README.rst +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/bin/bootstrap +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/bin/parallel_sync +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/bin/pgsync +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/docs/Makefile +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/docs/authors.rst +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/docs/changelog.rst +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/docs/conf.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/docs/contributing.rst +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/docs/history.rst +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/docs/index.rst +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/docs/installation.rst +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/docs/logo.png +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/docs/make.bat +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/docs/readme.rst +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/docs/usage.rst +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/exc.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/helper.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/node.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/plugin.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/redisqueue.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/search_client.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/settings.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/singleton.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/transform.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/trigger.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/urls.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/utils.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/view.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/pgsync.egg-info/SOURCES.txt +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/pgsync.egg-info/dependency_links.txt +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/pgsync.egg-info/not-zip-safe +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/pgsync.egg-info/top_level.txt +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/pyproject.toml +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/setup.cfg +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/setup.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/tests/__init__.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/tests/conftest.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/tests/fixtures/schema.json +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_env_vars.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_helper.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_log_handlers.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_node.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_query_builder.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_redisqueue.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_search_client.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_settings.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_sync_nested_children.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_sync_root.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_sync_single_child_fk_on_child.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_sync_single_child_fk_on_parent.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_transform.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_trigger.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_unique_behaviour.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_urls.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_utils.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_view.py +0 -0
- {pgsync-7.0.1 → pgsync-7.0.2}/tests/testing_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pgsync
|
|
3
|
-
Version: 7.0.
|
|
3
|
+
Version: 7.0.2
|
|
4
4
|
Summary: Postgres/MySQL/MariaDB to Elasticsearch/OpenSearch sync
|
|
5
5
|
Home-page: https://github.com/toluaina/pgsync
|
|
6
6
|
Author: Tolu Aina
|
|
@@ -33,8 +33,8 @@ License-File: LICENSE
|
|
|
33
33
|
License-File: AUTHORS.rst
|
|
34
34
|
Requires-Dist: async-timeout==5.0.1
|
|
35
35
|
Requires-Dist: backports-datetime-fromisoformat==2.0.3
|
|
36
|
-
Requires-Dist: boto3==1.42.
|
|
37
|
-
Requires-Dist: botocore==1.42.
|
|
36
|
+
Requires-Dist: boto3==1.42.13
|
|
37
|
+
Requires-Dist: botocore==1.42.13
|
|
38
38
|
Requires-Dist: certifi==2025.11.12
|
|
39
39
|
Requires-Dist: charset-normalizer==3.4.4
|
|
40
40
|
Requires-Dist: click==8.1.8
|
|
@@ -43,7 +43,6 @@ Requires-Dist: elasticsearch==7.17.12
|
|
|
43
43
|
Requires-Dist: elasticsearch-dsl==7.4.1
|
|
44
44
|
Requires-Dist: environs==14.4.0
|
|
45
45
|
Requires-Dist: events==0.5
|
|
46
|
-
Requires-Dist: greenlet==3.2.4
|
|
47
46
|
Requires-Dist: idna==3.11
|
|
48
47
|
Requires-Dist: jmespath==1.0.1
|
|
49
48
|
Requires-Dist: marshmallow==4.0.1
|
|
@@ -61,7 +60,7 @@ Requires-Dist: requests-aws4auth==1.3.1
|
|
|
61
60
|
Requires-Dist: s3transfer==0.16.0
|
|
62
61
|
Requires-Dist: six==1.17.0
|
|
63
62
|
Requires-Dist: sqlalchemy==2.0.45
|
|
64
|
-
Requires-Dist: sqlparse==0.5.
|
|
63
|
+
Requires-Dist: sqlparse==0.5.5
|
|
65
64
|
Requires-Dist: typing-extensions==4.15.0
|
|
66
65
|
Requires-Dist: urllib3==1.26.20
|
|
67
66
|
Dynamic: author
|
|
@@ -749,21 +749,6 @@ class Base(object):
|
|
|
749
749
|
)
|
|
750
750
|
)[0]
|
|
751
751
|
|
|
752
|
-
def get_replication_connection(
|
|
753
|
-
self, engine: sa.engine.Engine
|
|
754
|
-
) -> psycopg2.extensions.connection:
|
|
755
|
-
url: sa.engine.URL = make_url(str(engine.url))
|
|
756
|
-
# Build a libpq-style connection by keyword args
|
|
757
|
-
conn: psycopg2.extensions.connection = psycopg2.connect(
|
|
758
|
-
host=url.host,
|
|
759
|
-
port=url.port or 5432,
|
|
760
|
-
user=url.username,
|
|
761
|
-
password=url.password,
|
|
762
|
-
dbname=url.database,
|
|
763
|
-
connection_factory=LogicalReplicationConnection,
|
|
764
|
-
)
|
|
765
|
-
return conn
|
|
766
|
-
|
|
767
752
|
def logical_slot_get_changes(
|
|
768
753
|
self,
|
|
769
754
|
slot_name: str,
|
|
@@ -1340,18 +1325,23 @@ def pg_engine(
|
|
|
1340
1325
|
)
|
|
1341
1326
|
|
|
1342
1327
|
|
|
1343
|
-
def
|
|
1328
|
+
def _pg_connect_config(
|
|
1329
|
+
*,
|
|
1344
1330
|
database: str,
|
|
1345
1331
|
user: t.Optional[str] = None,
|
|
1346
1332
|
host: t.Optional[str] = None,
|
|
1347
1333
|
password: t.Optional[str] = None,
|
|
1348
1334
|
port: t.Optional[int] = None,
|
|
1349
|
-
echo: bool = False,
|
|
1350
1335
|
sslmode: t.Optional[str] = None,
|
|
1351
1336
|
sslrootcert: t.Optional[str] = None,
|
|
1352
1337
|
url: t.Optional[str] = None,
|
|
1353
|
-
) ->
|
|
1338
|
+
) -> tuple[str, dict]:
|
|
1339
|
+
"""
|
|
1340
|
+
Shared config builder for both SQLAlchemy engines and direct psycopg2 connects.
|
|
1341
|
+
Returns (url, connect_args).
|
|
1342
|
+
"""
|
|
1354
1343
|
connect_args: dict = {}
|
|
1344
|
+
|
|
1355
1345
|
sslmode = sslmode or PG_SSLMODE
|
|
1356
1346
|
sslrootcert = sslrootcert or PG_SSLROOTCERT
|
|
1357
1347
|
|
|
@@ -1370,7 +1360,7 @@ def _pg_engine(
|
|
|
1370
1360
|
connect_args["sslrootcert"] = sslrootcert
|
|
1371
1361
|
|
|
1372
1362
|
if url is None:
|
|
1373
|
-
url
|
|
1363
|
+
url = get_database_url(
|
|
1374
1364
|
database,
|
|
1375
1365
|
user=user,
|
|
1376
1366
|
host=host,
|
|
@@ -1378,6 +1368,31 @@ def _pg_engine(
|
|
|
1378
1368
|
port=port,
|
|
1379
1369
|
)
|
|
1380
1370
|
|
|
1371
|
+
return url, connect_args
|
|
1372
|
+
|
|
1373
|
+
|
|
1374
|
+
def _pg_engine(
|
|
1375
|
+
database: str,
|
|
1376
|
+
user: t.Optional[str] = None,
|
|
1377
|
+
host: t.Optional[str] = None,
|
|
1378
|
+
password: t.Optional[str] = None,
|
|
1379
|
+
port: t.Optional[int] = None,
|
|
1380
|
+
echo: bool = False,
|
|
1381
|
+
sslmode: t.Optional[str] = None,
|
|
1382
|
+
sslrootcert: t.Optional[str] = None,
|
|
1383
|
+
url: t.Optional[str] = None,
|
|
1384
|
+
) -> sa.engine.Engine:
|
|
1385
|
+
url, connect_args = _pg_connect_config(
|
|
1386
|
+
database=database,
|
|
1387
|
+
user=user,
|
|
1388
|
+
host=host,
|
|
1389
|
+
password=password,
|
|
1390
|
+
port=port,
|
|
1391
|
+
sslmode=sslmode,
|
|
1392
|
+
sslrootcert=sslrootcert,
|
|
1393
|
+
url=url,
|
|
1394
|
+
)
|
|
1395
|
+
|
|
1381
1396
|
# Use NullPool for testing to avoid connection exhaustion
|
|
1382
1397
|
if SQLALCHEMY_USE_NULLPOOL:
|
|
1383
1398
|
from sqlalchemy.pool import NullPool
|
|
@@ -1401,6 +1416,43 @@ def _pg_engine(
|
|
|
1401
1416
|
)
|
|
1402
1417
|
|
|
1403
1418
|
|
|
1419
|
+
def pg_logical_repl_conn(
|
|
1420
|
+
*,
|
|
1421
|
+
database: str,
|
|
1422
|
+
user: t.Optional[str] = None,
|
|
1423
|
+
host: t.Optional[str] = None,
|
|
1424
|
+
password: t.Optional[str] = None,
|
|
1425
|
+
port: t.Optional[int] = None,
|
|
1426
|
+
sslmode: t.Optional[str] = None,
|
|
1427
|
+
sslrootcert: t.Optional[str] = None,
|
|
1428
|
+
url: t.Optional[str] = None,
|
|
1429
|
+
) -> psycopg2.extensions.connection:
|
|
1430
|
+
url, connect_args = _pg_connect_config(
|
|
1431
|
+
database=database,
|
|
1432
|
+
user=user,
|
|
1433
|
+
host=host,
|
|
1434
|
+
password=password,
|
|
1435
|
+
port=port,
|
|
1436
|
+
sslmode=sslmode,
|
|
1437
|
+
sslrootcert=sslrootcert,
|
|
1438
|
+
url=url,
|
|
1439
|
+
)
|
|
1440
|
+
|
|
1441
|
+
url_: sa.engine.URL = make_url(url)
|
|
1442
|
+
|
|
1443
|
+
conn: psycopg2.extensions.connection = psycopg2.connect(
|
|
1444
|
+
host=url_.host,
|
|
1445
|
+
port=url_.port or 5432,
|
|
1446
|
+
user=url_.username,
|
|
1447
|
+
password=url_.password,
|
|
1448
|
+
dbname=url_.database,
|
|
1449
|
+
connection_factory=LogicalReplicationConnection,
|
|
1450
|
+
**connect_args,
|
|
1451
|
+
)
|
|
1452
|
+
|
|
1453
|
+
return conn
|
|
1454
|
+
|
|
1455
|
+
|
|
1404
1456
|
def pg_execute(
|
|
1405
1457
|
engine: sa.engine.Engine,
|
|
1406
1458
|
statement: sa.sql.Select,
|
|
@@ -207,8 +207,21 @@ PRIMARY_KEY_DELIMITER = "|"
|
|
|
207
207
|
|
|
208
208
|
# Replication slot patterns
|
|
209
209
|
LOGICAL_SLOT_PREFIX = re.compile(
|
|
210
|
-
r"table\s
|
|
210
|
+
r"^table\s+"
|
|
211
|
+
r'"?(?P<schema>[^"]+?)"?\.' # schema up to the dot
|
|
212
|
+
r'"?(?P<table>[^"]+?)"?' # table up to the colon
|
|
213
|
+
r"\s*:\s*(?P<tg_op>[A-Z]+):"
|
|
211
214
|
)
|
|
212
215
|
LOGICAL_SLOT_SUFFIX = re.compile(
|
|
213
|
-
r
|
|
216
|
+
r"\s"
|
|
217
|
+
r'(?P<key>"(?:[^"]|"")+"|[\w$-]+)' # "Weird Key" or my_col or my-col or my$col
|
|
218
|
+
r"\[(?P<type>[^\]]+)\]" # anything until ]
|
|
219
|
+
r":"
|
|
220
|
+
r"(?P<value>"
|
|
221
|
+
r"null|true|false|NaN|Infinity|-Infinity|"
|
|
222
|
+
r'"(?:[^"]|"")*"|' # double-quoted, supports "" escape
|
|
223
|
+
r"'(?:[^']|'')*'|" # single-quoted, supports '' escape
|
|
224
|
+
r"-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?|" # numbers: -1, 3.14, 9e-3, 9E+2
|
|
225
|
+
r"[\w$-]+" # bare tokens like uuid-ish or identifiers
|
|
226
|
+
r")"
|
|
214
227
|
)
|
|
@@ -30,6 +30,7 @@ from pymysqlreplication.row_event import (
|
|
|
30
30
|
)
|
|
31
31
|
from redis.exceptions import ConnectionError
|
|
32
32
|
|
|
33
|
+
from pgsync.base import pg_logical_repl_conn
|
|
33
34
|
from pgsync.settings import IS_MYSQL_COMPAT
|
|
34
35
|
|
|
35
36
|
from . import __version__, settings
|
|
@@ -1983,7 +1984,7 @@ class Sync(Base, metaclass=Singleton):
|
|
|
1983
1984
|
|
|
1984
1985
|
def wal_consumer(self) -> None:
|
|
1985
1986
|
# open a replication‐mode connection
|
|
1986
|
-
conn =
|
|
1987
|
+
conn = pg_logical_repl_conn(database=self.database)
|
|
1987
1988
|
cursor = conn.cursor()
|
|
1988
1989
|
# start streaming; include XIDs so you see BEGIN/COMMIT markers
|
|
1989
1990
|
cursor.start_replication(
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pgsync
|
|
3
|
-
Version: 7.0.
|
|
3
|
+
Version: 7.0.2
|
|
4
4
|
Summary: Postgres/MySQL/MariaDB to Elasticsearch/OpenSearch sync
|
|
5
5
|
Home-page: https://github.com/toluaina/pgsync
|
|
6
6
|
Author: Tolu Aina
|
|
@@ -33,8 +33,8 @@ License-File: LICENSE
|
|
|
33
33
|
License-File: AUTHORS.rst
|
|
34
34
|
Requires-Dist: async-timeout==5.0.1
|
|
35
35
|
Requires-Dist: backports-datetime-fromisoformat==2.0.3
|
|
36
|
-
Requires-Dist: boto3==1.42.
|
|
37
|
-
Requires-Dist: botocore==1.42.
|
|
36
|
+
Requires-Dist: boto3==1.42.13
|
|
37
|
+
Requires-Dist: botocore==1.42.13
|
|
38
38
|
Requires-Dist: certifi==2025.11.12
|
|
39
39
|
Requires-Dist: charset-normalizer==3.4.4
|
|
40
40
|
Requires-Dist: click==8.1.8
|
|
@@ -43,7 +43,6 @@ Requires-Dist: elasticsearch==7.17.12
|
|
|
43
43
|
Requires-Dist: elasticsearch-dsl==7.4.1
|
|
44
44
|
Requires-Dist: environs==14.4.0
|
|
45
45
|
Requires-Dist: events==0.5
|
|
46
|
-
Requires-Dist: greenlet==3.2.4
|
|
47
46
|
Requires-Dist: idna==3.11
|
|
48
47
|
Requires-Dist: jmespath==1.0.1
|
|
49
48
|
Requires-Dist: marshmallow==4.0.1
|
|
@@ -61,7 +60,7 @@ Requires-Dist: requests-aws4auth==1.3.1
|
|
|
61
60
|
Requires-Dist: s3transfer==0.16.0
|
|
62
61
|
Requires-Dist: six==1.17.0
|
|
63
62
|
Requires-Dist: sqlalchemy==2.0.45
|
|
64
|
-
Requires-Dist: sqlparse==0.5.
|
|
63
|
+
Requires-Dist: sqlparse==0.5.5
|
|
65
64
|
Requires-Dist: typing-extensions==4.15.0
|
|
66
65
|
Requires-Dist: urllib3==1.26.20
|
|
67
66
|
Dynamic: author
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
async-timeout==5.0.1
|
|
2
2
|
backports-datetime-fromisoformat==2.0.3
|
|
3
|
-
boto3==1.42.
|
|
4
|
-
botocore==1.42.
|
|
3
|
+
boto3==1.42.13
|
|
4
|
+
botocore==1.42.13
|
|
5
5
|
certifi==2025.11.12
|
|
6
6
|
charset-normalizer==3.4.4
|
|
7
7
|
click==8.1.8
|
|
@@ -10,7 +10,6 @@ elasticsearch==7.17.12
|
|
|
10
10
|
elasticsearch-dsl==7.4.1
|
|
11
11
|
environs==14.4.0
|
|
12
12
|
events==0.5
|
|
13
|
-
greenlet==3.2.4
|
|
14
13
|
idna==3.11
|
|
15
14
|
jmespath==1.0.1
|
|
16
15
|
marshmallow==4.0.1
|
|
@@ -28,6 +27,6 @@ requests-aws4auth==1.3.1
|
|
|
28
27
|
s3transfer==0.16.0
|
|
29
28
|
six==1.17.0
|
|
30
29
|
sqlalchemy==2.0.45
|
|
31
|
-
sqlparse==0.5.
|
|
30
|
+
sqlparse==0.5.5
|
|
32
31
|
typing-extensions==4.15.0
|
|
33
32
|
urllib3==1.26.20
|
|
@@ -488,7 +488,7 @@ class TestBase(object):
|
|
|
488
488
|
row = """
|
|
489
489
|
table public."B1_XYZ": INSERT: "ID"[integer]:5 "CREATED_TIMESTAMP"[bigint]:222 "ADDRESS"[character varying]:'from3' "SOME_FIELD_KEY"[character varying]:'key3' "SOME_OTHER_FIELD_KEY"[character varying]:'issue to handle' "CHANNEL_ID"[integer]:3 "CHANNEL_NAME"[character varying]:'channel 45' "ITEM_ID"[integer]:3 "MESSAGE"[character varying]:'message3' "RETRY"[integer]:4 "STATUS"[character varying]:'status' "SUBJECT"[character varying]:'sub3' "TIMESTAMP"[bigint]:33
|
|
490
490
|
""" # noqa E501
|
|
491
|
-
payload = pg_base.parse_logical_slot(row)
|
|
491
|
+
payload = pg_base.parse_logical_slot(row.strip())
|
|
492
492
|
assert payload.data == {
|
|
493
493
|
"CHANNEL_ID": 3,
|
|
494
494
|
"CHANNEL_NAME": "channel 45",
|
|
@@ -527,7 +527,7 @@ class TestBase(object):
|
|
|
527
527
|
row = """
|
|
528
528
|
table public.book: UPDATE: id[integer]:1 isbn[character varying]:'001' title[character varying]:'It' description[character varying]:'Stephens Kings It' copyright[character varying]:null tags[jsonb]:'["a", "b", "c"]' doc[jsonb]:'{"a": {"b": {"c": [0, 1, 2, 3, 4]}}, "i": 73, "x": [{"y": 0, "z": 5}, {"y": 1, "z": 6}], "bool": true, "lastname": "Judye", "firstname": "Glenda", "generation": {"name": "X"}, "nick_names": ["Beatriz", "Jean", "Carilyn", "Carol-Jean", "Sara-Ann"], "coordinates": {"lat": 21.1, "lon": 32.9}}' publisher_id[integer]:1 publish_date[timestamp without time zone]:'1980-01-01 00:00:00' quad[double precision]:2e+58
|
|
529
529
|
""" # noqa E501
|
|
530
|
-
payload = pg_base.parse_logical_slot(row)
|
|
530
|
+
payload = pg_base.parse_logical_slot(row.strip())
|
|
531
531
|
assert payload.data == {
|
|
532
532
|
"copyright": None,
|
|
533
533
|
"description": "Stephens Kings It",
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
"""Constants tests."""
|
|
2
2
|
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
3
5
|
from pgsync.constants import (
|
|
4
6
|
ELASTICSEARCH_MAPPING_PARAMETERS,
|
|
5
7
|
ELASTICSEARCH_TYPES,
|
|
@@ -11,6 +13,52 @@ from pgsync.constants import (
|
|
|
11
13
|
class TestConstants(object):
|
|
12
14
|
"""Constants tests."""
|
|
13
15
|
|
|
16
|
+
def test_row_complex_update_prefix_and_suffix(self):
|
|
17
|
+
row = """
|
|
18
|
+
table public.book: UPDATE: id[integer]:1 isbn[character varying]:'001' title[character varying]:'It' description[character varying]:'Stephens Kings It' copyright[character varying]:null tags[jsonb]:'["a", "b", "c"]' doc[jsonb]:'{"a": {"b": {"c": [0, 1, 2, 3, 4]}}, "i": 73, "x": [{"y": 0, "z": 5}, {"y": 1, "z": 6}], "bool": true, "lastname": "Judye", "firstname": "Glenda", "generation": {"name": "X"}, "nick_names": ["Beatriz", "Jean", "Carilyn", "Carol-Jean", "Sara-Ann"], "coordinates": {"lat": 21.1, "lon": 32.9}}' publisher_id[integer]:1 publish_date[timestamp without time zone]:'1980-01-01 00:00:00' quad[double precision]:2e+58
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
m = LOGICAL_SLOT_PREFIX.search(row.strip())
|
|
22
|
+
assert m is not None
|
|
23
|
+
assert m.groupdict() == {
|
|
24
|
+
"schema": "public",
|
|
25
|
+
"table": "book",
|
|
26
|
+
"tg_op": "UPDATE",
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
fields = [
|
|
30
|
+
mm.groupdict() for mm in LOGICAL_SLOT_SUFFIX.finditer(row.strip())
|
|
31
|
+
]
|
|
32
|
+
# sanity: first field
|
|
33
|
+
assert fields[0] == {"key": "id", "type": "integer", "value": "1"}
|
|
34
|
+
|
|
35
|
+
# spot-check a few tricky ones
|
|
36
|
+
assert {
|
|
37
|
+
"key": "copyright",
|
|
38
|
+
"type": "character varying",
|
|
39
|
+
"value": "null",
|
|
40
|
+
} in fields
|
|
41
|
+
assert {
|
|
42
|
+
"key": "tags",
|
|
43
|
+
"type": "jsonb",
|
|
44
|
+
"value": '\'["a", "b", "c"]\'',
|
|
45
|
+
} in fields
|
|
46
|
+
assert {
|
|
47
|
+
"key": "publisher_id",
|
|
48
|
+
"type": "integer",
|
|
49
|
+
"value": "1",
|
|
50
|
+
} in fields
|
|
51
|
+
assert {
|
|
52
|
+
"key": "publish_date",
|
|
53
|
+
"type": "timestamp without time zone",
|
|
54
|
+
"value": "'1980-01-01 00:00:00'",
|
|
55
|
+
} in fields
|
|
56
|
+
assert {
|
|
57
|
+
"key": "quad",
|
|
58
|
+
"type": "double precision",
|
|
59
|
+
"value": "2e+58",
|
|
60
|
+
} in fields
|
|
61
|
+
|
|
14
62
|
def test_logical_slot_prefix_insert(self):
|
|
15
63
|
insert = "table public.book: INSERT: id[integer]:9 isbn[character varying]:'978-0-924595-91-2a51f2c9f-930d-403c-8687-eeffd0fbfe6f' title[character varying]:'Certainly state million dog son night.' description[character varying]:'Idea prepare how push candidate page. Physical easy sister by let.' copyright[character varying]:null tags[jsonb]:null doc[jsonb]:null point[geometry]:null polygon[geometry]:null publisher_id[integer]:1" # noqa E501
|
|
16
64
|
match = LOGICAL_SLOT_PREFIX.search(insert)
|
|
@@ -89,6 +137,186 @@ class TestConstants(object):
|
|
|
89
137
|
"value": "12",
|
|
90
138
|
}
|
|
91
139
|
|
|
140
|
+
@pytest.mark.parametrize(
|
|
141
|
+
"line, expected",
|
|
142
|
+
[
|
|
143
|
+
(
|
|
144
|
+
"table public.book: INSERT: id[integer]:9",
|
|
145
|
+
{"schema": "public", "table": "book", "tg_op": "INSERT"},
|
|
146
|
+
),
|
|
147
|
+
(
|
|
148
|
+
'table public."cars": INSERT: brand[character varying]:' "'a'",
|
|
149
|
+
{"schema": "public", "table": "cars", "tg_op": "INSERT"},
|
|
150
|
+
),
|
|
151
|
+
(
|
|
152
|
+
"table public.bo-ok: UPDATE: id[integer]:1",
|
|
153
|
+
{"schema": "public", "table": "bo-ok", "tg_op": "UPDATE"},
|
|
154
|
+
),
|
|
155
|
+
(
|
|
156
|
+
"table public.book-: DELETE: id[integer]:12",
|
|
157
|
+
{"schema": "public", "table": "book-", "tg_op": "DELETE"},
|
|
158
|
+
),
|
|
159
|
+
# Quoted schema/table
|
|
160
|
+
(
|
|
161
|
+
'table "public"."book": INSERT: id[integer]:9',
|
|
162
|
+
{"schema": "public", "table": "book", "tg_op": "INSERT"},
|
|
163
|
+
),
|
|
164
|
+
# Schema with dash (works only if your schema group allows "-")
|
|
165
|
+
(
|
|
166
|
+
'table "my-schema"."book": INSERT: id[integer]:9',
|
|
167
|
+
{"schema": "my-schema", "table": "book", "tg_op": "INSERT"},
|
|
168
|
+
),
|
|
169
|
+
pytest.param(
|
|
170
|
+
'table public."cars$xxx": INSERT: brand[character varying]:'
|
|
171
|
+
"'a' model[character varying]:'b' year[integer]:1",
|
|
172
|
+
{"schema": "public", "table": "cars$xxx", "tg_op": "INSERT"},
|
|
173
|
+
),
|
|
174
|
+
# Unquoted table with dollar (Postgres allows $ unquoted too)
|
|
175
|
+
pytest.param(
|
|
176
|
+
"table public.cars$xxx: INSERT: id[integer]:1",
|
|
177
|
+
{"schema": "public", "table": "cars$xxx", "tg_op": "INSERT"},
|
|
178
|
+
),
|
|
179
|
+
],
|
|
180
|
+
)
|
|
181
|
+
def test_logical_slot_prefix_variants(self, line, expected):
|
|
182
|
+
match = LOGICAL_SLOT_PREFIX.search(line)
|
|
183
|
+
assert match is not None
|
|
184
|
+
assert match.groupdict() == expected
|
|
185
|
+
|
|
186
|
+
def test_logical_slot_suffix_first_field_insert(self):
|
|
187
|
+
insert = (
|
|
188
|
+
"table public.book: INSERT: id[integer]:9 "
|
|
189
|
+
"isbn[character varying]:'978-0-924595-91-2a51f2c9f-930d-403c-8687-eeffd0fbfe6f' "
|
|
190
|
+
"title[character varying]:'Certainly state million dog son night.' "
|
|
191
|
+
"copyright[character varying]:null"
|
|
192
|
+
)
|
|
193
|
+
match = LOGICAL_SLOT_SUFFIX.search(insert)
|
|
194
|
+
assert match is not None
|
|
195
|
+
assert match.groupdict() == {
|
|
196
|
+
"key": "id",
|
|
197
|
+
"type": "integer",
|
|
198
|
+
"value": "9",
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
def test_logical_slot_suffix_find_multiple_fields(self):
|
|
202
|
+
line = (
|
|
203
|
+
"table public.book: INSERT: "
|
|
204
|
+
"id[integer]:9 "
|
|
205
|
+
"title[character varying]:'Hello world' "
|
|
206
|
+
"is_active[boolean]:true "
|
|
207
|
+
"copyright[character varying]:null"
|
|
208
|
+
)
|
|
209
|
+
matches = [m.groupdict() for m in LOGICAL_SLOT_SUFFIX.finditer(line)]
|
|
210
|
+
assert matches == [
|
|
211
|
+
{"key": "id", "type": "integer", "value": "9"},
|
|
212
|
+
{
|
|
213
|
+
"key": "title",
|
|
214
|
+
"type": "character varying",
|
|
215
|
+
"value": "'Hello world'",
|
|
216
|
+
},
|
|
217
|
+
{"key": "is_active", "type": "boolean", "value": "true"},
|
|
218
|
+
{"key": "copyright", "type": "character varying", "value": "null"},
|
|
219
|
+
]
|
|
220
|
+
|
|
221
|
+
def test_logical_slot_suffix_key_can_be_quoted_simple(self):
|
|
222
|
+
line = 'table public.book: INSERT: "id"[integer]:9'
|
|
223
|
+
match = LOGICAL_SLOT_SUFFIX.search(line)
|
|
224
|
+
assert match is not None
|
|
225
|
+
assert match.groupdict() == {
|
|
226
|
+
"key": '"id"',
|
|
227
|
+
"type": "integer",
|
|
228
|
+
"value": "9",
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
def test_logical_slot_suffix_empty_strings(self):
|
|
232
|
+
line = (
|
|
233
|
+
"table public.book: INSERT: "
|
|
234
|
+
"empty_single[character varying]:'' "
|
|
235
|
+
'empty_double[character varying]:""'
|
|
236
|
+
)
|
|
237
|
+
matches = [m.groupdict() for m in LOGICAL_SLOT_SUFFIX.finditer(line)]
|
|
238
|
+
assert matches == [
|
|
239
|
+
{
|
|
240
|
+
"key": "empty_single",
|
|
241
|
+
"type": "character varying",
|
|
242
|
+
"value": "''",
|
|
243
|
+
},
|
|
244
|
+
{
|
|
245
|
+
"key": "empty_double",
|
|
246
|
+
"type": "character varying",
|
|
247
|
+
"value": '""',
|
|
248
|
+
},
|
|
249
|
+
]
|
|
250
|
+
|
|
251
|
+
def test_logical_slot_suffix_json_in_single_quotes(self):
|
|
252
|
+
line = (
|
|
253
|
+
"table public.book: UPDATE: "
|
|
254
|
+
'tags[jsonb]:\'["a", "b", "c"]\' '
|
|
255
|
+
'doc[jsonb]:\'{"a": {"b": 1}, "ok": true}\''
|
|
256
|
+
)
|
|
257
|
+
matches = [m.groupdict() for m in LOGICAL_SLOT_SUFFIX.finditer(line)]
|
|
258
|
+
assert matches == [
|
|
259
|
+
{"key": "tags", "type": "jsonb", "value": '\'["a", "b", "c"]\''},
|
|
260
|
+
{
|
|
261
|
+
"key": "doc",
|
|
262
|
+
"type": "jsonb",
|
|
263
|
+
"value": '\'{"a": {"b": 1}, "ok": true}\'',
|
|
264
|
+
},
|
|
265
|
+
]
|
|
266
|
+
|
|
267
|
+
def test_logical_slot_suffix_type_with_spaces(self):
|
|
268
|
+
line = (
|
|
269
|
+
"table public.book: INSERT: "
|
|
270
|
+
"created_at[timestamp without time zone]:'2025-12-17 10:11:12'"
|
|
271
|
+
)
|
|
272
|
+
match = LOGICAL_SLOT_SUFFIX.search(line)
|
|
273
|
+
assert match is not None
|
|
274
|
+
assert match.groupdict() == {
|
|
275
|
+
"key": "created_at",
|
|
276
|
+
"type": "timestamp without time zone",
|
|
277
|
+
"value": "'2025-12-17 10:11:12'",
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
def test_logical_slot_suffix_scientific_lower_e(self):
|
|
281
|
+
line = "table public.book: INSERT: ratio[double precision]:9e-3"
|
|
282
|
+
match = LOGICAL_SLOT_SUFFIX.search(line)
|
|
283
|
+
assert match is not None
|
|
284
|
+
assert match.groupdict() == {
|
|
285
|
+
"key": "ratio",
|
|
286
|
+
"type": "double precision",
|
|
287
|
+
"value": "9e-3",
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
def test_logical_slot_suffix_scientific_upper_E(self):
|
|
291
|
+
line = "table public.book: INSERT: ratio[double precision]:9E-3"
|
|
292
|
+
match = LOGICAL_SLOT_SUFFIX.search(line)
|
|
293
|
+
assert match is not None
|
|
294
|
+
|
|
295
|
+
def test_logical_slot_suffix_negative_int(self):
|
|
296
|
+
line = "table public.book: INSERT: delta[integer]:-1"
|
|
297
|
+
match = LOGICAL_SLOT_SUFFIX.search(line)
|
|
298
|
+
assert match is not None
|
|
299
|
+
|
|
300
|
+
def test_logical_slot_suffix_float(self):
|
|
301
|
+
line = "table public.book: INSERT: price[numeric]:3.14"
|
|
302
|
+
match = LOGICAL_SLOT_SUFFIX.search(line)
|
|
303
|
+
assert match is not None
|
|
304
|
+
|
|
305
|
+
def test_logical_slot_suffix_type_with_parens(self):
|
|
306
|
+
line = "table public.book: INSERT: price[numeric(10,2)]:123.45"
|
|
307
|
+
match = LOGICAL_SLOT_SUFFIX.search(line)
|
|
308
|
+
assert match is not None
|
|
309
|
+
|
|
310
|
+
def test_logical_slot_suffix_quoted_key_with_space(self):
|
|
311
|
+
line = 'table public.book: INSERT: "Weird Key"[integer]:1'
|
|
312
|
+
match = LOGICAL_SLOT_SUFFIX.search(line)
|
|
313
|
+
assert match is not None
|
|
314
|
+
|
|
315
|
+
def test_logical_slot_suffix_unquoted_key_with_dash_or_dollar(self):
|
|
316
|
+
line = "table public.book: INSERT: my-col[integer]:1 my$col[integer]:2"
|
|
317
|
+
matches = list(LOGICAL_SLOT_SUFFIX.finditer(line))
|
|
318
|
+
assert len(matches) == 2
|
|
319
|
+
|
|
92
320
|
def test_elasticsearch_types(self):
|
|
93
321
|
assert ELASTICSEARCH_TYPES == sorted(
|
|
94
322
|
[
|
|
@@ -234,17 +234,13 @@ class TestSync(object):
|
|
|
234
234
|
ROW("COMMIT 76474", 76474),
|
|
235
235
|
ROW("BEGIN 76475", 76475),
|
|
236
236
|
ROW(
|
|
237
|
-
"""
|
|
238
|
-
table public.book: UPDATE: id[integer]:1 isbn[character varying]:'001' title[character varying]:'xyz' description[character varying]:'de' copyright[character varying]:null tags[jsonb]:'["a", "b", "c"]' doc[jsonb]:'{"a": {"b": {"c": [0, 1, 2, 3, 4]}}, "i": 73, "x": [{"y": 0, "z": 5}, {"y": 1, "z": 6}], "bool": true, "lastname": "Judye", "firstname": "Glenda", "generation": {"name": "X"}, "nick_names": ["Beatriz", "Jean", "Carilyn", "Carol-Jean", "Sara-Ann"], "coordinates": {"lat": 21.1, "lon": 32.9}}' publisher_id[integer]:1 publish_date[timestamp without time zone]:'1980-01-01 00:00:00'
|
|
239
|
-
""",
|
|
237
|
+
"""table public.book: UPDATE: id[integer]:1 isbn[character varying]:'001' title[character varying]:'xyz' description[character varying]:'de' copyright[character varying]:null tags[jsonb]:'["a", "b", "c"]' doc[jsonb]:'{"a": {"b": {"c": [0, 1, 2, 3, 4]}}, "i": 73, "x": [{"y": 0, "z": 5}, {"y": 1, "z": 6}], "bool": true, "lastname": "Judye", "firstname": "Glenda", "generation": {"name": "X"}, "nick_names": ["Beatriz", "Jean", "Carilyn", "Carol-Jean", "Sara-Ann"], "coordinates": {"lat": 21.1, "lon": 32.9}}' publisher_id[integer]:1 publish_date[timestamp without time zone]:'1980-01-01 00:00:00'""",
|
|
240
238
|
76475,
|
|
241
239
|
),
|
|
242
240
|
ROW("COMMIT 76475", 76472),
|
|
243
241
|
ROW("BEGIN 76476", 76472),
|
|
244
242
|
ROW(
|
|
245
|
-
"""
|
|
246
|
-
table public.book: UPDATE: id[integer]:2 isbn[character varying]:'002' title[character varying]:'abc' description[character varying]:'Lodsdcsdrem ipsum dodscdslor sit amet' copyright[character varying]:null tags[jsonb]:'["d", "e", "f"]' doc[jsonb]:'{"a": {"b": {"c": [2, 3, 4, 5, 6]}}, "i": 99, "x": [{"y": 2, "z": 3}, {"y": 7, "z": 2}], "bool": false, "lastname": "Jones", "firstname": "Jack", "generation": {"name": "X"}, "nick_names": ["Jack", "Jones", "Jay", "Jay-Jay", "Jackie"], "coordinates": {"lat": 25.1, "lon": 52.2}}' publisher_id[integer]:1 publish_date[timestamp without time zone]:'infinity'
|
|
247
|
-
""",
|
|
243
|
+
"""table public.book: UPDATE: id[integer]:2 isbn[character varying]:'002' title[character varying]:'abc' description[character varying]:'Lodsdcsdrem ipsum dodscdslor sit amet' copyright[character varying]:null tags[jsonb]:'["d", "e", "f"]' doc[jsonb]:'{"a": {"b": {"c": [2, 3, 4, 5, 6]}}, "i": 99, "x": [{"y": 2, "z": 3}, {"y": 7, "z": 2}], "bool": false, "lastname": "Jones", "firstname": "Jack", "generation": {"name": "X"}, "nick_names": ["Jack", "Jones", "Jay", "Jay-Jay", "Jackie"], "coordinates": {"lat": 25.1, "lon": 52.2}}' publisher_id[integer]:1 publish_date[timestamp without time zone]:'infinity'""",
|
|
248
244
|
76472,
|
|
249
245
|
),
|
|
250
246
|
ROW("COMMIT 76476", 76472),
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|