pgsync 7.0.1__tar.gz → 7.0.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. {pgsync-7.0.1 → pgsync-7.0.2}/PKG-INFO +4 -5
  2. {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/__init__.py +1 -1
  3. {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/base.py +71 -19
  4. {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/constants.py +15 -2
  5. {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/querybuilder.py +1 -1
  6. {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/sync.py +2 -1
  7. {pgsync-7.0.1 → pgsync-7.0.2}/pgsync.egg-info/PKG-INFO +4 -5
  8. {pgsync-7.0.1 → pgsync-7.0.2}/pgsync.egg-info/requires.txt +3 -4
  9. {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_base.py +2 -2
  10. {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_constants.py +228 -0
  11. {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_sync.py +2 -6
  12. {pgsync-7.0.1 → pgsync-7.0.2}/AUTHORS.rst +0 -0
  13. {pgsync-7.0.1 → pgsync-7.0.2}/CONTRIBUTING.rst +0 -0
  14. {pgsync-7.0.1 → pgsync-7.0.2}/HISTORY.rst +0 -0
  15. {pgsync-7.0.1 → pgsync-7.0.2}/LICENSE +0 -0
  16. {pgsync-7.0.1 → pgsync-7.0.2}/MANIFEST.in +0 -0
  17. {pgsync-7.0.1 → pgsync-7.0.2}/README.md +0 -0
  18. {pgsync-7.0.1 → pgsync-7.0.2}/README.rst +0 -0
  19. {pgsync-7.0.1 → pgsync-7.0.2}/bin/bootstrap +0 -0
  20. {pgsync-7.0.1 → pgsync-7.0.2}/bin/parallel_sync +0 -0
  21. {pgsync-7.0.1 → pgsync-7.0.2}/bin/pgsync +0 -0
  22. {pgsync-7.0.1 → pgsync-7.0.2}/docs/Makefile +0 -0
  23. {pgsync-7.0.1 → pgsync-7.0.2}/docs/authors.rst +0 -0
  24. {pgsync-7.0.1 → pgsync-7.0.2}/docs/changelog.rst +0 -0
  25. {pgsync-7.0.1 → pgsync-7.0.2}/docs/conf.py +0 -0
  26. {pgsync-7.0.1 → pgsync-7.0.2}/docs/contributing.rst +0 -0
  27. {pgsync-7.0.1 → pgsync-7.0.2}/docs/history.rst +0 -0
  28. {pgsync-7.0.1 → pgsync-7.0.2}/docs/index.rst +0 -0
  29. {pgsync-7.0.1 → pgsync-7.0.2}/docs/installation.rst +0 -0
  30. {pgsync-7.0.1 → pgsync-7.0.2}/docs/logo.png +0 -0
  31. {pgsync-7.0.1 → pgsync-7.0.2}/docs/make.bat +0 -0
  32. {pgsync-7.0.1 → pgsync-7.0.2}/docs/readme.rst +0 -0
  33. {pgsync-7.0.1 → pgsync-7.0.2}/docs/usage.rst +0 -0
  34. {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/exc.py +0 -0
  35. {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/helper.py +0 -0
  36. {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/node.py +0 -0
  37. {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/plugin.py +0 -0
  38. {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/redisqueue.py +0 -0
  39. {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/search_client.py +0 -0
  40. {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/settings.py +0 -0
  41. {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/singleton.py +0 -0
  42. {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/transform.py +0 -0
  43. {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/trigger.py +0 -0
  44. {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/urls.py +0 -0
  45. {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/utils.py +0 -0
  46. {pgsync-7.0.1 → pgsync-7.0.2}/pgsync/view.py +0 -0
  47. {pgsync-7.0.1 → pgsync-7.0.2}/pgsync.egg-info/SOURCES.txt +0 -0
  48. {pgsync-7.0.1 → pgsync-7.0.2}/pgsync.egg-info/dependency_links.txt +0 -0
  49. {pgsync-7.0.1 → pgsync-7.0.2}/pgsync.egg-info/not-zip-safe +0 -0
  50. {pgsync-7.0.1 → pgsync-7.0.2}/pgsync.egg-info/top_level.txt +0 -0
  51. {pgsync-7.0.1 → pgsync-7.0.2}/pyproject.toml +0 -0
  52. {pgsync-7.0.1 → pgsync-7.0.2}/setup.cfg +0 -0
  53. {pgsync-7.0.1 → pgsync-7.0.2}/setup.py +0 -0
  54. {pgsync-7.0.1 → pgsync-7.0.2}/tests/__init__.py +0 -0
  55. {pgsync-7.0.1 → pgsync-7.0.2}/tests/conftest.py +0 -0
  56. {pgsync-7.0.1 → pgsync-7.0.2}/tests/fixtures/schema.json +0 -0
  57. {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_env_vars.py +0 -0
  58. {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_helper.py +0 -0
  59. {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_log_handlers.py +0 -0
  60. {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_node.py +0 -0
  61. {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_query_builder.py +0 -0
  62. {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_redisqueue.py +0 -0
  63. {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_search_client.py +0 -0
  64. {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_settings.py +0 -0
  65. {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_sync_nested_children.py +0 -0
  66. {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_sync_root.py +0 -0
  67. {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_sync_single_child_fk_on_child.py +0 -0
  68. {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_sync_single_child_fk_on_parent.py +0 -0
  69. {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_transform.py +0 -0
  70. {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_trigger.py +0 -0
  71. {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_unique_behaviour.py +0 -0
  72. {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_urls.py +0 -0
  73. {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_utils.py +0 -0
  74. {pgsync-7.0.1 → pgsync-7.0.2}/tests/test_view.py +0 -0
  75. {pgsync-7.0.1 → pgsync-7.0.2}/tests/testing_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pgsync
3
- Version: 7.0.1
3
+ Version: 7.0.2
4
4
  Summary: Postgres/MySQL/MariaDB to Elasticsearch/OpenSearch sync
5
5
  Home-page: https://github.com/toluaina/pgsync
6
6
  Author: Tolu Aina
@@ -33,8 +33,8 @@ License-File: LICENSE
33
33
  License-File: AUTHORS.rst
34
34
  Requires-Dist: async-timeout==5.0.1
35
35
  Requires-Dist: backports-datetime-fromisoformat==2.0.3
36
- Requires-Dist: boto3==1.42.8
37
- Requires-Dist: botocore==1.42.8
36
+ Requires-Dist: boto3==1.42.13
37
+ Requires-Dist: botocore==1.42.13
38
38
  Requires-Dist: certifi==2025.11.12
39
39
  Requires-Dist: charset-normalizer==3.4.4
40
40
  Requires-Dist: click==8.1.8
@@ -43,7 +43,6 @@ Requires-Dist: elasticsearch==7.17.12
43
43
  Requires-Dist: elasticsearch-dsl==7.4.1
44
44
  Requires-Dist: environs==14.4.0
45
45
  Requires-Dist: events==0.5
46
- Requires-Dist: greenlet==3.2.4
47
46
  Requires-Dist: idna==3.11
48
47
  Requires-Dist: jmespath==1.0.1
49
48
  Requires-Dist: marshmallow==4.0.1
@@ -61,7 +60,7 @@ Requires-Dist: requests-aws4auth==1.3.1
61
60
  Requires-Dist: s3transfer==0.16.0
62
61
  Requires-Dist: six==1.17.0
63
62
  Requires-Dist: sqlalchemy==2.0.45
64
- Requires-Dist: sqlparse==0.5.4
63
+ Requires-Dist: sqlparse==0.5.5
65
64
  Requires-Dist: typing-extensions==4.15.0
66
65
  Requires-Dist: urllib3==1.26.20
67
66
  Dynamic: author
@@ -2,4 +2,4 @@
2
2
 
3
3
  __author__ = "Tolu Aina"
4
4
  __email__ = "tolu@pgsync.com"
5
- __version__ = "7.0.1"
5
+ __version__ = "7.0.2"
@@ -749,21 +749,6 @@ class Base(object):
749
749
  )
750
750
  )[0]
751
751
 
752
- def get_replication_connection(
753
- self, engine: sa.engine.Engine
754
- ) -> psycopg2.extensions.connection:
755
- url: sa.engine.URL = make_url(str(engine.url))
756
- # Build a libpq-style connection by keyword args
757
- conn: psycopg2.extensions.connection = psycopg2.connect(
758
- host=url.host,
759
- port=url.port or 5432,
760
- user=url.username,
761
- password=url.password,
762
- dbname=url.database,
763
- connection_factory=LogicalReplicationConnection,
764
- )
765
- return conn
766
-
767
752
  def logical_slot_get_changes(
768
753
  self,
769
754
  slot_name: str,
@@ -1340,18 +1325,23 @@ def pg_engine(
1340
1325
  )
1341
1326
 
1342
1327
 
1343
- def _pg_engine(
1328
+ def _pg_connect_config(
1329
+ *,
1344
1330
  database: str,
1345
1331
  user: t.Optional[str] = None,
1346
1332
  host: t.Optional[str] = None,
1347
1333
  password: t.Optional[str] = None,
1348
1334
  port: t.Optional[int] = None,
1349
- echo: bool = False,
1350
1335
  sslmode: t.Optional[str] = None,
1351
1336
  sslrootcert: t.Optional[str] = None,
1352
1337
  url: t.Optional[str] = None,
1353
- ) -> sa.engine.Engine:
1338
+ ) -> tuple[str, dict]:
1339
+ """
1340
+ Shared config builder for both SQLAlchemy engines and direct psycopg2 connects.
1341
+ Returns (url, connect_args).
1342
+ """
1354
1343
  connect_args: dict = {}
1344
+
1355
1345
  sslmode = sslmode or PG_SSLMODE
1356
1346
  sslrootcert = sslrootcert or PG_SSLROOTCERT
1357
1347
 
@@ -1370,7 +1360,7 @@ def _pg_engine(
1370
1360
  connect_args["sslrootcert"] = sslrootcert
1371
1361
 
1372
1362
  if url is None:
1373
- url: str = get_database_url(
1363
+ url = get_database_url(
1374
1364
  database,
1375
1365
  user=user,
1376
1366
  host=host,
@@ -1378,6 +1368,31 @@ def _pg_engine(
1378
1368
  port=port,
1379
1369
  )
1380
1370
 
1371
+ return url, connect_args
1372
+
1373
+
1374
+ def _pg_engine(
1375
+ database: str,
1376
+ user: t.Optional[str] = None,
1377
+ host: t.Optional[str] = None,
1378
+ password: t.Optional[str] = None,
1379
+ port: t.Optional[int] = None,
1380
+ echo: bool = False,
1381
+ sslmode: t.Optional[str] = None,
1382
+ sslrootcert: t.Optional[str] = None,
1383
+ url: t.Optional[str] = None,
1384
+ ) -> sa.engine.Engine:
1385
+ url, connect_args = _pg_connect_config(
1386
+ database=database,
1387
+ user=user,
1388
+ host=host,
1389
+ password=password,
1390
+ port=port,
1391
+ sslmode=sslmode,
1392
+ sslrootcert=sslrootcert,
1393
+ url=url,
1394
+ )
1395
+
1381
1396
  # Use NullPool for testing to avoid connection exhaustion
1382
1397
  if SQLALCHEMY_USE_NULLPOOL:
1383
1398
  from sqlalchemy.pool import NullPool
@@ -1401,6 +1416,43 @@ def _pg_engine(
1401
1416
  )
1402
1417
 
1403
1418
 
1419
+ def pg_logical_repl_conn(
1420
+ *,
1421
+ database: str,
1422
+ user: t.Optional[str] = None,
1423
+ host: t.Optional[str] = None,
1424
+ password: t.Optional[str] = None,
1425
+ port: t.Optional[int] = None,
1426
+ sslmode: t.Optional[str] = None,
1427
+ sslrootcert: t.Optional[str] = None,
1428
+ url: t.Optional[str] = None,
1429
+ ) -> psycopg2.extensions.connection:
1430
+ url, connect_args = _pg_connect_config(
1431
+ database=database,
1432
+ user=user,
1433
+ host=host,
1434
+ password=password,
1435
+ port=port,
1436
+ sslmode=sslmode,
1437
+ sslrootcert=sslrootcert,
1438
+ url=url,
1439
+ )
1440
+
1441
+ url_: sa.engine.URL = make_url(url)
1442
+
1443
+ conn: psycopg2.extensions.connection = psycopg2.connect(
1444
+ host=url_.host,
1445
+ port=url_.port or 5432,
1446
+ user=url_.username,
1447
+ password=url_.password,
1448
+ dbname=url_.database,
1449
+ connection_factory=LogicalReplicationConnection,
1450
+ **connect_args,
1451
+ )
1452
+
1453
+ return conn
1454
+
1455
+
1404
1456
  def pg_execute(
1405
1457
  engine: sa.engine.Engine,
1406
1458
  statement: sa.sql.Select,
@@ -207,8 +207,21 @@ PRIMARY_KEY_DELIMITER = "|"
207
207
 
208
208
  # Replication slot patterns
209
209
  LOGICAL_SLOT_PREFIX = re.compile(
210
- r"table\s\"?(?P<schema>[\w-]+)\"?.\"?(?P<table>[\w-]+)\"?:\s(?P<tg_op>[A-Z]+):" # noqa E501
210
+ r"^table\s+"
211
+ r'"?(?P<schema>[^"]+?)"?\.' # schema up to the dot
212
+ r'"?(?P<table>[^"]+?)"?' # table up to the colon
213
+ r"\s*:\s*(?P<tg_op>[A-Z]+):"
211
214
  )
212
215
  LOGICAL_SLOT_SUFFIX = re.compile(
213
- r'\s(?P<key>"?\w+"?)\[(?P<type>[\w\s]+)\]:(?P<value>(?:"[^"]*"|\'[^\']*\'|null|\d+e[+-]?\d+|\w+))'
216
+ r"\s"
217
+ r'(?P<key>"(?:[^"]|"")+"|[\w$-]+)' # "Weird Key" or my_col or my-col or my$col
218
+ r"\[(?P<type>[^\]]+)\]" # anything until ]
219
+ r":"
220
+ r"(?P<value>"
221
+ r"null|true|false|NaN|Infinity|-Infinity|"
222
+ r'"(?:[^"]|"")*"|' # double-quoted, supports "" escape
223
+ r"'(?:[^']|'')*'|" # single-quoted, supports '' escape
224
+ r"-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?|" # numbers: -1, 3.14, 9e-3, 9E+2
225
+ r"[\w$-]+" # bare tokens like uuid-ish or identifiers
226
+ r")"
214
227
  )
@@ -400,7 +400,7 @@ class QueryBuilder(threading.local):
400
400
  subquery.append(
401
401
  sa.select(
402
402
  *[
403
- JSON_CAST(
403
+ sa.cast(
404
404
  sa.literal_column(f"'({page},'")
405
405
  .concat(sa.column("s"))
406
406
  .concat(")"),
@@ -30,6 +30,7 @@ from pymysqlreplication.row_event import (
30
30
  )
31
31
  from redis.exceptions import ConnectionError
32
32
 
33
+ from pgsync.base import pg_logical_repl_conn
33
34
  from pgsync.settings import IS_MYSQL_COMPAT
34
35
 
35
36
  from . import __version__, settings
@@ -1983,7 +1984,7 @@ class Sync(Base, metaclass=Singleton):
1983
1984
 
1984
1985
  def wal_consumer(self) -> None:
1985
1986
  # open a replication‐mode connection
1986
- conn = self.get_replication_connection(self.engine)
1987
+ conn = pg_logical_repl_conn(database=self.database)
1987
1988
  cursor = conn.cursor()
1988
1989
  # start streaming; include XIDs so you see BEGIN/COMMIT markers
1989
1990
  cursor.start_replication(
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pgsync
3
- Version: 7.0.1
3
+ Version: 7.0.2
4
4
  Summary: Postgres/MySQL/MariaDB to Elasticsearch/OpenSearch sync
5
5
  Home-page: https://github.com/toluaina/pgsync
6
6
  Author: Tolu Aina
@@ -33,8 +33,8 @@ License-File: LICENSE
33
33
  License-File: AUTHORS.rst
34
34
  Requires-Dist: async-timeout==5.0.1
35
35
  Requires-Dist: backports-datetime-fromisoformat==2.0.3
36
- Requires-Dist: boto3==1.42.8
37
- Requires-Dist: botocore==1.42.8
36
+ Requires-Dist: boto3==1.42.13
37
+ Requires-Dist: botocore==1.42.13
38
38
  Requires-Dist: certifi==2025.11.12
39
39
  Requires-Dist: charset-normalizer==3.4.4
40
40
  Requires-Dist: click==8.1.8
@@ -43,7 +43,6 @@ Requires-Dist: elasticsearch==7.17.12
43
43
  Requires-Dist: elasticsearch-dsl==7.4.1
44
44
  Requires-Dist: environs==14.4.0
45
45
  Requires-Dist: events==0.5
46
- Requires-Dist: greenlet==3.2.4
47
46
  Requires-Dist: idna==3.11
48
47
  Requires-Dist: jmespath==1.0.1
49
48
  Requires-Dist: marshmallow==4.0.1
@@ -61,7 +60,7 @@ Requires-Dist: requests-aws4auth==1.3.1
61
60
  Requires-Dist: s3transfer==0.16.0
62
61
  Requires-Dist: six==1.17.0
63
62
  Requires-Dist: sqlalchemy==2.0.45
64
- Requires-Dist: sqlparse==0.5.4
63
+ Requires-Dist: sqlparse==0.5.5
65
64
  Requires-Dist: typing-extensions==4.15.0
66
65
  Requires-Dist: urllib3==1.26.20
67
66
  Dynamic: author
@@ -1,7 +1,7 @@
1
1
  async-timeout==5.0.1
2
2
  backports-datetime-fromisoformat==2.0.3
3
- boto3==1.42.8
4
- botocore==1.42.8
3
+ boto3==1.42.13
4
+ botocore==1.42.13
5
5
  certifi==2025.11.12
6
6
  charset-normalizer==3.4.4
7
7
  click==8.1.8
@@ -10,7 +10,6 @@ elasticsearch==7.17.12
10
10
  elasticsearch-dsl==7.4.1
11
11
  environs==14.4.0
12
12
  events==0.5
13
- greenlet==3.2.4
14
13
  idna==3.11
15
14
  jmespath==1.0.1
16
15
  marshmallow==4.0.1
@@ -28,6 +27,6 @@ requests-aws4auth==1.3.1
28
27
  s3transfer==0.16.0
29
28
  six==1.17.0
30
29
  sqlalchemy==2.0.45
31
- sqlparse==0.5.4
30
+ sqlparse==0.5.5
32
31
  typing-extensions==4.15.0
33
32
  urllib3==1.26.20
@@ -488,7 +488,7 @@ class TestBase(object):
488
488
  row = """
489
489
  table public."B1_XYZ": INSERT: "ID"[integer]:5 "CREATED_TIMESTAMP"[bigint]:222 "ADDRESS"[character varying]:'from3' "SOME_FIELD_KEY"[character varying]:'key3' "SOME_OTHER_FIELD_KEY"[character varying]:'issue to handle' "CHANNEL_ID"[integer]:3 "CHANNEL_NAME"[character varying]:'channel 45' "ITEM_ID"[integer]:3 "MESSAGE"[character varying]:'message3' "RETRY"[integer]:4 "STATUS"[character varying]:'status' "SUBJECT"[character varying]:'sub3' "TIMESTAMP"[bigint]:33
490
490
  """ # noqa E501
491
- payload = pg_base.parse_logical_slot(row)
491
+ payload = pg_base.parse_logical_slot(row.strip())
492
492
  assert payload.data == {
493
493
  "CHANNEL_ID": 3,
494
494
  "CHANNEL_NAME": "channel 45",
@@ -527,7 +527,7 @@ class TestBase(object):
527
527
  row = """
528
528
  table public.book: UPDATE: id[integer]:1 isbn[character varying]:'001' title[character varying]:'It' description[character varying]:'Stephens Kings It' copyright[character varying]:null tags[jsonb]:'["a", "b", "c"]' doc[jsonb]:'{"a": {"b": {"c": [0, 1, 2, 3, 4]}}, "i": 73, "x": [{"y": 0, "z": 5}, {"y": 1, "z": 6}], "bool": true, "lastname": "Judye", "firstname": "Glenda", "generation": {"name": "X"}, "nick_names": ["Beatriz", "Jean", "Carilyn", "Carol-Jean", "Sara-Ann"], "coordinates": {"lat": 21.1, "lon": 32.9}}' publisher_id[integer]:1 publish_date[timestamp without time zone]:'1980-01-01 00:00:00' quad[double precision]:2e+58
529
529
  """ # noqa E501
530
- payload = pg_base.parse_logical_slot(row)
530
+ payload = pg_base.parse_logical_slot(row.strip())
531
531
  assert payload.data == {
532
532
  "copyright": None,
533
533
  "description": "Stephens Kings It",
@@ -1,5 +1,7 @@
1
1
  """Constants tests."""
2
2
 
3
+ import pytest
4
+
3
5
  from pgsync.constants import (
4
6
  ELASTICSEARCH_MAPPING_PARAMETERS,
5
7
  ELASTICSEARCH_TYPES,
@@ -11,6 +13,52 @@ from pgsync.constants import (
11
13
  class TestConstants(object):
12
14
  """Constants tests."""
13
15
 
16
+ def test_row_complex_update_prefix_and_suffix(self):
17
+ row = """
18
+ table public.book: UPDATE: id[integer]:1 isbn[character varying]:'001' title[character varying]:'It' description[character varying]:'Stephens Kings It' copyright[character varying]:null tags[jsonb]:'["a", "b", "c"]' doc[jsonb]:'{"a": {"b": {"c": [0, 1, 2, 3, 4]}}, "i": 73, "x": [{"y": 0, "z": 5}, {"y": 1, "z": 6}], "bool": true, "lastname": "Judye", "firstname": "Glenda", "generation": {"name": "X"}, "nick_names": ["Beatriz", "Jean", "Carilyn", "Carol-Jean", "Sara-Ann"], "coordinates": {"lat": 21.1, "lon": 32.9}}' publisher_id[integer]:1 publish_date[timestamp without time zone]:'1980-01-01 00:00:00' quad[double precision]:2e+58
19
+ """
20
+
21
+ m = LOGICAL_SLOT_PREFIX.search(row.strip())
22
+ assert m is not None
23
+ assert m.groupdict() == {
24
+ "schema": "public",
25
+ "table": "book",
26
+ "tg_op": "UPDATE",
27
+ }
28
+
29
+ fields = [
30
+ mm.groupdict() for mm in LOGICAL_SLOT_SUFFIX.finditer(row.strip())
31
+ ]
32
+ # sanity: first field
33
+ assert fields[0] == {"key": "id", "type": "integer", "value": "1"}
34
+
35
+ # spot-check a few tricky ones
36
+ assert {
37
+ "key": "copyright",
38
+ "type": "character varying",
39
+ "value": "null",
40
+ } in fields
41
+ assert {
42
+ "key": "tags",
43
+ "type": "jsonb",
44
+ "value": '\'["a", "b", "c"]\'',
45
+ } in fields
46
+ assert {
47
+ "key": "publisher_id",
48
+ "type": "integer",
49
+ "value": "1",
50
+ } in fields
51
+ assert {
52
+ "key": "publish_date",
53
+ "type": "timestamp without time zone",
54
+ "value": "'1980-01-01 00:00:00'",
55
+ } in fields
56
+ assert {
57
+ "key": "quad",
58
+ "type": "double precision",
59
+ "value": "2e+58",
60
+ } in fields
61
+
14
62
  def test_logical_slot_prefix_insert(self):
15
63
  insert = "table public.book: INSERT: id[integer]:9 isbn[character varying]:'978-0-924595-91-2a51f2c9f-930d-403c-8687-eeffd0fbfe6f' title[character varying]:'Certainly state million dog son night.' description[character varying]:'Idea prepare how push candidate page. Physical easy sister by let.' copyright[character varying]:null tags[jsonb]:null doc[jsonb]:null point[geometry]:null polygon[geometry]:null publisher_id[integer]:1" # noqa E501
16
64
  match = LOGICAL_SLOT_PREFIX.search(insert)
@@ -89,6 +137,186 @@ class TestConstants(object):
89
137
  "value": "12",
90
138
  }
91
139
 
140
+ @pytest.mark.parametrize(
141
+ "line, expected",
142
+ [
143
+ (
144
+ "table public.book: INSERT: id[integer]:9",
145
+ {"schema": "public", "table": "book", "tg_op": "INSERT"},
146
+ ),
147
+ (
148
+ 'table public."cars": INSERT: brand[character varying]:' "'a'",
149
+ {"schema": "public", "table": "cars", "tg_op": "INSERT"},
150
+ ),
151
+ (
152
+ "table public.bo-ok: UPDATE: id[integer]:1",
153
+ {"schema": "public", "table": "bo-ok", "tg_op": "UPDATE"},
154
+ ),
155
+ (
156
+ "table public.book-: DELETE: id[integer]:12",
157
+ {"schema": "public", "table": "book-", "tg_op": "DELETE"},
158
+ ),
159
+ # Quoted schema/table
160
+ (
161
+ 'table "public"."book": INSERT: id[integer]:9',
162
+ {"schema": "public", "table": "book", "tg_op": "INSERT"},
163
+ ),
164
+ # Schema with dash (works only if your schema group allows "-")
165
+ (
166
+ 'table "my-schema"."book": INSERT: id[integer]:9',
167
+ {"schema": "my-schema", "table": "book", "tg_op": "INSERT"},
168
+ ),
169
+ pytest.param(
170
+ 'table public."cars$xxx": INSERT: brand[character varying]:'
171
+ "'a' model[character varying]:'b' year[integer]:1",
172
+ {"schema": "public", "table": "cars$xxx", "tg_op": "INSERT"},
173
+ ),
174
+ # Unquoted table with dollar (Postgres allows $ unquoted too)
175
+ pytest.param(
176
+ "table public.cars$xxx: INSERT: id[integer]:1",
177
+ {"schema": "public", "table": "cars$xxx", "tg_op": "INSERT"},
178
+ ),
179
+ ],
180
+ )
181
+ def test_logical_slot_prefix_variants(self, line, expected):
182
+ match = LOGICAL_SLOT_PREFIX.search(line)
183
+ assert match is not None
184
+ assert match.groupdict() == expected
185
+
186
+ def test_logical_slot_suffix_first_field_insert(self):
187
+ insert = (
188
+ "table public.book: INSERT: id[integer]:9 "
189
+ "isbn[character varying]:'978-0-924595-91-2a51f2c9f-930d-403c-8687-eeffd0fbfe6f' "
190
+ "title[character varying]:'Certainly state million dog son night.' "
191
+ "copyright[character varying]:null"
192
+ )
193
+ match = LOGICAL_SLOT_SUFFIX.search(insert)
194
+ assert match is not None
195
+ assert match.groupdict() == {
196
+ "key": "id",
197
+ "type": "integer",
198
+ "value": "9",
199
+ }
200
+
201
+ def test_logical_slot_suffix_find_multiple_fields(self):
202
+ line = (
203
+ "table public.book: INSERT: "
204
+ "id[integer]:9 "
205
+ "title[character varying]:'Hello world' "
206
+ "is_active[boolean]:true "
207
+ "copyright[character varying]:null"
208
+ )
209
+ matches = [m.groupdict() for m in LOGICAL_SLOT_SUFFIX.finditer(line)]
210
+ assert matches == [
211
+ {"key": "id", "type": "integer", "value": "9"},
212
+ {
213
+ "key": "title",
214
+ "type": "character varying",
215
+ "value": "'Hello world'",
216
+ },
217
+ {"key": "is_active", "type": "boolean", "value": "true"},
218
+ {"key": "copyright", "type": "character varying", "value": "null"},
219
+ ]
220
+
221
+ def test_logical_slot_suffix_key_can_be_quoted_simple(self):
222
+ line = 'table public.book: INSERT: "id"[integer]:9'
223
+ match = LOGICAL_SLOT_SUFFIX.search(line)
224
+ assert match is not None
225
+ assert match.groupdict() == {
226
+ "key": '"id"',
227
+ "type": "integer",
228
+ "value": "9",
229
+ }
230
+
231
+ def test_logical_slot_suffix_empty_strings(self):
232
+ line = (
233
+ "table public.book: INSERT: "
234
+ "empty_single[character varying]:'' "
235
+ 'empty_double[character varying]:""'
236
+ )
237
+ matches = [m.groupdict() for m in LOGICAL_SLOT_SUFFIX.finditer(line)]
238
+ assert matches == [
239
+ {
240
+ "key": "empty_single",
241
+ "type": "character varying",
242
+ "value": "''",
243
+ },
244
+ {
245
+ "key": "empty_double",
246
+ "type": "character varying",
247
+ "value": '""',
248
+ },
249
+ ]
250
+
251
+ def test_logical_slot_suffix_json_in_single_quotes(self):
252
+ line = (
253
+ "table public.book: UPDATE: "
254
+ 'tags[jsonb]:\'["a", "b", "c"]\' '
255
+ 'doc[jsonb]:\'{"a": {"b": 1}, "ok": true}\''
256
+ )
257
+ matches = [m.groupdict() for m in LOGICAL_SLOT_SUFFIX.finditer(line)]
258
+ assert matches == [
259
+ {"key": "tags", "type": "jsonb", "value": '\'["a", "b", "c"]\''},
260
+ {
261
+ "key": "doc",
262
+ "type": "jsonb",
263
+ "value": '\'{"a": {"b": 1}, "ok": true}\'',
264
+ },
265
+ ]
266
+
267
+ def test_logical_slot_suffix_type_with_spaces(self):
268
+ line = (
269
+ "table public.book: INSERT: "
270
+ "created_at[timestamp without time zone]:'2025-12-17 10:11:12'"
271
+ )
272
+ match = LOGICAL_SLOT_SUFFIX.search(line)
273
+ assert match is not None
274
+ assert match.groupdict() == {
275
+ "key": "created_at",
276
+ "type": "timestamp without time zone",
277
+ "value": "'2025-12-17 10:11:12'",
278
+ }
279
+
280
+ def test_logical_slot_suffix_scientific_lower_e(self):
281
+ line = "table public.book: INSERT: ratio[double precision]:9e-3"
282
+ match = LOGICAL_SLOT_SUFFIX.search(line)
283
+ assert match is not None
284
+ assert match.groupdict() == {
285
+ "key": "ratio",
286
+ "type": "double precision",
287
+ "value": "9e-3",
288
+ }
289
+
290
+ def test_logical_slot_suffix_scientific_upper_E(self):
291
+ line = "table public.book: INSERT: ratio[double precision]:9E-3"
292
+ match = LOGICAL_SLOT_SUFFIX.search(line)
293
+ assert match is not None
294
+
295
+ def test_logical_slot_suffix_negative_int(self):
296
+ line = "table public.book: INSERT: delta[integer]:-1"
297
+ match = LOGICAL_SLOT_SUFFIX.search(line)
298
+ assert match is not None
299
+
300
+ def test_logical_slot_suffix_float(self):
301
+ line = "table public.book: INSERT: price[numeric]:3.14"
302
+ match = LOGICAL_SLOT_SUFFIX.search(line)
303
+ assert match is not None
304
+
305
+ def test_logical_slot_suffix_type_with_parens(self):
306
+ line = "table public.book: INSERT: price[numeric(10,2)]:123.45"
307
+ match = LOGICAL_SLOT_SUFFIX.search(line)
308
+ assert match is not None
309
+
310
+ def test_logical_slot_suffix_quoted_key_with_space(self):
311
+ line = 'table public.book: INSERT: "Weird Key"[integer]:1'
312
+ match = LOGICAL_SLOT_SUFFIX.search(line)
313
+ assert match is not None
314
+
315
+ def test_logical_slot_suffix_unquoted_key_with_dash_or_dollar(self):
316
+ line = "table public.book: INSERT: my-col[integer]:1 my$col[integer]:2"
317
+ matches = list(LOGICAL_SLOT_SUFFIX.finditer(line))
318
+ assert len(matches) == 2
319
+
92
320
  def test_elasticsearch_types(self):
93
321
  assert ELASTICSEARCH_TYPES == sorted(
94
322
  [
@@ -234,17 +234,13 @@ class TestSync(object):
234
234
  ROW("COMMIT 76474", 76474),
235
235
  ROW("BEGIN 76475", 76475),
236
236
  ROW(
237
- """
238
- table public.book: UPDATE: id[integer]:1 isbn[character varying]:'001' title[character varying]:'xyz' description[character varying]:'de' copyright[character varying]:null tags[jsonb]:'["a", "b", "c"]' doc[jsonb]:'{"a": {"b": {"c": [0, 1, 2, 3, 4]}}, "i": 73, "x": [{"y": 0, "z": 5}, {"y": 1, "z": 6}], "bool": true, "lastname": "Judye", "firstname": "Glenda", "generation": {"name": "X"}, "nick_names": ["Beatriz", "Jean", "Carilyn", "Carol-Jean", "Sara-Ann"], "coordinates": {"lat": 21.1, "lon": 32.9}}' publisher_id[integer]:1 publish_date[timestamp without time zone]:'1980-01-01 00:00:00'
239
- """,
237
+ """table public.book: UPDATE: id[integer]:1 isbn[character varying]:'001' title[character varying]:'xyz' description[character varying]:'de' copyright[character varying]:null tags[jsonb]:'["a", "b", "c"]' doc[jsonb]:'{"a": {"b": {"c": [0, 1, 2, 3, 4]}}, "i": 73, "x": [{"y": 0, "z": 5}, {"y": 1, "z": 6}], "bool": true, "lastname": "Judye", "firstname": "Glenda", "generation": {"name": "X"}, "nick_names": ["Beatriz", "Jean", "Carilyn", "Carol-Jean", "Sara-Ann"], "coordinates": {"lat": 21.1, "lon": 32.9}}' publisher_id[integer]:1 publish_date[timestamp without time zone]:'1980-01-01 00:00:00'""",
240
238
  76475,
241
239
  ),
242
240
  ROW("COMMIT 76475", 76472),
243
241
  ROW("BEGIN 76476", 76472),
244
242
  ROW(
245
- """
246
- table public.book: UPDATE: id[integer]:2 isbn[character varying]:'002' title[character varying]:'abc' description[character varying]:'Lodsdcsdrem ipsum dodscdslor sit amet' copyright[character varying]:null tags[jsonb]:'["d", "e", "f"]' doc[jsonb]:'{"a": {"b": {"c": [2, 3, 4, 5, 6]}}, "i": 99, "x": [{"y": 2, "z": 3}, {"y": 7, "z": 2}], "bool": false, "lastname": "Jones", "firstname": "Jack", "generation": {"name": "X"}, "nick_names": ["Jack", "Jones", "Jay", "Jay-Jay", "Jackie"], "coordinates": {"lat": 25.1, "lon": 52.2}}' publisher_id[integer]:1 publish_date[timestamp without time zone]:'infinity'
247
- """,
243
+ """table public.book: UPDATE: id[integer]:2 isbn[character varying]:'002' title[character varying]:'abc' description[character varying]:'Lodsdcsdrem ipsum dodscdslor sit amet' copyright[character varying]:null tags[jsonb]:'["d", "e", "f"]' doc[jsonb]:'{"a": {"b": {"c": [2, 3, 4, 5, 6]}}, "i": 99, "x": [{"y": 2, "z": 3}, {"y": 7, "z": 2}], "bool": false, "lastname": "Jones", "firstname": "Jack", "generation": {"name": "X"}, "nick_names": ["Jack", "Jones", "Jay", "Jay-Jay", "Jackie"], "coordinates": {"lat": 25.1, "lon": 52.2}}' publisher_id[integer]:1 publish_date[timestamp without time zone]:'infinity'""",
248
244
  76472,
249
245
  ),
250
246
  ROW("COMMIT 76476", 76472),
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes