pgsync 3.2.0__tar.gz → 3.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. {pgsync-3.2.0 → pgsync-3.2.1}/PKG-INFO +13 -13
  2. {pgsync-3.2.0 → pgsync-3.2.1}/bin/bootstrap +9 -1
  3. {pgsync-3.2.0 → pgsync-3.2.1}/bin/parallel_sync +1 -1
  4. {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/__init__.py +1 -1
  5. {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/base.py +2 -0
  6. {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/constants.py +1 -1
  7. {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/node.py +6 -0
  8. {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/plugin.py +5 -0
  9. {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/search_client.py +9 -8
  10. {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/settings.py +1 -0
  11. {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/sync.py +56 -22
  12. {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/urls.py +8 -0
  13. {pgsync-3.2.0 → pgsync-3.2.1}/pgsync.egg-info/PKG-INFO +13 -13
  14. {pgsync-3.2.0 → pgsync-3.2.1}/pgsync.egg-info/requires.txt +12 -12
  15. {pgsync-3.2.0 → pgsync-3.2.1}/setup.py +2 -1
  16. {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_base.py +13 -9
  17. {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_search_client.py +1 -1
  18. {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_sync_nested_children.py +23 -5
  19. {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_sync_root.py +205 -49
  20. {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_sync_single_child_fk_on_child.py +71 -62
  21. {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_sync_single_child_fk_on_parent.py +81 -60
  22. {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_unique_behaviour.py +1 -0
  23. {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_urls.py +18 -0
  24. {pgsync-3.2.0 → pgsync-3.2.1}/AUTHORS.rst +0 -0
  25. {pgsync-3.2.0 → pgsync-3.2.1}/CONTRIBUTING.rst +0 -0
  26. {pgsync-3.2.0 → pgsync-3.2.1}/HISTORY.rst +0 -0
  27. {pgsync-3.2.0 → pgsync-3.2.1}/LICENSE +0 -0
  28. {pgsync-3.2.0 → pgsync-3.2.1}/MANIFEST.in +0 -0
  29. {pgsync-3.2.0 → pgsync-3.2.1}/README.md +0 -0
  30. {pgsync-3.2.0 → pgsync-3.2.1}/README.rst +0 -0
  31. {pgsync-3.2.0 → pgsync-3.2.1}/bin/pgsync +0 -0
  32. {pgsync-3.2.0 → pgsync-3.2.1}/docs/Makefile +0 -0
  33. {pgsync-3.2.0 → pgsync-3.2.1}/docs/authors.rst +0 -0
  34. {pgsync-3.2.0 → pgsync-3.2.1}/docs/changelog.rst +0 -0
  35. {pgsync-3.2.0 → pgsync-3.2.1}/docs/conf.py +0 -0
  36. {pgsync-3.2.0 → pgsync-3.2.1}/docs/contributing.rst +0 -0
  37. {pgsync-3.2.0 → pgsync-3.2.1}/docs/history.rst +0 -0
  38. {pgsync-3.2.0 → pgsync-3.2.1}/docs/index.rst +0 -0
  39. {pgsync-3.2.0 → pgsync-3.2.1}/docs/installation.rst +0 -0
  40. {pgsync-3.2.0 → pgsync-3.2.1}/docs/logo.png +0 -0
  41. {pgsync-3.2.0 → pgsync-3.2.1}/docs/make.bat +0 -0
  42. {pgsync-3.2.0 → pgsync-3.2.1}/docs/readme.rst +0 -0
  43. {pgsync-3.2.0 → pgsync-3.2.1}/docs/usage.rst +0 -0
  44. {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/exc.py +0 -0
  45. {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/helper.py +0 -0
  46. {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/querybuilder.py +0 -0
  47. {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/redisqueue.py +0 -0
  48. {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/singleton.py +0 -0
  49. {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/transform.py +0 -0
  50. {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/trigger.py +0 -0
  51. {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/utils.py +0 -0
  52. {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/view.py +0 -0
  53. {pgsync-3.2.0 → pgsync-3.2.1}/pgsync.egg-info/SOURCES.txt +0 -0
  54. {pgsync-3.2.0 → pgsync-3.2.1}/pgsync.egg-info/dependency_links.txt +0 -0
  55. {pgsync-3.2.0 → pgsync-3.2.1}/pgsync.egg-info/not-zip-safe +0 -0
  56. {pgsync-3.2.0 → pgsync-3.2.1}/pgsync.egg-info/top_level.txt +0 -0
  57. {pgsync-3.2.0 → pgsync-3.2.1}/pyproject.toml +0 -0
  58. {pgsync-3.2.0 → pgsync-3.2.1}/setup.cfg +0 -0
  59. {pgsync-3.2.0 → pgsync-3.2.1}/tests/__init__.py +0 -0
  60. {pgsync-3.2.0 → pgsync-3.2.1}/tests/conftest.py +0 -0
  61. {pgsync-3.2.0 → pgsync-3.2.1}/tests/fixtures/schema.json +0 -0
  62. {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_constants.py +0 -0
  63. {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_env_vars.py +0 -0
  64. {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_helper.py +0 -0
  65. {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_log_handlers.py +0 -0
  66. {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_node.py +0 -0
  67. {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_query_builder.py +0 -0
  68. {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_redisqueue.py +0 -0
  69. {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_settings.py +0 -0
  70. {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_sync.py +0 -0
  71. {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_transform.py +0 -0
  72. {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_trigger.py +0 -0
  73. {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_utils.py +0 -0
  74. {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_view.py +0 -0
  75. {pgsync-3.2.0 → pgsync-3.2.1}/tests/testing_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: pgsync
3
- Version: 3.2.0
3
+ Version: 3.2.1
4
4
  Summary: Postgres to Elasticsearch/OpenSearch sync
5
5
  Home-page: https://github.com/toluaina/pgsync
6
6
  Author: Tolu Aina
@@ -31,33 +31,33 @@ Description-Content-Type: text/markdown
31
31
  License-File: LICENSE
32
32
  License-File: AUTHORS.rst
33
33
  Requires-Dist: async-timeout==4.0.3
34
- Requires-Dist: boto3==1.34.142
35
- Requires-Dist: botocore==1.34.142
34
+ Requires-Dist: boto3==1.35.5
35
+ Requires-Dist: botocore==1.35.5
36
36
  Requires-Dist: certifi==2024.7.4
37
37
  Requires-Dist: charset-normalizer==3.3.2
38
38
  Requires-Dist: click==8.1.7
39
- Requires-Dist: elastic-transport==8.13.1
40
- Requires-Dist: elasticsearch==8.14.0
41
- Requires-Dist: elasticsearch-dsl==8.14.0
39
+ Requires-Dist: elastic-transport==8.15.0
40
+ Requires-Dist: elasticsearch==8.15.0
41
+ Requires-Dist: elasticsearch-dsl==8.15.1
42
42
  Requires-Dist: environs==11.0.0
43
43
  Requires-Dist: events==0.5
44
44
  Requires-Dist: greenlet==3.0.3
45
- Requires-Dist: idna==3.7
45
+ Requires-Dist: idna==3.8
46
46
  Requires-Dist: jmespath==1.0.1
47
- Requires-Dist: marshmallow==3.21.3
47
+ Requires-Dist: marshmallow==3.22.0
48
48
  Requires-Dist: opensearch-dsl==2.1.0
49
- Requires-Dist: opensearch-py==2.6.0
49
+ Requires-Dist: opensearch-py==2.7.1
50
50
  Requires-Dist: packaging==24.1
51
51
  Requires-Dist: psycopg2-binary==2.9.9
52
52
  Requires-Dist: python-dateutil==2.9.0.post0
53
53
  Requires-Dist: python-dotenv==1.0.1
54
- Requires-Dist: redis==5.0.7
54
+ Requires-Dist: redis==5.0.8
55
55
  Requires-Dist: requests==2.32.3
56
- Requires-Dist: requests-aws4auth==1.2.3
56
+ Requires-Dist: requests-aws4auth==1.3.1
57
57
  Requires-Dist: s3transfer==0.10.2
58
58
  Requires-Dist: six==1.16.0
59
- Requires-Dist: sqlalchemy==2.0.31
60
- Requires-Dist: sqlparse==0.5.0
59
+ Requires-Dist: sqlalchemy==2.0.32
60
+ Requires-Dist: sqlparse==0.5.1
61
61
  Requires-Dist: typing-extensions==4.12.2
62
62
  Requires-Dist: urllib3==1.26.19
63
63
 
@@ -35,7 +35,15 @@ logger = logging.getLogger(__name__)
35
35
  default=False,
36
36
  help="Turn on verbosity",
37
37
  )
38
- def main(teardown, config, user, password, host, port, verbose):
38
+ def main(
39
+ teardown: bool,
40
+ config: str,
41
+ user: str,
42
+ password: bool,
43
+ host: str,
44
+ port: int,
45
+ verbose: bool,
46
+ ) -> None:
39
47
  """Application onetime Bootstrap."""
40
48
  kwargs: dict = {
41
49
  "user": user,
@@ -409,7 +409,7 @@ def run_task(
409
409
  ),
410
410
  default="multiprocess_async",
411
411
  )
412
- def main(config, nprocs, mode, verbose):
412
+ def main(config: str, nprocs: int, mode: str, verbose: bool) -> None:
413
413
  """
414
414
  TODO:
415
415
  - Track progress across cpus/threads
@@ -2,4 +2,4 @@
2
2
 
3
3
  __author__ = "Tolu Aina"
4
4
  __email__ = "tolu@pgsync.com"
5
- __version__ = "3.2.0"
5
+ __version__ = "3.2.1"
@@ -778,6 +778,8 @@ class Base(object):
778
778
 
779
779
  NB: All integers are long in python3 and call to convert is just int
780
780
  """
781
+ if self.verbose:
782
+ logger.debug(f"type: {type_} value: {value}")
781
783
  if value.lower() == "null":
782
784
  return None
783
785
  if type_.lower() in self.INT_TYPES:
@@ -208,5 +208,5 @@ LOGICAL_SLOT_PREFIX = re.compile(
208
208
  r"table\s\"?(?P<schema>[\w-]+)\"?.\"?(?P<table>[\w-]+)\"?:\s(?P<tg_op>[A-Z]+):" # noqa E501
209
209
  )
210
210
  LOGICAL_SLOT_SUFFIX = re.compile(
211
- '\s(?P<key>"?\w+"?)\[(?P<type>[\w\s]+)\]:(?P<value>[\w\'"\-\\.\\+]+)'
211
+ r'\s(?P<key>"?\w+"?)\[(?P<type>[\w\s]+)\]:(?P<value>(?:"[^"]*"|\'[^\']*\'|null|\d+e[+-]?\d+|\w+))'
212
212
  )
@@ -284,6 +284,7 @@ class Tree(threading.local):
284
284
  def __post_init__(self):
285
285
  self.tables: t.Set[str] = set()
286
286
  self.__nodes: t.Dict[Node] = {}
287
+ self.__schemas: t.Set[str] = set()
287
288
  self.root: t.Optional[Node] = None
288
289
  self.build(self.nodes)
289
290
 
@@ -334,6 +335,7 @@ class Tree(threading.local):
334
335
  node.add_child(self.build(child))
335
336
 
336
337
  self.__nodes[key] = node
338
+ self.__schemas.add(schema)
337
339
  return node
338
340
 
339
341
  def get_node(self, table: str, schema: str) -> Node:
@@ -352,3 +354,7 @@ class Tree(threading.local):
352
354
  else:
353
355
  raise RuntimeError(f"Node for {schema}.{table} not found")
354
356
  return self.__nodes[key]
357
+
358
+ @property
359
+ def schemas(self) -> t.Set[str]:
360
+ return self.__schemas
@@ -48,6 +48,11 @@ class Plugins(object):
48
48
  if "test" not in sys.argv[0]:
49
49
  self.walk(self.package)
50
50
 
51
+ # main plugin ordering
52
+ self.plugins = sorted(
53
+ self.plugins, key=lambda x: self.names.index(x.name)
54
+ )
55
+
51
56
  def walk(self, package: str) -> None:
52
57
  """Recursively walk the supplied package and fetch all plugins."""
53
58
  module = import_module(package)
@@ -174,7 +174,7 @@ class SearchClient(object):
174
174
  ):
175
175
  """Bulk index, update, delete docs to Elasticsearch/OpenSearch."""
176
176
  if settings.ELASTICSEARCH_STREAMING_BULK:
177
- for ok, _ in self.streaming_bulk(
177
+ for ok, info in self.streaming_bulk(
178
178
  self.__client,
179
179
  actions,
180
180
  index=index,
@@ -189,10 +189,12 @@ class SearchClient(object):
189
189
  ):
190
190
  if ok:
191
191
  self.doc_count += 1
192
+ else:
193
+ logger.error(f"Document failed to index: {info}")
192
194
  else:
193
195
  # parallel bulk consumes more memory and is also more likely
194
196
  # to result in 429 errors.
195
- for ok, _ in self.parallel_bulk(
197
+ for ok, info in self.parallel_bulk(
196
198
  self.__client,
197
199
  actions,
198
200
  thread_count=thread_count,
@@ -206,6 +208,8 @@ class SearchClient(object):
206
208
  ):
207
209
  if ok:
208
210
  self.doc_count += 1
211
+ else:
212
+ logger.error(f"Document failed to index: {info}")
209
213
 
210
214
  def refresh(self, indices: t.List[str]) -> None:
211
215
  """Refresh the Elasticsearch/OpenSearch index."""
@@ -384,9 +388,9 @@ def get_search_client(
384
388
  service,
385
389
  session_token=credentials.token,
386
390
  ),
387
- use_ssl=True,
388
391
  verify_certs=True,
389
392
  connection_class=connection_class,
393
+ timeout=settings.ELASTICSEARCH_TIMEOUT,
390
394
  )
391
395
  elif settings.ELASTICSEARCH:
392
396
  return client(
@@ -398,9 +402,9 @@ def get_search_client(
398
402
  service,
399
403
  session_token=credentials.token,
400
404
  ),
401
- use_ssl=True,
402
405
  verify_certs=True,
403
406
  node_class=node_class,
407
+ timeout=settings.ELASTICSEARCH_TIMEOUT,
404
408
  )
405
409
  else:
406
410
  hosts: t.List[str] = [url]
@@ -436,8 +440,6 @@ def get_search_client(
436
440
  ssl_version: t.Optional[int] = settings.ELASTICSEARCH_SSL_VERSION
437
441
  ssl_context: t.Optional[t.Any] = settings.ELASTICSEARCH_SSL_CONTEXT
438
442
  ssl_show_warn: bool = settings.ELASTICSEARCH_SSL_SHOW_WARN
439
- # Transport
440
- timeout: float = settings.ELASTICSEARCH_TIMEOUT
441
443
  return client(
442
444
  hosts=hosts,
443
445
  http_auth=http_auth,
@@ -456,6 +458,5 @@ def get_search_client(
456
458
  ssl_version=ssl_version,
457
459
  ssl_context=ssl_context,
458
460
  ssl_show_warn=ssl_show_warn,
459
- # use_ssl=use_ssl,
460
- timeout=timeout,
461
+ timeout=settings.ELASTICSEARCH_TIMEOUT,
461
462
  )
@@ -161,6 +161,7 @@ PG_USER = env.str("PG_USER")
161
161
 
162
162
  # Redis:
163
163
  REDIS_AUTH = env.str("REDIS_AUTH", default=None)
164
+ REDIS_USER = env.str("REDIS_USER", default=None)
164
165
  REDIS_DB = env.int("REDIS_DB", default=0)
165
166
  REDIS_HOST = env.str("REDIS_HOST", default="localhost")
166
167
  # redis poll interval (in secs)
@@ -109,6 +109,7 @@ class Sync(Base, metaclass=Singleton):
109
109
  self._plugins: Plugins = Plugins("plugins", self.plugins)
110
110
  self.query_builder: QueryBuilder = QueryBuilder(verbose=verbose)
111
111
  self.count: dict = dict(xlog=0, db=0, redis=0)
112
+ self.tasks: t.List[asyncio.Task] = []
112
113
 
113
114
  def validate(self, repl_slots: bool = True) -> None:
114
115
  """Perform all validation right away."""
@@ -409,6 +410,11 @@ class Sync(Base, metaclass=Singleton):
409
410
  f"Error parsing row: {e}\nRow data: {row.data}"
410
411
  )
411
412
  raise
413
+
414
+ # filter out unknown schemas
415
+ if payload.schema not in self.tree.schemas:
416
+ continue
417
+
412
418
  payloads.append(payload)
413
419
 
414
420
  j: int = i + 1
@@ -810,7 +816,10 @@ class Sync(Base, metaclass=Singleton):
810
816
  # e.g a through table which we need to react to.
811
817
  # in this case, we find the parent of the through
812
818
  # table and force a re-sync.
813
- if payload.table not in self.tree.tables:
819
+ if (
820
+ payload.table not in self.tree.tables
821
+ or payload.schema not in self.tree.schemas
822
+ ):
814
823
  return
815
824
 
816
825
  node: Node = self.tree.get_node(payload.table, payload.schema)
@@ -974,6 +983,18 @@ class Sync(Base, metaclass=Singleton):
974
983
 
975
984
  row[META] = Transform.get_primary_keys(keys)
976
985
 
986
+ if node.is_root:
987
+ primary_key_values: t.List[str] = list(
988
+ map(str, primary_keys)
989
+ )
990
+ primary_key_names: t.List[str] = [
991
+ primary_key.name for primary_key in node.primary_keys
992
+ ]
993
+ # TODO: add support for composite pkeys
994
+ row[META][node.table] = {
995
+ primary_key_names[0]: [primary_key_values[0]],
996
+ }
997
+
977
998
  if self.verbose:
978
999
  print(f"{(i+1)})")
979
1000
  print(f"pkeys: {primary_keys}")
@@ -1111,7 +1132,11 @@ class Sync(Base, metaclass=Singleton):
1111
1132
  notification: t.AnyStr = conn.notifies.pop(0)
1112
1133
  if notification.channel == self.database:
1113
1134
  payload = json.loads(notification.payload)
1114
- if payload["indices"] and self.index in payload["indices"]:
1135
+ if (
1136
+ payload["indices"]
1137
+ and self.index in payload["indices"]
1138
+ and payload["schema"] in self.tree.schemas
1139
+ ):
1115
1140
  payloads.append(payload)
1116
1141
  logger.debug(f"poll_db: {payload}")
1117
1142
  self.count["db"] += 1
@@ -1133,7 +1158,11 @@ class Sync(Base, metaclass=Singleton):
1133
1158
  notification: t.AnyStr = self.conn.notifies.pop(0)
1134
1159
  if notification.channel == self.database:
1135
1160
  payload = json.loads(notification.payload)
1136
- if payload["indices"] and self.index in payload["indices"]:
1161
+ if (
1162
+ payload["indices"]
1163
+ and self.index in payload["indices"]
1164
+ and payload["schema"] in self.tree.schemas
1165
+ ):
1137
1166
  self.redis.push([payload])
1138
1167
  logger.debug(f"async_poll: {payload}")
1139
1168
  self.count["db"] += 1
@@ -1295,13 +1324,11 @@ class Sync(Base, metaclass=Singleton):
1295
1324
  cursor.execute(f'LISTEN "{self.database}"')
1296
1325
  event_loop = asyncio.get_event_loop()
1297
1326
  event_loop.add_reader(self.conn, self.async_poll_db)
1298
- tasks: list = [
1327
+ self.tasks: t.List[asyncio.Task] = [
1299
1328
  event_loop.create_task(self.async_poll_redis()),
1300
1329
  event_loop.create_task(self.async_truncate_slots()),
1301
1330
  event_loop.create_task(self.async_status()),
1302
1331
  ]
1303
- event_loop.run_until_complete(asyncio.wait(tasks))
1304
- event_loop.close()
1305
1332
 
1306
1333
  else:
1307
1334
  # sync up to and produce items in the Redis cache
@@ -1414,22 +1441,22 @@ class Sync(Base, metaclass=Singleton):
1414
1441
  default=settings.NUM_WORKERS,
1415
1442
  )
1416
1443
  def main(
1417
- config,
1418
- daemon,
1419
- host,
1420
- password,
1421
- port,
1422
- sslmode,
1423
- sslrootcert,
1424
- user,
1425
- verbose,
1426
- version,
1427
- analyze,
1428
- num_workers,
1429
- polling,
1430
- producer,
1431
- consumer,
1432
- ):
1444
+ config: str,
1445
+ daemon: bool,
1446
+ host: str,
1447
+ password: bool,
1448
+ port: int,
1449
+ sslmode: str,
1450
+ sslrootcert: str,
1451
+ user: str,
1452
+ verbose: bool,
1453
+ version: bool,
1454
+ analyze: bool,
1455
+ num_workers: int,
1456
+ polling: bool,
1457
+ producer: bool,
1458
+ consumer: bool,
1459
+ ) -> None:
1433
1460
  """Main application syncer."""
1434
1461
  if version:
1435
1462
  sys.stdout.write(f"Version: {__version__}\n")
@@ -1477,6 +1504,7 @@ def main(
1477
1504
  time.sleep(settings.POLL_INTERVAL)
1478
1505
 
1479
1506
  else:
1507
+ tasks: t.List[asyncio.Task] = []
1480
1508
  for doc in config_loader(config):
1481
1509
  sync: Sync = Sync(
1482
1510
  doc,
@@ -1489,6 +1517,12 @@ def main(
1489
1517
  sync.pull()
1490
1518
  if daemon:
1491
1519
  sync.receive()
1520
+ tasks.extend(sync.tasks)
1521
+
1522
+ if settings.USE_ASYNC:
1523
+ event_loop = asyncio.get_event_loop()
1524
+ event_loop.run_until_complete(asyncio.gather(*tasks))
1525
+ event_loop.close()
1492
1526
 
1493
1527
 
1494
1528
  if __name__ == "__main__":
@@ -21,6 +21,7 @@ from .settings import (
21
21
  REDIS_HOST,
22
22
  REDIS_PORT,
23
23
  REDIS_SCHEME,
24
+ REDIS_USER,
24
25
  )
25
26
 
26
27
  logger = logging.getLogger(__name__)
@@ -117,6 +118,7 @@ def get_postgres_url(
117
118
  def get_redis_url(
118
119
  scheme: t.Optional[str] = None,
119
120
  host: t.Optional[str] = None,
121
+ username: t.Optional[str] = None,
120
122
  password: t.Optional[str] = None,
121
123
  port: t.Optional[int] = None,
122
124
  db: t.Optional[str] = None,
@@ -127,6 +129,7 @@ def get_redis_url(
127
129
  Args:
128
130
  scheme (Optional[str]): The scheme to use for the Redis connection. Defaults to None.
129
131
  host (Optional[str]): The Redis host to connect to. Defaults to None.
132
+ username (Optional[str]): The Redis username to use for authentication. Defaults to None.
130
133
  password (Optional[str]): The Redis password to use for authentication. Defaults to None.
131
134
  port (Optional[int]): The Redis port to connect to. Defaults to None.
132
135
  db (Optional[str]): The Redis database to connect to. Defaults to None.
@@ -135,11 +138,16 @@ def get_redis_url(
135
138
  str: The Redis connection URL.
136
139
  """
137
140
  host = host or REDIS_HOST
141
+ username = username or REDIS_USER
138
142
  password = _get_auth("REDIS_AUTH") or password or REDIS_AUTH
139
143
  port = port or REDIS_PORT
140
144
  db = db or REDIS_DB
141
145
  scheme = scheme or REDIS_SCHEME
146
+ if username and password:
147
+ logger.debug("Connecting to Redis with custom username and password.")
148
+ return f"{scheme}://{quote_plus(username)}:{quote_plus(password)}@{host}:{port}/{db}"
142
149
  if password:
150
+ logger.debug("Connecting to Redis with default password.")
143
151
  return f"{scheme}://:{quote_plus(password)}@{host}:{port}/{db}"
144
152
  logger.debug("Connecting to Redis without password.")
145
153
  return f"{scheme}://{host}:{port}/{db}"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: pgsync
3
- Version: 3.2.0
3
+ Version: 3.2.1
4
4
  Summary: Postgres to Elasticsearch/OpenSearch sync
5
5
  Home-page: https://github.com/toluaina/pgsync
6
6
  Author: Tolu Aina
@@ -31,33 +31,33 @@ Description-Content-Type: text/markdown
31
31
  License-File: LICENSE
32
32
  License-File: AUTHORS.rst
33
33
  Requires-Dist: async-timeout==4.0.3
34
- Requires-Dist: boto3==1.34.142
35
- Requires-Dist: botocore==1.34.142
34
+ Requires-Dist: boto3==1.35.5
35
+ Requires-Dist: botocore==1.35.5
36
36
  Requires-Dist: certifi==2024.7.4
37
37
  Requires-Dist: charset-normalizer==3.3.2
38
38
  Requires-Dist: click==8.1.7
39
- Requires-Dist: elastic-transport==8.13.1
40
- Requires-Dist: elasticsearch==8.14.0
41
- Requires-Dist: elasticsearch-dsl==8.14.0
39
+ Requires-Dist: elastic-transport==8.15.0
40
+ Requires-Dist: elasticsearch==8.15.0
41
+ Requires-Dist: elasticsearch-dsl==8.15.1
42
42
  Requires-Dist: environs==11.0.0
43
43
  Requires-Dist: events==0.5
44
44
  Requires-Dist: greenlet==3.0.3
45
- Requires-Dist: idna==3.7
45
+ Requires-Dist: idna==3.8
46
46
  Requires-Dist: jmespath==1.0.1
47
- Requires-Dist: marshmallow==3.21.3
47
+ Requires-Dist: marshmallow==3.22.0
48
48
  Requires-Dist: opensearch-dsl==2.1.0
49
- Requires-Dist: opensearch-py==2.6.0
49
+ Requires-Dist: opensearch-py==2.7.1
50
50
  Requires-Dist: packaging==24.1
51
51
  Requires-Dist: psycopg2-binary==2.9.9
52
52
  Requires-Dist: python-dateutil==2.9.0.post0
53
53
  Requires-Dist: python-dotenv==1.0.1
54
- Requires-Dist: redis==5.0.7
54
+ Requires-Dist: redis==5.0.8
55
55
  Requires-Dist: requests==2.32.3
56
- Requires-Dist: requests-aws4auth==1.2.3
56
+ Requires-Dist: requests-aws4auth==1.3.1
57
57
  Requires-Dist: s3transfer==0.10.2
58
58
  Requires-Dist: six==1.16.0
59
- Requires-Dist: sqlalchemy==2.0.31
60
- Requires-Dist: sqlparse==0.5.0
59
+ Requires-Dist: sqlalchemy==2.0.32
60
+ Requires-Dist: sqlparse==0.5.1
61
61
  Requires-Dist: typing-extensions==4.12.2
62
62
  Requires-Dist: urllib3==1.26.19
63
63
 
@@ -1,30 +1,30 @@
1
1
  async-timeout==4.0.3
2
- boto3==1.34.142
3
- botocore==1.34.142
2
+ boto3==1.35.5
3
+ botocore==1.35.5
4
4
  certifi==2024.7.4
5
5
  charset-normalizer==3.3.2
6
6
  click==8.1.7
7
- elastic-transport==8.13.1
8
- elasticsearch==8.14.0
9
- elasticsearch-dsl==8.14.0
7
+ elastic-transport==8.15.0
8
+ elasticsearch==8.15.0
9
+ elasticsearch-dsl==8.15.1
10
10
  environs==11.0.0
11
11
  events==0.5
12
12
  greenlet==3.0.3
13
- idna==3.7
13
+ idna==3.8
14
14
  jmespath==1.0.1
15
- marshmallow==3.21.3
15
+ marshmallow==3.22.0
16
16
  opensearch-dsl==2.1.0
17
- opensearch-py==2.6.0
17
+ opensearch-py==2.7.1
18
18
  packaging==24.1
19
19
  psycopg2-binary==2.9.9
20
20
  python-dateutil==2.9.0.post0
21
21
  python-dotenv==1.0.1
22
- redis==5.0.7
22
+ redis==5.0.8
23
23
  requests==2.32.3
24
- requests-aws4auth==1.2.3
24
+ requests-aws4auth==1.3.1
25
25
  s3transfer==0.10.2
26
26
  six==1.16.0
27
- sqlalchemy==2.0.31
28
- sqlparse==0.5.0
27
+ sqlalchemy==2.0.32
28
+ sqlparse==0.5.1
29
29
  typing-extensions==4.12.2
30
30
  urllib3==1.26.19
@@ -34,6 +34,7 @@ KEYWORDS = [
34
34
  "pgsync",
35
35
  "postgres",
36
36
  ]
37
+ LICENSE = "MIT"
37
38
  CLASSIFIERS = [
38
39
  "Development Status :: 5 - Production/Stable",
39
40
  "Intended Audience :: Developers",
@@ -64,7 +65,7 @@ with open("requirements/base.txt") as fp:
64
65
  setup(
65
66
  name=NAME,
66
67
  author=AUTHOR,
67
- license="MIT",
68
+ license=LICENSE,
68
69
  maintainer=MAINTAINER,
69
70
  maintainer_email=MAINTAINER_EMAIL,
70
71
  author_email=AUTHOR_EMAIL,
@@ -433,12 +433,12 @@ class TestBase(object):
433
433
  assert "No match for row:" in str(excinfo.value)
434
434
 
435
435
  row = """
436
- table public."B1_XYZ": INSERT: "ID"[integer]:5 "CREATED_TIMESTAMP"[bigint]:222 "ADDRESS"[character varying]:'from3' "SOME_FIELD_KEY"[character varying]:'key3' "SOME_OTHER_FIELD_KEY"[character varying]:'issue3' "CHANNEL_ID"[integer]:3 "CHANNEL_NAME"[character varying]:'channel3' "ITEM_ID"[integer]:3 "MESSAGE"[character varying]:'message3' "RETRY"[integer]:4 "STATUS"[character varying]:'status' "SUBJECT"[character varying]:'sub3' "TIMESTAMP"[bigint]:33
436
+ table public."B1_XYZ": INSERT: "ID"[integer]:5 "CREATED_TIMESTAMP"[bigint]:222 "ADDRESS"[character varying]:'from3' "SOME_FIELD_KEY"[character varying]:'key3' "SOME_OTHER_FIELD_KEY"[character varying]:'issue to handle' "CHANNEL_ID"[integer]:3 "CHANNEL_NAME"[character varying]:'channel 45' "ITEM_ID"[integer]:3 "MESSAGE"[character varying]:'message3' "RETRY"[integer]:4 "STATUS"[character varying]:'status' "SUBJECT"[character varying]:'sub3' "TIMESTAMP"[bigint]:33
437
437
  """ # noqa E501
438
438
  payload = pg_base.parse_logical_slot(row)
439
439
  assert payload.data == {
440
440
  "CHANNEL_ID": 3,
441
- "CHANNEL_NAME": "channel3",
441
+ "CHANNEL_NAME": "channel 45",
442
442
  "CREATED_TIMESTAMP": 222,
443
443
  "ADDRESS": "from3",
444
444
  "ID": 5,
@@ -446,7 +446,7 @@ class TestBase(object):
446
446
  "MESSAGE": "message3",
447
447
  "RETRY": 4,
448
448
  "SOME_FIELD_KEY": "key3",
449
- "SOME_OTHER_FIELD_KEY": "issue3",
449
+ "SOME_OTHER_FIELD_KEY": "issue to handle",
450
450
  "STATUS": "status",
451
451
  "SUBJECT": "sub3",
452
452
  "TIMESTAMP": 33,
@@ -472,16 +472,20 @@ class TestBase(object):
472
472
  """ # noqa E501
473
473
  payload = pg_base.parse_logical_slot(row)
474
474
  assert payload.data == {
475
+ "copyright": None,
476
+ "description": "Stephens Kings It",
477
+ "doc": '\'{"a": {"b": {"c": [0, 1, 2, 3, 4]}}, "i": 73, "x": [{"y": 0, "z": '
478
+ '5}, {"y": 1, "z": 6}], "bool": true, "lastname": "Judye", '
479
+ '"firstname": "Glenda", "generation": {"name": "X"}, "nick_names": '
480
+ '["Beatriz", "Jean", "Carilyn", "Carol-Jean", "Sara-Ann"], '
481
+ '"coordinates": {"lat": 21.1, "lon": 32.9}}\'',
475
482
  "id": 1,
476
483
  "isbn": "001",
477
- "title": "It",
478
- "description": "Stephens",
479
- "copyright": None,
480
- "tags": "'",
481
- "doc": "'",
484
+ "publish_date": "'1980-01-01 00:00:00'",
482
485
  "publisher_id": 1,
483
- "publish_date": "'1980-01-01",
484
486
  "quad": 2e58,
487
+ "tags": '\'["a", "b", "c"]\'',
488
+ "title": "It",
485
489
  }
486
490
  assert payload.old == {}
487
491
  assert payload.schema == "public"
@@ -106,7 +106,7 @@ class TestSearchClient(object):
106
106
  mock_search_client.assert_called_once_with(
107
107
  hosts=[url],
108
108
  http_auth=ANY,
109
- use_ssl=True,
110
109
  verify_certs=True,
111
110
  node_class=elastic_transport.RequestsHttpNode,
111
+ timeout=settings.ELASTICSEARCH_TIMEOUT,
112
112
  )