pgsync 3.2.0__tar.gz → 3.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pgsync-3.2.0 → pgsync-3.2.1}/PKG-INFO +13 -13
- {pgsync-3.2.0 → pgsync-3.2.1}/bin/bootstrap +9 -1
- {pgsync-3.2.0 → pgsync-3.2.1}/bin/parallel_sync +1 -1
- {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/__init__.py +1 -1
- {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/base.py +2 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/constants.py +1 -1
- {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/node.py +6 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/plugin.py +5 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/search_client.py +9 -8
- {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/settings.py +1 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/sync.py +56 -22
- {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/urls.py +8 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/pgsync.egg-info/PKG-INFO +13 -13
- {pgsync-3.2.0 → pgsync-3.2.1}/pgsync.egg-info/requires.txt +12 -12
- {pgsync-3.2.0 → pgsync-3.2.1}/setup.py +2 -1
- {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_base.py +13 -9
- {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_search_client.py +1 -1
- {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_sync_nested_children.py +23 -5
- {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_sync_root.py +205 -49
- {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_sync_single_child_fk_on_child.py +71 -62
- {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_sync_single_child_fk_on_parent.py +81 -60
- {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_unique_behaviour.py +1 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_urls.py +18 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/AUTHORS.rst +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/CONTRIBUTING.rst +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/HISTORY.rst +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/LICENSE +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/MANIFEST.in +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/README.md +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/README.rst +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/bin/pgsync +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/docs/Makefile +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/docs/authors.rst +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/docs/changelog.rst +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/docs/conf.py +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/docs/contributing.rst +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/docs/history.rst +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/docs/index.rst +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/docs/installation.rst +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/docs/logo.png +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/docs/make.bat +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/docs/readme.rst +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/docs/usage.rst +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/exc.py +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/helper.py +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/querybuilder.py +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/redisqueue.py +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/singleton.py +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/transform.py +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/trigger.py +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/utils.py +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/pgsync/view.py +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/pgsync.egg-info/SOURCES.txt +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/pgsync.egg-info/dependency_links.txt +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/pgsync.egg-info/not-zip-safe +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/pgsync.egg-info/top_level.txt +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/pyproject.toml +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/setup.cfg +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/tests/__init__.py +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/tests/conftest.py +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/tests/fixtures/schema.json +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_constants.py +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_env_vars.py +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_helper.py +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_log_handlers.py +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_node.py +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_query_builder.py +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_redisqueue.py +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_settings.py +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_sync.py +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_transform.py +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_trigger.py +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_utils.py +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/tests/test_view.py +0 -0
- {pgsync-3.2.0 → pgsync-3.2.1}/tests/testing_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: pgsync
|
|
3
|
-
Version: 3.2.
|
|
3
|
+
Version: 3.2.1
|
|
4
4
|
Summary: Postgres to Elasticsearch/OpenSearch sync
|
|
5
5
|
Home-page: https://github.com/toluaina/pgsync
|
|
6
6
|
Author: Tolu Aina
|
|
@@ -31,33 +31,33 @@ Description-Content-Type: text/markdown
|
|
|
31
31
|
License-File: LICENSE
|
|
32
32
|
License-File: AUTHORS.rst
|
|
33
33
|
Requires-Dist: async-timeout==4.0.3
|
|
34
|
-
Requires-Dist: boto3==1.
|
|
35
|
-
Requires-Dist: botocore==1.
|
|
34
|
+
Requires-Dist: boto3==1.35.5
|
|
35
|
+
Requires-Dist: botocore==1.35.5
|
|
36
36
|
Requires-Dist: certifi==2024.7.4
|
|
37
37
|
Requires-Dist: charset-normalizer==3.3.2
|
|
38
38
|
Requires-Dist: click==8.1.7
|
|
39
|
-
Requires-Dist: elastic-transport==8.
|
|
40
|
-
Requires-Dist: elasticsearch==8.
|
|
41
|
-
Requires-Dist: elasticsearch-dsl==8.
|
|
39
|
+
Requires-Dist: elastic-transport==8.15.0
|
|
40
|
+
Requires-Dist: elasticsearch==8.15.0
|
|
41
|
+
Requires-Dist: elasticsearch-dsl==8.15.1
|
|
42
42
|
Requires-Dist: environs==11.0.0
|
|
43
43
|
Requires-Dist: events==0.5
|
|
44
44
|
Requires-Dist: greenlet==3.0.3
|
|
45
|
-
Requires-Dist: idna==3.
|
|
45
|
+
Requires-Dist: idna==3.8
|
|
46
46
|
Requires-Dist: jmespath==1.0.1
|
|
47
|
-
Requires-Dist: marshmallow==3.
|
|
47
|
+
Requires-Dist: marshmallow==3.22.0
|
|
48
48
|
Requires-Dist: opensearch-dsl==2.1.0
|
|
49
|
-
Requires-Dist: opensearch-py==2.
|
|
49
|
+
Requires-Dist: opensearch-py==2.7.1
|
|
50
50
|
Requires-Dist: packaging==24.1
|
|
51
51
|
Requires-Dist: psycopg2-binary==2.9.9
|
|
52
52
|
Requires-Dist: python-dateutil==2.9.0.post0
|
|
53
53
|
Requires-Dist: python-dotenv==1.0.1
|
|
54
|
-
Requires-Dist: redis==5.0.
|
|
54
|
+
Requires-Dist: redis==5.0.8
|
|
55
55
|
Requires-Dist: requests==2.32.3
|
|
56
|
-
Requires-Dist: requests-aws4auth==1.
|
|
56
|
+
Requires-Dist: requests-aws4auth==1.3.1
|
|
57
57
|
Requires-Dist: s3transfer==0.10.2
|
|
58
58
|
Requires-Dist: six==1.16.0
|
|
59
|
-
Requires-Dist: sqlalchemy==2.0.
|
|
60
|
-
Requires-Dist: sqlparse==0.5.
|
|
59
|
+
Requires-Dist: sqlalchemy==2.0.32
|
|
60
|
+
Requires-Dist: sqlparse==0.5.1
|
|
61
61
|
Requires-Dist: typing-extensions==4.12.2
|
|
62
62
|
Requires-Dist: urllib3==1.26.19
|
|
63
63
|
|
|
@@ -35,7 +35,15 @@ logger = logging.getLogger(__name__)
|
|
|
35
35
|
default=False,
|
|
36
36
|
help="Turn on verbosity",
|
|
37
37
|
)
|
|
38
|
-
def main(
|
|
38
|
+
def main(
|
|
39
|
+
teardown: bool,
|
|
40
|
+
config: str,
|
|
41
|
+
user: str,
|
|
42
|
+
password: bool,
|
|
43
|
+
host: str,
|
|
44
|
+
port: int,
|
|
45
|
+
verbose: bool,
|
|
46
|
+
) -> None:
|
|
39
47
|
"""Application onetime Bootstrap."""
|
|
40
48
|
kwargs: dict = {
|
|
41
49
|
"user": user,
|
|
@@ -778,6 +778,8 @@ class Base(object):
|
|
|
778
778
|
|
|
779
779
|
NB: All integers are long in python3 and call to convert is just int
|
|
780
780
|
"""
|
|
781
|
+
if self.verbose:
|
|
782
|
+
logger.debug(f"type: {type_} value: {value}")
|
|
781
783
|
if value.lower() == "null":
|
|
782
784
|
return None
|
|
783
785
|
if type_.lower() in self.INT_TYPES:
|
|
@@ -208,5 +208,5 @@ LOGICAL_SLOT_PREFIX = re.compile(
|
|
|
208
208
|
r"table\s\"?(?P<schema>[\w-]+)\"?.\"?(?P<table>[\w-]+)\"?:\s(?P<tg_op>[A-Z]+):" # noqa E501
|
|
209
209
|
)
|
|
210
210
|
LOGICAL_SLOT_SUFFIX = re.compile(
|
|
211
|
-
'\s(?P<key>"?\w+"?)\[(?P<type>[\w\s]+)\]:(?P<value>[
|
|
211
|
+
r'\s(?P<key>"?\w+"?)\[(?P<type>[\w\s]+)\]:(?P<value>(?:"[^"]*"|\'[^\']*\'|null|\d+e[+-]?\d+|\w+))'
|
|
212
212
|
)
|
|
@@ -284,6 +284,7 @@ class Tree(threading.local):
|
|
|
284
284
|
def __post_init__(self):
|
|
285
285
|
self.tables: t.Set[str] = set()
|
|
286
286
|
self.__nodes: t.Dict[Node] = {}
|
|
287
|
+
self.__schemas: t.Set[str] = set()
|
|
287
288
|
self.root: t.Optional[Node] = None
|
|
288
289
|
self.build(self.nodes)
|
|
289
290
|
|
|
@@ -334,6 +335,7 @@ class Tree(threading.local):
|
|
|
334
335
|
node.add_child(self.build(child))
|
|
335
336
|
|
|
336
337
|
self.__nodes[key] = node
|
|
338
|
+
self.__schemas.add(schema)
|
|
337
339
|
return node
|
|
338
340
|
|
|
339
341
|
def get_node(self, table: str, schema: str) -> Node:
|
|
@@ -352,3 +354,7 @@ class Tree(threading.local):
|
|
|
352
354
|
else:
|
|
353
355
|
raise RuntimeError(f"Node for {schema}.{table} not found")
|
|
354
356
|
return self.__nodes[key]
|
|
357
|
+
|
|
358
|
+
@property
|
|
359
|
+
def schemas(self) -> t.Set[str]:
|
|
360
|
+
return self.__schemas
|
|
@@ -48,6 +48,11 @@ class Plugins(object):
|
|
|
48
48
|
if "test" not in sys.argv[0]:
|
|
49
49
|
self.walk(self.package)
|
|
50
50
|
|
|
51
|
+
# main plugin ordering
|
|
52
|
+
self.plugins = sorted(
|
|
53
|
+
self.plugins, key=lambda x: self.names.index(x.name)
|
|
54
|
+
)
|
|
55
|
+
|
|
51
56
|
def walk(self, package: str) -> None:
|
|
52
57
|
"""Recursively walk the supplied package and fetch all plugins."""
|
|
53
58
|
module = import_module(package)
|
|
@@ -174,7 +174,7 @@ class SearchClient(object):
|
|
|
174
174
|
):
|
|
175
175
|
"""Bulk index, update, delete docs to Elasticsearch/OpenSearch."""
|
|
176
176
|
if settings.ELASTICSEARCH_STREAMING_BULK:
|
|
177
|
-
for ok,
|
|
177
|
+
for ok, info in self.streaming_bulk(
|
|
178
178
|
self.__client,
|
|
179
179
|
actions,
|
|
180
180
|
index=index,
|
|
@@ -189,10 +189,12 @@ class SearchClient(object):
|
|
|
189
189
|
):
|
|
190
190
|
if ok:
|
|
191
191
|
self.doc_count += 1
|
|
192
|
+
else:
|
|
193
|
+
logger.error(f"Document failed to index: {info}")
|
|
192
194
|
else:
|
|
193
195
|
# parallel bulk consumes more memory and is also more likely
|
|
194
196
|
# to result in 429 errors.
|
|
195
|
-
for ok,
|
|
197
|
+
for ok, info in self.parallel_bulk(
|
|
196
198
|
self.__client,
|
|
197
199
|
actions,
|
|
198
200
|
thread_count=thread_count,
|
|
@@ -206,6 +208,8 @@ class SearchClient(object):
|
|
|
206
208
|
):
|
|
207
209
|
if ok:
|
|
208
210
|
self.doc_count += 1
|
|
211
|
+
else:
|
|
212
|
+
logger.error(f"Document failed to index: {info}")
|
|
209
213
|
|
|
210
214
|
def refresh(self, indices: t.List[str]) -> None:
|
|
211
215
|
"""Refresh the Elasticsearch/OpenSearch index."""
|
|
@@ -384,9 +388,9 @@ def get_search_client(
|
|
|
384
388
|
service,
|
|
385
389
|
session_token=credentials.token,
|
|
386
390
|
),
|
|
387
|
-
use_ssl=True,
|
|
388
391
|
verify_certs=True,
|
|
389
392
|
connection_class=connection_class,
|
|
393
|
+
timeout=settings.ELASTICSEARCH_TIMEOUT,
|
|
390
394
|
)
|
|
391
395
|
elif settings.ELASTICSEARCH:
|
|
392
396
|
return client(
|
|
@@ -398,9 +402,9 @@ def get_search_client(
|
|
|
398
402
|
service,
|
|
399
403
|
session_token=credentials.token,
|
|
400
404
|
),
|
|
401
|
-
use_ssl=True,
|
|
402
405
|
verify_certs=True,
|
|
403
406
|
node_class=node_class,
|
|
407
|
+
timeout=settings.ELASTICSEARCH_TIMEOUT,
|
|
404
408
|
)
|
|
405
409
|
else:
|
|
406
410
|
hosts: t.List[str] = [url]
|
|
@@ -436,8 +440,6 @@ def get_search_client(
|
|
|
436
440
|
ssl_version: t.Optional[int] = settings.ELASTICSEARCH_SSL_VERSION
|
|
437
441
|
ssl_context: t.Optional[t.Any] = settings.ELASTICSEARCH_SSL_CONTEXT
|
|
438
442
|
ssl_show_warn: bool = settings.ELASTICSEARCH_SSL_SHOW_WARN
|
|
439
|
-
# Transport
|
|
440
|
-
timeout: float = settings.ELASTICSEARCH_TIMEOUT
|
|
441
443
|
return client(
|
|
442
444
|
hosts=hosts,
|
|
443
445
|
http_auth=http_auth,
|
|
@@ -456,6 +458,5 @@ def get_search_client(
|
|
|
456
458
|
ssl_version=ssl_version,
|
|
457
459
|
ssl_context=ssl_context,
|
|
458
460
|
ssl_show_warn=ssl_show_warn,
|
|
459
|
-
|
|
460
|
-
timeout=timeout,
|
|
461
|
+
timeout=settings.ELASTICSEARCH_TIMEOUT,
|
|
461
462
|
)
|
|
@@ -161,6 +161,7 @@ PG_USER = env.str("PG_USER")
|
|
|
161
161
|
|
|
162
162
|
# Redis:
|
|
163
163
|
REDIS_AUTH = env.str("REDIS_AUTH", default=None)
|
|
164
|
+
REDIS_USER = env.str("REDIS_USER", default=None)
|
|
164
165
|
REDIS_DB = env.int("REDIS_DB", default=0)
|
|
165
166
|
REDIS_HOST = env.str("REDIS_HOST", default="localhost")
|
|
166
167
|
# redis poll interval (in secs)
|
|
@@ -109,6 +109,7 @@ class Sync(Base, metaclass=Singleton):
|
|
|
109
109
|
self._plugins: Plugins = Plugins("plugins", self.plugins)
|
|
110
110
|
self.query_builder: QueryBuilder = QueryBuilder(verbose=verbose)
|
|
111
111
|
self.count: dict = dict(xlog=0, db=0, redis=0)
|
|
112
|
+
self.tasks: t.List[asyncio.Task] = []
|
|
112
113
|
|
|
113
114
|
def validate(self, repl_slots: bool = True) -> None:
|
|
114
115
|
"""Perform all validation right away."""
|
|
@@ -409,6 +410,11 @@ class Sync(Base, metaclass=Singleton):
|
|
|
409
410
|
f"Error parsing row: {e}\nRow data: {row.data}"
|
|
410
411
|
)
|
|
411
412
|
raise
|
|
413
|
+
|
|
414
|
+
# filter out unknown schemas
|
|
415
|
+
if payload.schema not in self.tree.schemas:
|
|
416
|
+
continue
|
|
417
|
+
|
|
412
418
|
payloads.append(payload)
|
|
413
419
|
|
|
414
420
|
j: int = i + 1
|
|
@@ -810,7 +816,10 @@ class Sync(Base, metaclass=Singleton):
|
|
|
810
816
|
# e.g a through table which we need to react to.
|
|
811
817
|
# in this case, we find the parent of the through
|
|
812
818
|
# table and force a re-sync.
|
|
813
|
-
if
|
|
819
|
+
if (
|
|
820
|
+
payload.table not in self.tree.tables
|
|
821
|
+
or payload.schema not in self.tree.schemas
|
|
822
|
+
):
|
|
814
823
|
return
|
|
815
824
|
|
|
816
825
|
node: Node = self.tree.get_node(payload.table, payload.schema)
|
|
@@ -974,6 +983,18 @@ class Sync(Base, metaclass=Singleton):
|
|
|
974
983
|
|
|
975
984
|
row[META] = Transform.get_primary_keys(keys)
|
|
976
985
|
|
|
986
|
+
if node.is_root:
|
|
987
|
+
primary_key_values: t.List[str] = list(
|
|
988
|
+
map(str, primary_keys)
|
|
989
|
+
)
|
|
990
|
+
primary_key_names: t.List[str] = [
|
|
991
|
+
primary_key.name for primary_key in node.primary_keys
|
|
992
|
+
]
|
|
993
|
+
# TODO: add support for composite pkeys
|
|
994
|
+
row[META][node.table] = {
|
|
995
|
+
primary_key_names[0]: [primary_key_values[0]],
|
|
996
|
+
}
|
|
997
|
+
|
|
977
998
|
if self.verbose:
|
|
978
999
|
print(f"{(i+1)})")
|
|
979
1000
|
print(f"pkeys: {primary_keys}")
|
|
@@ -1111,7 +1132,11 @@ class Sync(Base, metaclass=Singleton):
|
|
|
1111
1132
|
notification: t.AnyStr = conn.notifies.pop(0)
|
|
1112
1133
|
if notification.channel == self.database:
|
|
1113
1134
|
payload = json.loads(notification.payload)
|
|
1114
|
-
if
|
|
1135
|
+
if (
|
|
1136
|
+
payload["indices"]
|
|
1137
|
+
and self.index in payload["indices"]
|
|
1138
|
+
and payload["schema"] in self.tree.schemas
|
|
1139
|
+
):
|
|
1115
1140
|
payloads.append(payload)
|
|
1116
1141
|
logger.debug(f"poll_db: {payload}")
|
|
1117
1142
|
self.count["db"] += 1
|
|
@@ -1133,7 +1158,11 @@ class Sync(Base, metaclass=Singleton):
|
|
|
1133
1158
|
notification: t.AnyStr = self.conn.notifies.pop(0)
|
|
1134
1159
|
if notification.channel == self.database:
|
|
1135
1160
|
payload = json.loads(notification.payload)
|
|
1136
|
-
if
|
|
1161
|
+
if (
|
|
1162
|
+
payload["indices"]
|
|
1163
|
+
and self.index in payload["indices"]
|
|
1164
|
+
and payload["schema"] in self.tree.schemas
|
|
1165
|
+
):
|
|
1137
1166
|
self.redis.push([payload])
|
|
1138
1167
|
logger.debug(f"async_poll: {payload}")
|
|
1139
1168
|
self.count["db"] += 1
|
|
@@ -1295,13 +1324,11 @@ class Sync(Base, metaclass=Singleton):
|
|
|
1295
1324
|
cursor.execute(f'LISTEN "{self.database}"')
|
|
1296
1325
|
event_loop = asyncio.get_event_loop()
|
|
1297
1326
|
event_loop.add_reader(self.conn, self.async_poll_db)
|
|
1298
|
-
tasks:
|
|
1327
|
+
self.tasks: t.List[asyncio.Task] = [
|
|
1299
1328
|
event_loop.create_task(self.async_poll_redis()),
|
|
1300
1329
|
event_loop.create_task(self.async_truncate_slots()),
|
|
1301
1330
|
event_loop.create_task(self.async_status()),
|
|
1302
1331
|
]
|
|
1303
|
-
event_loop.run_until_complete(asyncio.wait(tasks))
|
|
1304
|
-
event_loop.close()
|
|
1305
1332
|
|
|
1306
1333
|
else:
|
|
1307
1334
|
# sync up to and produce items in the Redis cache
|
|
@@ -1414,22 +1441,22 @@ class Sync(Base, metaclass=Singleton):
|
|
|
1414
1441
|
default=settings.NUM_WORKERS,
|
|
1415
1442
|
)
|
|
1416
1443
|
def main(
|
|
1417
|
-
config,
|
|
1418
|
-
daemon,
|
|
1419
|
-
host,
|
|
1420
|
-
password,
|
|
1421
|
-
port,
|
|
1422
|
-
sslmode,
|
|
1423
|
-
sslrootcert,
|
|
1424
|
-
user,
|
|
1425
|
-
verbose,
|
|
1426
|
-
version,
|
|
1427
|
-
analyze,
|
|
1428
|
-
num_workers,
|
|
1429
|
-
polling,
|
|
1430
|
-
producer,
|
|
1431
|
-
consumer,
|
|
1432
|
-
):
|
|
1444
|
+
config: str,
|
|
1445
|
+
daemon: bool,
|
|
1446
|
+
host: str,
|
|
1447
|
+
password: bool,
|
|
1448
|
+
port: int,
|
|
1449
|
+
sslmode: str,
|
|
1450
|
+
sslrootcert: str,
|
|
1451
|
+
user: str,
|
|
1452
|
+
verbose: bool,
|
|
1453
|
+
version: bool,
|
|
1454
|
+
analyze: bool,
|
|
1455
|
+
num_workers: int,
|
|
1456
|
+
polling: bool,
|
|
1457
|
+
producer: bool,
|
|
1458
|
+
consumer: bool,
|
|
1459
|
+
) -> None:
|
|
1433
1460
|
"""Main application syncer."""
|
|
1434
1461
|
if version:
|
|
1435
1462
|
sys.stdout.write(f"Version: {__version__}\n")
|
|
@@ -1477,6 +1504,7 @@ def main(
|
|
|
1477
1504
|
time.sleep(settings.POLL_INTERVAL)
|
|
1478
1505
|
|
|
1479
1506
|
else:
|
|
1507
|
+
tasks: t.List[asyncio.Task] = []
|
|
1480
1508
|
for doc in config_loader(config):
|
|
1481
1509
|
sync: Sync = Sync(
|
|
1482
1510
|
doc,
|
|
@@ -1489,6 +1517,12 @@ def main(
|
|
|
1489
1517
|
sync.pull()
|
|
1490
1518
|
if daemon:
|
|
1491
1519
|
sync.receive()
|
|
1520
|
+
tasks.extend(sync.tasks)
|
|
1521
|
+
|
|
1522
|
+
if settings.USE_ASYNC:
|
|
1523
|
+
event_loop = asyncio.get_event_loop()
|
|
1524
|
+
event_loop.run_until_complete(asyncio.gather(*tasks))
|
|
1525
|
+
event_loop.close()
|
|
1492
1526
|
|
|
1493
1527
|
|
|
1494
1528
|
if __name__ == "__main__":
|
|
@@ -21,6 +21,7 @@ from .settings import (
|
|
|
21
21
|
REDIS_HOST,
|
|
22
22
|
REDIS_PORT,
|
|
23
23
|
REDIS_SCHEME,
|
|
24
|
+
REDIS_USER,
|
|
24
25
|
)
|
|
25
26
|
|
|
26
27
|
logger = logging.getLogger(__name__)
|
|
@@ -117,6 +118,7 @@ def get_postgres_url(
|
|
|
117
118
|
def get_redis_url(
|
|
118
119
|
scheme: t.Optional[str] = None,
|
|
119
120
|
host: t.Optional[str] = None,
|
|
121
|
+
username: t.Optional[str] = None,
|
|
120
122
|
password: t.Optional[str] = None,
|
|
121
123
|
port: t.Optional[int] = None,
|
|
122
124
|
db: t.Optional[str] = None,
|
|
@@ -127,6 +129,7 @@ def get_redis_url(
|
|
|
127
129
|
Args:
|
|
128
130
|
scheme (Optional[str]): The scheme to use for the Redis connection. Defaults to None.
|
|
129
131
|
host (Optional[str]): The Redis host to connect to. Defaults to None.
|
|
132
|
+
username (Optional[str]): The Redis username to use for authentication. Defaults to None.
|
|
130
133
|
password (Optional[str]): The Redis password to use for authentication. Defaults to None.
|
|
131
134
|
port (Optional[int]): The Redis port to connect to. Defaults to None.
|
|
132
135
|
db (Optional[str]): The Redis database to connect to. Defaults to None.
|
|
@@ -135,11 +138,16 @@ def get_redis_url(
|
|
|
135
138
|
str: The Redis connection URL.
|
|
136
139
|
"""
|
|
137
140
|
host = host or REDIS_HOST
|
|
141
|
+
username = username or REDIS_USER
|
|
138
142
|
password = _get_auth("REDIS_AUTH") or password or REDIS_AUTH
|
|
139
143
|
port = port or REDIS_PORT
|
|
140
144
|
db = db or REDIS_DB
|
|
141
145
|
scheme = scheme or REDIS_SCHEME
|
|
146
|
+
if username and password:
|
|
147
|
+
logger.debug("Connecting to Redis with custom username and password.")
|
|
148
|
+
return f"{scheme}://{quote_plus(username)}:{quote_plus(password)}@{host}:{port}/{db}"
|
|
142
149
|
if password:
|
|
150
|
+
logger.debug("Connecting to Redis with default password.")
|
|
143
151
|
return f"{scheme}://:{quote_plus(password)}@{host}:{port}/{db}"
|
|
144
152
|
logger.debug("Connecting to Redis without password.")
|
|
145
153
|
return f"{scheme}://{host}:{port}/{db}"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: pgsync
|
|
3
|
-
Version: 3.2.
|
|
3
|
+
Version: 3.2.1
|
|
4
4
|
Summary: Postgres to Elasticsearch/OpenSearch sync
|
|
5
5
|
Home-page: https://github.com/toluaina/pgsync
|
|
6
6
|
Author: Tolu Aina
|
|
@@ -31,33 +31,33 @@ Description-Content-Type: text/markdown
|
|
|
31
31
|
License-File: LICENSE
|
|
32
32
|
License-File: AUTHORS.rst
|
|
33
33
|
Requires-Dist: async-timeout==4.0.3
|
|
34
|
-
Requires-Dist: boto3==1.
|
|
35
|
-
Requires-Dist: botocore==1.
|
|
34
|
+
Requires-Dist: boto3==1.35.5
|
|
35
|
+
Requires-Dist: botocore==1.35.5
|
|
36
36
|
Requires-Dist: certifi==2024.7.4
|
|
37
37
|
Requires-Dist: charset-normalizer==3.3.2
|
|
38
38
|
Requires-Dist: click==8.1.7
|
|
39
|
-
Requires-Dist: elastic-transport==8.
|
|
40
|
-
Requires-Dist: elasticsearch==8.
|
|
41
|
-
Requires-Dist: elasticsearch-dsl==8.
|
|
39
|
+
Requires-Dist: elastic-transport==8.15.0
|
|
40
|
+
Requires-Dist: elasticsearch==8.15.0
|
|
41
|
+
Requires-Dist: elasticsearch-dsl==8.15.1
|
|
42
42
|
Requires-Dist: environs==11.0.0
|
|
43
43
|
Requires-Dist: events==0.5
|
|
44
44
|
Requires-Dist: greenlet==3.0.3
|
|
45
|
-
Requires-Dist: idna==3.
|
|
45
|
+
Requires-Dist: idna==3.8
|
|
46
46
|
Requires-Dist: jmespath==1.0.1
|
|
47
|
-
Requires-Dist: marshmallow==3.
|
|
47
|
+
Requires-Dist: marshmallow==3.22.0
|
|
48
48
|
Requires-Dist: opensearch-dsl==2.1.0
|
|
49
|
-
Requires-Dist: opensearch-py==2.
|
|
49
|
+
Requires-Dist: opensearch-py==2.7.1
|
|
50
50
|
Requires-Dist: packaging==24.1
|
|
51
51
|
Requires-Dist: psycopg2-binary==2.9.9
|
|
52
52
|
Requires-Dist: python-dateutil==2.9.0.post0
|
|
53
53
|
Requires-Dist: python-dotenv==1.0.1
|
|
54
|
-
Requires-Dist: redis==5.0.
|
|
54
|
+
Requires-Dist: redis==5.0.8
|
|
55
55
|
Requires-Dist: requests==2.32.3
|
|
56
|
-
Requires-Dist: requests-aws4auth==1.
|
|
56
|
+
Requires-Dist: requests-aws4auth==1.3.1
|
|
57
57
|
Requires-Dist: s3transfer==0.10.2
|
|
58
58
|
Requires-Dist: six==1.16.0
|
|
59
|
-
Requires-Dist: sqlalchemy==2.0.
|
|
60
|
-
Requires-Dist: sqlparse==0.5.
|
|
59
|
+
Requires-Dist: sqlalchemy==2.0.32
|
|
60
|
+
Requires-Dist: sqlparse==0.5.1
|
|
61
61
|
Requires-Dist: typing-extensions==4.12.2
|
|
62
62
|
Requires-Dist: urllib3==1.26.19
|
|
63
63
|
|
|
@@ -1,30 +1,30 @@
|
|
|
1
1
|
async-timeout==4.0.3
|
|
2
|
-
boto3==1.
|
|
3
|
-
botocore==1.
|
|
2
|
+
boto3==1.35.5
|
|
3
|
+
botocore==1.35.5
|
|
4
4
|
certifi==2024.7.4
|
|
5
5
|
charset-normalizer==3.3.2
|
|
6
6
|
click==8.1.7
|
|
7
|
-
elastic-transport==8.
|
|
8
|
-
elasticsearch==8.
|
|
9
|
-
elasticsearch-dsl==8.
|
|
7
|
+
elastic-transport==8.15.0
|
|
8
|
+
elasticsearch==8.15.0
|
|
9
|
+
elasticsearch-dsl==8.15.1
|
|
10
10
|
environs==11.0.0
|
|
11
11
|
events==0.5
|
|
12
12
|
greenlet==3.0.3
|
|
13
|
-
idna==3.
|
|
13
|
+
idna==3.8
|
|
14
14
|
jmespath==1.0.1
|
|
15
|
-
marshmallow==3.
|
|
15
|
+
marshmallow==3.22.0
|
|
16
16
|
opensearch-dsl==2.1.0
|
|
17
|
-
opensearch-py==2.
|
|
17
|
+
opensearch-py==2.7.1
|
|
18
18
|
packaging==24.1
|
|
19
19
|
psycopg2-binary==2.9.9
|
|
20
20
|
python-dateutil==2.9.0.post0
|
|
21
21
|
python-dotenv==1.0.1
|
|
22
|
-
redis==5.0.
|
|
22
|
+
redis==5.0.8
|
|
23
23
|
requests==2.32.3
|
|
24
|
-
requests-aws4auth==1.
|
|
24
|
+
requests-aws4auth==1.3.1
|
|
25
25
|
s3transfer==0.10.2
|
|
26
26
|
six==1.16.0
|
|
27
|
-
sqlalchemy==2.0.
|
|
28
|
-
sqlparse==0.5.
|
|
27
|
+
sqlalchemy==2.0.32
|
|
28
|
+
sqlparse==0.5.1
|
|
29
29
|
typing-extensions==4.12.2
|
|
30
30
|
urllib3==1.26.19
|
|
@@ -34,6 +34,7 @@ KEYWORDS = [
|
|
|
34
34
|
"pgsync",
|
|
35
35
|
"postgres",
|
|
36
36
|
]
|
|
37
|
+
LICENSE = "MIT"
|
|
37
38
|
CLASSIFIERS = [
|
|
38
39
|
"Development Status :: 5 - Production/Stable",
|
|
39
40
|
"Intended Audience :: Developers",
|
|
@@ -64,7 +65,7 @@ with open("requirements/base.txt") as fp:
|
|
|
64
65
|
setup(
|
|
65
66
|
name=NAME,
|
|
66
67
|
author=AUTHOR,
|
|
67
|
-
license=
|
|
68
|
+
license=LICENSE,
|
|
68
69
|
maintainer=MAINTAINER,
|
|
69
70
|
maintainer_email=MAINTAINER_EMAIL,
|
|
70
71
|
author_email=AUTHOR_EMAIL,
|
|
@@ -433,12 +433,12 @@ class TestBase(object):
|
|
|
433
433
|
assert "No match for row:" in str(excinfo.value)
|
|
434
434
|
|
|
435
435
|
row = """
|
|
436
|
-
table public."B1_XYZ": INSERT: "ID"[integer]:5 "CREATED_TIMESTAMP"[bigint]:222 "ADDRESS"[character varying]:'from3' "SOME_FIELD_KEY"[character varying]:'key3' "SOME_OTHER_FIELD_KEY"[character varying]:'
|
|
436
|
+
table public."B1_XYZ": INSERT: "ID"[integer]:5 "CREATED_TIMESTAMP"[bigint]:222 "ADDRESS"[character varying]:'from3' "SOME_FIELD_KEY"[character varying]:'key3' "SOME_OTHER_FIELD_KEY"[character varying]:'issue to handle' "CHANNEL_ID"[integer]:3 "CHANNEL_NAME"[character varying]:'channel 45' "ITEM_ID"[integer]:3 "MESSAGE"[character varying]:'message3' "RETRY"[integer]:4 "STATUS"[character varying]:'status' "SUBJECT"[character varying]:'sub3' "TIMESTAMP"[bigint]:33
|
|
437
437
|
""" # noqa E501
|
|
438
438
|
payload = pg_base.parse_logical_slot(row)
|
|
439
439
|
assert payload.data == {
|
|
440
440
|
"CHANNEL_ID": 3,
|
|
441
|
-
"CHANNEL_NAME": "
|
|
441
|
+
"CHANNEL_NAME": "channel 45",
|
|
442
442
|
"CREATED_TIMESTAMP": 222,
|
|
443
443
|
"ADDRESS": "from3",
|
|
444
444
|
"ID": 5,
|
|
@@ -446,7 +446,7 @@ class TestBase(object):
|
|
|
446
446
|
"MESSAGE": "message3",
|
|
447
447
|
"RETRY": 4,
|
|
448
448
|
"SOME_FIELD_KEY": "key3",
|
|
449
|
-
"SOME_OTHER_FIELD_KEY": "
|
|
449
|
+
"SOME_OTHER_FIELD_KEY": "issue to handle",
|
|
450
450
|
"STATUS": "status",
|
|
451
451
|
"SUBJECT": "sub3",
|
|
452
452
|
"TIMESTAMP": 33,
|
|
@@ -472,16 +472,20 @@ class TestBase(object):
|
|
|
472
472
|
""" # noqa E501
|
|
473
473
|
payload = pg_base.parse_logical_slot(row)
|
|
474
474
|
assert payload.data == {
|
|
475
|
+
"copyright": None,
|
|
476
|
+
"description": "Stephens Kings It",
|
|
477
|
+
"doc": '\'{"a": {"b": {"c": [0, 1, 2, 3, 4]}}, "i": 73, "x": [{"y": 0, "z": '
|
|
478
|
+
'5}, {"y": 1, "z": 6}], "bool": true, "lastname": "Judye", '
|
|
479
|
+
'"firstname": "Glenda", "generation": {"name": "X"}, "nick_names": '
|
|
480
|
+
'["Beatriz", "Jean", "Carilyn", "Carol-Jean", "Sara-Ann"], '
|
|
481
|
+
'"coordinates": {"lat": 21.1, "lon": 32.9}}\'',
|
|
475
482
|
"id": 1,
|
|
476
483
|
"isbn": "001",
|
|
477
|
-
"
|
|
478
|
-
"description": "Stephens",
|
|
479
|
-
"copyright": None,
|
|
480
|
-
"tags": "'",
|
|
481
|
-
"doc": "'",
|
|
484
|
+
"publish_date": "'1980-01-01 00:00:00'",
|
|
482
485
|
"publisher_id": 1,
|
|
483
|
-
"publish_date": "'1980-01-01",
|
|
484
486
|
"quad": 2e58,
|
|
487
|
+
"tags": '\'["a", "b", "c"]\'',
|
|
488
|
+
"title": "It",
|
|
485
489
|
}
|
|
486
490
|
assert payload.old == {}
|
|
487
491
|
assert payload.schema == "public"
|
|
@@ -106,7 +106,7 @@ class TestSearchClient(object):
|
|
|
106
106
|
mock_search_client.assert_called_once_with(
|
|
107
107
|
hosts=[url],
|
|
108
108
|
http_auth=ANY,
|
|
109
|
-
use_ssl=True,
|
|
110
109
|
verify_certs=True,
|
|
111
110
|
node_class=elastic_transport.RequestsHttpNode,
|
|
111
|
+
timeout=settings.ELASTICSEARCH_TIMEOUT,
|
|
112
112
|
)
|