pgsync 6.1.0__tar.gz → 6.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pgsync-6.1.0 → pgsync-6.1.1}/PKG-INFO +5 -4
- {pgsync-6.1.0 → pgsync-6.1.1}/pgsync/__init__.py +1 -1
- {pgsync-6.1.0 → pgsync-6.1.1}/pgsync/sync.py +12 -4
- {pgsync-6.1.0 → pgsync-6.1.1}/pgsync/utils.py +14 -10
- {pgsync-6.1.0 → pgsync-6.1.1}/pgsync.egg-info/PKG-INFO +5 -4
- {pgsync-6.1.0 → pgsync-6.1.1}/pgsync.egg-info/requires.txt +4 -3
- {pgsync-6.1.0 → pgsync-6.1.1}/AUTHORS.rst +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/CONTRIBUTING.rst +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/HISTORY.rst +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/LICENSE +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/MANIFEST.in +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/README.md +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/README.rst +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/bin/bootstrap +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/bin/parallel_sync +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/bin/pgsync +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/docs/Makefile +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/docs/authors.rst +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/docs/changelog.rst +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/docs/conf.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/docs/contributing.rst +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/docs/history.rst +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/docs/index.rst +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/docs/installation.rst +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/docs/logo.png +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/docs/make.bat +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/docs/readme.rst +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/docs/usage.rst +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/pgsync/base.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/pgsync/constants.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/pgsync/exc.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/pgsync/helper.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/pgsync/node.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/pgsync/plugin.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/pgsync/querybuilder.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/pgsync/redisqueue.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/pgsync/search_client.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/pgsync/settings.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/pgsync/singleton.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/pgsync/transform.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/pgsync/trigger.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/pgsync/urls.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/pgsync/view.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/pgsync.egg-info/SOURCES.txt +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/pgsync.egg-info/dependency_links.txt +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/pgsync.egg-info/not-zip-safe +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/pgsync.egg-info/top_level.txt +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/pyproject.toml +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/setup.cfg +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/setup.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/tests/__init__.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/tests/conftest.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/tests/fixtures/schema.json +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/tests/test_base.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/tests/test_constants.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/tests/test_env_vars.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/tests/test_helper.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/tests/test_log_handlers.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/tests/test_node.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/tests/test_query_builder.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/tests/test_redisqueue.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/tests/test_search_client.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/tests/test_settings.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/tests/test_sync.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/tests/test_sync_nested_children.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/tests/test_sync_root.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/tests/test_sync_single_child_fk_on_child.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/tests/test_sync_single_child_fk_on_parent.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/tests/test_transform.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/tests/test_trigger.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/tests/test_unique_behaviour.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/tests/test_urls.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/tests/test_utils.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/tests/test_view.py +0 -0
- {pgsync-6.1.0 → pgsync-6.1.1}/tests/testing_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pgsync
|
|
3
|
-
Version: 6.1.
|
|
3
|
+
Version: 6.1.1
|
|
4
4
|
Summary: Postgres/MySQL/MariaDB to Elasticsearch/OpenSearch sync
|
|
5
5
|
Home-page: https://github.com/toluaina/pgsync
|
|
6
6
|
Author: Tolu Aina
|
|
@@ -33,8 +33,8 @@ License-File: LICENSE
|
|
|
33
33
|
License-File: AUTHORS.rst
|
|
34
34
|
Requires-Dist: async-timeout==5.0.1
|
|
35
35
|
Requires-Dist: backports-datetime-fromisoformat==2.0.3
|
|
36
|
-
Requires-Dist: boto3==1.41.
|
|
37
|
-
Requires-Dist: botocore==1.41.
|
|
36
|
+
Requires-Dist: boto3==1.41.5
|
|
37
|
+
Requires-Dist: botocore==1.41.5
|
|
38
38
|
Requires-Dist: certifi==2025.11.12
|
|
39
39
|
Requires-Dist: charset-normalizer==3.4.4
|
|
40
40
|
Requires-Dist: click==8.1.8
|
|
@@ -43,6 +43,7 @@ Requires-Dist: elasticsearch==7.17.12
|
|
|
43
43
|
Requires-Dist: elasticsearch-dsl==7.4.1
|
|
44
44
|
Requires-Dist: environs==14.4.0
|
|
45
45
|
Requires-Dist: events==0.5
|
|
46
|
+
Requires-Dist: greenlet==3.2.4
|
|
46
47
|
Requires-Dist: idna==3.11
|
|
47
48
|
Requires-Dist: jmespath==1.0.1
|
|
48
49
|
Requires-Dist: marshmallow==4.0.1
|
|
@@ -60,7 +61,7 @@ Requires-Dist: requests-aws4auth==1.3.1
|
|
|
60
61
|
Requires-Dist: s3transfer==0.15.0
|
|
61
62
|
Requires-Dist: six==1.17.0
|
|
62
63
|
Requires-Dist: sqlalchemy==2.0.44
|
|
63
|
-
Requires-Dist: sqlparse==0.5.
|
|
64
|
+
Requires-Dist: sqlparse==0.5.4
|
|
64
65
|
Requires-Dist: typing-extensions==4.15.0
|
|
65
66
|
Requires-Dist: urllib3==1.26.20
|
|
66
67
|
Dynamic: author
|
|
@@ -126,7 +126,7 @@ class Sync(Base, metaclass=Singleton):
|
|
|
126
126
|
self.models, nodes=self.nodes, database=doc["database"]
|
|
127
127
|
)
|
|
128
128
|
if bootstrap:
|
|
129
|
-
self.setup(wal, polling)
|
|
129
|
+
self.setup(wal=wal, polling=polling)
|
|
130
130
|
|
|
131
131
|
if validate:
|
|
132
132
|
self.validate(repl_slots=repl_slots, polling=polling)
|
|
@@ -413,7 +413,7 @@ class Sync(Base, metaclass=Singleton):
|
|
|
413
413
|
if_not_exists=if_not_exists,
|
|
414
414
|
)
|
|
415
415
|
|
|
416
|
-
if not
|
|
416
|
+
if not polling:
|
|
417
417
|
if if_not_exists or not self.replication_slots(self.__name):
|
|
418
418
|
|
|
419
419
|
self.create_replication_slot(self.__name)
|
|
@@ -2291,7 +2291,13 @@ def main(
|
|
|
2291
2291
|
schema_url=schema_url,
|
|
2292
2292
|
s3_schema_url=s3_schema_url,
|
|
2293
2293
|
):
|
|
2294
|
-
sync: Sync = Sync(
|
|
2294
|
+
sync: Sync = Sync(
|
|
2295
|
+
doc,
|
|
2296
|
+
verbose=verbose,
|
|
2297
|
+
wal=wal,
|
|
2298
|
+
bootstrap=bootstrap,
|
|
2299
|
+
**kwargs,
|
|
2300
|
+
)
|
|
2295
2301
|
sync.wal_consumer()
|
|
2296
2302
|
else:
|
|
2297
2303
|
tasks: t.List[asyncio.Task] = []
|
|
@@ -2315,7 +2321,9 @@ def main(
|
|
|
2315
2321
|
tasks.extend(sync.tasks)
|
|
2316
2322
|
|
|
2317
2323
|
if settings.USE_ASYNC:
|
|
2318
|
-
event_loop: asyncio.AbstractEventLoop =
|
|
2324
|
+
event_loop: asyncio.AbstractEventLoop = (
|
|
2325
|
+
asyncio.get_event_loop()
|
|
2326
|
+
)
|
|
2319
2327
|
event_loop.run_until_complete(asyncio.gather(*tasks))
|
|
2320
2328
|
event_loop.close()
|
|
2321
2329
|
|
|
@@ -112,10 +112,10 @@ def get_redacted_url(url: str) -> str:
|
|
|
112
112
|
"""
|
|
113
113
|
parsed_url: ParseResult = urlparse(url)
|
|
114
114
|
if parsed_url.password:
|
|
115
|
-
username = parsed_url.username or ""
|
|
116
|
-
hostname = parsed_url.hostname or ""
|
|
117
|
-
port = f":{parsed_url.port}" if parsed_url.port else ""
|
|
118
|
-
redacted_password = "*" * len(parsed_url.password)
|
|
115
|
+
username: str = parsed_url.username or ""
|
|
116
|
+
hostname: str = parsed_url.hostname or ""
|
|
117
|
+
port: str = f":{parsed_url.port}" if parsed_url.port else ""
|
|
118
|
+
redacted_password: str = "*" * len(parsed_url.password)
|
|
119
119
|
netloc: str = f"{username}:{redacted_password}@{hostname}{port}"
|
|
120
120
|
parsed_url = parsed_url._replace(netloc=netloc)
|
|
121
121
|
return parsed_url.geturl()
|
|
@@ -187,7 +187,7 @@ def validate_config(
|
|
|
187
187
|
raise FileNotFoundError(f'Schema config "{config}" not found')
|
|
188
188
|
|
|
189
189
|
if schema_url:
|
|
190
|
-
parsed = urlparse(schema_url)
|
|
190
|
+
parsed: ParseResult = urlparse(schema_url)
|
|
191
191
|
if parsed.scheme not in ("http", "https"):
|
|
192
192
|
raise ValueError(f'Invalid URL: "{schema_url}"')
|
|
193
193
|
|
|
@@ -217,10 +217,12 @@ def config_loader(
|
|
|
217
217
|
parsed = urlparse(s3_url)
|
|
218
218
|
if not parsed.netloc or not parsed.path:
|
|
219
219
|
raise ValueError(f"Invalid S3 URL: {s3_url}")
|
|
220
|
-
bucket = parsed.netloc
|
|
221
|
-
key = parsed.path.lstrip("/")
|
|
222
|
-
s3 = boto3.client("s3")
|
|
223
|
-
temp_file = tempfile.NamedTemporaryFile(
|
|
220
|
+
bucket: str = parsed.netloc
|
|
221
|
+
key: str = parsed.path.lstrip("/")
|
|
222
|
+
s3: boto3.client = boto3.client("s3")
|
|
223
|
+
temp_file: tempfile.NamedTemporaryFile = tempfile.NamedTemporaryFile(
|
|
224
|
+
delete=False, suffix=".json"
|
|
225
|
+
)
|
|
224
226
|
s3.download_file(bucket, key, temp_file.name)
|
|
225
227
|
return temp_file.name
|
|
226
228
|
|
|
@@ -283,7 +285,9 @@ def config_loader(
|
|
|
283
285
|
try:
|
|
284
286
|
data = json.load(f)
|
|
285
287
|
except json.JSONDecodeError as e:
|
|
286
|
-
raise ValueError(
|
|
288
|
+
raise ValueError(
|
|
289
|
+
f"{config_path} is not valid JSON: {e}"
|
|
290
|
+
) from e
|
|
287
291
|
for doc in data:
|
|
288
292
|
for key, value in doc.items():
|
|
289
293
|
try:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pgsync
|
|
3
|
-
Version: 6.1.
|
|
3
|
+
Version: 6.1.1
|
|
4
4
|
Summary: Postgres/MySQL/MariaDB to Elasticsearch/OpenSearch sync
|
|
5
5
|
Home-page: https://github.com/toluaina/pgsync
|
|
6
6
|
Author: Tolu Aina
|
|
@@ -33,8 +33,8 @@ License-File: LICENSE
|
|
|
33
33
|
License-File: AUTHORS.rst
|
|
34
34
|
Requires-Dist: async-timeout==5.0.1
|
|
35
35
|
Requires-Dist: backports-datetime-fromisoformat==2.0.3
|
|
36
|
-
Requires-Dist: boto3==1.41.
|
|
37
|
-
Requires-Dist: botocore==1.41.
|
|
36
|
+
Requires-Dist: boto3==1.41.5
|
|
37
|
+
Requires-Dist: botocore==1.41.5
|
|
38
38
|
Requires-Dist: certifi==2025.11.12
|
|
39
39
|
Requires-Dist: charset-normalizer==3.4.4
|
|
40
40
|
Requires-Dist: click==8.1.8
|
|
@@ -43,6 +43,7 @@ Requires-Dist: elasticsearch==7.17.12
|
|
|
43
43
|
Requires-Dist: elasticsearch-dsl==7.4.1
|
|
44
44
|
Requires-Dist: environs==14.4.0
|
|
45
45
|
Requires-Dist: events==0.5
|
|
46
|
+
Requires-Dist: greenlet==3.2.4
|
|
46
47
|
Requires-Dist: idna==3.11
|
|
47
48
|
Requires-Dist: jmespath==1.0.1
|
|
48
49
|
Requires-Dist: marshmallow==4.0.1
|
|
@@ -60,7 +61,7 @@ Requires-Dist: requests-aws4auth==1.3.1
|
|
|
60
61
|
Requires-Dist: s3transfer==0.15.0
|
|
61
62
|
Requires-Dist: six==1.17.0
|
|
62
63
|
Requires-Dist: sqlalchemy==2.0.44
|
|
63
|
-
Requires-Dist: sqlparse==0.5.
|
|
64
|
+
Requires-Dist: sqlparse==0.5.4
|
|
64
65
|
Requires-Dist: typing-extensions==4.15.0
|
|
65
66
|
Requires-Dist: urllib3==1.26.20
|
|
66
67
|
Dynamic: author
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
async-timeout==5.0.1
|
|
2
2
|
backports-datetime-fromisoformat==2.0.3
|
|
3
|
-
boto3==1.41.
|
|
4
|
-
botocore==1.41.
|
|
3
|
+
boto3==1.41.5
|
|
4
|
+
botocore==1.41.5
|
|
5
5
|
certifi==2025.11.12
|
|
6
6
|
charset-normalizer==3.4.4
|
|
7
7
|
click==8.1.8
|
|
@@ -10,6 +10,7 @@ elasticsearch==7.17.12
|
|
|
10
10
|
elasticsearch-dsl==7.4.1
|
|
11
11
|
environs==14.4.0
|
|
12
12
|
events==0.5
|
|
13
|
+
greenlet==3.2.4
|
|
13
14
|
idna==3.11
|
|
14
15
|
jmespath==1.0.1
|
|
15
16
|
marshmallow==4.0.1
|
|
@@ -27,6 +28,6 @@ requests-aws4auth==1.3.1
|
|
|
27
28
|
s3transfer==0.15.0
|
|
28
29
|
six==1.17.0
|
|
29
30
|
sqlalchemy==2.0.44
|
|
30
|
-
sqlparse==0.5.
|
|
31
|
+
sqlparse==0.5.4
|
|
31
32
|
typing-extensions==4.15.0
|
|
32
33
|
urllib3==1.26.20
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|