pgsync 4.2.1__tar.gz → 6.0.0__tar.gz

{pgsync-4.2.1 → pgsync-6.0.0}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: pgsync
-Version: 4.2.1
-Summary: Postgres to Elasticsearch/OpenSearch sync
+Version: 6.0.0
+Summary: Postgres/MySQL/MariaDB to Elasticsearch/OpenSearch sync
 Home-page: https://github.com/toluaina/pgsync
 Author: Tolu Aina
 Author-email: tolu@pgsync.com
@@ -13,7 +13,7 @@ Project-URL: Funding, https://github.com/sponsors/toluaina
 Project-URL: Source, https://github.com/toluaina/pgsync
 Project-URL: Web, https://pgsync.com
 Project-URL: Documentation, https://pgsync.com
-Keywords: change data capture,elasticsearch,opensearch,pgsync,postgres
+Keywords: change data capture,elasticsearch,opensearch,pgsync,postgres,mysql,mariadb
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Natural Language :: English
@@ -22,6 +22,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Classifier: License :: OSI Approved :: MIT License
@@ -32,31 +33,33 @@ License-File: LICENSE
 License-File: AUTHORS.rst
 Requires-Dist: async-timeout==5.0.1
 Requires-Dist: backports-datetime-fromisoformat==2.0.3
-Requires-Dist: boto3==1.40.35
-Requires-Dist: botocore==1.40.35
-Requires-Dist: certifi==2025.8.3
-Requires-Dist: charset-normalizer==3.4.3
+Requires-Dist: boto3==1.40.64
+Requires-Dist: botocore==1.40.64
+Requires-Dist: certifi==2025.10.5
+Requires-Dist: charset-normalizer==3.4.4
 Requires-Dist: click==8.1.8
-Requires-Dist: elastic-transport==8.17.1
-Requires-Dist: elasticsearch==8.19.1
-Requires-Dist: elasticsearch-dsl==8.15.4
-Requires-Dist: environs==14.3.0
+Requires-Dist: elastic-transport==9.1.0
+Requires-Dist: elasticsearch==7.17.12
+Requires-Dist: elasticsearch-dsl==7.4.1
+Requires-Dist: environs==14.4.0
 Requires-Dist: events==0.5
-Requires-Dist: greenlet==3.2.4
-Requires-Dist: idna==3.10
+Requires-Dist: idna==3.11
 Requires-Dist: jmespath==1.0.1
 Requires-Dist: marshmallow==4.0.1
+Requires-Dist: mysql-replication==1.0.9
 Requires-Dist: opensearch-dsl==2.1.0
 Requires-Dist: opensearch-py==3.0.0
-Requires-Dist: psycopg2-binary==2.9.10
+Requires-Dist: packaging==25.0
+Requires-Dist: psycopg2-binary==2.9.11
+Requires-Dist: pymysql==1.1.2
 Requires-Dist: python-dateutil==2.9.0.post0
-Requires-Dist: python-dotenv==1.1.1
-Requires-Dist: redis==6.4.0
+Requires-Dist: python-dotenv==1.2.1
+Requires-Dist: redis==7.0.1
 Requires-Dist: requests==2.32.5
 Requires-Dist: requests-aws4auth==1.3.1
 Requires-Dist: s3transfer==0.14.0
 Requires-Dist: six==1.17.0
-Requires-Dist: sqlalchemy==2.0.43
+Requires-Dist: sqlalchemy==2.0.44
 Requires-Dist: sqlparse==0.5.3
 Requires-Dist: typing-extensions==4.15.0
 Requires-Dist: urllib3==1.26.20
@@ -87,12 +90,12 @@ expose structured denormalized documents in [Elasticsearch](https://www.elastic.
 ### Requirements
 
 - [Python](https://www.python.org) 3.9+
-- [Postgres](https://www.postgresql.org) 9.6+
+- [Postgres](https://www.postgresql.org) 9.6+ or [MySQL](https://www.mysql.com/) 8.0.0+ or [MariaDB](https://mariadb.org/) 12.0.0+ 
 - [Redis](https://redis.io) 3.1.0+ or [Valkey](https://valkey.io) 7.2.0+
 - [Elasticsearch](https://www.elastic.co/products/elastic-stack) 6.3.1+ or [OpenSearch](https://opensearch.org/) 1.3.7+
 - [SQLAlchemy](https://www.sqlalchemy.org) 1.3.4+
 
-### Postgres setup
+### Postgres Setup
   
   Enable [logical decoding](https://www.postgresql.org/docs/current/logicaldecoding.html) in your 
   Postgres setting.
@@ -103,6 +106,30 @@ expose structured denormalized documents in [Elasticsearch](https://www.elastic.
 
     ```max_replication_slots = 1```
 
+
+### MySQL / MariaDB setup
+
+- Enable binary logging in your MySQL / MariaDB setting.
+
+- You also need to set up the following parameters in your MySQL / MariaDB config my.cnf, then restart the database server.
+
+  ```server-id = 1``` # any non-zero unique ID
+
+  ```log_bin = mysql-bin```
+
+  ```binlog_row_image = FULL``` # recommended; if not supported on older MariaDB, omit
+
+- optional housekeeping:
+  ```binlog_expire_logs_seconds = 604800```   # 7 days
+
+- You need to create a replication user with REPLICATION SLAVE and REPLICATION CLIENT privileges
+    
+  ```sql
+  CREATE USER 'replicator'@'%' IDENTIFIED WITH mysql_native_password BY 'password';
+  GRANT REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'replicator'@'%';
+  FLUSH PRIVILEGES;
+  ```
+
 ### Installation
 
 You can install PGSync from [PyPI](https://pypi.org):

{pgsync-4.2.1 → pgsync-6.0.0}/README.md

@@ -5,12 +5,13 @@
 [![Python versions](https://img.shields.io/pypi/pyversions/pgsync)](https://pypi.org/project/pgsync)
 [![Downloads](https://img.shields.io/pypi/dm/pgsync)](https://pypi.org/project/pgsync)
 [![codecov](https://codecov.io/gh/toluaina/pgsync/branch/main/graph/badge.svg?token=cvQzYkz6CV)](https://codecov.io/gh/toluaina/pgsync)
+[![Sponsored by DigitalOcean](https://img.shields.io/badge/Sponsored%20by-DigitalOcean-0080FF?logo=digitalocean&logoColor=white)](https://www.digitalocean.com/?utm_medium=opensource&utm_source=pgsync)
 
 
-## PostgreSQL to Elasticsearch/OpenSearch sync
+## PostgreSQL/MySQL/MariaDB to Elasticsearch/OpenSearch sync
 
-[PGSync](https://pgsync.com) is a middleware for syncing data from [Postgres](https://www.postgresql.org) to [Elasticsearch](https://www.elastic.co/products/elastic-stack)/[OpenSearch](https://opensearch.org/) effortlessly.
-It allows you to keep [Postgres](https://www.postgresql.org) as your source of truth and
+[PGSync](https://pgsync.com) is a middleware for syncing data from [Postgres](https://www.postgresql.org) or [MySQL](https://www.mysql.com/) or [MariaDB](https://mariadb.org/) to [Elasticsearch](https://www.elastic.co/products/elastic-stack)/[OpenSearch](https://opensearch.org/) effortlessly.
+It allows you to keep [Postgres](https://www.postgresql.org) or [MySQL](https://www.mysql.com/) or [MariaDB](https://mariadb.org/) as your source of truth and
 expose structured denormalized documents in [Elasticsearch](https://www.elastic.co/products/elastic-stack)/[OpenSearch](https://opensearch.org/).
 
 Changes to nested entities are propagated to [Elasticsearch](https://www.elastic.co/products/elastic-stack)/[OpenSearch](https://opensearch.org/).
@@ -24,7 +25,7 @@ without writing any code.
 [PGSync](https://pgsync.com) transforms your relational data into a structured document format.
 
 It allows you to take advantage of the expressive power and scalability of 
-[Elasticsearch](https://www.elastic.co/products/elastic-stack)/[OpenSearch](https://opensearch.org/) directly from [Postgres](https://www.postgresql.org). 
+[Elasticsearch](https://www.elastic.co/products/elastic-stack)/[OpenSearch](https://opensearch.org/) directly from [Postgres](https://www.postgresql.org) or [MySQL](https://www.mysql.com/) or [MariaDB](https://mariadb.org/). 
 You don't have to write complex queries and transformation pipelines.
 PGSync is lightweight, flexible and fast.
 
@@ -43,9 +44,26 @@ Other benefits of PGSync include:
 - Scale on-demand (multiple consumers)
 - Easily join multiple nested tables
 
+## Sponsors
+
+[PGSync](https://pgsync.com) is made possible with support from [DigitalOcean](https://www.digitalocean.com/?utm_medium=opensource&utm_source=pgsync).
+
+<p>
+  <a href="https://www.digitalocean.com/?utm_medium=opensource&utm_source=pgsync" rel="sponsored noopener noreferrer">
+    <img
+      src="https://opensource.nyc3.cdn.digitaloceanspaces.com/attribution/assets/SVG/DO_Logo_horizontal_blue.svg"
+      alt="DigitalOcean"
+      width="210"
+      loading="lazy"
+      decoding="async"
+    >
+  </a>
+</p>
+
+
 #### Why?
 
-At a high level, you have data in a Postgres database and you want to mirror it in Elasticsearch/OpenSearch.  
+At a high level, you have data in a PostgreSQL/MySQL/MariaDB database and you want to mirror it in Elasticsearch/OpenSearch.  
 This means every change to your data (***Insert***, ***Update***, ***Delete*** and ***Truncate*** statements) needs to be replicated to Elasticsearch/OpenSearch. 
 At first, this seems easy and then it's not. Simply add some code to copy the data to Elasticsearch/OpenSearch after updating the database (or so called dual writes).
 Writing SQL queries spanning multiple tables and involving multiple relationships are hard to write.
@@ -53,12 +71,11 @@ Detecting changes within a nested document can also be quite hard.
 Of course, if your data never changed, then you could just take a snapshot in time and load it into Elasticsearch/OpenSearch as a one-off operation.
 
 PGSync is appropriate for you if:
-- [Postgres](https://www.postgresql.org) is your read/write source of truth whilst [Elasticsearch](https://www.elastic.co/products/elastic-stack)/[OpenSearch](https://opensearch.org/) is your 
+- [Postgres](https://www.postgresql.org) or [MySQL](https://www.mysql.com/) or [MariaDB](https://mariadb.org/) is your read/write source of truth whilst [Elasticsearch](https://www.elastic.co/products/elastic-stack)/[OpenSearch](https://opensearch.org/) is your 
 read-only search layer.
 - You need to denormalize relational data into a NoSQL data source.
 - Your data is constantly changing.
-- You have existing data in a relational database such as [Postgres](https://www.postgresql.org) and you need
-a secondary NoSQL database like [Elasticsearch](https://www.elastic.co/products/elastic-stack)/[OpenSearch](https://opensearch.org/) for text-based queries or autocomplete queries to mirror the existing data without having your application perform dual writes.
+- You have existing data in a relational database such as [Postgres](https://www.postgresql.org) or [MySQL](https://www.mysql.com/) or [MariaDB](https://mariadb.org/) and you need a secondary NoSQL database like [Elasticsearch](https://www.elastic.co/products/elastic-stack)/[OpenSearch](https://opensearch.org/) for text-based queries or autocomplete queries to mirror the existing data without having your application perform dual writes.
 - You want to keep your existing data untouched whilst taking advantage of
 the search capabilities of [Elasticsearch](https://www.elastic.co/products/elastic-stack)/[OpenSearch](https://opensearch.org/) by exposing a view of your data without compromising the security of your relational data.
 - Or you simply want to expose a view of your relational data for search purposes.
@@ -66,7 +83,7 @@ the search capabilities of [Elasticsearch](https://www.elastic.co/products/elast
 
 #### How it works
 
-PGSync is written in Python (supporting version 3.9 onwards) and the stack is composed of: [Redis](https://redis.io)/[Valkey](https://valkey.io), [Elasticsearch](https://www.elastic.co/products/elastic-stack)/[OpenSearch](https://opensearch.org/), [Postgres](https://www.postgresql.org), and [SQLAlchemy](https://www.sqlalchemy.org).
+PGSync is written in Python (supporting version 3.9 onwards) and the stack is composed of: [Redis](https://redis.io)/[Valkey](https://valkey.io), [Elasticsearch](https://www.elastic.co/products/elastic-stack)/[OpenSearch](https://opensearch.org/), [Postgres](https://www.postgresql.org)/[MySQL](https://www.mysql.com/)/[MariaDB](https://mariadb.org/), and [SQLAlchemy](https://www.sqlalchemy.org).
 
 PGSync leverages the [logical decoding](https://www.postgresql.org/docs/current/logicaldecoding.html) feature of [Postgres](https://www.postgresql.org) (introduced in PostgreSQL 9.4) to capture a continuous stream of change events.
 This feature needs to be enabled in your [Postgres](https://www.postgresql.org) configuration file by setting in the postgresql.conf file:
@@ -92,6 +109,14 @@ There are several ways of installing and trying PGSync
  - [Running in Docker](#running-in-docker) is the easiest way to get up and running.
  - [Manual configuration](#manual-configuration) 
 
+#### Book Demo Example (requires a DigitalOcean account)
+
+[![Deploy to DO](https://www.deploytodo.com/do-btn-blue.svg)](https://cloud.digitalocean.com/apps/new?repo=https://github.com/toluaina/pgsync/tree/main)
+
+Fill in the following during the setup
+ - `ELASTICSEARCH_URL` e.g. https://user:pass@os-host:443
+ - `REDIS_URL` e.g. rediss://default:pass@host:port/0
+
 
 ##### Running in Docker (Using Github Repository)
 
@@ -137,16 +162,16 @@ To start all services with Docker, follow these steps:
 Environment variable placeholders - full list [here](https://pgsync.com/env-vars):
 
 - redis_host_address — Address of the Redis/Valkey server (e.g., host.docker.internal for local Docker setup)
-- username — PostgreSQL username
-- password — PostgreSQL password
-- postgres_host — Host address for PostgreSQL instance (e.g., host.docker.internal)
-- database — Name of PostgreSQL database
+- username — PostgreSQL/MySQL/MariaDB username
+- password — PostgreSQL/MySQL/MariaDB password
+- postgres_host — Host address for PostgreSQL/MySQL/MariaDB instance (e.g., host.docker.internal)
+- database — Name of PostgreSQL/MySQL/MariaDB database
 - elasticsearch_host — Address of Elasticsearch/OpenSearch instance (e.g., host.docker.internal)
 
 
 ##### Manual configuration
 
-- Setup
+### Postgres Setup
   - Ensure the database user is a superuser 
   - Enable logical decoding. You would also need to set up at least two parameters at postgresql.conf
 
@@ -159,7 +184,31 @@ Environment variable placeholders - full list [here](https://pgsync.com/env-vars
 
     ```max_slot_wal_keep_size = 100GB```
 
-- Installation
+### MySQL / MariaDB setup
+
+- Enable binary logging in your MySQL / MariaDB setting.
+
+- You also need to set up the following parameters in your MySQL / MariaDB config my.cnf, then restart the database server.
+
+  ```server-id = 1``` # any non-zero unique ID
+
+  ```log_bin = mysql-bin```
+
+  ```binlog_row_image = FULL``` # recommended; if not supported on older MariaDB, omit
+
+- optional housekeeping:
+  ```binlog_expire_logs_seconds = 604800```   # 7 days
+
+- You need to create a replication user with REPLICATION SLAVE and REPLICATION CLIENT privileges
+    
+  ```sql
+  CREATE USER 'replicator'@'%' IDENTIFIED WITH mysql_native_password BY 'password';
+  GRANT REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'replicator'@'%';
+  FLUSH PRIVILEGES;
+  ```
+
+### Installation
+
   - Install PGSync from pypi using pip
     - ```$ pip install pgsync``` 
   - Create a [schema.json](https://github.com/toluaina/pgsync/blob/main/examples/airbnb/schema.json) for your document representation
@@ -180,18 +229,18 @@ Key features of PGSync are:
 - Negligible impact on database performance.
 - Transactionally consistent output in Elasticsearch/OpenSearch. This means: writes appear only when they are committed to the database, insert, update and delete operations appear in the same order as they were committed (as opposed to eventual consistency).
 - Fault-tolerant: does not lose data, even if processes crash or a network interruption occurs, etc. The process can be recovered from the last checkpoint.
-- Returns the data directly as Postgres JSON from the database for speed.
+- Returns the data directly as Postgres/MySQL/MariaDB JSON from the database for speed.
 - Supports composite primary and foreign keys.
 - Supports Views and Materialized views.
 - Supports an arbitrary depth of nested entities i.e Tables having long chain of relationship dependencies.
-- Supports Postgres JSON data fields. This means: we can extract JSON fields in a database table as a separate field in the resulting document.
+- Supports PostgreSQL/MySQL/MariaDB JSON data fields. This means: we can extract JSON fields in a database table as a separate field in the resulting document.
 - Customizable document structure.
 
 
 #### Requirements
 
 - [Python](https://www.python.org) 3.9+
-- [Postgres](https://www.postgresql.org) 9.6+
+- [Postgres](https://www.postgresql.org) 9.6+ or [MySQL](https://www.mysql.com/) 5.7.22+ or [MariaDB](https://mariadb.org/) 10.5.0+ 
 - [Redis](https://redis.io) 3.1.0+ or [Valkey](https://valkey.io) 7.2.0+
 - [Elasticsearch](https://www.elastic.co/products/elastic-stack) 6.3.1+ or [OpenSearch](https://opensearch.org/) 1.3.7+
 - [SQLAlchemy](https://www.sqlalchemy.org) 1.3.4+
@@ -327,7 +376,7 @@ PGSync addresses the following challenges:
 - PGSync generates advanced queries matching your schema directly.
 - PGSync allows you to easily rebuild your indexes in case of a schema change.
 - You can expose only the data you require in Elasticsearch/OpenSearch.
-- Supports multiple Postgres schemas for multi-tennant applications.
+- Supports multiple Postgres/MySQL/MariaDB schemas for multi-tennant applications.
 
 
 #### Contributing
@@ -335,16 +384,10 @@ PGSync addresses the following challenges:
 Contributions are very welcome! Check out the [Contribution](CONTRIBUTING.rst) Guidelines for instructions.
 
 
-#### Credits
-
-- This package was created with [Cookiecutter](https://github.com/audreyr/cookiecutter)
-- Elasticsearch is a trademark of Elasticsearch BV, registered in the U.S. and in other countries.
-
-
 #### License
 
 This project is licensed under the terms of the [MIT](https://opensource.org/license/mit/) license.
 Please see [LICENSE](LICENSE) for more details.
 
-You should have received a copy of the MIT License along with PGSync.  
+You should have received a copy of the MIT License along with **PGSync**.  
 If not, see https://opensource.org/license/mit/.

{pgsync-4.2.1 → pgsync-6.0.0}/README.rst

@@ -9,12 +9,12 @@ expose structured denormalized documents in [Elasticsearch](https://www.elastic.
 ### Requirements
 
 - [Python](https://www.python.org) 3.9+
-- [Postgres](https://www.postgresql.org) 9.6+
+- [Postgres](https://www.postgresql.org) 9.6+ or [MySQL](https://www.mysql.com/) 8.0.0+ or [MariaDB](https://mariadb.org/) 12.0.0+ 
 - [Redis](https://redis.io) 3.1.0+ or [Valkey](https://valkey.io) 7.2.0+
 - [Elasticsearch](https://www.elastic.co/products/elastic-stack) 6.3.1+ or [OpenSearch](https://opensearch.org/) 1.3.7+
 - [SQLAlchemy](https://www.sqlalchemy.org) 1.3.4+
 
-### Postgres setup
+### Postgres Setup
   
   Enable [logical decoding](https://www.postgresql.org/docs/current/logicaldecoding.html) in your 
   Postgres setting.
@@ -25,6 +25,30 @@ expose structured denormalized documents in [Elasticsearch](https://www.elastic.
 
     ```max_replication_slots = 1```
 
+
+### MySQL / MariaDB setup
+
+- Enable binary logging in your MySQL / MariaDB setting.
+
+- You also need to set up the following parameters in your MySQL / MariaDB config my.cnf, then restart the database server.
+
+  ```server-id = 1``` # any non-zero unique ID
+
+  ```log_bin = mysql-bin```
+
+  ```binlog_row_image = FULL``` # recommended; if not supported on older MariaDB, omit
+
+- optional housekeeping:
+  ```binlog_expire_logs_seconds = 604800```   # 7 days
+
+- You need to create a replication user with REPLICATION SLAVE and REPLICATION CLIENT privileges
+    
+  ```sql
+  CREATE USER 'replicator'@'%' IDENTIFIED WITH mysql_native_password BY 'password';
+  GRANT REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'replicator'@'%';
+  FLUSH PRIVILEGES;
+  ```
+
 ### Installation
 
 You can install PGSync from [PyPI](https://pypi.org):

{pgsync-4.2.1 → pgsync-6.0.0}/bin/bootstrap

@@ -26,7 +26,16 @@ logger = logging.getLogger(__name__)
     default=settings.SCHEMA,
     show_default=True,
     cls=MutuallyExclusiveOption,
-    mutually_exclusive=["s3_schema_url"],
+    mutually_exclusive=["s3_schema_url", "schema_url"],
+)
+@click.option(
+    "--schema_url",
+    help="URL for schema config",
+    type=click.STRING,
+    default=settings.SCHEMA_URL,
+    show_default=True,
+    cls=MutuallyExclusiveOption,
+    mutually_exclusive=["config", "s3_schema_url"],
 )
 @click.option(
     "--s3_schema_url",
@@ -35,7 +44,7 @@ logger = logging.getLogger(__name__)
     default=settings.S3_SCHEMA_URL,
     show_default=True,
     cls=MutuallyExclusiveOption,
-    mutually_exclusive=["config"],
+    mutually_exclusive=["config", "schema_url"],
 )
 @click.option("--host", "-h", help="PG_HOST override")
 @click.option("--password", is_flag=True, help="Prompt for database password")
@@ -67,6 +76,7 @@ logger = logging.getLogger(__name__)
 def main(
     teardown: bool,
     config: str,
+    schema_url: str,
     s3_schema_url: str,
     user: str,
     password: bool,
@@ -75,7 +85,7 @@ def main(
     verbose: bool,
     no_create: bool = False,
 ) -> None:
-    """Application onetime Bootstrap."""
+    """Application onetime bootstrap."""
     kwargs: dict = {
         "user": user,
         "host": host,
@@ -89,13 +99,24 @@ def main(
         )
     kwargs = {key: value for key, value in kwargs.items() if value is not None}
 
-    validate_config(config=config, s3_schema_url=s3_schema_url)
+    validate_config(
+        config=config,
+        schema_url=schema_url,
+        s3_schema_url=s3_schema_url,
+    )
 
-    show_settings(config=config, s3_schema_url=s3_schema_url)
+    show_settings(
+        config=config,
+        schema_url=schema_url,
+        s3_schema_url=s3_schema_url,
+        **kwargs,
+    )
 
     validate: bool = False if teardown else True
 
-    for doc in config_loader(config=config, s3_schema_url=s3_schema_url):
+    for doc in config_loader(
+        config=config, schema_url=schema_url, s3_schema_url=s3_schema_url
+    ):
         sync: Sync = Sync(
             doc,
             verbose=verbose,

{pgsync-4.2.1 → pgsync-6.0.0}/bin/parallel_sync

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Parallel sync is an innovative, experimental feature designed to optimize
+Parallel sync is a feature designed to optimize
 throughput by utilizing available CPUs/threads, particularly beneficial
 in environments experiencing high network latency.
 
@@ -53,7 +53,13 @@ from threading import Thread
 import click
 import sqlalchemy as sa
 
-from pgsync.settings import BLOCK_SIZE, CHECKPOINT_PATH, S3_SCHEMA_URL, SCHEMA
+from pgsync.settings import (
+    BLOCK_SIZE,
+    CHECKPOINT_PATH,
+    S3_SCHEMA_URL,
+    SCHEMA,
+    SCHEMA_URL,
+)
 from pgsync.sync import Sync
 from pgsync.utils import (
     config_loader,
@@ -106,7 +112,7 @@ def logical_slot_changes(
     txmin: int = sync.checkpoint
     txmax: int = sync.txid_current
     sync.logical_slot_changes(txmin=txmin, txmax=txmax)
-    sync.checkpoint: int = txmax or sync.txid_current
+    sync.checkpoint = txmax or sync.txid_current
 
 
 @dataclass
@@ -386,7 +392,16 @@ def run_task(
     default=SCHEMA,
     show_default=True,
     cls=MutuallyExclusiveOption,
-    mutually_exclusive=["s3_schema_url"],
+    mutually_exclusive=["s3_schema_url", "schema_url"],
+)
+@click.option(
+    "--schema_url",
+    help="URL for schema config",
+    type=click.STRING,
+    default=SCHEMA_URL,
+    show_default=True,
+    cls=MutuallyExclusiveOption,
+    mutually_exclusive=["config", "s3_schema_url"],
 )
 @click.option(
     "--s3_schema_url",
@@ -395,7 +410,7 @@ def run_task(
     default=S3_SCHEMA_URL,
     show_default=True,
     cls=MutuallyExclusiveOption,
-    mutually_exclusive=["config"],
+    mutually_exclusive=["config", "schema_url"],
 )
 @click.option(
     "--verbose",
@@ -428,7 +443,12 @@ def run_task(
     default="multiprocess_async",
 )
 def main(
-    config: str, s3_schema_url: str, nprocs: int, mode: str, verbose: bool
+    config: str,
+    schema_url: str,
+    s3_schema_url: str,
+    nprocs: int,
+    mode: str,
+    verbose: bool,
 ) -> None:
     """
     TODO:
@@ -436,11 +456,17 @@ def main(
     - Handle KeyboardInterrupt Exception
     """
 
-    validate_config(config=config, s3_schema_url=s3_schema_url)
+    validate_config(
+        config=config, schema_url=schema_url, s3_schema_url=s3_schema_url
+    )
 
-    show_settings(config=config, s3_schema_url=s3_schema_url)
+    show_settings(
+        config=config, schema_url=schema_url, s3_schema_url=s3_schema_url
+    )
 
-    for doc in config_loader(config=config, s3_schema_url=s3_schema_url):
+    for doc in config_loader(
+        config=config, schema_url=schema_url, s3_schema_url=s3_schema_url
+    ):
         tasks: t.Generator = fetch_tasks(doc)
         if mode == "synchronous":
             synchronous(tasks, doc, verbose=verbose)

{pgsync-4.2.1 → pgsync-6.0.0}/pgsync/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = "Tolu Aina"
 __email__ = "tolu@pgsync.com"
-__version__ = "4.2.1"
+__version__ = "6.0.0"