pgsync 4.0.0__tar.gz → 4.2.0__tar.gz

{pgsync-4.0.0 → pgsync-4.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: pgsync
-Version: 4.0.0
+Version: 4.2.0
 Summary: Postgres to Elasticsearch/OpenSearch sync
 Home-page: https://github.com/toluaina/pgsync
 Author: Tolu Aina
@@ -31,34 +31,34 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 License-File: AUTHORS.rst
 Requires-Dist: async-timeout==5.0.1
-Requires-Dist: boto3==1.37.11
-Requires-Dist: botocore==1.37.11
-Requires-Dist: certifi==2025.1.31
-Requires-Dist: charset-normalizer==3.4.1
+Requires-Dist: backports-datetime-fromisoformat==2.0.3
+Requires-Dist: boto3==1.40.1
+Requires-Dist: botocore==1.40.1
+Requires-Dist: certifi==2025.8.3
+Requires-Dist: charset-normalizer==3.4.2
 Requires-Dist: click==8.1.8
-Requires-Dist: elastic-transport==8.17.0
-Requires-Dist: elasticsearch==8.17.2
-Requires-Dist: elasticsearch-dsl==8.17.1
-Requires-Dist: environs==14.1.1
+Requires-Dist: elastic-transport==8.17.1
+Requires-Dist: elasticsearch==8.19.0
+Requires-Dist: elasticsearch-dsl==8.15.4
+Requires-Dist: environs==14.3.0
 Requires-Dist: events==0.5
-Requires-Dist: greenlet==3.1.1
+Requires-Dist: greenlet==3.2.3
 Requires-Dist: idna==3.10
 Requires-Dist: jmespath==1.0.1
-Requires-Dist: marshmallow==3.26.1
+Requires-Dist: marshmallow==4.0.0
 Requires-Dist: opensearch-dsl==2.1.0
-Requires-Dist: opensearch-py==2.8.0
-Requires-Dist: packaging==24.2
+Requires-Dist: opensearch-py==3.0.0
 Requires-Dist: psycopg2-binary==2.9.10
 Requires-Dist: python-dateutil==2.9.0.post0
-Requires-Dist: python-dotenv==1.0.1
-Requires-Dist: redis==5.2.1
-Requires-Dist: requests==2.32.3
+Requires-Dist: python-dotenv==1.1.1
+Requires-Dist: redis==6.2.0
+Requires-Dist: requests==2.32.4
 Requires-Dist: requests-aws4auth==1.3.1
-Requires-Dist: s3transfer==0.11.4
+Requires-Dist: s3transfer==0.13.1
 Requires-Dist: six==1.17.0
-Requires-Dist: sqlalchemy==2.0.39
+Requires-Dist: sqlalchemy==2.0.42
 Requires-Dist: sqlparse==0.5.3
-Requires-Dist: typing-extensions==4.12.2
+Requires-Dist: typing-extensions==4.14.1
 Requires-Dist: urllib3==1.26.20
 Dynamic: author
 Dynamic: author-email
@@ -68,6 +68,7 @@ Dynamic: description-content-type
 Dynamic: home-page
 Dynamic: keywords
 Dynamic: license
+Dynamic: license-file
 Dynamic: maintainer
 Dynamic: maintainer-email
 Dynamic: project-url
@@ -89,7 +90,7 @@ expose structured denormalized documents in [Elasticsearch](https://www.elastic.
 - [Postgres](https://www.postgresql.org) 9.6+
 - [Redis](https://redis.io) 3.1.0+
 - [Elasticsearch](https://www.elastic.co/products/elastic-stack) 6.3.1+ or [OpenSearch](https://opensearch.org/) 1.3.7+
-- [SQlAlchemy](https://www.sqlalchemy.org) 1.3.4+
+- [SQLAlchemy](https://www.sqlalchemy.org) 1.3.4+
 
 ### Postgres setup
   

{pgsync-4.0.0 → pgsync-4.2.0}/README.md

@@ -40,7 +40,7 @@ of engineering and development.
 Other benefits of PGSync include:
 - Real-time analytics
 - Reliable primary datastore/source of truth
-- Scale on-demand
+- Scale on-demand (multiple consumers)
 - Easily join multiple nested tables
 
 #### Why?
@@ -66,7 +66,7 @@ the search capabilities of [Elasticsearch](https://www.elastic.co/products/elast
 
 #### How it works
 
-PGSync is written in Python (supporting version 3.9 onwards) and the stack is composed of: [Redis](https://redis.io), [Elasticsearch](https://www.elastic.co/products/elastic-stack)/[OpenSearch](https://opensearch.org/), [Postgres](https://www.postgresql.org), and [SQlAlchemy](https://www.sqlalchemy.org).
+PGSync is written in Python (supporting version 3.9 onwards) and the stack is composed of: [Redis](https://redis.io), [Elasticsearch](https://www.elastic.co/products/elastic-stack)/[OpenSearch](https://opensearch.org/), [Postgres](https://www.postgresql.org), and [SQLAlchemy](https://www.sqlalchemy.org).
 
 PGSync leverages the [logical decoding](https://www.postgresql.org/docs/current/logicaldecoding.html) feature of [Postgres](https://www.postgresql.org) (introduced in PostgreSQL 9.4) to capture a continuous stream of change events.
 This feature needs to be enabled in your [Postgres](https://www.postgresql.org) configuration file by setting in the postgresql.conf file:
@@ -93,9 +93,14 @@ There are several ways of installing and trying PGSync
  - [Manual configuration](#manual-configuration) 
 
 
-##### Running in Docker
+##### Running in Docker (Using Github Repository)
 
 To startup all services with docker.
+
+```
+$ git checkout https://github.com/toluaina/pgsync
+```
+
 Run:
 ```
 $ docker-compose up
@@ -106,6 +111,39 @@ Show the content in Elasticsearch/OpenSearch
 $ curl -X GET http://[Elasticsearch/OpenSearch host]:9201/reservations/_search?pretty=true
 ```
 
+
+##### Running with Docker (Using Image Repository)
+
+To start all services with Docker, follow these steps:
+
+1. Pull the Docker image:
+
+  ```
+  $ docker pull toluaina1/pgsync:latest
+  ```
+
+2. Run the container:
+
+  ```
+  $ docker run --rm -it \
+  -e REDIS_CHECKPOINT=true \
+  -e REDIS_HOST=<redis_host_address> \
+  -e PG_URL=postgres://<username>:<password>@<postgres_host>/<database> \
+  -e ELASTICSEARCH_URL=http://<elasticsearch_host>:9200 \
+  -v "$(pwd)/schema.json:/app/schema.json" \
+  toluaina1/pgsync:latest -c schema.json -d -b
+  ```
+
+Environment variable placeholders - full list [here](https://pgsync.com/env-vars):
+
+- redis_host_address — Address of the Redis server (e.g., host.docker.internal for local Docker setup)
+- username — PostgreSQL username
+- password — PostgreSQL password
+- postgres_host — Host address for PostgreSQL instance (e.g., host.docker.internal)
+- database — Name of PostgreSQL database
+- elasticsearch_host — Address of Elasticsearch/OpenSearch instance (e.g., host.docker.internal)
+
+
 ##### Manual configuration
 
 - Setup
@@ -156,7 +194,7 @@ Key features of PGSync are:
 - [Postgres](https://www.postgresql.org) 9.6+
 - [Redis](https://redis.io) 3.1.0+
 - [Elasticsearch](https://www.elastic.co/products/elastic-stack) 6.3.1+ or [OpenSearch](https://opensearch.org/) 1.3.7+
-- [SQlAlchemy](https://www.sqlalchemy.org) 1.3.4+
+- [SQLAlchemy](https://www.sqlalchemy.org) 1.3.4+
 
 
 #### Example

{pgsync-4.0.0 → pgsync-4.2.0}/README.rst

@@ -12,7 +12,7 @@ expose structured denormalized documents in [Elasticsearch](https://www.elastic.
 - [Postgres](https://www.postgresql.org) 9.6+
 - [Redis](https://redis.io) 3.1.0+
 - [Elasticsearch](https://www.elastic.co/products/elastic-stack) 6.3.1+ or [OpenSearch](https://opensearch.org/) 1.3.7+
-- [SQlAlchemy](https://www.sqlalchemy.org) 1.3.4+
+- [SQLAlchemy](https://www.sqlalchemy.org) 1.3.4+
 
 ### Postgres setup
   

{pgsync-4.0.0 → pgsync-4.2.0}/bin/bootstrap

@@ -5,8 +5,14 @@ import logging
 
 import click
 
+from pgsync import settings
 from pgsync.sync import Sync
-from pgsync.utils import config_loader, get_config, show_settings
+from pgsync.utils import (
+    config_loader,
+    MutuallyExclusiveOption,
+    show_settings,
+    validate_config,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -17,17 +23,40 @@ logger = logging.getLogger(__name__)
     "-c",
     help="Schema config",
     type=click.Path(exists=True),
+    default=settings.SCHEMA,
+    show_default=True,
+    cls=MutuallyExclusiveOption,
+    mutually_exclusive=["s3_schema_url"],
+)
+@click.option(
+    "--s3_schema_url",
+    help="S3 URL for schema config",
+    type=click.STRING,
+    default=settings.S3_SCHEMA_URL,
+    show_default=True,
+    cls=MutuallyExclusiveOption,
+    mutually_exclusive=["config"],
 )
-@click.option("--host", "-h", help="PG_HOST overide")
+@click.option("--host", "-h", help="PG_HOST override")
 @click.option("--password", is_flag=True, help="Prompt for database password")
-@click.option("--port", "-p", help="PG_PORT overide", type=int)
+@click.option("--port", "-p", help="PG_PORT override", type=int)
 @click.option(
     "--teardown",
     "-t",
     is_flag=True,
     help="Teardown database triggers and replication slots",
 )
-@click.option("--user", "-u", help="PG_USER overide")
+@click.option(
+    "--no-create",
+    "-nc",
+    is_flag=True,
+    help=(
+        "Skip DDL statement for objects "
+        "(Functions, Views & Replication slots) creation"
+    ),
+    default=False,
+)
+@click.option("--user", "-u", help="PG_USER override")
 @click.option(
     "--verbose",
     "-v",
@@ -38,11 +67,13 @@ logger = logging.getLogger(__name__)
 def main(
     teardown: bool,
     config: str,
+    s3_schema_url: str,
     user: str,
     password: bool,
     host: str,
     port: int,
     verbose: bool,
+    no_create: bool = False,
 ) -> None:
     """Application onetime Bootstrap."""
     kwargs: dict = {
@@ -58,13 +89,13 @@ def main(
         )
     kwargs = {key: value for key, value in kwargs.items() if value is not None}
 
-    config: str = get_config(config)
+    validate_config(config=config, s3_schema_url=s3_schema_url)
 
-    show_settings(config)
+    show_settings(config=config, s3_schema_url=s3_schema_url)
 
     validate: bool = False if teardown else True
 
-    for doc in config_loader(config):
+    for doc in config_loader(config=config, s3_schema_url=s3_schema_url):
         sync: Sync = Sync(
             doc,
             verbose=verbose,
@@ -75,7 +106,7 @@ def main(
         if teardown:
             sync.teardown()
             continue
-        sync.setup()
+        sync.setup(no_create=no_create)
         logger.info(f"Bootstrap: {sync.database}")
 
 

{pgsync-4.0.0 → pgsync-4.2.0}/bin/parallel_sync

@@ -39,7 +39,6 @@ filtered based on the page number and row numbers.
 This systematic and parallel approach optimizes the synchronization process,
 especially in environments challenged by network latency.
 """
-
 import asyncio
 import multiprocessing
 import os
@@ -54,9 +53,15 @@ from threading import Thread
 import click
 import sqlalchemy as sa
 
-from pgsync.settings import BLOCK_SIZE, CHECKPOINT_PATH
+from pgsync.settings import BLOCK_SIZE, CHECKPOINT_PATH, S3_SCHEMA_URL, SCHEMA
 from pgsync.sync import Sync
-from pgsync.utils import config_loader, get_config, show_settings, timeit
+from pgsync.utils import (
+    config_loader,
+    MutuallyExclusiveOption,
+    show_settings,
+    timeit,
+    validate_config,
+)
 
 
 def save_ctid(page: int, row: int, filename: str) -> None:
@@ -378,6 +383,19 @@ def run_task(
     "-c",
     help="Schema config",
     type=click.Path(exists=True),
+    default=SCHEMA,
+    show_default=True,
+    cls=MutuallyExclusiveOption,
+    mutually_exclusive=["s3_schema_url"],
+)
+@click.option(
+    "--s3_schema_url",
+    help="S3 URL for schema config",
+    type=click.STRING,
+    default=S3_SCHEMA_URL,
+    show_default=True,
+    cls=MutuallyExclusiveOption,
+    mutually_exclusive=["config"],
 )
 @click.option(
     "--verbose",
@@ -409,17 +427,20 @@ def run_task(
     ),
     default="multiprocess_async",
 )
-def main(config: str, nprocs: int, mode: str, verbose: bool) -> None:
+def main(
+    config: str, s3_schema_url: str, nprocs: int, mode: str, verbose: bool
+) -> None:
     """
     TODO:
     - Track progress across cpus/threads
     - Handle KeyboardInterrupt Exception
     """
 
-    show_settings()
-    config: str = get_config(config)
+    validate_config(config=config, s3_schema_url=s3_schema_url)
+
+    show_settings(config=config, s3_schema_url=s3_schema_url)
 
-    for doc in config_loader(config):
+    for doc in config_loader(config=config, s3_schema_url=s3_schema_url):
         tasks: t.Generator = fetch_tasks(doc)
         if mode == "synchronous":
             synchronous(tasks, doc, verbose=verbose)

{pgsync-4.0.0 → pgsync-4.2.0}/pgsync/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = "Tolu Aina"
 __email__ = "tolu@pgsync.com"
-__version__ = "4.0.0"
+__version__ = "4.2.0"