pgbelt 0.7.10__tar.gz → 0.8.0__tar.gz

{pgbelt-0.7.10 → pgbelt-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pgbelt
-Version: 0.7.10
+Version: 0.8.0
 Summary: A CLI tool used to manage Postgres data migrations from beginning to end, for a single database or a fleet, leveraging pglogical replication.
 Author: Varjitt Jeeva
 Author-email: varjitt.jeeva@autodesk.com
@@ -10,11 +10,12 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: aiofiles (>=0.8,<24.2)
-Requires-Dist: asyncpg (>=0.27,<0.30)
+Requires-Dist: asyncpg (>=0.27,<0.31)
 Requires-Dist: pydantic (>=2.0,<3.0)
 Requires-Dist: tabulate (>=0.9.0,<0.10.0)
-Requires-Dist: typer (>=0.9,<0.13)
+Requires-Dist: typer (>=0.9,<0.14)
 Description-Content-Type: text/markdown
 
 # Pgbelt

{pgbelt-0.7.10 → pgbelt-0.8.0}/pgbelt/cmd/schema.py

@@ -1,4 +1,5 @@
 from collections.abc import Awaitable
+from asyncpg import create_pool
 
 from pgbelt.cmd.helpers import run_with_configs
 from pgbelt.config.models import DbupgradeConfig
@@ -11,6 +12,7 @@ from pgbelt.util.dump import dump_dst_create_index
 from pgbelt.util.dump import remove_dst_not_valid_constraints
 from pgbelt.util.dump import remove_dst_indexes
 from pgbelt.util.logs import get_logger
+from pgbelt.util.postgres import run_analyze
 
 
 @run_with_configs
@@ -109,11 +111,24 @@ async def create_indexes(config_future: Awaitable[DbupgradeConfig]) -> None:
     as the owner user. This must only be done after most data is synchronized
     (at minimum after the initializing phase) from the source to the destination
     database.
+
+    After creating indexes, the destination database should be analyzed to ensure
+    the query planner has the most up-to-date statistics for the indexes.
     """
     conf = await config_future
     logger = get_logger(conf.db, conf.dc, "schema.dst")
     await create_target_indexes(conf, logger, during_sync=False)
 
+    # Run ANALYZE after creating indexes (without statement timeout)
+    async with create_pool(
+        conf.dst.root_uri,
+        min_size=1,
+        server_settings={
+            "statement_timeout": "0",
+        },
+    ) as dst_pool:
+        await run_analyze(dst_pool, logger)
+
 
 COMMANDS = [
     dump_schema,

{pgbelt-0.7.10 → pgbelt-0.8.0}/pgbelt/cmd/status.py

@@ -116,10 +116,20 @@ async def status(conf_future: Awaitable[DbupgradeConfig]) -> dict[str, str]:
 
         result[0].update(result[1])
         result[0]["db"] = conf.db
-        if result[0]["pg1_pg2"] == "replicating":
+
+        # We should hide the progress in the following cases:
+        # 1. When src -> dst is replicating and dst -> src is any state (replicating, unconfigured, down)
+        #    a. We do this because the size when done still will be a tad smaller than SRC, showing <100%
+        # 2. When src -> dst is unconfigured and dst -> src is replicating (not down or unconfigured)
+        #    a. We do this because reverse-only occurs at the start of cutover and onwards, and seeing the progress at that stage is not useful.
+        if (result[0]["pg1_pg2"] == "replicating") or (  # 1
+            result[0]["pg1_pg2"] == "unconfigured"
+            and result[0]["pg2_pg1"] == "replicating"
+        ):  # 2
             result[2]["src_dataset_size"] = "n/a"
             result[2]["dst_dataset_size"] = "n/a"
             result[2]["progress"] = "n/a"
+
         result[0].update(result[2])
         return result[0]
     finally:

{pgbelt-0.7.10 → pgbelt-0.8.0}/pgbelt/cmd/sync.py

@@ -108,40 +108,6 @@ async def load_tables(
     await load_dumped_tables(conf, tables, logger)
 
 
-@run_with_configs
-async def sync_tables(
-    config_future: Awaitable[DbupgradeConfig],
-    tables: list[str] = Option([], help="Specific tables to sync"),
-):
-    """
-    Dump and load all tables from the source database to the destination database.
-    Equivalent to running dump-tables followed by load-tables. Table data will be
-    saved locally in files.
-
-    You may also provide a list of tables to sync with the
-    --tables option and only these tables will be synced.
-    """
-    conf = await config_future
-    src_logger = get_logger(conf.db, conf.dc, "sync.src")
-    dst_logger = get_logger(conf.db, conf.dc, "sync.dst")
-
-    if tables:
-        dump_tables = tables.split(",")
-    else:
-        async with create_pool(conf.src.pglogical_uri, min_size=1) as src_pool:
-            _, dump_tables, _ = await analyze_table_pkeys(
-                src_pool, conf.schema_name, src_logger
-            )
-
-        if conf.tables:
-            dump_tables = [t for t in dump_tables if t in conf.tables]
-
-    await dump_source_tables(conf, dump_tables)
-    await load_dumped_tables(
-        conf, [] if not tables and not conf.tables else dump_tables, dst_logger
-    )
-
-
 @run_with_configs(skip_src=True)
 async def analyze(config_future: Awaitable[DbupgradeConfig]) -> None:
     """
@@ -150,7 +116,13 @@ async def analyze(config_future: Awaitable[DbupgradeConfig]) -> None:
     """
     conf = await config_future
     logger = get_logger(conf.db, conf.dc, "sync.dst")
-    async with create_pool(conf.dst.root_uri, min_size=1) as dst_pool:
+    async with create_pool(
+        conf.dst.root_uri,
+        min_size=1,
+        server_settings={
+            "statement_timeout": "0",
+        },
+    ) as dst_pool:
         await run_analyze(dst_pool, logger)
 
 
@@ -208,8 +180,15 @@ async def sync(
         create_pool(conf.src.pglogical_uri, min_size=1),
         create_pool(conf.dst.root_uri, min_size=1),
         create_pool(conf.dst.owner_uri, min_size=1),
+        create_pool(
+            conf.dst.root_uri,
+            min_size=1,
+            server_settings={
+                "statement_timeout": "0",
+            },
+        ),
     )
-    src_pool, dst_root_pool, dst_owner_pool = pools
+    src_pool, dst_root_pool, dst_owner_pool, dst_root_no_timeout_pool = pools
 
     try:
         src_logger = get_logger(conf.db, conf.dc, "sync.src")
@@ -253,7 +232,7 @@ async def sync(
                 conf.schema_name,
                 validation_logger,
             ),
-            run_analyze(dst_owner_pool, dst_logger),
+            run_analyze(dst_root_no_timeout_pool, dst_logger),
         )
     finally:
         await gather(*[p.close() for p in pools])
@@ -263,7 +242,6 @@ COMMANDS = [
     sync_sequences,
     dump_tables,
     load_tables,
-    sync_tables,
     analyze,
     validate_data,
     sync,

{pgbelt-0.7.10 → pgbelt-0.8.0}/pgbelt/util/postgres.py

@@ -1,5 +1,6 @@
 from logging import Logger
 
+from decimal import Decimal
 from asyncpg import Pool
 from asyncpg import Record
 from asyncpg.exceptions import UndefinedObjectError
@@ -171,12 +172,33 @@ async def compare_data(
         # Check each row for exact match
         for src_row, dst_row in zip(src_rows, dst_rows):
             if src_row != dst_row:
-                raise AssertionError(
-                    "Row match failure between source and destination.\n"
-                    f"Table: {full_table_name}\n"
-                    f"Source Row: {src_row}\n"
-                    f"Dest Row: {dst_row}"
-                )
+
+                # Addresses #571, AsyncPG is decoding numeric NaN as Python Decimal('NaN').
+                # Decimal('NaN') != Decimal('NaN'), breaks comparison. Convert those NaNs to None.
+                src_row_d = {
+                    key: (
+                        value
+                        if not (isinstance(value, Decimal) and value.is_nan())
+                        else None
+                    )
+                    for key, value in row.items()
+                }
+                dst_row_d = {
+                    key: (
+                        value
+                        if not (isinstance(value, Decimal) and value.is_nan())
+                        else None
+                    )
+                    for key, value in row.items()
+                }
+
+                if src_row_d != dst_row_d:
+                    raise AssertionError(
+                        "Row match failure between source and destination.\n"
+                        f"Table: {full_table_name}\n"
+                        f"Source Row: {src_row}\n"
+                        f"Dest Row: {dst_row}"
+                    )
 
     # Just a paranoia check. If this throws, then it's possible pgbelt didn't migrate any data.
     # This was found in issue #420, and previous commands threw errors before this issue could arise.

{pgbelt-0.7.10 → pgbelt-0.8.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pgbelt"
-version = "0.7.10"
+version = "0.8.0"
 description = "A CLI tool used to manage Postgres data migrations from beginning to end, for a single database or a fleet, leveraging pglogical replication."
 authors = ["Varjitt Jeeva <varjitt.jeeva@autodesk.com>"]
 readme = "README.md"
@@ -12,20 +12,20 @@ packages = [
 [tool.poetry.dependencies]
 python = ">=3.9,<4.0"
 aiofiles = ">=0.8,<24.2"
-asyncpg = ">=0.27,<0.30"
+asyncpg = ">=0.27,<0.31"
 pydantic = ">=2.0,<3.0"
 tabulate = "^0.9.0"
-typer = ">=0.9,<0.13"
+typer = ">=0.9,<0.14"
 
 [tool.poetry.dev-dependencies]
-black = "~24.8.0"
-pre-commit = "~3.8.0"
+black = "~24.10.0"
+pre-commit = "~4.0.1"
 flake8 = "^7.1.1"
-pytest-cov = "~5.0.0"
+pytest-cov = "~6.0.0"
 pytest = "^8.3.3"
 coverage = {extras = ["toml"], version = "^7.6"}
-safety = "^3.2.7"
-mypy = "^1.11"
+safety = "^3.2.11"
+mypy = "^1.13"
 xdoctest = {extras = ["colors"], version = "^1.2.0"}
 flake8-bandit = "~4.1.1"
 flake8-bugbear = ">=21.9.2"
@@ -33,10 +33,10 @@ flake8-docstrings = "^1.6.0"
 flake8-rst-docstrings = "^0.3.0"
 pep8-naming = "^0.14.1"
 darglint = "^1.8.1"
-reorder-python-imports = "^3.13.0"
-pre-commit-hooks = "^4.6.0"
+reorder-python-imports = "^3.14.0"
+pre-commit-hooks = "^5.0.0"
 Pygments = "^2.18.0"
-pyupgrade = "^3.17.0"
+pyupgrade = "^3.19.0"
 pylint = "^3.3.1"
 pytest-asyncio = "~0.24.0"