pgbelt 0.7.11__tar.gz → 0.8.1__tar.gz

{pgbelt-0.7.11 → pgbelt-0.8.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pgbelt
-Version: 0.7.11
+Version: 0.8.1
 Summary: A CLI tool used to manage Postgres data migrations from beginning to end, for a single database or a fleet, leveraging pglogical replication.
 Author: Varjitt Jeeva
 Author-email: varjitt.jeeva@autodesk.com
@@ -10,11 +10,12 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: aiofiles (>=0.8,<24.2)
-Requires-Dist: asyncpg (>=0.27,<0.30)
+Requires-Dist: asyncpg (>=0.27,<0.31)
 Requires-Dist: pydantic (>=2.0,<3.0)
 Requires-Dist: tabulate (>=0.9.0,<0.10.0)
-Requires-Dist: typer (>=0.9,<0.13)
+Requires-Dist: typer (>=0.9,<0.14)
 Description-Content-Type: text/markdown
 
 # Pgbelt

{pgbelt-0.7.11 → pgbelt-0.8.1}/pgbelt/cmd/schema.py

@@ -1,16 +1,16 @@
 from collections.abc import Awaitable
+from asyncpg import create_pool
 
 from pgbelt.cmd.helpers import run_with_configs
 from pgbelt.config.models import DbupgradeConfig
 from pgbelt.util.dump import apply_target_constraints
 from pgbelt.util.dump import apply_target_schema
 from pgbelt.util.dump import create_target_indexes
-from pgbelt.util.dump import dump_dst_not_valid_constraints
 from pgbelt.util.dump import dump_source_schema
-from pgbelt.util.dump import dump_dst_create_index
 from pgbelt.util.dump import remove_dst_not_valid_constraints
 from pgbelt.util.dump import remove_dst_indexes
 from pgbelt.util.logs import get_logger
+from pgbelt.util.postgres import run_analyze
 
 
 @run_with_configs
@@ -56,17 +56,6 @@ async def load_constraints(config_future: Awaitable[DbupgradeConfig]) -> None:
     await apply_target_constraints(conf, logger)
 
 
-@run_with_configs(skip_src=True)
-async def dump_constraints(config_future: Awaitable[DbupgradeConfig]) -> None:
-    """
-    Dumps the NOT VALID constraints from the target database onto disk, in
-    the schemas directory.
-    """
-    conf = await config_future
-    logger = get_logger(conf.db, conf.dc, "schema.dst")
-    await dump_dst_not_valid_constraints(conf, logger)
-
-
 @run_with_configs(skip_src=True)
 async def remove_constraints(config_future: Awaitable[DbupgradeConfig]) -> None:
     """
@@ -79,17 +68,6 @@ async def remove_constraints(config_future: Awaitable[DbupgradeConfig]) -> None:
     await remove_dst_not_valid_constraints(conf, logger)
 
 
-@run_with_configs(skip_src=True)
-async def dump_indexes(config_future: Awaitable[DbupgradeConfig]) -> None:
-    """
-    Dumps the CREATE INDEX statements from the target database onto disk, in
-    the schemas directory.
-    """
-    conf = await config_future
-    logger = get_logger(conf.db, conf.dc, "schema.dst")
-    await dump_dst_create_index(conf, logger)
-
-
 @run_with_configs(skip_src=True)
 async def remove_indexes(config_future: Awaitable[DbupgradeConfig]) -> None:
     """
@@ -109,19 +87,30 @@ async def create_indexes(config_future: Awaitable[DbupgradeConfig]) -> None:
     as the owner user. This must only be done after most data is synchronized
     (at minimum after the initializing phase) from the source to the destination
     database.
+
+    After creating indexes, the destination database should be analyzed to ensure
+    the query planner has the most up-to-date statistics for the indexes.
     """
     conf = await config_future
     logger = get_logger(conf.db, conf.dc, "schema.dst")
     await create_target_indexes(conf, logger, during_sync=False)
 
+    # Run ANALYZE after creating indexes (without statement timeout)
+    async with create_pool(
+        conf.dst.root_uri,
+        min_size=1,
+        server_settings={
+            "statement_timeout": "0",
+        },
+    ) as dst_pool:
+        await run_analyze(dst_pool, logger)
+
 
 COMMANDS = [
     dump_schema,
     load_schema,
     load_constraints,
-    dump_constraints,
     remove_constraints,
-    dump_indexes,
     remove_indexes,
     create_indexes,
 ]

{pgbelt-0.7.11 → pgbelt-0.8.1}/pgbelt/cmd/status.py

@@ -116,10 +116,20 @@ async def status(conf_future: Awaitable[DbupgradeConfig]) -> dict[str, str]:
 
         result[0].update(result[1])
         result[0]["db"] = conf.db
-        if result[0]["pg1_pg2"] == "replicating":
+
+        # We should hide the progress in the following cases:
+        # 1. When src -> dst is replicating and dst -> src is any state (replicating, unconfigured, down)
+        #    a. We do this because the size when done still will be a tad smaller than SRC, showing <100%
+        # 2. When src -> dst is unconfigured and dst -> src is replicating (not down or unconfigured)
+        #    a. We do this because reverse-only occurs at the start of cutover and onwards, and seeing the progress at that stage is not useful.
+        if (result[0]["pg1_pg2"] == "replicating") or (  # 1
+            result[0]["pg1_pg2"] == "unconfigured"
+            and result[0]["pg2_pg1"] == "replicating"
+        ):  # 2
             result[2]["src_dataset_size"] = "n/a"
             result[2]["dst_dataset_size"] = "n/a"
             result[2]["progress"] = "n/a"
+
         result[0].update(result[2])
         return result[0]
     finally:

{pgbelt-0.7.11 → pgbelt-0.8.1}/pgbelt/cmd/sync.py

@@ -108,40 +108,6 @@ async def load_tables(
     await load_dumped_tables(conf, tables, logger)
 
 
-@run_with_configs
-async def sync_tables(
-    config_future: Awaitable[DbupgradeConfig],
-    tables: list[str] = Option([], help="Specific tables to sync"),
-):
-    """
-    Dump and load all tables from the source database to the destination database.
-    Equivalent to running dump-tables followed by load-tables. Table data will be
-    saved locally in files.
-
-    You may also provide a list of tables to sync with the
-    --tables option and only these tables will be synced.
-    """
-    conf = await config_future
-    src_logger = get_logger(conf.db, conf.dc, "sync.src")
-    dst_logger = get_logger(conf.db, conf.dc, "sync.dst")
-
-    if tables:
-        dump_tables = tables.split(",")
-    else:
-        async with create_pool(conf.src.pglogical_uri, min_size=1) as src_pool:
-            _, dump_tables, _ = await analyze_table_pkeys(
-                src_pool, conf.schema_name, src_logger
-            )
-
-        if conf.tables:
-            dump_tables = [t for t in dump_tables if t in conf.tables]
-
-    await dump_source_tables(conf, dump_tables)
-    await load_dumped_tables(
-        conf, [] if not tables and not conf.tables else dump_tables, dst_logger
-    )
-
-
 @run_with_configs(skip_src=True)
 async def analyze(config_future: Awaitable[DbupgradeConfig]) -> None:
     """
@@ -150,7 +116,13 @@ async def analyze(config_future: Awaitable[DbupgradeConfig]) -> None:
     """
     conf = await config_future
     logger = get_logger(conf.db, conf.dc, "sync.dst")
-    async with create_pool(conf.dst.root_uri, min_size=1) as dst_pool:
+    async with create_pool(
+        conf.dst.root_uri,
+        min_size=1,
+        server_settings={
+            "statement_timeout": "0",
+        },
+    ) as dst_pool:
         await run_analyze(dst_pool, logger)
 
 
@@ -208,8 +180,15 @@ async def sync(
         create_pool(conf.src.pglogical_uri, min_size=1),
         create_pool(conf.dst.root_uri, min_size=1),
         create_pool(conf.dst.owner_uri, min_size=1),
+        create_pool(
+            conf.dst.root_uri,
+            min_size=1,
+            server_settings={
+                "statement_timeout": "0",
+            },
+        ),
     )
-    src_pool, dst_root_pool, dst_owner_pool = pools
+    src_pool, dst_root_pool, dst_owner_pool, dst_root_no_timeout_pool = pools
 
     try:
         src_logger = get_logger(conf.db, conf.dc, "sync.src")
@@ -253,7 +232,7 @@ async def sync(
                 conf.schema_name,
                 validation_logger,
             ),
-            run_analyze(dst_owner_pool, dst_logger),
+            run_analyze(dst_root_no_timeout_pool, dst_logger),
         )
     finally:
         await gather(*[p.close() for p in pools])
@@ -263,7 +242,6 @@ COMMANDS = [
     sync_sequences,
     dump_tables,
     load_tables,
-    sync_tables,
     analyze,
     validate_data,
     sync,

{pgbelt-0.7.11 → pgbelt-0.8.1}/pgbelt/util/dump.py

@@ -310,11 +310,14 @@ async def remove_dst_not_valid_constraints(
         if (config.tables and table in config.tables) or not config.tables:
             queries = queries + f"ALTER TABLE {table} DROP CONSTRAINT {constraint};"
 
-    command = ["psql", config.dst.owner_dsn, "-c", f"'{queries}'"]
+    if queries != "":
+        command = ["psql", config.dst.owner_dsn, "-c", f"'{queries}'"]
 
-    await _execute_subprocess(
-        command, "Finished removing NOT VALID constraints from the target.", logger
-    )
+        await _execute_subprocess(
+            command, "Finished removing NOT VALID constraints from the target.", logger
+        )
+    else:
+        logger.info("No NOT VALID detected for removal.")
 
 
 async def apply_target_constraints(config: DbupgradeConfig, logger: Logger) -> None:
@@ -336,53 +339,6 @@ async def apply_target_constraints(config: DbupgradeConfig, logger: Logger) -> N
     )
 
 
-async def dump_dst_create_index(config: DbupgradeConfig, logger: Logger) -> None:
-    """
-    Dump CREATE INDEX statements from the target database.
-    Used when schema is loaded in outside of pgbelt.
-    """
-
-    logger.info("Dumping target CREATE INDEX statements...")
-
-    command = [
-        "pg_dump",
-        "--schema-only",
-        "--no-owner",
-        "-n",
-        config.schema_name,
-        config.dst.pglogical_dsn,
-    ]
-
-    out = await _execute_subprocess(command, "Retrieved target schema", logger)
-
-    # No username replacement needs to be done, so replace dst user with the same.
-    commands_raw = _parse_dump_commands(
-        out.decode("utf-8"), config.dst.owner_user.name, config.dst.owner_user.name
-    )
-
-    commands = []
-    for c in commands_raw:
-        if "CREATE" in command and "INDEX" in command:
-            regex_matches = search(
-                r"CREATE [UNIQUE ]*INDEX (?P<index>[a-zA-Z0-9._]+)+.*",
-                c,
-            )
-            if not regex_matches:
-                continue
-            commands.append(c)
-
-    try:
-        await makedirs(schema_dir(config.db, config.dc))
-    except FileExistsError:
-        pass
-
-    async with aopen(schema_file(config.db, config.dc, ONLY_INDEXES), "w") as out:
-        for command in commands:
-            await out.write(command)
-
-    logger.debug("Finished dumping CREATE INDEX statements from the target.")
-
-
 async def remove_dst_indexes(config: DbupgradeConfig, logger: Logger) -> None:
     """
     Remove the INDEXes from the schema of the target database.
@@ -395,7 +351,6 @@ async def remove_dst_indexes(config: DbupgradeConfig, logger: Logger) -> None:
 
     logger.info("Removing Indexes from the target...")
 
-    queries = ""
     for c in create_index_statements.split(";"):
         regex_matches = search(
             r"CREATE [UNIQUE ]*INDEX (?P<index>[a-zA-Z0-9._]+)+.*",
@@ -404,14 +359,20 @@ async def remove_dst_indexes(config: DbupgradeConfig, logger: Logger) -> None:
         if not regex_matches:
             continue
         index = regex_matches.groupdict()["index"]
-
-        queries = queries + f"DROP INDEX {index};"
-
-    command = ["psql", config.dst.owner_dsn, "-c", f"'{queries}'"]
-
-    await _execute_subprocess(
-        command, "Finished removing indexes from the target.", logger
-    )
+        if config.schema_name:
+            index = f"{config.schema_name}.{index}"
+
+        # DROP the index
+        # Note that the host DSN must have a statement timeout of 0.
+        # Example DSN: `host=server-hostname user=user dbname=db_name options='-c statement_timeout=3600000'`
+        host_dsn = config.dst.owner_dsn + " options='-c statement_timeout=0'"
+
+        # DROP INDEX IF EXISTS so no need to catch exceptions
+        command = ["psql", host_dsn, "-c", f"DROP INDEX IF EXISTS {index};"]
+        logger.info(f"Dropping index {index} on the target...")
+        await _execute_subprocess(
+            command, f"Finished dropping index {index} on the target.", logger
+        )
 
 
 async def create_target_indexes(
@@ -448,7 +409,10 @@ async def create_target_indexes(
         index = regex_matches.groupdict()["index"]
 
         # Create the index
-        command = ["psql", config.dst.owner_dsn, "-c", f"{c};"]
+        # Note that the host DSN must have a statement timeout of 0.
+        # Example DSN: `host=server-hostname user=user dbname=db_name options='-c statement_timeout=3600000'`
+        host_dsn = config.dst.owner_dsn + " options='-c statement_timeout=0'"
+        command = ["psql", host_dsn, "-c", f"{c};"]
         logger.info(f"Creating index {index} on the target...")
         try:
             await _execute_subprocess(

{pgbelt-0.7.11 → pgbelt-0.8.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pgbelt"
-version = "0.7.11"
+version = "0.8.1"
 description = "A CLI tool used to manage Postgres data migrations from beginning to end, for a single database or a fleet, leveraging pglogical replication."
 authors = ["Varjitt Jeeva <varjitt.jeeva@autodesk.com>"]
 readme = "README.md"
@@ -12,20 +12,20 @@ packages = [
 [tool.poetry.dependencies]
 python = ">=3.9,<4.0"
 aiofiles = ">=0.8,<24.2"
-asyncpg = ">=0.27,<0.30"
+asyncpg = ">=0.27,<0.31"
 pydantic = ">=2.0,<3.0"
 tabulate = "^0.9.0"
-typer = ">=0.9,<0.13"
+typer = ">=0.9,<0.14"
 
 [tool.poetry.dev-dependencies]
-black = "~24.8.0"
-pre-commit = "~3.8.0"
+black = "~24.10.0"
+pre-commit = "~4.0.1"
 flake8 = "^7.1.1"
-pytest-cov = "~5.0.0"
+pytest-cov = "~6.0.0"
 pytest = "^8.3.3"
 coverage = {extras = ["toml"], version = "^7.6"}
-safety = "^3.2.7"
-mypy = "^1.11"
+safety = "^3.2.11"
+mypy = "^1.13"
 xdoctest = {extras = ["colors"], version = "^1.2.0"}
 flake8-bandit = "~4.1.1"
 flake8-bugbear = ">=21.9.2"
@@ -33,10 +33,10 @@ flake8-docstrings = "^1.6.0"
 flake8-rst-docstrings = "^0.3.0"
 pep8-naming = "^0.14.1"
 darglint = "^1.8.1"
-reorder-python-imports = "^3.13.0"
-pre-commit-hooks = "^4.6.0"
+reorder-python-imports = "^3.14.0"
+pre-commit-hooks = "^5.0.0"
 Pygments = "^2.18.0"
-pyupgrade = "^3.17.0"
+pyupgrade = "^3.19.0"
 pylint = "^3.3.1"
 pytest-asyncio = "~0.24.0"