pgbelt 0.7.11__py3-none-any.whl → 0.8.1__py3-none-any.whl

pgbelt/cmd/schema.py

@@ -1,16 +1,16 @@
 from collections.abc import Awaitable
+from asyncpg import create_pool
 
 from pgbelt.cmd.helpers import run_with_configs
 from pgbelt.config.models import DbupgradeConfig
 from pgbelt.util.dump import apply_target_constraints
 from pgbelt.util.dump import apply_target_schema
 from pgbelt.util.dump import create_target_indexes
-from pgbelt.util.dump import dump_dst_not_valid_constraints
 from pgbelt.util.dump import dump_source_schema
-from pgbelt.util.dump import dump_dst_create_index
 from pgbelt.util.dump import remove_dst_not_valid_constraints
 from pgbelt.util.dump import remove_dst_indexes
 from pgbelt.util.logs import get_logger
+from pgbelt.util.postgres import run_analyze
 
 
 @run_with_configs
@@ -56,17 +56,6 @@ async def load_constraints(config_future: Awaitable[DbupgradeConfig]) -> None:
     await apply_target_constraints(conf, logger)
 
 
-@run_with_configs(skip_src=True)
-async def dump_constraints(config_future: Awaitable[DbupgradeConfig]) -> None:
-    """
-    Dumps the NOT VALID constraints from the target database onto disk, in
-    the schemas directory.
-    """
-    conf = await config_future
-    logger = get_logger(conf.db, conf.dc, "schema.dst")
-    await dump_dst_not_valid_constraints(conf, logger)
-
-
 @run_with_configs(skip_src=True)
 async def remove_constraints(config_future: Awaitable[DbupgradeConfig]) -> None:
     """
@@ -79,17 +68,6 @@ async def remove_constraints(config_future: Awaitable[DbupgradeConfig]) -> None:
     await remove_dst_not_valid_constraints(conf, logger)
 
 
-@run_with_configs(skip_src=True)
-async def dump_indexes(config_future: Awaitable[DbupgradeConfig]) -> None:
-    """
-    Dumps the CREATE INDEX statements from the target database onto disk, in
-    the schemas directory.
-    """
-    conf = await config_future
-    logger = get_logger(conf.db, conf.dc, "schema.dst")
-    await dump_dst_create_index(conf, logger)
-
-
 @run_with_configs(skip_src=True)
 async def remove_indexes(config_future: Awaitable[DbupgradeConfig]) -> None:
     """
@@ -109,19 +87,30 @@ async def create_indexes(config_future: Awaitable[DbupgradeConfig]) -> None:
     as the owner user. This must only be done after most data is synchronized
     (at minimum after the initializing phase) from the source to the destination
     database.
+
+    After creating indexes, the destination database should be analyzed to ensure
+    the query planner has the most up-to-date statistics for the indexes.
     """
     conf = await config_future
     logger = get_logger(conf.db, conf.dc, "schema.dst")
     await create_target_indexes(conf, logger, during_sync=False)
 
+    # Run ANALYZE after creating indexes (without statement timeout)
+    async with create_pool(
+        conf.dst.root_uri,
+        min_size=1,
+        server_settings={
+            "statement_timeout": "0",
+        },
+    ) as dst_pool:
+        await run_analyze(dst_pool, logger)
+
 
 COMMANDS = [
     dump_schema,
     load_schema,
     load_constraints,
-    dump_constraints,
     remove_constraints,
-    dump_indexes,
     remove_indexes,
     create_indexes,
 ]

pgbelt/cmd/status.py

@@ -116,10 +116,20 @@ async def status(conf_future: Awaitable[DbupgradeConfig]) -> dict[str, str]:
 
         result[0].update(result[1])
         result[0]["db"] = conf.db
-        if result[0]["pg1_pg2"] == "replicating":
+
+        # We should hide the progress in the following cases:
+        # 1. When src -> dst is replicating and dst -> src is any state (replicating, unconfigured, down)
+        #    a. We do this because the size when done still will be a tad smaller than SRC, showing <100%
+        # 2. When src -> dst is unconfigured and dst -> src is replicating (not down or unconfigured)
+        #    a. We do this because reverse-only occurs at the start of cutover and onwards, and seeing the progress at that stage is not useful.
+        if (result[0]["pg1_pg2"] == "replicating") or (  # 1
+            result[0]["pg1_pg2"] == "unconfigured"
+            and result[0]["pg2_pg1"] == "replicating"
+        ):  # 2
             result[2]["src_dataset_size"] = "n/a"
             result[2]["dst_dataset_size"] = "n/a"
             result[2]["progress"] = "n/a"
+
         result[0].update(result[2])
         return result[0]
     finally:

pgbelt/cmd/sync.py

@@ -108,40 +108,6 @@ async def load_tables(
     await load_dumped_tables(conf, tables, logger)
 
 
-@run_with_configs
-async def sync_tables(
-    config_future: Awaitable[DbupgradeConfig],
-    tables: list[str] = Option([], help="Specific tables to sync"),
-):
-    """
-    Dump and load all tables from the source database to the destination database.
-    Equivalent to running dump-tables followed by load-tables. Table data will be
-    saved locally in files.
-
-    You may also provide a list of tables to sync with the
-    --tables option and only these tables will be synced.
-    """
-    conf = await config_future
-    src_logger = get_logger(conf.db, conf.dc, "sync.src")
-    dst_logger = get_logger(conf.db, conf.dc, "sync.dst")
-
-    if tables:
-        dump_tables = tables.split(",")
-    else:
-        async with create_pool(conf.src.pglogical_uri, min_size=1) as src_pool:
-            _, dump_tables, _ = await analyze_table_pkeys(
-                src_pool, conf.schema_name, src_logger
-            )
-
-        if conf.tables:
-            dump_tables = [t for t in dump_tables if t in conf.tables]
-
-    await dump_source_tables(conf, dump_tables)
-    await load_dumped_tables(
-        conf, [] if not tables and not conf.tables else dump_tables, dst_logger
-    )
-
-
 @run_with_configs(skip_src=True)
 async def analyze(config_future: Awaitable[DbupgradeConfig]) -> None:
     """
@@ -150,7 +116,13 @@ async def analyze(config_future: Awaitable[DbupgradeConfig]) -> None:
     """
     conf = await config_future
     logger = get_logger(conf.db, conf.dc, "sync.dst")
-    async with create_pool(conf.dst.root_uri, min_size=1) as dst_pool:
+    async with create_pool(
+        conf.dst.root_uri,
+        min_size=1,
+        server_settings={
+            "statement_timeout": "0",
+        },
+    ) as dst_pool:
         await run_analyze(dst_pool, logger)
 
 
@@ -208,8 +180,15 @@ async def sync(
         create_pool(conf.src.pglogical_uri, min_size=1),
         create_pool(conf.dst.root_uri, min_size=1),
         create_pool(conf.dst.owner_uri, min_size=1),
+        create_pool(
+            conf.dst.root_uri,
+            min_size=1,
+            server_settings={
+                "statement_timeout": "0",
+            },
+        ),
     )
-    src_pool, dst_root_pool, dst_owner_pool = pools
+    src_pool, dst_root_pool, dst_owner_pool, dst_root_no_timeout_pool = pools
 
     try:
         src_logger = get_logger(conf.db, conf.dc, "sync.src")
@@ -253,7 +232,7 @@ async def sync(
                 conf.schema_name,
                 validation_logger,
             ),
-            run_analyze(dst_owner_pool, dst_logger),
+            run_analyze(dst_root_no_timeout_pool, dst_logger),
         )
     finally:
         await gather(*[p.close() for p in pools])
@@ -263,7 +242,6 @@ COMMANDS = [
     sync_sequences,
     dump_tables,
     load_tables,
-    sync_tables,
     analyze,
     validate_data,
     sync,

pgbelt/util/dump.py

@@ -310,11 +310,14 @@ async def remove_dst_not_valid_constraints(
         if (config.tables and table in config.tables) or not config.tables:
             queries = queries + f"ALTER TABLE {table} DROP CONSTRAINT {constraint};"
 
-    command = ["psql", config.dst.owner_dsn, "-c", f"'{queries}'"]
+    if queries != "":
+        command = ["psql", config.dst.owner_dsn, "-c", f"'{queries}'"]
 
-    await _execute_subprocess(
-        command, "Finished removing NOT VALID constraints from the target.", logger
-    )
+        await _execute_subprocess(
+            command, "Finished removing NOT VALID constraints from the target.", logger
+        )
+    else:
+        logger.info("No NOT VALID detected for removal.")
 
 
 async def apply_target_constraints(config: DbupgradeConfig, logger: Logger) -> None:
@@ -336,53 +339,6 @@ async def apply_target_constraints(config: DbupgradeConfig, logger: Logger) -> N
     )
 
 
-async def dump_dst_create_index(config: DbupgradeConfig, logger: Logger) -> None:
-    """
-    Dump CREATE INDEX statements from the target database.
-    Used when schema is loaded in outside of pgbelt.
-    """
-
-    logger.info("Dumping target CREATE INDEX statements...")
-
-    command = [
-        "pg_dump",
-        "--schema-only",
-        "--no-owner",
-        "-n",
-        config.schema_name,
-        config.dst.pglogical_dsn,
-    ]
-
-    out = await _execute_subprocess(command, "Retrieved target schema", logger)
-
-    # No username replacement needs to be done, so replace dst user with the same.
-    commands_raw = _parse_dump_commands(
-        out.decode("utf-8"), config.dst.owner_user.name, config.dst.owner_user.name
-    )
-
-    commands = []
-    for c in commands_raw:
-        if "CREATE" in command and "INDEX" in command:
-            regex_matches = search(
-                r"CREATE [UNIQUE ]*INDEX (?P<index>[a-zA-Z0-9._]+)+.*",
-                c,
-            )
-            if not regex_matches:
-                continue
-            commands.append(c)
-
-    try:
-        await makedirs(schema_dir(config.db, config.dc))
-    except FileExistsError:
-        pass
-
-    async with aopen(schema_file(config.db, config.dc, ONLY_INDEXES), "w") as out:
-        for command in commands:
-            await out.write(command)
-
-    logger.debug("Finished dumping CREATE INDEX statements from the target.")
-
-
 async def remove_dst_indexes(config: DbupgradeConfig, logger: Logger) -> None:
     """
     Remove the INDEXes from the schema of the target database.
@@ -395,7 +351,6 @@ async def remove_dst_indexes(config: DbupgradeConfig, logger: Logger) -> None:
 
     logger.info("Removing Indexes from the target...")
 
-    queries = ""
     for c in create_index_statements.split(";"):
         regex_matches = search(
             r"CREATE [UNIQUE ]*INDEX (?P<index>[a-zA-Z0-9._]+)+.*",
@@ -404,14 +359,20 @@ async def remove_dst_indexes(config: DbupgradeConfig, logger: Logger) -> None:
         if not regex_matches:
             continue
         index = regex_matches.groupdict()["index"]
-
-        queries = queries + f"DROP INDEX {index};"
-
-    command = ["psql", config.dst.owner_dsn, "-c", f"'{queries}'"]
-
-    await _execute_subprocess(
-        command, "Finished removing indexes from the target.", logger
-    )
+        if config.schema_name:
+            index = f"{config.schema_name}.{index}"
+
+        # DROP the index
+        # Note that the host DSN must have a statement timeout of 0.
+        # Example DSN: `host=server-hostname user=user dbname=db_name options='-c statement_timeout=3600000'`
+        host_dsn = config.dst.owner_dsn + " options='-c statement_timeout=0'"
+
+        # DROP INDEX IF EXISTS so no need to catch exceptions
+        command = ["psql", host_dsn, "-c", f"DROP INDEX IF EXISTS {index};"]
+        logger.info(f"Dropping index {index} on the target...")
+        await _execute_subprocess(
+            command, f"Finished dropping index {index} on the target.", logger
+        )
 
 
 async def create_target_indexes(
@@ -448,7 +409,10 @@ async def create_target_indexes(
         index = regex_matches.groupdict()["index"]
 
         # Create the index
-        command = ["psql", config.dst.owner_dsn, "-c", f"{c};"]
+        # Note that the host DSN must have a statement timeout of 0.
+        # Example DSN: `host=server-hostname user=user dbname=db_name options='-c statement_timeout=3600000'`
+        host_dsn = config.dst.owner_dsn + " options='-c statement_timeout=0'"
+        command = ["psql", host_dsn, "-c", f"{c};"]
         logger.info(f"Creating index {index} on the target...")
         try:
             await _execute_subprocess(

{pgbelt-0.7.11.dist-info → pgbelt-0.8.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pgbelt
-Version: 0.7.11
+Version: 0.8.1
 Summary: A CLI tool used to manage Postgres data migrations from beginning to end, for a single database or a fleet, leveraging pglogical replication.
 Author: Varjitt Jeeva
 Author-email: varjitt.jeeva@autodesk.com
@@ -10,11 +10,12 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: aiofiles (>=0.8,<24.2)
-Requires-Dist: asyncpg (>=0.27,<0.30)
+Requires-Dist: asyncpg (>=0.27,<0.31)
 Requires-Dist: pydantic (>=2.0,<3.0)
 Requires-Dist: tabulate (>=0.9.0,<0.10.0)
-Requires-Dist: typer (>=0.9,<0.13)
+Requires-Dist: typer (>=0.9,<0.14)
 Description-Content-Type: text/markdown
 
 # Pgbelt

{pgbelt-0.7.11.dist-info → pgbelt-0.8.1.dist-info}/RECORD

@@ -4,10 +4,10 @@ pgbelt/cmd/convenience.py,sha256=RY762BYoyOccmhHqSOaHWIPVBzY4kU0qZautTprGps0,585
 pgbelt/cmd/helpers.py,sha256=anMAkZjJRy7n0EwewyMNJGJ4Jx9keSNZdzWqi3ICNgI,5292
 pgbelt/cmd/login.py,sha256=FNg7shEIC2iwiDjOMza2ZvBv_33U1GMKRh51vsGMTb8,3351
 pgbelt/cmd/preflight.py,sha256=-78Puqqf1rCxNQEyn4bQIAORez_tsy6zJAbSuO_de9s,20676
-pgbelt/cmd/schema.py,sha256=XAAj2BH6HVoA1LxuWw2kheesGDu41L3GAd5UahNp3nE,4760
+pgbelt/cmd/schema.py,sha256=OuxJdUILxlNCvaBV71Si8035pX3LVwHm5Urt0vftO8E,4389
 pgbelt/cmd/setup.py,sha256=Jp5sqT9_whoVBiOzAlOzX1ubtXQADYBkBrJldch_fKk,6627
-pgbelt/cmd/status.py,sha256=LTvteFwA1jcfnGJdlA2siJn5TUtlf14ffMkGVSi5gi0,4730
-pgbelt/cmd/sync.py,sha256=JOOVn7nIIOpQ0bGFqHIXRq87pHOOAD6U8kdQzgngGxs,9035
+pgbelt/cmd/status.py,sha256=8K1c2OMZ3uHNmEh-5a2a0fhTDmCU0RohzgjjVfXoKGo,5385
+pgbelt/cmd/sync.py,sha256=MrrBqCtsBqhL4nyzp78gKYp90xXxlzuHcqWQ5Nt6_hs,8156
 pgbelt/cmd/teardown.py,sha256=Nl37vpugxO7QHO0tKpTfZDV0KtsSm0SbWUbzb3k2N-0,3545
 pgbelt/config/__init__.py,sha256=SXok1aZcpMYJpX_hk5cuKO33CJ5s8IESkswNN9KsVSo,35
 pgbelt/config/config.py,sha256=Kw2H-G1Evfj0TXIbh3k06gE72dZEp_wXWJ2Icq_T54c,3817
@@ -16,12 +16,12 @@ pgbelt/config/remote.py,sha256=D9bOekVfMU1xX2Wy0OiJwSXetxJUdt9Tn5Fukwn9rnE,5307
 pgbelt/main.py,sha256=YiagBiGt8pbNlukkRxROXnQX1Tx6ax7c6riuHRCrPYU,186
 pgbelt/util/__init__.py,sha256=-6KkvVMz-yGNQfeoo4CZZrgWKXYmFd4CMyoiao8OnFE,40
 pgbelt/util/asyncfuncs.py,sha256=7i_GpBmUNNZ8RUGvU-q5nclsoaCm6Lx8jLP8usYvmZc,583
-pgbelt/util/dump.py,sha256=AwyOAd9CP014gvsl-qlo1lbnXZfxoeN4ujZWUIq7KM8,14715
+pgbelt/util/dump.py,sha256=IardD6ESz1Vbet3dEQfAv6IXxLMKQRkaS5z1I6VMmnM,14107
 pgbelt/util/logs.py,sha256=E6doEhi8vrusmYUTZRdjKHP-h2dDiExCjgEtMl7epPw,1793
 pgbelt/util/pglogical.py,sha256=Y6KZBeiH85zhNSvhATqh0xozhfUMyQnPWN1HwRosZFo,13613
 pgbelt/util/postgres.py,sha256=pDl5CTlLUeuWdQgC_wHbO9Ywd69hHYNaasstog0uzvw,20074
-pgbelt-0.7.11.dist-info/LICENSE,sha256=FQ5cFkW02dKK3LmKH8z-rwn93tWSCh7lsxfNUiWcFsg,10758
-pgbelt-0.7.11.dist-info/METADATA,sha256=-zFOMHxif_8fEFDrHPjwCIR9MPqvHWGwNmUBfQj3yVA,2961
-pgbelt-0.7.11.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-pgbelt-0.7.11.dist-info/entry_points.txt,sha256=SCz_poPjkaVnWpJ-CeytAnDzbVc6l0WalOwitIqW_3g,40
-pgbelt-0.7.11.dist-info/RECORD,,
+pgbelt-0.8.1.dist-info/LICENSE,sha256=FQ5cFkW02dKK3LmKH8z-rwn93tWSCh7lsxfNUiWcFsg,10758
+pgbelt-0.8.1.dist-info/METADATA,sha256=AvVEfcvJsWUntrELrs16hu3ewwfgZZXzyHjSazopuDc,3011
+pgbelt-0.8.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+pgbelt-0.8.1.dist-info/entry_points.txt,sha256=SCz_poPjkaVnWpJ-CeytAnDzbVc6l0WalOwitIqW_3g,40
+pgbelt-0.8.1.dist-info/RECORD,,

{pgbelt-0.7.11.dist-info → pgbelt-0.8.1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.0
+Generator: poetry-core 1.9.1
 Root-Is-Purelib: true
 Tag: py3-none-any