pgbelt 0.8.0__py3-none-any.whl → 0.8.2__py3-none-any.whl

pgbelt/cmd/schema.py

@@ -6,9 +6,7 @@ from pgbelt.config.models import DbupgradeConfig
 from pgbelt.util.dump import apply_target_constraints
 from pgbelt.util.dump import apply_target_schema
 from pgbelt.util.dump import create_target_indexes
-from pgbelt.util.dump import dump_dst_not_valid_constraints
 from pgbelt.util.dump import dump_source_schema
-from pgbelt.util.dump import dump_dst_create_index
 from pgbelt.util.dump import remove_dst_not_valid_constraints
 from pgbelt.util.dump import remove_dst_indexes
 from pgbelt.util.logs import get_logger
@@ -58,17 +56,6 @@ async def load_constraints(config_future: Awaitable[DbupgradeConfig]) -> None:
     await apply_target_constraints(conf, logger)
 
 
-@run_with_configs(skip_src=True)
-async def dump_constraints(config_future: Awaitable[DbupgradeConfig]) -> None:
-    """
-    Dumps the NOT VALID constraints from the target database onto disk, in
-    the schemas directory.
-    """
-    conf = await config_future
-    logger = get_logger(conf.db, conf.dc, "schema.dst")
-    await dump_dst_not_valid_constraints(conf, logger)
-
-
 @run_with_configs(skip_src=True)
 async def remove_constraints(config_future: Awaitable[DbupgradeConfig]) -> None:
     """
@@ -81,17 +68,6 @@ async def remove_constraints(config_future: Awaitable[DbupgradeConfig]) -> None:
     await remove_dst_not_valid_constraints(conf, logger)
 
 
-@run_with_configs(skip_src=True)
-async def dump_indexes(config_future: Awaitable[DbupgradeConfig]) -> None:
-    """
-    Dumps the CREATE INDEX statements from the target database onto disk, in
-    the schemas directory.
-    """
-    conf = await config_future
-    logger = get_logger(conf.db, conf.dc, "schema.dst")
-    await dump_dst_create_index(conf, logger)
-
-
 @run_with_configs(skip_src=True)
 async def remove_indexes(config_future: Awaitable[DbupgradeConfig]) -> None:
     """
@@ -134,9 +110,7 @@ COMMANDS = [
     dump_schema,
     load_schema,
     load_constraints,
-    dump_constraints,
     remove_constraints,
-    dump_indexes,
     remove_indexes,
     create_indexes,
 ]

pgbelt/util/dump.py

@@ -93,7 +93,7 @@ async def dump_source_tables(
                 [
                     "pg_dump",
                     "--data-only",
-                    f"--table={config.schema_name}.{table}",
+                    f'--table={config.schema_name}."{table}"',
                     "-Fc",
                     "-f",
                     table_file(config.db, config.dc, table),
@@ -310,11 +310,14 @@ async def remove_dst_not_valid_constraints(
         if (config.tables and table in config.tables) or not config.tables:
             queries = queries + f"ALTER TABLE {table} DROP CONSTRAINT {constraint};"
 
-    command = ["psql", config.dst.owner_dsn, "-c", f"'{queries}'"]
+    if queries != "":
+        command = ["psql", config.dst.owner_dsn, "-c", f"'{queries}'"]
 
-    await _execute_subprocess(
-        command, "Finished removing NOT VALID constraints from the target.", logger
-    )
+        await _execute_subprocess(
+            command, "Finished removing NOT VALID constraints from the target.", logger
+        )
+    else:
+        logger.info("No NOT VALID detected for removal.")
 
 
 async def apply_target_constraints(config: DbupgradeConfig, logger: Logger) -> None:
@@ -336,53 +339,6 @@ async def apply_target_constraints(config: DbupgradeConfig, logger: Logger) -> N
     )
 
 
-async def dump_dst_create_index(config: DbupgradeConfig, logger: Logger) -> None:
-    """
-    Dump CREATE INDEX statements from the target database.
-    Used when schema is loaded in outside of pgbelt.
-    """
-
-    logger.info("Dumping target CREATE INDEX statements...")
-
-    command = [
-        "pg_dump",
-        "--schema-only",
-        "--no-owner",
-        "-n",
-        config.schema_name,
-        config.dst.pglogical_dsn,
-    ]
-
-    out = await _execute_subprocess(command, "Retrieved target schema", logger)
-
-    # No username replacement needs to be done, so replace dst user with the same.
-    commands_raw = _parse_dump_commands(
-        out.decode("utf-8"), config.dst.owner_user.name, config.dst.owner_user.name
-    )
-
-    commands = []
-    for c in commands_raw:
-        if "CREATE" in command and "INDEX" in command:
-            regex_matches = search(
-                r"CREATE [UNIQUE ]*INDEX (?P<index>[a-zA-Z0-9._]+)+.*",
-                c,
-            )
-            if not regex_matches:
-                continue
-            commands.append(c)
-
-    try:
-        await makedirs(schema_dir(config.db, config.dc))
-    except FileExistsError:
-        pass
-
-    async with aopen(schema_file(config.db, config.dc, ONLY_INDEXES), "w") as out:
-        for command in commands:
-            await out.write(command)
-
-    logger.debug("Finished dumping CREATE INDEX statements from the target.")
-
-
 async def remove_dst_indexes(config: DbupgradeConfig, logger: Logger) -> None:
     """
     Remove the INDEXes from the schema of the target database.
@@ -395,7 +351,6 @@ async def remove_dst_indexes(config: DbupgradeConfig, logger: Logger) -> None:
 
     logger.info("Removing Indexes from the target...")
 
-    queries = ""
     for c in create_index_statements.split(";"):
         regex_matches = search(
             r"CREATE [UNIQUE ]*INDEX (?P<index>[a-zA-Z0-9._]+)+.*",
@@ -404,14 +359,20 @@ async def remove_dst_indexes(config: DbupgradeConfig, logger: Logger) -> None:
         if not regex_matches:
             continue
         index = regex_matches.groupdict()["index"]
-
-        queries = queries + f"DROP INDEX {index};"
-
-    command = ["psql", config.dst.owner_dsn, "-c", f"'{queries}'"]
-
-    await _execute_subprocess(
-        command, "Finished removing indexes from the target.", logger
-    )
+        if config.schema_name:
+            index = f"{config.schema_name}.{index}"
+
+        # DROP the index
+        # Note that the host DSN must have a statement timeout of 0.
+        # Example DSN: `host=server-hostname user=user dbname=db_name options='-c statement_timeout=3600000'`
+        host_dsn = config.dst.owner_dsn + " options='-c statement_timeout=0'"
+
+        # DROP INDEX IF EXISTS so no need to catch exceptions
+        command = ["psql", host_dsn, "-c", f"DROP INDEX IF EXISTS {index};"]
+        logger.info(f"Dropping index {index} on the target...")
+        await _execute_subprocess(
+            command, f"Finished dropping index {index} on the target.", logger
+        )
 
 
 async def create_target_indexes(
@@ -440,15 +401,21 @@ async def create_target_indexes(
     for c in create_index_statements.split(";"):
         # Get the Index Name
         regex_matches = search(
-            r"CREATE [UNIQUE ]*INDEX (?P<index>[a-zA-Z0-9._]+)+.*",
+            r"CREATE [UNIQUE ]*INDEX (?P<index>[a-zA-Z0-9._\"]+)+.*",
             c,
         )
         if not regex_matches:
             continue
         index = regex_matches.groupdict()["index"]
 
+        # Sometimes the index name is quoted, so remove the quotes
+        index = index.replace('"', "")
+
         # Create the index
-        command = ["psql", config.dst.owner_dsn, "-c", f"{c};"]
+        # Note that the host DSN must have a statement timeout of 0.
+        # Example DSN: `host=server-hostname user=user dbname=db_name options='-c statement_timeout=3600000'`
+        host_dsn = config.dst.owner_dsn + " options='-c statement_timeout=0'"
+        command = ["psql", host_dsn, "-c", f"{c};"]
         logger.info(f"Creating index {index} on the target...")
         try:
             await _execute_subprocess(

pgbelt/util/postgres.py

@@ -264,7 +264,7 @@ async def table_empty(pool: Pool, table: str, schema: str, logger: Logger) -> bo
     return true if the table is empty
     """
     logger.info(f"Checking if table {table} is empty...")
-    result = await pool.fetch(f"SELECT * FROM {schema}.{table} LIMIT 1;")
+    result = await pool.fetch(f'SELECT * FROM {schema}."{table}" LIMIT 1;')
     return len(result) == 0
 
 

{pgbelt-0.8.0.dist-info → pgbelt-0.8.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pgbelt
-Version: 0.8.0
+Version: 0.8.2
 Summary: A CLI tool used to manage Postgres data migrations from beginning to end, for a single database or a fleet, leveraging pglogical replication.
 Author: Varjitt Jeeva
 Author-email: varjitt.jeeva@autodesk.com
@@ -15,7 +15,7 @@ Requires-Dist: aiofiles (>=0.8,<24.2)
 Requires-Dist: asyncpg (>=0.27,<0.31)
 Requires-Dist: pydantic (>=2.0,<3.0)
 Requires-Dist: tabulate (>=0.9.0,<0.10.0)
-Requires-Dist: typer (>=0.9,<0.14)
+Requires-Dist: typer (>=0.9,<0.16)
 Description-Content-Type: text/markdown
 
 # Pgbelt

{pgbelt-0.8.0.dist-info → pgbelt-0.8.2.dist-info}/RECORD

@@ -4,7 +4,7 @@ pgbelt/cmd/convenience.py,sha256=RY762BYoyOccmhHqSOaHWIPVBzY4kU0qZautTprGps0,585
 pgbelt/cmd/helpers.py,sha256=anMAkZjJRy7n0EwewyMNJGJ4Jx9keSNZdzWqi3ICNgI,5292
 pgbelt/cmd/login.py,sha256=FNg7shEIC2iwiDjOMza2ZvBv_33U1GMKRh51vsGMTb8,3351
 pgbelt/cmd/preflight.py,sha256=-78Puqqf1rCxNQEyn4bQIAORez_tsy6zJAbSuO_de9s,20676
-pgbelt/cmd/schema.py,sha256=NT0ccfzykEezxGcUDDgbAN_jBXovGSWWac5VOvb9TGs,5277
+pgbelt/cmd/schema.py,sha256=OuxJdUILxlNCvaBV71Si8035pX3LVwHm5Urt0vftO8E,4389
 pgbelt/cmd/setup.py,sha256=Jp5sqT9_whoVBiOzAlOzX1ubtXQADYBkBrJldch_fKk,6627
 pgbelt/cmd/status.py,sha256=8K1c2OMZ3uHNmEh-5a2a0fhTDmCU0RohzgjjVfXoKGo,5385
 pgbelt/cmd/sync.py,sha256=MrrBqCtsBqhL4nyzp78gKYp90xXxlzuHcqWQ5Nt6_hs,8156
@@ -16,12 +16,12 @@ pgbelt/config/remote.py,sha256=D9bOekVfMU1xX2Wy0OiJwSXetxJUdt9Tn5Fukwn9rnE,5307
 pgbelt/main.py,sha256=YiagBiGt8pbNlukkRxROXnQX1Tx6ax7c6riuHRCrPYU,186
 pgbelt/util/__init__.py,sha256=-6KkvVMz-yGNQfeoo4CZZrgWKXYmFd4CMyoiao8OnFE,40
 pgbelt/util/asyncfuncs.py,sha256=7i_GpBmUNNZ8RUGvU-q5nclsoaCm6Lx8jLP8usYvmZc,583
-pgbelt/util/dump.py,sha256=AwyOAd9CP014gvsl-qlo1lbnXZfxoeN4ujZWUIq7KM8,14715
+pgbelt/util/dump.py,sha256=R5eI1MFwbv5M6ICnkmjhugj3ZhpWu5sjT20ixEXNfC4,14218
 pgbelt/util/logs.py,sha256=E6doEhi8vrusmYUTZRdjKHP-h2dDiExCjgEtMl7epPw,1793
 pgbelt/util/pglogical.py,sha256=Y6KZBeiH85zhNSvhATqh0xozhfUMyQnPWN1HwRosZFo,13613
-pgbelt/util/postgres.py,sha256=pDl5CTlLUeuWdQgC_wHbO9Ywd69hHYNaasstog0uzvw,20074
-pgbelt-0.8.0.dist-info/LICENSE,sha256=FQ5cFkW02dKK3LmKH8z-rwn93tWSCh7lsxfNUiWcFsg,10758
-pgbelt-0.8.0.dist-info/METADATA,sha256=SzxroPhFZmph0yzRtV2FqEPqU30NCIfaNsk18R2LTIE,3011
-pgbelt-0.8.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-pgbelt-0.8.0.dist-info/entry_points.txt,sha256=SCz_poPjkaVnWpJ-CeytAnDzbVc6l0WalOwitIqW_3g,40
-pgbelt-0.8.0.dist-info/RECORD,,
+pgbelt/util/postgres.py,sha256=0Jx0auJZsSigDX57M1J9ntN1booL1AUxPc8oBYzsbUE,20076
+pgbelt-0.8.2.dist-info/LICENSE,sha256=FQ5cFkW02dKK3LmKH8z-rwn93tWSCh7lsxfNUiWcFsg,10758
+pgbelt-0.8.2.dist-info/METADATA,sha256=dhrSqkEUjn642o_hu0sZn8zUhCQLAMDZmUZSqrBmY4s,3011
+pgbelt-0.8.2.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+pgbelt-0.8.2.dist-info/entry_points.txt,sha256=SCz_poPjkaVnWpJ-CeytAnDzbVc6l0WalOwitIqW_3g,40
+pgbelt-0.8.2.dist-info/RECORD,,