pgbelt 0.7.1__tar.gz → 0.7.2__tar.gz

{pgbelt-0.7.1 → pgbelt-0.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pgbelt
-Version: 0.7.1
+Version: 0.7.2
 Summary: A CLI tool used to manage Postgres data migrations from beginning to end, for a single database or a fleet, leveraging pglogical replication.
 Author: Varjitt Jeeva
 Author-email: varjitt.jeeva@autodesk.com

{pgbelt-0.7.1 → pgbelt-0.7.2}/pgbelt/cmd/setup.py

@@ -41,7 +41,23 @@ async def _setup_src_node(
 
     pglogical_tables = pkey_tables
     if conf.tables:
-        pglogical_tables = [t for t in pkey_tables if t in conf.tables]
+        pglogical_tables = [
+            t
+            for t in pkey_tables
+            if t
+            in list(
+                map(str.lower, conf.tables)
+            )  # Postgres returns table names in lowercase (in analyze_table_pkeys)
+        ]
+
+    # Intentionally throw an error if no tables are found, so that the user can correct their config.
+    # When reported by a certain user, errors showed when running the status command, but it was ignored,
+    # then the user ran setup and since that DIDN'T throw an error, they assumed everything was fine.
+
+    if not pglogical_tables:
+        raise ValueError(
+            f"No tables were targeted to replicate. Please check your config's schema and tables. DB: {conf.db} DC: {conf.dc}, SCHEMA: {conf.schema_name} TABLES: {conf.tables}.\nIf TABLES is [], all tables in the schema should be replicated, but pgbelt still found no tables.\nCheck the schema name or reach out to the pgbelt team for help."
+        )
 
     await configure_replication_set(
         src_root_pool, pglogical_tables, conf.schema_name, src_logger
@@ -145,7 +161,14 @@ async def setup_back_replication(config_future: Awaitable[DbupgradeConfig]) -> N
 
         pglogical_tables = pkeys
         if conf.tables:
-            pglogical_tables = [t for t in pkeys if t in conf.tables]
+            pglogical_tables = [
+                t
+                for t in pkeys
+                if t
+                in list(
+                    map(str.lower, conf.tables)
+                )  # Postgres returns table names in lowercase (in analyze_table_pkeys)
+            ]
 
         await configure_replication_set(
             dst_root_pool, pglogical_tables, conf.schema_name, dst_logger

{pgbelt-0.7.1 → pgbelt-0.7.2}/pgbelt/cmd/status.py

@@ -92,7 +92,14 @@ async def status(conf_future: Awaitable[DbupgradeConfig]) -> dict[str, str]:
     all_tables = pkey_tables + non_pkey_tables
     target_tables = all_tables
     if conf.tables:
-        target_tables = [t for t in all_tables if t in conf.tables]
+        target_tables = [
+            t
+            for t in all_tables
+            if t
+            in list(
+                map(str.lower, conf.tables)
+            )  # Postgres gave us lowercase table names in analyze_table_pkeys
+        ]
 
     if not target_tables:
         raise ValueError(

{pgbelt-0.7.1 → pgbelt-0.7.2}/pgbelt/cmd/sync.py

@@ -77,7 +77,14 @@ async def dump_tables(
             _, tables, _ = await analyze_table_pkeys(src_pool, conf.schema_name, logger)
 
         if conf.tables:
-            tables = [t for t in tables if t in conf.tables]
+            tables = [
+                t
+                for t in tables
+                if t
+                in list(
+                    map(str.lower, conf.tables)
+                )  # Postgres returns table names in lowercase (in analyze_table_pkeys)
+            ]
 
     await dump_source_tables(conf, tables, logger)
 
@@ -185,7 +192,14 @@ async def _dump_and_load_all_tables(
 ) -> None:
     _, tables, _ = await analyze_table_pkeys(src_pool, conf.schema_name, src_logger)
     if conf.tables:
-        tables = [t for t in tables if t in conf.tables]
+        tables = [
+            t
+            for t in tables
+            if t
+            in list(
+                map(str.lower, conf.tables)
+            )  # Postgres returns table names in lowercase (in analyze_table_pkeys)
+        ]
     await dump_source_tables(conf, tables, src_logger)
     await load_dumped_tables(conf, tables, dst_logger)
 

{pgbelt-0.7.1 → pgbelt-0.7.2}/pgbelt/util/postgres.py

@@ -97,10 +97,16 @@ async def compare_data(
     dst_old_extra_float_digits = await dst_pool.fetchval("SHOW extra_float_digits;")
     await dst_pool.execute("SET extra_float_digits TO 0;")
 
+    has_run = False
     for table in set(pkeys):
-        # If specific table list is defined and iterated table is not in that list, skip.
-        if tables and (table not in tables):
+        # If specific table list is defined and the iterated table is not in that list, skip.
+        # Note that the pkeys tables returned from Postgres are all lowercased, so we need to
+        # map the passed conf tables to lowercase.
+        if tables and (table not in list(map(str.lower, tables))):
             continue
+
+        has_run = True  # If this runs, we have at least one table to compare. We will use this flag to throw an error if no tables are found.
+
         full_table_name = f"{schema}.{table}"
 
         logger.debug(f"Validating table {full_table_name}...")
@@ -171,6 +177,13 @@ async def compare_data(
                     f"Dest Row: {dst_row}"
                 )
 
+    # Just a paranoia check. If this throws, then it's possible pgbelt didn't migrate any data.
+    # This was found in issue #420, and previous commands threw errors before this issue could arise.
+    if not has_run:
+        raise ValueError(
+            "No tables were found to compare. Please reach out to the pgbelt for help, and check if your data was migrated."
+        )
+
     await src_pool.execute(f"SET extra_float_digits TO {src_old_extra_float_digits};")
     await dst_pool.execute(f"SET extra_float_digits TO {dst_old_extra_float_digits};")
     logger.info(
@@ -372,9 +385,22 @@ async def precheck_info(
               AND n.nspname <> 'pglogical'
         ORDER BY 1,2;"""
     )
+
     # We filter the table list if the user has specified a list of tables to target.
+    # Note, from issue #420, the above query will return the table names in lowercase,
+    # so we need to map the target_tables to lowercase.
     if target_tables:
-        result["tables"] = [t for t in result["tables"] if t["Name"] in target_tables]
+
+        result["tables"] = [
+            t
+            for t in result["tables"]
+            if t["Name"] in list(map(str.lower, target_tables))
+        ]
+
+        # We will not recapitalize the table names in the result["tables"] list,
+        # to preserve how Postgres sees those tables in its system catalog. Easy
+        # rabbit hole later if we keep patching the table names to match the user's
+        # input.
 
     result["sequences"] = await pool.fetch(
         """
@@ -392,12 +418,22 @@ async def precheck_info(
         ORDER BY 1,2;"""
     )
 
-    # We filter the sequence list if the user has specified a list of sequences to target.
+    # We filter the table list if the user has specified a list of tables to target.
+    # Note, from issue #420, the above query will return the table names in lowercase,
+    # so we need to map the target_tables to lowercase.
     if target_sequences:
+
         result["sequences"] = [
-            s for s in result["sequences"] if s["Name"] in target_sequences
+            t
+            for t in result["sequences"]
+            if t["Name"] in list(map(str.lower, target_sequences))
         ]
 
+        # We will not recapitalize the table names in the result["tables"] list,
+        # to preserve how Postgres sees those tables in its system catalog. Easy
+        # rabbit hole later if we keep patching the table names to match the user's
+        # input.
+
     users = await pool.fetch(
         f"""
         SELECT r.rolname, r.rolsuper, r.rolinherit,

{pgbelt-0.7.1 → pgbelt-0.7.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pgbelt"
-version = "0.7.1"
+version = "0.7.2"
 description = "A CLI tool used to manage Postgres data migrations from beginning to end, for a single database or a fleet, leveraging pglogical replication."
 authors = ["Varjitt Jeeva <varjitt.jeeva@autodesk.com>"]
 readme = "README.md"
@@ -18,14 +18,14 @@ tabulate = "^0.9.0"
 typer = "^0.9.0"
 
 [tool.poetry.dev-dependencies]
-black = "~24.2.0"
+black = "~24.3.0"
 pre-commit = "~3.6.2"
 flake8 = "^7.0.0"
 pytest-cov = "~4.1.0"
-pytest = "^8.0.2"
+pytest = "^8.1.1"
 coverage = {extras = ["toml"], version = "^7.4"}
 safety = "^2.3.1"
-mypy = "^1.8"
+mypy = "^1.9"
 xdoctest = {extras = ["colors"], version = "^1.1.3"}
 flake8-bandit = "~4.1.1"
 flake8-bugbear = ">=21.9.2"