pgbelt 0.7.0__tar.gz → 0.7.2__tar.gz

{pgbelt-0.7.0 → pgbelt-0.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pgbelt
-Version: 0.7.0
+Version: 0.7.2
 Summary: A CLI tool used to manage Postgres data migrations from beginning to end, for a single database or a fleet, leveraging pglogical replication.
 Author: Varjitt Jeeva
 Author-email: varjitt.jeeva@autodesk.com

{pgbelt-0.7.0 → pgbelt-0.7.2}/pgbelt/cmd/setup.py

@@ -41,7 +41,23 @@ async def _setup_src_node(
 
     pglogical_tables = pkey_tables
     if conf.tables:
-        pglogical_tables = [t for t in pkey_tables if t in conf.tables]
+        pglogical_tables = [
+            t
+            for t in pkey_tables
+            if t
+            in list(
+                map(str.lower, conf.tables)
+            )  # Postgres returns table names in lowercase (in analyze_table_pkeys)
+        ]
+
+    # Intentionally throw an error if no tables are found, so that the user can correct their config.
+    # When reported by a certain user, errors showed when running the status command, but it was ignored,
+    # then the user ran setup and since that DIDN'T throw an error, they assumed everything was fine.
+
+    if not pglogical_tables:
+        raise ValueError(
+            f"No tables were targeted to replicate. Please check your config's schema and tables. DB: {conf.db} DC: {conf.dc}, SCHEMA: {conf.schema_name} TABLES: {conf.tables}.\nIf TABLES is [], all tables in the schema should be replicated, but pgbelt still found no tables.\nCheck the schema name or reach out to the pgbelt team for help."
+        )
 
     await configure_replication_set(
         src_root_pool, pglogical_tables, conf.schema_name, src_logger
@@ -145,7 +161,14 @@ async def setup_back_replication(config_future: Awaitable[DbupgradeConfig]) -> N
 
         pglogical_tables = pkeys
         if conf.tables:
-            pglogical_tables = [t for t in pkeys if t in conf.tables]
+            pglogical_tables = [
+                t
+                for t in pkeys
+                if t
+                in list(
+                    map(str.lower, conf.tables)
+                )  # Postgres returns table names in lowercase (in analyze_table_pkeys)
+            ]
 
         await configure_replication_set(
             dst_root_pool, pglogical_tables, conf.schema_name, dst_logger

{pgbelt-0.7.0 → pgbelt-0.7.2}/pgbelt/cmd/status.py

@@ -92,7 +92,14 @@ async def status(conf_future: Awaitable[DbupgradeConfig]) -> dict[str, str]:
     all_tables = pkey_tables + non_pkey_tables
     target_tables = all_tables
     if conf.tables:
-        target_tables = [t for t in all_tables if t in conf.tables]
+        target_tables = [
+            t
+            for t in all_tables
+            if t
+            in list(
+                map(str.lower, conf.tables)
+            )  # Postgres gave us lowercase table names in analyze_table_pkeys
+        ]
 
     if not target_tables:
         raise ValueError(

{pgbelt-0.7.0 → pgbelt-0.7.2}/pgbelt/cmd/sync.py

@@ -28,6 +28,7 @@ async def _sync_sequences(
     src_logger: Logger,
     dst_logger: Logger,
 ) -> None:
+
     seq_vals = await dump_sequences(src_pool, targeted_sequences, schema, src_logger)
     await load_sequences(dst_pool, seq_vals, dst_logger)
 
@@ -76,7 +77,14 @@ async def dump_tables(
             _, tables, _ = await analyze_table_pkeys(src_pool, conf.schema_name, logger)
 
         if conf.tables:
-            tables = [t for t in tables if t in conf.tables]
+            tables = [
+                t
+                for t in tables
+                if t
+                in list(
+                    map(str.lower, conf.tables)
+                )  # Postgres returns table names in lowercase (in analyze_table_pkeys)
+            ]
 
     await dump_source_tables(conf, tables, logger)
 
@@ -184,7 +192,14 @@ async def _dump_and_load_all_tables(
 ) -> None:
     _, tables, _ = await analyze_table_pkeys(src_pool, conf.schema_name, src_logger)
     if conf.tables:
-        tables = [t for t in tables if t in conf.tables]
+        tables = [
+            t
+            for t in tables
+            if t
+            in list(
+                map(str.lower, conf.tables)
+            )  # Postgres returns table names in lowercase (in analyze_table_pkeys)
+        ]
     await dump_source_tables(conf, tables, src_logger)
     await load_dumped_tables(conf, tables, dst_logger)
 

{pgbelt-0.7.0 → pgbelt-0.7.2}/pgbelt/util/pglogical.py

@@ -78,8 +78,9 @@ async def grant_pgl(pool: Pool, tables: list[str], schema: str, logger: Logger)
     async with pool.acquire() as conn:
         async with conn.transaction():
             if tables:
+                tables_with_schema = [f"{schema}.{table}" for table in tables]
                 await conn.execute(
-                    f"GRANT ALL ON TABLE {','.join(tables)} TO pglogical;"
+                    f"GRANT ALL ON TABLE {','.join(tables_with_schema)} TO pglogical;"
                 )
             else:
                 await conn.execute(

{pgbelt-0.7.0 → pgbelt-0.7.2}/pgbelt/util/postgres.py

@@ -12,20 +12,38 @@ async def dump_sequences(
     return a dictionary of sequence names mapped to their last values
     """
     logger.info("Dumping sequence values...")
-    seqs = await pool.fetch("SELECT sequence_name FROM information_schema.sequences;")
+    # Get all sequences in the schema
+    seqs = await pool.fetch(
+        f"""
+        SELECT '{schema}' || '.' || sequence_name
+        FROM information_schema.sequences
+        WHERE sequence_schema = '{schema}';
+        """
+    )
+
+    # Note: When in an exodus migration with a non-public schema, the sequence names must be prefixed with the schema name.
+    # This may not be done by the user, so we must do it here.
+    proper_sequence_names = None
+    if targeted_sequences is not None:
+        proper_sequence_names = []
+        for seq in targeted_sequences:
+            if f"{schema}." not in seq:
+                proper_sequence_names.append(f"{schema}.{seq}")
+            else:
+                proper_sequence_names.append(seq)
+    targeted_sequences = proper_sequence_names
 
     seq_vals = {}
+    final_seqs = []
+    # If we get a list of targeted sequences, we only want to dump whichever of those are found in the database and schema.
     if targeted_sequences:
-        for seq in [r[0] for r in seqs if r[0] in targeted_sequences]:
-            seq_vals[seq.strip()] = await pool.fetchval(
-                f"SELECT last_value FROM {schema}.{seq};"
-            )
-    else:
-        for seq in [r[0] for r in seqs]:
-            seq_stripped = seq.strip()
-            seq_vals[f"{schema}.{seq_stripped}"] = await pool.fetchval(
-                f"SELECT last_value FROM {schema}.{seq};"
-            )
+        final_seqs = [r[0] for r in seqs if r[0] in targeted_sequences]
+    else:  # Otherwise, we want to dump all sequences found in the schema.
+        final_seqs = [r[0] for r in seqs]
+
+    for seq in final_seqs:
+        res = await pool.fetchval(f"SELECT last_value FROM {seq};")
+        seq_vals[seq.strip()] = res
 
     logger.debug(f"Dumped sequences: {seq_vals}")
     return seq_vals
@@ -79,10 +97,16 @@ async def compare_data(
     dst_old_extra_float_digits = await dst_pool.fetchval("SHOW extra_float_digits;")
     await dst_pool.execute("SET extra_float_digits TO 0;")
 
+    has_run = False
     for table in set(pkeys):
-        # If specific table list is defined and iterated table is not in that list, skip.
-        if tables and (table not in tables):
+        # If specific table list is defined and the iterated table is not in that list, skip.
+        # Note that the pkeys tables returned from Postgres are all lowercased, so we need to
+        # map the passed conf tables to lowercase.
+        if tables and (table not in list(map(str.lower, tables))):
             continue
+
+        has_run = True  # If this runs, we have at least one table to compare. We will use this flag to throw an error if no tables are found.
+
         full_table_name = f"{schema}.{table}"
 
         logger.debug(f"Validating table {full_table_name}...")
@@ -153,6 +177,13 @@ async def compare_data(
                     f"Dest Row: {dst_row}"
                 )
 
+    # Just a paranoia check. If this throws, then it's possible pgbelt didn't migrate any data.
+    # This was found in issue #420, and previous commands threw errors before this issue could arise.
+    if not has_run:
+        raise ValueError(
+            "No tables were found to compare. Please reach out to the pgbelt for help, and check if your data was migrated."
+        )
+
     await src_pool.execute(f"SET extra_float_digits TO {src_old_extra_float_digits};")
     await dst_pool.execute(f"SET extra_float_digits TO {dst_old_extra_float_digits};")
     logger.info(
@@ -354,9 +385,22 @@ async def precheck_info(
               AND n.nspname <> 'pglogical'
         ORDER BY 1,2;"""
     )
+
     # We filter the table list if the user has specified a list of tables to target.
+    # Note, from issue #420, the above query will return the table names in lowercase,
+    # so we need to map the target_tables to lowercase.
     if target_tables:
-        result["tables"] = [t for t in result["tables"] if t["Name"] in target_tables]
+
+        result["tables"] = [
+            t
+            for t in result["tables"]
+            if t["Name"] in list(map(str.lower, target_tables))
+        ]
+
+        # We will not recapitalize the table names in the result["tables"] list,
+        # to preserve how Postgres sees those tables in its system catalog. Easy
+        # rabbit hole later if we keep patching the table names to match the user's
+        # input.
 
     result["sequences"] = await pool.fetch(
         """
@@ -374,12 +418,22 @@ async def precheck_info(
         ORDER BY 1,2;"""
     )
 
-    # We filter the sequence list if the user has specified a list of sequences to target.
+    # We filter the table list if the user has specified a list of tables to target.
+    # Note, from issue #420, the above query will return the table names in lowercase,
+    # so we need to map the target_tables to lowercase.
     if target_sequences:
+
         result["sequences"] = [
-            s for s in result["sequences"] if s["Name"] in target_sequences
+            t
+            for t in result["sequences"]
+            if t["Name"] in list(map(str.lower, target_sequences))
         ]
 
+        # We will not recapitalize the table names in the result["tables"] list,
+        # to preserve how Postgres sees those tables in its system catalog. Easy
+        # rabbit hole later if we keep patching the table names to match the user's
+        # input.
+
     users = await pool.fetch(
         f"""
         SELECT r.rolname, r.rolsuper, r.rolinherit,

{pgbelt-0.7.0 → pgbelt-0.7.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pgbelt"
-version = "0.7.0"
+version = "0.7.2"
 description = "A CLI tool used to manage Postgres data migrations from beginning to end, for a single database or a fleet, leveraging pglogical replication."
 authors = ["Varjitt Jeeva <varjitt.jeeva@autodesk.com>"]
 readme = "README.md"
@@ -18,14 +18,14 @@ tabulate = "^0.9.0"
 typer = "^0.9.0"
 
 [tool.poetry.dev-dependencies]
-black = "~24.2.0"
+black = "~24.3.0"
 pre-commit = "~3.6.2"
 flake8 = "^7.0.0"
 pytest-cov = "~4.1.0"
-pytest = "^8.0.2"
+pytest = "^8.1.1"
 coverage = {extras = ["toml"], version = "^7.4"}
 safety = "^2.3.1"
-mypy = "^1.8"
+mypy = "^1.9"
 xdoctest = {extras = ["colors"], version = "^1.1.3"}
 flake8-bandit = "~4.1.1"
 flake8-bugbear = ">=21.9.2"