pgbelt 0.9.2__py3-none-any.whl → 0.9.4__py3-none-any.whl

pgbelt/cmd/sync.py

@@ -12,8 +12,9 @@ from pgbelt.util.dump import dump_source_tables
 from pgbelt.util.dump import load_dumped_tables
 from pgbelt.util.logs import get_logger
 from pgbelt.util.postgres import analyze_table_pkeys
-from pgbelt.util.postgres import compare_100_rows
+from pgbelt.util.postgres import compare_100_random_rows
 from pgbelt.util.postgres import compare_latest_100_rows
+from pgbelt.util.postgres import compare_tables_without_pkeys
 from pgbelt.util.postgres import dump_sequences
 from pgbelt.util.postgres import load_sequences
 from pgbelt.util.postgres import run_analyze
@@ -143,10 +144,15 @@ async def validate_data(config_future: Awaitable[DbupgradeConfig]) -> None:
     try:
         logger = get_logger(conf.db, conf.dc, "sync")
         await gather(
-            compare_100_rows(src_pool, dst_pool, conf.tables, conf.schema_name, logger),
+            compare_100_random_rows(
+                src_pool, dst_pool, conf.tables, conf.schema_name, logger
+            ),
             compare_latest_100_rows(
                 src_pool, dst_pool, conf.tables, conf.schema_name, logger
             ),
+            compare_tables_without_pkeys(
+                src_pool, dst_pool, conf.tables, conf.schema_name, logger
+            ),
         )
     finally:
         await gather(*[p.close() for p in pools])
@@ -218,7 +224,7 @@ async def sync(
             )
 
         await gather(
-            compare_100_rows(
+            compare_100_random_rows(
                 src_pool,
                 dst_owner_pool,
                 conf.tables,

pgbelt/util/dump.py

@@ -90,8 +90,22 @@ async def _dump_table(config: DbupgradeConfig, table: str, logger: Logger) -> No
     out = await _execute_subprocess(command, f"dumped {table}", logger)
     content = out.decode("utf-8")
 
-    # Strip out unwanted lines, stupid PG17 adding transaction_timeout lines.
-    keywords = ["transaction_timeout"]
+    # Strip out unwanted lines, stupid PG17
+    keywords = [
+        "transaction_timeout",
+        # "SET statement_timeout", # This one is fine
+        # "SET lock_timeout", # This one is fine
+        # "SET idle_in_transaction_session_timeout", # This one is fine
+        "SET client_encoding",
+        "SET standard_conforming_strings",
+        "SET check_function_bodies",
+        "SET xmloption",
+        "SET client_min_messages",
+        "SET row_security",
+        "pg_catalog.set_config",  # Stupid search path, this should not be run.
+        "\\restrict",
+        "\\unrestrict",
+    ]
     lines = content.split("\n")
     filtered_lines = [
         line for line in lines if not any(keyword in line for keyword in keywords)

pgbelt/util/postgres.py

@@ -214,22 +214,23 @@ async def compare_data(
     )
 
 
-async def compare_100_rows(
+async def compare_100_random_rows(
     src_pool: Pool, dst_pool: Pool, tables: list[str], schema: str, logger: Logger
 ) -> None:
     """
     Validate data between source and destination databases by doing the following:
     1. Get all tables with primary keys
-    2. For each of those tables, select * limit 100
+    2. For each of those tables, select 100 random rows
     3. For each row, ensure the row in the destination is identical
     """
-    logger.info("Comparing 100 rows...")
+    logger.info("Comparing 100 random rows...")
 
     query = """
     SELECT * FROM
     (
         SELECT *
         FROM {table}
+        ORDER BY RANDOM()
         LIMIT 100
     ) AS T1
     ORDER BY {order_by_pkeys};
@@ -259,6 +260,113 @@ async def compare_latest_100_rows(
     await compare_data(src_pool, dst_pool, query, tables, schema, logger)
 
 
+async def compare_tables_without_pkeys(
+    src_pool: Pool,
+    dst_pool: Pool,
+    tables: list[str],
+    schema: str,
+    logger: Logger,
+) -> None:
+    """
+    Validate data for tables without primary keys by:
+    1. Getting the list of tables without primary keys
+    2. For each table, selecting 100 random rows from source
+    3. For each row, verifying it exists in destination by matching all columns
+    """
+    logger.info("Comparing tables without primary keys...")
+
+    _, no_pkeys, _ = await analyze_table_pkeys(src_pool, schema, logger)
+
+    # Filter by tables list if provided
+    if tables:
+        no_pkeys = [t for t in no_pkeys if t in tables]
+
+    if not no_pkeys:
+        logger.info("No tables without primary keys to compare.")
+        return
+
+    src_old_extra_float_digits = await src_pool.fetchval("SHOW extra_float_digits;")
+    await src_pool.execute("SET extra_float_digits TO 0;")
+
+    dst_old_extra_float_digits = await dst_pool.fetchval("SHOW extra_float_digits;")
+    await dst_pool.execute("SET extra_float_digits TO 0;")
+
+    for table in no_pkeys:
+        full_table_name = f'{schema}."{table}"'
+        logger.debug(f"Validating table without primary key: {full_table_name}...")
+
+        # Select 100 random rows from source
+        query = f"""
+        SELECT * FROM {full_table_name}
+        ORDER BY RANDOM()
+        LIMIT 100;
+        """
+
+        src_rows = await src_pool.fetch(query)
+
+        if len(src_rows) == 0:
+            logger.debug(f"Table {full_table_name} is empty in source.")
+            continue
+
+        # For each source row, check if it exists in destination
+        for src_row in src_rows:
+            # Build WHERE clause matching all columns
+            where_clauses = []
+            for key, value in src_row.items():
+                # Handle Decimal NaN values
+                if isinstance(value, Decimal) and value.is_nan():
+                    value = None
+
+                if value is None:
+                    where_clauses.append(f'"{key}" IS NULL')
+                elif isinstance(value, (int, float, Decimal)):
+                    where_clauses.append(f'"{key}" = {value}')
+                elif isinstance(value, bool):
+                    where_clauses.append(f'"{key}" = {str(value).upper()}')
+                elif isinstance(value, bytes):
+                    hex_val = value.hex()
+                    where_clauses.append(f"\"{key}\" = '\\x{hex_val}'")
+                elif isinstance(value, list):
+                    # Handle PostgreSQL arrays - format as '{val1, val2, ...}'
+                    # Need this and not in the PK comparison because these rows are compared by row here only, not by row there also broken by column.
+                    escaped_elements = []
+                    for elem in value:
+                        if elem is None:
+                            escaped_elements.append("NULL")
+                        else:
+                            # Escape double quotes and backslashes in array elements
+                            escaped_elem = (
+                                str(elem).replace("\\", "\\\\").replace('"', '\\"')
+                            )
+                            escaped_elements.append(f'"{escaped_elem}"')
+                    array_literal = "{" + ",".join(escaped_elements) + "}"
+                    where_clauses.append(f"\"{key}\" = '{array_literal}'")
+                else:
+                    # Escape single quotes in string values
+                    escaped_val = str(value).replace("'", "''")
+                    where_clauses.append(f"\"{key}\" = '{escaped_val}'")
+
+            where_clause = " AND ".join(where_clauses)
+            check_query = (
+                f"SELECT 1 FROM {full_table_name} WHERE {where_clause} LIMIT 1;"
+            )
+
+            dst_result = await dst_pool.fetch(check_query)
+
+            if len(dst_result) == 0:
+                raise AssertionError(
+                    f"Row from source not found in destination.\n"
+                    f"Table: {full_table_name}\n"
+                    f"Source Row: {dict(src_row)}"
+                )
+
+        logger.debug(f"Table {full_table_name} validated successfully.")
+
+    await src_pool.execute(f"SET extra_float_digits TO {src_old_extra_float_digits};")
+    await dst_pool.execute(f"SET extra_float_digits TO {dst_old_extra_float_digits};")
+    logger.info("Tables without primary keys validation complete!")
+
+
 async def table_empty(pool: Pool, table: str, schema: str, logger: Logger) -> bool:
     """
     return true if the table is empty

{pgbelt-0.9.2.dist-info → pgbelt-0.9.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pgbelt
-Version: 0.9.2
+Version: 0.9.4
 Summary: A CLI tool used to manage Postgres data migrations from beginning to end, for a single database or a fleet, leveraging pglogical replication.
 License-File: LICENSE
 Author: Varjitt Jeeva

{pgbelt-0.9.2.dist-info → pgbelt-0.9.4.dist-info}/RECORD

@@ -7,7 +7,7 @@ pgbelt/cmd/preflight.py,sha256=GH2IE-yb1HvbpwSwm-jJG1dnRO9CXfSsTH5OqYEVyv4,21557
 pgbelt/cmd/schema.py,sha256=OuxJdUILxlNCvaBV71Si8035pX3LVwHm5Urt0vftO8E,4389
 pgbelt/cmd/setup.py,sha256=Jp5sqT9_whoVBiOzAlOzX1ubtXQADYBkBrJldch_fKk,6627
 pgbelt/cmd/status.py,sha256=8K1c2OMZ3uHNmEh-5a2a0fhTDmCU0RohzgjjVfXoKGo,5385
-pgbelt/cmd/sync.py,sha256=MrrBqCtsBqhL4nyzp78gKYp90xXxlzuHcqWQ5Nt6_hs,8156
+pgbelt/cmd/sync.py,sha256=iLFrVNx5_rmhlSbLOMuPTgNmPhIdVx3DBT-QKjcNIpc,8400
 pgbelt/cmd/teardown.py,sha256=Nl37vpugxO7QHO0tKpTfZDV0KtsSm0SbWUbzb3k2N-0,3545
 pgbelt/config/__init__.py,sha256=SXok1aZcpMYJpX_hk5cuKO33CJ5s8IESkswNN9KsVSo,35
 pgbelt/config/config.py,sha256=Kw2H-G1Evfj0TXIbh3k06gE72dZEp_wXWJ2Icq_T54c,3817
@@ -16,12 +16,12 @@ pgbelt/config/remote.py,sha256=D9bOekVfMU1xX2Wy0OiJwSXetxJUdt9Tn5Fukwn9rnE,5307
 pgbelt/main.py,sha256=YiagBiGt8pbNlukkRxROXnQX1Tx6ax7c6riuHRCrPYU,186
 pgbelt/util/__init__.py,sha256=-6KkvVMz-yGNQfeoo4CZZrgWKXYmFd4CMyoiao8OnFE,40
 pgbelt/util/asyncfuncs.py,sha256=7i_GpBmUNNZ8RUGvU-q5nclsoaCm6Lx8jLP8usYvmZc,583
-pgbelt/util/dump.py,sha256=W814ULJPM9IBEhmvMFRD2E8DkqTuXZRuRtLXT8k4xhU,14691
+pgbelt/util/dump.py,sha256=J0TXPv4LfsrRnt4ppVgxFK_E9tTueLcsAY4PQLVKXAI,15172
 pgbelt/util/logs.py,sha256=3Kk_dERXLsTwLxFS6tzI8fE0sRi5QrBYHJc4Al6ZCMA,2110
 pgbelt/util/pglogical.py,sha256=Y6KZBeiH85zhNSvhATqh0xozhfUMyQnPWN1HwRosZFo,13613
-pgbelt/util/postgres.py,sha256=ftBontoWkU7XaDXaBnsSJYYx7s2Su0xkcTyVCzwKJm0,20080
-pgbelt-0.9.2.dist-info/METADATA,sha256=we33iVu7QH6GjgfHHqzL9ISEfq73y2S8gf5wySkPHHQ,2933
-pgbelt-0.9.2.dist-info/WHEEL,sha256=kJCRJT_g0adfAJzTx2GUMmS80rTJIVHRCfG0DQgLq3o,88
-pgbelt-0.9.2.dist-info/entry_points.txt,sha256=SCz_poPjkaVnWpJ-CeytAnDzbVc6l0WalOwitIqW_3g,40
-pgbelt-0.9.2.dist-info/licenses/LICENSE,sha256=FQ5cFkW02dKK3LmKH8z-rwn93tWSCh7lsxfNUiWcFsg,10758
-pgbelt-0.9.2.dist-info/RECORD,,
+pgbelt/util/postgres.py,sha256=poUA8PJ71OJZupfCh7k-7qBAwVxocJYAzhZ2NajSfxk,24551
+pgbelt-0.9.4.dist-info/METADATA,sha256=1MA0qdMJ1lDd_LQ1DO9xzJLybhVcC3AH9iUU0MjSgC4,2933
+pgbelt-0.9.4.dist-info/WHEEL,sha256=kJCRJT_g0adfAJzTx2GUMmS80rTJIVHRCfG0DQgLq3o,88
+pgbelt-0.9.4.dist-info/entry_points.txt,sha256=SCz_poPjkaVnWpJ-CeytAnDzbVc6l0WalOwitIqW_3g,40
+pgbelt-0.9.4.dist-info/licenses/LICENSE,sha256=FQ5cFkW02dKK3LmKH8z-rwn93tWSCh7lsxfNUiWcFsg,10758
+pgbelt-0.9.4.dist-info/RECORD,,