pgbelt 0.7.1__py3-none-any.whl → 0.7.3__py3-none-any.whl

pgbelt/cmd/setup.py

@@ -43,6 +43,15 @@ async def _setup_src_node(
     if conf.tables:
         pglogical_tables = [t for t in pkey_tables if t in conf.tables]
 
+    # Intentionally throw an error if no tables are found, so that the user can correct their config.
+    # When reported by a certain user, errors showed when running the status command, but it was ignored,
+    # then the user ran setup and since that DIDN'T throw an error, they assumed everything was fine.
+
+    if not pglogical_tables:
+        raise ValueError(
+            f"No tables were targeted to replicate. Please check your config's schema and tables. DB: {conf.db} DC: {conf.dc}, SCHEMA: {conf.schema_name} TABLES: {conf.tables}.\nIf TABLES is [], all tables in the schema should be replicated, but pgbelt still found no tables.\nCheck the schema name or reach out to the pgbelt team for help."
+        )
+
     await configure_replication_set(
         src_root_pool, pglogical_tables, conf.schema_name, src_logger
     )

pgbelt/config/models.py

@@ -9,7 +9,7 @@ from pgbelt.util import get_logger
 from pgbelt.util.asyncfuncs import makedirs
 from pydantic import BaseModel
 from pydantic import ValidationError
-from pydantic import validator
+from pydantic import field_validator
 
 
 def config_dir(db: str, dc: str) -> str:
@@ -37,7 +37,7 @@ class User(BaseModel):
     name: str
     pw: Optional[str] = None
 
-    _not_empty = validator("name", "pw", allow_reuse=True)(not_empty)
+    _not_empty = field_validator("name", "pw")(not_empty)
 
 
 class DbConfig(BaseModel):
@@ -64,9 +64,9 @@ class DbConfig(BaseModel):
     pglogical_user: User
     other_users: Optional[list[User]] = None
 
-    _not_empty = validator("host", "ip", "db", "port", allow_reuse=True)(not_empty)
+    _not_empty = field_validator("host", "ip", "db", "port")(not_empty)
 
-    @validator("root_user", "owner_user", "pglogical_user")
+    @field_validator("root_user", "owner_user", "pglogical_user")
     def has_password(cls, v) -> User:  # noqa: N805
         if not v.pw:
             raise ValueError
@@ -118,7 +118,7 @@ class DbupgradeConfig(BaseModel):
     sequences: Optional[list[str]] = None
     schema_name: Optional[str] = "public"
 
-    _not_empty = validator("db", "dc", allow_reuse=True)(not_empty)
+    _not_empty = field_validator("db", "dc")(not_empty)
 
     @property
     def file(self) -> str:
@@ -167,7 +167,7 @@ class DbupgradeConfig(BaseModel):
             return None
 
         try:
-            out = cls.parse_raw(raw)
+            out = cls.model_validate_json(raw)
         except ValidationError:
             logger.info("Cached config was not a valid DbupgradeConfig")
             return None

pgbelt/config/remote.py

@@ -79,7 +79,7 @@ async def load_remote_conf_def(
         async with aopen(config_file, mode="r") as f:
             raw_json = await f.read()
 
-        return RemoteConfigDefinition.parse_raw(raw_json)
+        return RemoteConfigDefinition.model_validate_json(raw_json)
     except FileNotFoundError:
         logger.error(f"No remote config definition exists at {config_file}")
     except JSONDecodeError:

pgbelt/util/pglogical.py

@@ -78,7 +78,7 @@ async def grant_pgl(pool: Pool, tables: list[str], schema: str, logger: Logger)
     async with pool.acquire() as conn:
         async with conn.transaction():
             if tables:
-                tables_with_schema = [f"{schema}.{table}" for table in tables]
+                tables_with_schema = [f'{schema}."{table}"' for table in tables]
                 await conn.execute(
                     f"GRANT ALL ON TABLE {','.join(tables_with_schema)} TO pglogical;"
                 )

pgbelt/util/postgres.py

@@ -97,11 +97,15 @@ async def compare_data(
     dst_old_extra_float_digits = await dst_pool.fetchval("SHOW extra_float_digits;")
     await dst_pool.execute("SET extra_float_digits TO 0;")
 
+    has_run = False
     for table in set(pkeys):
-        # If specific table list is defined and iterated table is not in that list, skip.
+        # If specific table list is defined and the iterated table is not in that list, skip.
         if tables and (table not in tables):
             continue
-        full_table_name = f"{schema}.{table}"
+
+        has_run = True  # If this runs, we have at least one table to compare. We will use this flag to throw an error if no tables are found.
+
+        full_table_name = f'{schema}."{table}"'
 
         logger.debug(f"Validating table {full_table_name}...")
         order_by_pkeys = ",".join(pkeys_dict[table])
@@ -171,6 +175,13 @@ async def compare_data(
                     f"Dest Row: {dst_row}"
                 )
 
+    # Just a paranoia check. If this throws, then it's possible pgbelt didn't migrate any data.
+    # This was found in issue #420, and previous commands threw errors before this issue could arise.
+    if not has_run:
+        raise ValueError(
+            "No tables were found to compare. Please reach out to the pgbelt for help, and check if your data was migrated."
+        )
+
     await src_pool.execute(f"SET extra_float_digits TO {src_old_extra_float_digits};")
     await dst_pool.execute(f"SET extra_float_digits TO {dst_old_extra_float_digits};")
     logger.info(
@@ -372,10 +383,17 @@ async def precheck_info(
               AND n.nspname <> 'pglogical'
         ORDER BY 1,2;"""
     )
+
     # We filter the table list if the user has specified a list of tables to target.
     if target_tables:
+
         result["tables"] = [t for t in result["tables"] if t["Name"] in target_tables]
 
+        # We will not recapitalize the table names in the result["tables"] list,
+        # to preserve how Postgres sees those tables in its system catalog. Easy
+        # rabbit hole later if we keep patching the table names to match the user's
+        # input.
+
     result["sequences"] = await pool.fetch(
         """
         SELECT n.nspname as "Schema",
@@ -392,12 +410,18 @@ async def precheck_info(
         ORDER BY 1,2;"""
     )
 
-    # We filter the sequence list if the user has specified a list of sequences to target.
+    # We filter the table list if the user has specified a list of tables to target.
     if target_sequences:
+
         result["sequences"] = [
-            s for s in result["sequences"] if s["Name"] in target_sequences
+            t for t in result["sequences"] if t["Name"] in target_sequences
         ]
 
+        # We will not recapitalize the table names in the result["tables"] list,
+        # to preserve how Postgres sees those tables in its system catalog. Easy
+        # rabbit hole later if we keep patching the table names to match the user's
+        # input.
+
     users = await pool.fetch(
         f"""
         SELECT r.rolname, r.rolsuper, r.rolinherit,
@@ -449,7 +473,7 @@ async def get_dataset_size(
 
     query = f"""
     SELECT
-        sum(pg_total_relation_size(schemaname || '.' || tablename)) AS total_relation_size
+        sum(pg_total_relation_size(schemaname || '."' || tablename || '"')) AS total_relation_size
     FROM
         pg_tables
     WHERE
@@ -460,7 +484,7 @@ async def get_dataset_size(
     # Yes it's a duplicate, but it's a pretty one. Rather let Postgres do this than Python.
     pretty_query = f"""
     SELECT
-        pg_size_pretty(sum(pg_total_relation_size(schemaname || '.' || tablename))) AS total_relation_size
+        pg_size_pretty(sum(pg_total_relation_size(schemaname || '."' || tablename || '"'))) AS total_relation_size
     FROM
         pg_tables
     WHERE

{pgbelt-0.7.1.dist-info → pgbelt-0.7.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pgbelt
-Version: 0.7.1
+Version: 0.7.3
 Summary: A CLI tool used to manage Postgres data migrations from beginning to end, for a single database or a fleet, leveraging pglogical replication.
 Author: Varjitt Jeeva
 Author-email: varjitt.jeeva@autodesk.com
@@ -14,7 +14,7 @@ Requires-Dist: aiofiles (>=0.8,<23.3)
 Requires-Dist: asyncpg (>=0.27,<0.30)
 Requires-Dist: pydantic (>=2.0,<3.0)
 Requires-Dist: tabulate (>=0.9.0,<0.10.0)
-Requires-Dist: typer (>=0.9.0,<0.10.0)
+Requires-Dist: typer (>=0.9,<0.13)
 Description-Content-Type: text/markdown
 
 # Pgbelt

{pgbelt-0.7.1.dist-info → pgbelt-0.7.3.dist-info}/RECORD

@@ -5,23 +5,23 @@ pgbelt/cmd/helpers.py,sha256=anMAkZjJRy7n0EwewyMNJGJ4Jx9keSNZdzWqi3ICNgI,5292
 pgbelt/cmd/login.py,sha256=kyDT755YsScg3QA4ZlkyIq3J0WO4XqMepMchufjuaDE,3043
 pgbelt/cmd/preflight.py,sha256=-78Puqqf1rCxNQEyn4bQIAORez_tsy6zJAbSuO_de9s,20676
 pgbelt/cmd/schema.py,sha256=XAAj2BH6HVoA1LxuWw2kheesGDu41L3GAd5UahNp3nE,4760
-pgbelt/cmd/setup.py,sha256=CYaRGQG_ooz0BpP5g1YVmPvt8zcT4DSIdQM6nASr9Qc,5908
+pgbelt/cmd/setup.py,sha256=Jp5sqT9_whoVBiOzAlOzX1ubtXQADYBkBrJldch_fKk,6627
 pgbelt/cmd/status.py,sha256=LTvteFwA1jcfnGJdlA2siJn5TUtlf14ffMkGVSi5gi0,4730
 pgbelt/cmd/sync.py,sha256=8TSNbn1dpkyxlEMImY7FjOrzhriSXlorqE-HBjqbUfU,9027
 pgbelt/cmd/teardown.py,sha256=TTSmhmD1bqJ8-P5YyK5ZpCsmneczO0nvd2Q2s2oyXY4,3526
 pgbelt/config/__init__.py,sha256=SXok1aZcpMYJpX_hk5cuKO33CJ5s8IESkswNN9KsVSo,35
 pgbelt/config/config.py,sha256=Kw2H-G1Evfj0TXIbh3k06gE72dZEp_wXWJ2Icq_T54c,3817
-pgbelt/config/models.py,sha256=4qm23kdNNZiQOZrejD1yXP6h0Z38LNFSyI4d0i-IdUs,5830
-pgbelt/config/remote.py,sha256=RQ_dfL5g2ChVP6jeGWnpQMVKQZK1o2m9-ZPYBMaaGj4,5297
+pgbelt/config/models.py,sha256=4MeeQ_puAZtMVix8Cj5GNCtYTUWn5lHHijS82yhD4mg,5816
+pgbelt/config/remote.py,sha256=D9bOekVfMU1xX2Wy0OiJwSXetxJUdt9Tn5Fukwn9rnE,5307
 pgbelt/main.py,sha256=YiagBiGt8pbNlukkRxROXnQX1Tx6ax7c6riuHRCrPYU,186
 pgbelt/util/__init__.py,sha256=-6KkvVMz-yGNQfeoo4CZZrgWKXYmFd4CMyoiao8OnFE,40
 pgbelt/util/asyncfuncs.py,sha256=7i_GpBmUNNZ8RUGvU-q5nclsoaCm6Lx8jLP8usYvmZc,583
 pgbelt/util/dump.py,sha256=AwyOAd9CP014gvsl-qlo1lbnXZfxoeN4ujZWUIq7KM8,14715
 pgbelt/util/logs.py,sha256=l2jT-WKZ-33eNDw4S4W1_eE4ISo4rtDRXYLVf4QTV4Y,1699
-pgbelt/util/pglogical.py,sha256=r4I95CG4eRFsEPnN30P4yxt8HeGMzKBTEm3fecfw4ws,13611
-pgbelt/util/postgres.py,sha256=909eSLhnhAkGe_4IVt-j72I3wXHArT5V_YikJnl6gdc,18178
-pgbelt-0.7.1.dist-info/LICENSE,sha256=FQ5cFkW02dKK3LmKH8z-rwn93tWSCh7lsxfNUiWcFsg,10758
-pgbelt-0.7.1.dist-info/METADATA,sha256=uXZgpAXeisAYU3M89TVK_w18R3VaEKteXZVQgQB_Kg8,2964
-pgbelt-0.7.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-pgbelt-0.7.1.dist-info/entry_points.txt,sha256=SCz_poPjkaVnWpJ-CeytAnDzbVc6l0WalOwitIqW_3g,40
-pgbelt-0.7.1.dist-info/RECORD,,
+pgbelt/util/pglogical.py,sha256=Y6KZBeiH85zhNSvhATqh0xozhfUMyQnPWN1HwRosZFo,13613
+pgbelt/util/postgres.py,sha256=jNMYhkb4OHYhNzDu7Fppz8wrXogAw1XxVhxdBA2e6cI,19270
+pgbelt-0.7.3.dist-info/LICENSE,sha256=FQ5cFkW02dKK3LmKH8z-rwn93tWSCh7lsxfNUiWcFsg,10758
+pgbelt-0.7.3.dist-info/METADATA,sha256=DmeWJ6E8SdjC36sTp8qPHC73_6LWd5GSqN2tdW_BoWI,2960
+pgbelt-0.7.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+pgbelt-0.7.3.dist-info/entry_points.txt,sha256=SCz_poPjkaVnWpJ-CeytAnDzbVc6l0WalOwitIqW_3g,40
+pgbelt-0.7.3.dist-info/RECORD,,