pgbelt 0.1.1__tar.gz → 0.2.0__tar.gz

{pgbelt-0.1.1 → pgbelt-0.2.0}/PKG-INFO

@@ -1,18 +1,17 @@
 Metadata-Version: 2.1
 Name: pgbelt
-Version: 0.1.1
+Version: 0.2.0
 Summary: A CLI tool used to manage Postgres data migrations from beginning to end, for a single database or a fleet, leveraging pglogical replication.
 Author: Varjitt Jeeva
 Author-email: varjitt.jeeva@autodesk.com
-Requires-Python: >=3.9,<4.0
+Requires-Python: >=3.11,<4.0
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Requires-Dist: aiofiles (>=0.8.0,<0.9.0)
-Requires-Dist: asyncpg (>=0.26.0,<0.27.0)
-Requires-Dist: pydantic (>=1.10.1,<1.11.0)
-Requires-Dist: tabulate (>=0.8.9,<0.9.0)
-Requires-Dist: typer (>=0.6.1,<0.7.0)
+Classifier: Programming Language :: Python :: 3.11
+Requires-Dist: aiofiles (>=0.8,<23.2)
+Requires-Dist: asyncpg (>=0.27.0,<0.28.0)
+Requires-Dist: pydantic (>=1.10.0,<2.0.0)
+Requires-Dist: tabulate (>=0.9.0,<0.10.0)
+Requires-Dist: typer (>=0.7.0,<0.8.0)
 Description-Content-Type: text/markdown
 
 # Pgbelt

{pgbelt-0.1.1 → pgbelt-0.2.0}/pgbelt/cmd/convenience.py

@@ -1,15 +1,13 @@
 from asyncio import run
 from logging import Logger
-from typing import Tuple
 
 from asyncpg import create_pool
-from typer import echo
-from typer import Option
-
 from pgbelt.config.config import get_config
 from pgbelt.config.models import DbupgradeConfig
 from pgbelt.util.logs import get_logger
 from pgbelt.util.postgres import analyze_table_pkeys
+from typer import echo
+from typer import Option
 
 
 def src_dsn(
@@ -58,7 +56,7 @@ def dst_dsn(
 
 async def _check_pkeys(
     conf: DbupgradeConfig, logger: Logger
-) -> Tuple[list[str], list[str]]:
+) -> tuple[list[str], list[str]]:
     async with create_pool(conf.src.root_uri, min_size=1) as pool:
         pkey_tables, no_pkey_tables, _ = await analyze_table_pkeys(pool, logger)
     return pkey_tables, no_pkey_tables

{pgbelt-0.1.1 → pgbelt-0.2.0}/pgbelt/cmd/helpers.py

@@ -1,30 +1,28 @@
 from asyncio import gather
 from asyncio import run
+from collections.abc import Awaitable
+from collections.abc import Callable
 from functools import wraps
 from inspect import iscoroutinefunction
 from inspect import Parameter
 from inspect import signature
 from typing import Any
-from typing import Awaitable
-from typing import Callable
-from typing import Optional
 from typing import TypeVar
 
-from typer import Argument
-from typer import Typer
-
 from pgbelt.config import get_all_configs_async
 from pgbelt.config import get_config_async
+from typer import Argument
+from typer import Typer
 
 
 T = TypeVar("T")
 
 
 def run_with_configs(
-    decorated_func: Callable[..., Awaitable[Optional[T]]] = None,
+    decorated_func: Callable[..., Awaitable[T | None]] = None,
     skip_src: bool = False,
     skip_dst: bool = False,
-    results_callback: Optional[Callable[[list[T]], Awaitable[Optional[Any]]]] = None,
+    results_callback: Callable[[list[T]], Awaitable[Any | None]] | None = None,
 ) -> Callable:
     """
     Decorator for async commands. Implementations should take one Awaitable[DbupgradeConfig] arg
@@ -52,7 +50,7 @@ def run_with_configs(
 
         # The name, docstring, and signature of the implementation is preserved. Important for add_command
         @wraps(func)
-        async def wrapper(dc: str, db: Optional[str], **kwargs):
+        async def wrapper(dc: str, db: str | None, **kwargs):
             # If db is specified we only want to run on one of them
             if db is not None:
                 results = [
@@ -102,7 +100,7 @@ def add_command(app: Typer, command: Callable):
     if iscoroutinefunction(command):
 
         @app.command(name=name)
-        def cmdwrapper(dc: str, db: Optional[str] = Argument(None), **kwargs):
+        def cmdwrapper(dc: str, db: str | None = Argument(None), **kwargs):
             run(command(dc, db, **kwargs))
 
     # Synchronous commands can only be run on one db at a time

{pgbelt-0.1.1 → pgbelt-0.2.0}/pgbelt/cmd/login.py

@@ -1,10 +1,9 @@
 import asyncio
+from collections.abc import Awaitable
 from logging import Logger
-from typing import Awaitable
 
 from asyncpg import create_pool
 from asyncpg import Pool
-
 from pgbelt.cmd.helpers import run_with_configs
 from pgbelt.config.models import DbConfig
 from pgbelt.config.models import DbupgradeConfig

{pgbelt-0.1.1 → pgbelt-0.2.0}/pgbelt/cmd/preflight.py

@@ -1,12 +1,12 @@
 from asyncio import gather
+from collections.abc import Awaitable
+
+from asyncpg import create_pool
 from pgbelt.cmd.helpers import run_with_configs
 from pgbelt.config.models import DbupgradeConfig
 from pgbelt.util.logs import get_logger
 from pgbelt.util.postgres import analyze_table_pkeys
 from pgbelt.util.postgres import precheck_info
-from typing import Awaitable
-
-from asyncpg import create_pool
 from tabulate import tabulate
 from typer import echo
 from typer import style

{pgbelt-0.1.1 → pgbelt-0.2.0}/pgbelt/cmd/schema.py

@@ -1,4 +1,4 @@
-from typing import Awaitable
+from collections.abc import Awaitable
 
 from pgbelt.cmd.helpers import run_with_configs
 from pgbelt.config.models import DbupgradeConfig

{pgbelt-0.1.1 → pgbelt-0.2.0}/pgbelt/cmd/setup.py

@@ -1,12 +1,10 @@
 from asyncio import create_task
 from asyncio import gather
+from collections.abc import Awaitable
 from logging import Logger
-from typing import Awaitable
 
 from asyncpg import create_pool
 from asyncpg import Pool
-from typer import Option
-
 from pgbelt.cmd.helpers import run_with_configs
 from pgbelt.config.models import DbupgradeConfig
 from pgbelt.util.dump import apply_target_schema
@@ -18,6 +16,7 @@ from pgbelt.util.pglogical import configure_replication_set
 from pgbelt.util.pglogical import configure_subscription
 from pgbelt.util.pglogical import grant_pgl
 from pgbelt.util.postgres import analyze_table_pkeys
+from typer import Option
 
 
 async def _dump_and_load_schema(

{pgbelt-0.1.1 → pgbelt-0.2.0}/pgbelt/cmd/status.py

@@ -1,16 +1,15 @@
 from asyncio import gather
-from typing import Awaitable
+from collections.abc import Awaitable
 
 from asyncpg import create_pool
-from tabulate import tabulate
-from typer import echo
-from typer import style
-
 from pgbelt.cmd.helpers import run_with_configs
 from pgbelt.config.models import DbupgradeConfig
 from pgbelt.util import get_logger
 from pgbelt.util.pglogical import dst_status
 from pgbelt.util.pglogical import src_status
+from tabulate import tabulate
+from typer import echo
+from typer import style
 
 
 async def _print_status_table(results: list[dict[str, str]]) -> list[list[str]]:

{pgbelt-0.1.1 → pgbelt-0.2.0}/pgbelt/cmd/sync.py

@@ -1,11 +1,9 @@
 from asyncio import gather
+from collections.abc import Awaitable
 from logging import Logger
-from typing import Awaitable
 
 from asyncpg import create_pool
 from asyncpg import Pool
-from typer import Option
-
 from pgbelt.cmd.helpers import run_with_configs
 from pgbelt.config.models import DbupgradeConfig
 from pgbelt.util.dump import apply_target_constraints
@@ -18,6 +16,7 @@ from pgbelt.util.postgres import compare_latest_100_rows
 from pgbelt.util.postgres import dump_sequences
 from pgbelt.util.postgres import load_sequences
 from pgbelt.util.postgres import run_analyze
+from typer import Option
 
 
 async def _sync_sequences(
@@ -146,7 +145,7 @@ async def analyze(config_future: Awaitable[DbupgradeConfig]) -> None:
     """
     conf = await config_future
     logger = get_logger(conf.db, conf.dc, "sync.dst")
-    async with create_pool(conf.dst.owner_uri, min_size=1) as dst_pool:
+    async with create_pool(conf.dst.root_uri, min_size=1) as dst_pool:
         await run_analyze(dst_pool, logger)
 
 

{pgbelt-0.1.1 → pgbelt-0.2.0}/pgbelt/cmd/teardown.py

@@ -1,10 +1,8 @@
 from asyncio import gather
 from asyncio import sleep
-from typing import Awaitable
+from collections.abc import Awaitable
 
 from asyncpg import create_pool
-from typer import Option
-
 from pgbelt.cmd.helpers import run_with_configs
 from pgbelt.config.models import DbupgradeConfig
 from pgbelt.util.logs import get_logger
@@ -13,6 +11,7 @@ from pgbelt.util.pglogical import teardown_node
 from pgbelt.util.pglogical import teardown_pgl
 from pgbelt.util.pglogical import teardown_replication_set
 from pgbelt.util.pglogical import teardown_subscription
+from typer import Option
 
 
 @run_with_configs(skip_dst=True)

{pgbelt-0.1.1 → pgbelt-0.2.0}/pgbelt/config/config.py

@@ -1,8 +1,7 @@
 import asyncio
+from collections.abc import AsyncGenerator
+from collections.abc import Awaitable
 from os.path import join
-from typing import AsyncGenerator
-from typing import Awaitable
-from typing import Optional
 
 from pgbelt.config.models import DbupgradeConfig
 from pgbelt.config.remote import resolve_remote_config
@@ -14,7 +13,7 @@ from pgbelt.util.asyncfuncs import listdir
 
 def get_config(
     db: str, dc: str, skip_src: bool = False, skip_dst: bool = False
-) -> Optional[DbupgradeConfig]:
+) -> DbupgradeConfig | None:
     """
     Get a configuration for one database pair synchronously.
     """
@@ -28,7 +27,7 @@ def get_config(
 
 async def get_config_async(
     db: str, dc: str, skip_src: bool = False, skip_dst: bool = False
-) -> Optional[DbupgradeConfig]:
+) -> DbupgradeConfig | None:
     """
     Get configuration for one database pair asynchronously. Always prefers
     locally cached configuration but attempts to resolve any uncached configuration
@@ -84,7 +83,7 @@ async def find_available_configs(confdir: str, dc: str) -> set[str]:
 
 async def get_all_configs_async(
     dc: str, skip_src: bool = False, skip_dst: bool = False
-) -> AsyncGenerator[Awaitable[Optional[DbupgradeConfig]], None]:
+) -> AsyncGenerator[Awaitable[DbupgradeConfig | None], None]:
     """
     A generator that produces Awaitables that resolve to DbupgradeConfigs or None
 

{pgbelt-0.1.1 → pgbelt-0.2.0}/pgbelt/config/models.py

@@ -1,12 +1,11 @@
 from __future__ import annotations
 
 from os.path import join
-from pgbelt.util import get_logger
-from pgbelt.util.asyncfuncs import makedirs
-from typing import Optional
 
 from aiofiles import open as aopen
 from aiofiles.os import remove
+from pgbelt.util import get_logger
+from pgbelt.util.asyncfuncs import makedirs
 from pydantic import BaseModel
 from pydantic import ValidationError
 from pydantic import validator
@@ -20,7 +19,7 @@ def config_file(db: str, dc: str) -> str:
     return join(config_dir(db, dc), "config.json")
 
 
-def not_empty(v) -> Optional[str]:
+def not_empty(v) -> str | None:
     if v == "":
         raise ValueError
     return v
@@ -35,7 +34,7 @@ class User(BaseModel):
     """
 
     name: str
-    pw: Optional[str] = None
+    pw: str | None = None
 
     _not_empty = validator("name", "pw", allow_reuse=True)(not_empty)
 
@@ -62,7 +61,7 @@ class DbConfig(BaseModel):
     root_user: User
     owner_user: User
     pglogical_user: User
-    other_users: Optional[list[User]] = None
+    other_users: list[User] | None = None
 
     _not_empty = validator("host", "ip", "db", "port", allow_reuse=True)(not_empty)
 
@@ -109,10 +108,10 @@ class DbupgradeConfig(BaseModel):
 
     db: str
     dc: str
-    src: Optional[DbConfig] = None
-    dst: Optional[DbConfig] = None
-    tables: Optional[list[str]] = None
-    sequences: Optional[list[str]] = None
+    src: DbConfig | None = None
+    dst: DbConfig | None = None
+    tables: list[str] | None = None
+    sequences: list[str] | None = None
 
     _not_empty = validator("db", "dc", allow_reuse=True)(not_empty)
 
@@ -147,7 +146,7 @@ class DbupgradeConfig(BaseModel):
         logger.info("Cached config to disk.")
 
     @classmethod
-    async def load(cls, db: str, dc: str) -> Optional[DbupgradeConfig]:
+    async def load(cls, db: str, dc: str) -> DbupgradeConfig | None:
         """
         Load the specified configuration from disk if present.
         If the existing config is invalid or does not exist return None.

{pgbelt-0.1.1 → pgbelt-0.2.0}/pgbelt/config/remote.py

@@ -1,11 +1,10 @@
 from importlib import import_module
 from json import JSONDecodeError
 from logging import Logger
-from pgbelt.config.models import DbupgradeConfig
-from pgbelt.util.logs import get_logger
-from typing import Optional
 
 from aiofiles import open as aopen
+from pgbelt.config.models import DbupgradeConfig
+from pgbelt.util.logs import get_logger
 from pydantic import BaseModel
 from pydantic import ValidationError
 
@@ -60,7 +59,7 @@ class BaseResolver(BaseModel):
         arbitrary_types_allowed = True
         extra = "ignore"
 
-    async def resolve(self) -> Optional[DbupgradeConfig]:
+    async def resolve(self) -> DbupgradeConfig | None:
         """
         Called to retrieve the configuration from wherever your resolver gets it.
         Return configuration as a DbupgradeConfig.
@@ -73,7 +72,7 @@ class BaseResolver(BaseModel):
 
 async def load_remote_conf_def(
     config_file: str, logger: Logger
-) -> Optional[RemoteConfigDefinition]:
+) -> RemoteConfigDefinition | None:
     try:
         logger.debug(f"Reading remote config definition from file {config_file}")
         async with aopen(config_file, mode="r") as f:
@@ -90,7 +89,7 @@ async def load_remote_conf_def(
 
 async def resolve_remote_config(
     db: str, dc: str, skip_src: bool = False, skip_dst: bool = False
-) -> Optional[DbupgradeConfig]:
+) -> DbupgradeConfig | None:
     """
     Loads the referenced remote configuration json file, tries to import the
     specified resolver class, executes its resolve method, and returns the

{pgbelt-0.1.1 → pgbelt-0.2.0}/pgbelt/main.py

@@ -1,6 +1,5 @@
-from typer import Typer
-
 from pgbelt.cmd import add_commands
+from typer import Typer
 
 app = Typer(help="A tool to help manage postgres data migrations.")
 add_commands(app)

{pgbelt-0.1.1 → pgbelt-0.2.0}/pgbelt/util/dump.py

@@ -1,16 +1,15 @@
 import asyncio
 from logging import Logger
 from os.path import join
-from re import search
-
-from aiofiles import open as aopen
-from asyncpg import create_pool
-
 from pgbelt.config.models import DbupgradeConfig
 from pgbelt.util.asyncfuncs import isfile
 from pgbelt.util.asyncfuncs import listdir
 from pgbelt.util.asyncfuncs import makedirs
 from pgbelt.util.postgres import table_empty
+from re import search
+
+from aiofiles import open as aopen
+from asyncpg import create_pool
 
 RAW = "schema"
 NO_INVALID = "no_invalid_constraints"
@@ -33,7 +32,7 @@ def table_file(db: str, dc: str, name: str) -> str:
     return join(table_dir(db, dc), f"{name}.sql")
 
 
-def _parse_dump_commands(out: str, src_owner: str, dst_owner: str) -> list[str]:
+def _parse_dump_commands(out: str) -> list[str]:
     """
     Given a string containing output from pg_dump, return a list of strings where
     each is a complete postgres command. Commands may be multi-line.
@@ -47,9 +46,6 @@ def _parse_dump_commands(out: str, src_owner: str, dst_owner: str) -> list[str]:
         if not stripped or stripped.startswith("--"):
             continue
 
-        # replace source owner user with target owner user
-        line = line.replace(src_owner, dst_owner)
-
         # if the last command is terminated or we don't have any yet start a new one
         if not commands or commands[-1].endswith(";\n"):
             commands.append(line + "\n")
@@ -172,9 +168,7 @@ async def dump_source_schema(config: DbupgradeConfig, logger: Logger) -> None:
 
     out = await _execute_subprocess(command, "Retrieved source schema", logger)
 
-    commands_raw = _parse_dump_commands(
-        out.decode("utf-8"), config.src.owner_user.name, config.dst.owner_user.name
-    )
+    commands_raw = _parse_dump_commands(out.decode("utf-8"))
 
     commands = []
     for c in commands_raw:

{pgbelt-0.1.1 → pgbelt-0.2.0}/pgbelt/util/postgres.py

@@ -1,5 +1,4 @@
 from logging import Logger
-from typing import Tuple
 
 from asyncpg import Pool
 from asyncpg import Record
@@ -194,7 +193,7 @@ async def compare_latest_100_rows(
     SELECT *
     FROM {table}
     ORDER BY {order_by_pkeys} DESC
-    LIMIT 1;
+    LIMIT 100;
     """
 
     await compare_data(src_pool, dst_pool, query, tables, logger)
@@ -211,7 +210,7 @@ async def table_empty(pool: Pool, table: str, logger: Logger) -> bool:
 
 async def analyze_table_pkeys(
     pool: Pool, logger: Logger
-) -> Tuple[list[str], list[str], Record]:
+) -> tuple[list[str], list[str], Record]:
     """
     return three lists of table names. the first element is all tables
     with pkeys in public and the second is all tables without pkeys in public.

{pgbelt-0.1.1 → pgbelt-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pgbelt"
-version = "0.1.1"
+version = "0.2.0"
 description = "A CLI tool used to manage Postgres data migrations from beginning to end, for a single database or a fleet, leveraging pglogical replication."
 authors = ["Varjitt Jeeva <varjitt.jeeva@autodesk.com>"]
 readme = "README.md"
@@ -10,35 +10,35 @@ packages = [
 ]
 
 [tool.poetry.dependencies]
-python = "^3.9"
-aiofiles = "~0.8.0"
-asyncpg = "~0.26.0"
-pydantic = "~1.10.1"
-tabulate = "~0.8.9"
-typer = "~0.6.1"
+python = "^3.11"
+aiofiles = ">=0.8,<23.2"
+asyncpg = "^0.27.0"
+pydantic = "^1.10.0"
+tabulate = "^0.9.0"
+typer = "^0.7.0"
 
 [tool.poetry.dev-dependencies]
-black = "~22.6.0"
-pre-commit = "~2.20.0"
-flake8 = "~5.0.4"
-pytest-cov = "~3.0.0"
-pytest = "^7.1.2"
-coverage = {extras = ["toml"], version = "^6.4"}
-safety = "^2.1.1"
-mypy = "^0.971"
-xdoctest = {extras = ["colors"], version = "^1.0.2"}
+black = "~23.3.0"
+pre-commit = "~3.2.1"
+flake8 = "^6.0.0"
+pytest-cov = "~4.0.0"
+pytest = "^7.1.3"
+coverage = {extras = ["toml"], version = "^6.5"}
+safety = "^2.3.1"
+mypy = "^1.1"
+xdoctest = {extras = ["colors"], version = "^1.1.0"}
 flake8-bandit = "~4.1.1"
 flake8-bugbear = ">=21.9.2"
 flake8-docstrings = "^1.6.0"
-flake8-rst-docstrings = "^0.2.7"
+flake8-rst-docstrings = "^0.3.0"
 pep8-naming = "^0.13.2"
 darglint = "^1.8.1"
-reorder-python-imports = "^3.8.2"
+reorder-python-imports = "^3.9.0"
 pre-commit-hooks = "^4.2.0"
 Pygments = "^2.13.0"
-pyupgrade = "^2.37.3"
-pylint = "^2.15.0"
-pytest-asyncio = "~0.19.0"
+pyupgrade = "^3.3.1"
+pylint = "^2.17.2"
+pytest-asyncio = "~0.21.0"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

pgbelt-0.1.1/setup.py

@@ -1,42 +0,0 @@
-# -*- coding: utf-8 -*-
-from setuptools import setup
-
-package_dir = \
-{'': '.'}
-
-packages = \
-['pgbelt', 'pgbelt.cmd', 'pgbelt.config', 'pgbelt.util']
-
-package_data = \
-{'': ['*']}
-
-install_requires = \
-['aiofiles>=0.8.0,<0.9.0',
- 'asyncpg>=0.26.0,<0.27.0',
- 'pydantic>=1.10.1,<1.11.0',
- 'tabulate>=0.8.9,<0.9.0',
- 'typer>=0.6.1,<0.7.0']
-
-entry_points = \
-{'console_scripts': ['belt = pgbelt.main:app']}
-
-setup_kwargs = {
-    'name': 'pgbelt',
-    'version': '0.1.1',
-    'description': 'A CLI tool used to manage Postgres data migrations from beginning to end, for a single database or a fleet, leveraging pglogical replication.',
-    'long_description': '# Pgbelt\n\n<p align="center">\n    <img src="https://github.com/Autodesk/pgbelt/blob/main/pgbelt.png?raw=true" width="400">\n</p>\n\n<p align="center">\n    <a href="https://github.com/autodesk/pgbelt" target="_blank">\n        <img src="https://img.shields.io/github/last-commit/autodesk/pgbelt" alt="Latest Commit">\n    </a>\n    <img src="https://github.com/Autodesk/pgbelt/actions/workflows/ci.yml/badge.svg">\n    <a href="http://www.apache.org/licenses/LICENSE-2.0" target="_blank">\n        <img src="https://img.shields.io/github/license/Autodesk/pgbelt">\n    </a>\n</p>\n\nPgBelt is a CLI tool used to manage Postgres data migrations from beginning to end,\nfor a single database or a fleet, leveraging pglogical replication.\n\nIt was built to assist in migrating data between postgres databases with as\nlittle application downtime as possible. It works in databases running different versions\nof postgres and makes it easy to run many migrations in parallel during a single downtime.\n\n| :exclamation: This is very important                                                                                                            |\n| :---------------------------------------------------------------------------------------------------------------------------------------------- |\n| As with all Data Migration tasks, **there is a risk of data loss**. Please ensure you have backed up your data before attempting any migrations |\n\n## Installation\n\n### Install From PyPi\n\nIt is recommended to install pgbelt inside a virtual environment:\n\n- [pyenv](https://github.com/pyenv/pyenv)\n- [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv)\n\nYou must also have:\n\n- Postgres Client Tools (pg_dump, pg_restore). Mac: `brew install libpq`. Ubuntu: `sudo apt-get install postgresql-client`\n\nInstall pgbelt locally:\n\n    pip3 install pgbelt\n\n## Quickstart with Pgbelt\n\nSee [this doc](docs/quickstart.md)!\n\n## Contributing\n\nWe welcome contributions! See [this doc](CONTRIBUTING.md) on how to do so, including setting up your local development environment.\n',
-    'author': 'Varjitt Jeeva',
-    'author_email': 'varjitt.jeeva@autodesk.com',
-    'maintainer': 'None',
-    'maintainer_email': 'None',
-    'url': 'None',
-    'package_dir': package_dir,
-    'packages': packages,
-    'package_data': package_data,
-    'install_requires': install_requires,
-    'entry_points': entry_points,
-    'python_requires': '>=3.9,<4.0',
-}
-
-
-setup(**setup_kwargs)