pum 1.0.0__py3-none-any.whl

pum/cli.py

@@ -0,0 +1,402 @@
+#!/usr/bin/env python3
+
+import argparse
+import importlib.metadata
+import logging
+import sys
+from pathlib import Path
+
+import psycopg
+
+from .checker import Checker
+from .pum_config import PumConfig
+
+from .info import run_info
+from .upgrader import Upgrader
+from .parameter import ParameterType
+from .schema_migrations import SchemaMigrations
+from .dumper import DumpFormat
+
+
+def setup_logging(verbosity: int = 0):
+    """Setup logging based on verbosity level (0=WARNING, 1=INFO, 2+=DEBUG) with colored output."""
+    level = logging.WARNING  # default
+
+    if verbosity == 1:
+        level = logging.INFO
+    elif verbosity >= 2:
+        level = logging.DEBUG
+
+    class ColorFormatter(logging.Formatter):
+        COLORS = {
+            logging.ERROR: "\033[31m",  # Red
+            logging.WARNING: "\033[33m",  # Yellow
+            logging.INFO: "\033[36m",  # Cyan
+            logging.DEBUG: "\033[35m",  # Magenta
+        }
+        RESET = "\033[0m"
+
+        def format(self, record):
+            color = self.COLORS.get(record.levelno, "")
+            message = super().format(record)
+            if color:
+                message = f"{color}{message}{self.RESET}"
+            return message
+
+    handler = logging.StreamHandler()
+    handler.setFormatter(ColorFormatter("%(message)s"))
+    logging.basicConfig(
+        level=level,
+        handlers=[handler],
+        format="%(message)s",
+        force=True,
+    )
+
+
+class Pum:
+    def __init__(self, pg_service: str, config: str | PumConfig = None) -> None:
+        """Initialize the PUM class with a database connection and configuration.
+
+        Args:
+            pg_service (str): The name of the postgres service (defined in pg_service.conf)
+            config (str | PumConfig): The configuration file path or a PumConfig object.
+
+        """
+        self.pg_service = pg_service
+
+        if isinstance(config, str):
+            self.config = PumConfig.from_yaml(config)
+        else:
+            self.config = config
+
+    def run_check(
+        self,
+        pg_service1: str,
+        pg_service2: str,
+        ignore_list: list[str] | None,
+        exclude_schema: list[str] | None,
+        exclude_field_pattern: list[str] | None,
+        verbose_level: int = 1,
+        output_file: str | None = None,
+    ) -> bool:
+        """Run the check command.
+
+        Args:
+            pg_service1:
+                The name of the postgres service (defined in pg_service.conf)
+                related to the first db to be compared
+            pg_service2:
+                The name of the postgres service (defined in pg_service.conf)
+                related to the first db to be compared
+            ignore_list:
+                List of elements to be ignored in check (ex. tables, columns,
+                views, ...)
+            exclude_schema:
+                List of schemas to be ignored in check.
+            exclude_field_pattern:
+                List of field patterns to be ignored in check.
+            verbose_level:
+                verbose level, 0 -> nothing, 1 -> print first 80 char of each
+                difference, 2 -> print all the difference details
+            output_file:
+                a file path where write the differences
+
+        Returns:
+            True if no differences are found, False otherwise.
+
+        """
+        # self.__out("Check...")
+        verbose_level = verbose_level or 1
+        ignore_list = ignore_list or []
+        exclude_schema = exclude_schema or []
+        exclude_field_pattern = exclude_field_pattern or []
+        try:
+            checker = Checker(
+                pg_service1,
+                pg_service2,
+                exclude_schema=exclude_schema,
+                exclude_field_pattern=exclude_field_pattern,
+                ignore_list=ignore_list,
+                verbose_level=verbose_level,
+            )
+            result, differences = checker.run_checks()
+
+            if result:
+                self.__out("OK")
+            else:
+                self.__out("DIFFERENCES FOUND")
+
+            if differences:
+                if output_file:
+                    with open(output_file, "w") as f:
+                        for k, values in differences.items():
+                            f.write(k + "\n")
+                            f.writelines(f"{v}\n" for v in values)
+                else:
+                    for k, values in differences.items():
+                        print(k)
+                        for v in values:
+                            print(v)
+            return result
+
+        except psycopg.Error as e:
+            self.__out("ERROR")
+            self.__out(e.args[0] if e.args else str(e))
+            sys.exit(1)
+
+        except Exception as e:
+            self.__out("ERROR")
+            # if e.args is empty then use str(e)
+            self.__out(e.args[0] if e.args else str(e))
+            sys.exit(1)
+
+
+def create_parser() -> argparse.ArgumentParser:
+    """Creates the main parser with its sub-parsers"""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-c", "--config_file", help="set the config file. Default: .pum.yaml")
+    parser.add_argument("-s", "--pg-service", help="Name of the postgres service", required=True)
+
+    parser.add_argument(
+        "-d", "--dir", help="Directory or URL of the module. Default: .", default="."
+    )
+
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="count",
+        default=0,
+        help="Increase output verbosity (e.g. -v, -vv)",
+    )
+
+    version = importlib.metadata.version("pum")
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"pum {version}",
+        help="Show program's version number and exit.",
+    )
+
+    subparsers = parser.add_subparsers(
+        title="commands", description="valid pum commands", dest="command"
+    )
+
+    # Parser for the "info" command
+    parser_info = subparsers.add_parser("info", help="show info about schema migrations history.")  # NOQA
+
+    # Parser for the "install" command
+    parser_install = subparsers.add_parser("install", help="Installs the module.")
+    parser_install.add_argument(
+        "-p",
+        "--parameter",
+        nargs=2,
+        help="Assign variable for running SQL deltas. Format is name value.",
+        action="append",
+    )
+    parser_install.add_argument("--max-version", help="maximum version to install")
+    parser_install.add_argument("-r", "--roles", help="Create roles", action="store_true")
+    parser_install.add_argument(
+        "-g", "--grant", help="Grant permissions to roles", action="store_true"
+    )
+    parser_install.add_argument(
+        "-d", "--demo-data", help="Load demo data with the given name", type=str, default=None
+    )
+
+    # Role management parser
+    parser_role = subparsers.add_parser("role", help="manage roles in the database")
+    parser_role.add_argument(
+        "action", choices=["create", "grant", "revoke", "drop"], help="Action to perform"
+    )
+
+    # Parser for the "check" command
+    parser_check = subparsers.add_parser(
+        "check", help="check the differences between two databases"
+    )
+
+    parser_check.add_argument(
+        "-i",
+        "--ignore",
+        help="Elements to be ignored",
+        nargs="+",
+        choices=[
+            "tables",
+            "columns",
+            "constraints",
+            "views",
+            "sequences",
+            "indexes",
+            "triggers",
+            "functions",
+            "rules",
+        ],
+    )
+    parser_check.add_argument(
+        "-N", "--exclude-schema", help="Schema to be ignored.", action="append"
+    )
+    parser_check.add_argument(
+        "-P",
+        "--exclude-field-pattern",
+        help="Fields to be ignored based on a pattern compatible with SQL LIKE.",
+        action="append",
+    )
+
+    parser_check.add_argument("-o", "--output_file", help="Output file")
+
+    # Parser for the "dump" command
+    parser_dump = subparsers.add_parser("dump", help="dump a Postgres database")
+    parser_dump.add_argument(
+        "-f",
+        "--format",
+        type=lambda s: DumpFormat[s.upper()],
+        choices=list(DumpFormat),
+        default=DumpFormat.PLAIN,
+        help=f"Dump format. Choices: {[e.name.lower() for e in DumpFormat]}. Default: plain.",
+    )
+    parser_dump.add_argument(
+        "-N", "--exclude-schema", help="Schema to be ignored.", action="append"
+    )
+    parser_dump.add_argument("file", help="The backup file")
+
+    # Parser for the "restore" command
+    parser_restore = subparsers.add_parser(
+        "restore", help="restore a Postgres database from a dump file"
+    )
+    parser_restore.add_argument("-x", help="ignore pg_restore errors", action="store_true")
+    parser_restore.add_argument(
+        "-N", "--exclude-schema", help="Schema to be ignored.", action="append"
+    )
+    parser_restore.add_argument("file", help="The backup file")
+
+    # Parser for the "baseline" command
+    parser_baseline = subparsers.add_parser(
+        "baseline", help="Create upgrade information table and set baseline"
+    )
+    parser_baseline.add_argument(
+        "-b", "--baseline", help="Set baseline in the format x.x.x", required=True
+    )
+
+    # Parser for the "upgrade" command
+    parser_upgrade = subparsers.add_parser("upgrade", help="upgrade db")
+    parser_upgrade.add_argument("-u", "--max-version", help="upper bound limit version")
+    parser_upgrade.add_argument(
+        "-p",
+        "--parameter",
+        nargs=2,
+        help="Assign variable for running SQL deltas. Format is: name value.",
+        action="append",
+    )
+
+    return parser
+
+
+def cli() -> int:  # noqa: PLR0912
+    """Main function to run the command line interface."""
+    parser = create_parser()
+    args = parser.parse_args()
+
+    setup_logging(args.verbose)
+    logger = logging.getLogger(__name__)
+
+    # if no command is passed, print the help and exit
+    if not args.command:
+        parser.print_help()
+        parser.exit()
+
+    if args.config_file:
+        config = PumConfig.from_yaml(args.config_file)
+    else:
+        config = PumConfig.from_yaml(Path(args.dir) / ".pum.yaml")
+
+    with psycopg.connect(f"service={args.pg_service}") as conn:
+        # Check if the connection is successful
+        if not conn:
+            logger.error(f"Could not connect to the database using service: {args.pg_service}")
+            sys.exit(1)
+
+        # Build parameters dict for install and upgrade commands
+        parameters = {}
+        if args.command in ("install", "upgrade"):
+            for p in args.parameter or ():
+                param = config.parameter(p[0])
+                if not param:
+                    logger.error(f"Unknown parameter: {p[0]}")
+                    sys.exit(1)
+                if param.type == ParameterType.DECIMAL:
+                    parameters[p[0]] = float(p[1])
+                elif param.type == ParameterType.INTEGER:
+                    parameters[p[0]] = int(p[1])
+                elif param.type == ParameterType.BOOLEAN:
+                    parameters[p[0]] = p[1].lower() in ("true", "1", "yes")
+                elif param.type == ParameterType.TEXT:
+                    parameters[p[0]] = p[1]
+                else:
+                    raise ValueError(f"Unsupported parameter type for {p[0]}: {param.type}")
+            logger.debug(f"Parameters: {parameters}")
+
+        pum = Pum(args.pg_service, config)
+        exit_code = 0
+
+        if args.command == "info":
+            run_info(connection=conn, config=config)
+        elif args.command == "install":
+            upg = Upgrader(config=config)
+            upg.install(
+                connection=conn,
+                parameters=parameters,
+                max_version=args.max_version,
+                roles=args.roles,
+                grant=args.grant,
+                demo_data=args.demo_data,
+            )
+            conn.commit()
+            upg.install_demo_data(name=args.demo_data, connection=conn, parameters=parameters)
+        elif args.command == "role":
+            if not args.action:
+                logger.error(
+                    "You must specify an action for the role command (create, grant, revoke, drop)."
+                )
+                exit_code = 1
+            else:
+                if args.action == "create":
+                    config.role_manager().create_roles(connection=conn)
+                elif args.action == "grant":
+                    config.role_manager().grant_permissions(connection=conn)
+                elif args.action == "revoke":
+                    config.role_manager().revoke_permissions(connection=conn)
+                elif args.action == "drop":
+                    config.role_manager().drop_roles(connection=conn)
+                else:
+                    logger.error(f"Unknown action: {args.action}")
+                    exit_code = 1
+        elif args.command == "check":
+            success = pum.run_check(
+                args.pg_service1,
+                args.pg_service2,
+                ignore_list=args.ignore,
+                exclude_schema=args.exclude_schema,
+                exclude_field_pattern=args.exclude_field_pattern,
+                verbose_level=args.verbose_level,
+                output_file=args.output_file,
+            )
+            if not success:
+                exit_code = 1
+        elif args.command == "dump":
+            pass
+        elif args.command == "restore":
+            pum.run_restore(args.pg_service, args.file, args.x, args.exclude_schema)
+        elif args.command == "baseline":
+            SchemaMigrations(config=config).set_baseline(connection=conn, version=args.baseline)
+
+        elif args.command == "upgrade":
+            # TODO
+            logger.error("Upgrade is not implemented yet")
+        else:
+            logger.error(f"Unknown command: {args.command}")
+            logger.error("Use -h or --help for help.")
+            exit_code = 1
+
+    return exit_code
+
+
+if __name__ == "__main__":
+    sys.exit(cli())

pum/conf/pum_config_example.yaml

@@ -0,0 +1,19 @@
+#TODO comments
+upgrades_table: qwat_sys.upgrades
+delta_dir: ../update/delta/
+backup_file: /tmp/backup.dump
+ignore_elements:
+  - columns
+  - constraints
+  - views
+  - sequences
+  - indexes
+  - triggers
+  - functions
+  - rules
+
+# pg_dump_exe = 'C:\\Program Files\\PostgreSQL\\9.3\\bin\\pg_dump.exe'
+pg_dump_exe: pg_dump
+
+#  pg_restore_exe = 'C:\\Program Files\\PostgreSQL\\9.3\\bin\\pg_restore.exe'
+pg_restore_exe: pg_restore

pum/config_model.py

@@ -0,0 +1,152 @@
+import packaging
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+from typing import List, Optional, Any, Literal
+
+
+from .exceptions import PumConfigError
+from .parameter import ParameterType
+
+
+class PumCustomBaseModel(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+class ParameterDefinitionModel(PumCustomBaseModel):
+    """ParameterDefinitionModel represents a parameter definition in the configuration.
+
+    Attributes:
+        name: Name of the parameter.
+        type: Type of the parameter (default is TEXT).
+        default: Optional default value for the parameter.
+        description: Optional description of the parameter.
+    """
+
+    name: str
+    type: ParameterType = Field(default=ParameterType.TEXT, description="Type of the parameter")
+    default: Optional[Any] = None
+    description: Optional[str] = None
+
+
+class HookModel(PumCustomBaseModel):
+    """
+    HookModel represents a migration hook configuration.
+
+    Attributes:
+        file: Optional path to a SQL file to execute as a hook.
+        code: Optional Python code to execute as a hook.
+    """
+
+    file: Optional[str] = None
+    code: Optional[str] = None
+
+    @model_validator(mode="after")
+    def validate_args(self):
+        file, code = self.file, self.code
+        if (file and code) or (not file and not code):
+            raise PumConfigError("Exactly one of 'file' or 'code' must be set in a hook.")
+        return self
+
+
+class MigrationHooksModel(PumCustomBaseModel):
+    """
+    MigrationHooksModel holds the configuration for migration hooks.
+
+    Attributes:
+        pre: List of pre-migration hooks.
+        post: List of post-migration hooks.
+    """
+
+    pre: Optional[List[HookModel]] = []
+    post: Optional[List[HookModel]] = []
+
+
+class PumModel(PumCustomBaseModel):
+    """
+    PumModel holds some PUM specifics.
+
+    Attributes:
+        migration_table_schema: Name of schema for the migration table.
+        minimum_version: Minimum required version of PUM.
+    """
+
+    model_config = {"arbitrary_types_allowed": True}
+    migration_table_schema: Optional[str] = Field(
+        default="public", description="Name of schema for the migration table"
+    )
+
+    minimum_version: Optional[packaging.version.Version] = Field(
+        default=None,
+        description="Minimum required version of pum.",
+    )
+
+    @model_validator(mode="before")
+    def parse_minimum_version(cls, values):
+        min_ver = values.get("minimum_version")
+        if isinstance(min_ver, str):
+            values["minimum_version"] = packaging.version.Version(min_ver)
+        return values
+
+
+class PermissionModel(PumCustomBaseModel):
+    """
+    PermissionModel represents a permission for a database role.
+
+    Attributes:
+        type: Type of permission ('read' or 'write').
+        schemas: List of schemas this permission applies to.
+    """
+
+    type: Literal["read", "write"] = Field(..., description="Permission type ('read' or 'write').")
+    schemas: List[str] = Field(
+        default_factory=list, description="List of schemas this permission applies to."
+    )
+
+
+class RoleModel(PumCustomBaseModel):
+    """
+    RoleModel represents a database role with associated permissions.
+    Attributes:
+        name: Name of the role.
+        permissions: List of permissions associated with the role.
+        inherit: Optional name of another role to inherit permissions from.
+        description: Optional description of the role.
+    """
+
+    name: str = Field(..., description="Name of the role.")
+    permissions: List[PermissionModel] = Field(
+        default_factory=list, description="List of permissions for the role."
+    )
+    inherit: Optional[str] = Field(None, description="Name of the role to inherit from.")
+    description: Optional[str] = Field(None, description="Description of the role.")
+
+
+class DemoDataModel(PumCustomBaseModel):
+    """
+    DemoDataModel represents a configuration for demo data.
+
+    Attributes:
+        files: Optional list of named demo data files.
+    """
+
+    name: str = Field(..., description="Name of the demo data.")
+    file: str = Field(..., description="Path to the demo data file.")
+
+
+class ConfigModel(PumCustomBaseModel):
+    """
+    ConfigModel represents the main configuration schema for the application.
+
+    Attributes:
+        pum: The PUM (Project Update Manager) configuration. Defaults to a new PumModel instance.
+        parameters: List of parameter definitions. Defaults to an empty list.
+        migration_hooks: Configuration for migration hooks. Defaults to a new MigrationHooksModel instance.
+        changelogs_directory: Directory path for changelogs. Defaults to "changelogs".
+        roles: List of role definitions. Defaults to None.
+    """
+
+    pum: Optional[PumModel] = Field(default_factory=PumModel)
+    parameters: Optional[List[ParameterDefinitionModel]] = []
+    migration_hooks: Optional[MigrationHooksModel] = Field(default_factory=MigrationHooksModel)
+    changelogs_directory: Optional[str] = "changelogs"
+    roles: Optional[List[RoleModel]] = []
+    demo_data: Optional[List[DemoDataModel]] = []

pum/dumper.py

@@ -0,0 +1,110 @@
+import subprocess
+import logging
+from enum import Enum
+
+from .exceptions import (
+    PgDumpCommandError,
+    PgDumpFailed,
+    PgRestoreCommandError,
+    PgRestoreFailed,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class DumpFormat(Enum):
+    CUSTOM = "custom"
+    PLAIN = "plain"
+
+    def to_pg_dump_flag(self):
+        if self == DumpFormat.CUSTOM:
+            return "-Fc"
+        elif self == DumpFormat.PLAIN:
+            return "-Fp"
+        raise ValueError(f"Unknown dump format: {self}")
+
+
+class Dumper:
+    """This class is used to dump and restore a Postgres database."""
+
+    def __init__(self, pg_service: str, dump_path: str):
+        self.pg_service = pg_service
+        self.dump_path = dump_path
+
+    def pg_dump(
+        self,
+        dbname: str | None = None,
+        *,
+        pg_dump_exe: str = "pg_dump",
+        exclude_schema: list[str] | None = None,
+        format: DumpFormat = DumpFormat.CUSTOM,
+    ):
+        """
+        Call the pg_dump command to dump a db backup
+
+        Args:
+            dbname: Name of the database to dump.
+            pg_dump_exe: Path to the pg_dump executable.
+            exclude_schema: List of schemas to exclude from the dump.
+            format: DumpFormat, either custom (default) or plain
+        """
+
+        connection = f"service={self.pg_service}"
+        if dbname:
+            connection = f"{connection} dbname={dbname}"
+
+        command = [
+            pg_dump_exe,
+            format.to_pg_dump_flag(),
+            "--no-owner",
+            "--no-privileges",
+            "-f",
+            self.dump_path,
+        ]
+        if exclude_schema:
+            for schema in exclude_schema:
+                command.append(f"--exclude-schema={schema}")
+        command.extend(["-d", connection])
+
+        logger.debug("Running pg_dump command: %s", " ".join(command))
+
+        try:
+            output = subprocess.run(command, capture_output=True, text=True, check=False)
+            if output.returncode != 0:
+                logger.error("pg_dump failed: %s", output.stderr)
+                raise PgDumpFailed(output.stderr)
+        except TypeError:
+            logger.error("Invalid command: %s", " ".join(command))
+            raise PgDumpCommandError("invalid command: {}".format(" ".join(filter(None, command))))
+
+    def pg_restore(
+        self,
+        dbname: str | None = None,
+        pg_restore_exe: str = "pg_restore",
+        exclude_schema: list[str] | None = None,
+    ):
+        """ """
+
+        connection = f"service={self.pg_service}"
+        if dbname:
+            connection = f"{connection} dbname={dbname}"
+
+        command = [pg_restore_exe, "-d", connection, "--no-owner"]
+
+        if exclude_schema:
+            for schema in exclude_schema:
+                command.append(f"--exclude-schema={schema}")
+        command.append(self.dump_path)
+
+        logger.debug("Running pg_restore command: %s", " ".join(command))
+
+        try:
+            output = subprocess.run(command, capture_output=True, text=True, check=False)
+            if output.returncode != 0:
+                logger.error("pg_restore failed: %s", output.stderr)
+                raise PgRestoreFailed(output.stderr)
+        except TypeError:
+            logger.error("Invalid command: %s", " ".join(command))
+            raise PgRestoreCommandError(
+                "invalid command: {}".format(" ".join(filter(None, command)))
+            )