pum 1.0.0__py3-none-any.whl

pum/sql_content.py

@@ -0,0 +1,265 @@
+import logging
+import re
+from pathlib import Path
+
+import psycopg
+
+from .exceptions import PumSqlError
+
+logger = logging.getLogger(__name__)
+
+
+def sql_chunks_from_file(file: str | Path) -> list[psycopg.sql.SQL]:
+    """Read SQL from a file, remove comments, and split into chunks.
+
+    Args:
+        file (str | Path): Path to the SQL file.
+
+    Returns:
+        list: List of SQL statements.
+
+    Raises:
+        PumInvalidSqlFile: If the SQL file contains forbidden transaction statements.
+
+    """
+    file = Path(file) if not isinstance(file, Path) else file
+    sql_code = []
+    with Path.open(file) as file:
+        sql_content = file.read()
+
+        # Remove SQL comments
+        def remove_sql_comments(sql: str) -> str:
+            """Remove SQL comments from the SQL string.
+
+            Args:
+                sql (str): The SQL string to process.
+
+            Returns:
+                str: The SQL string without comments.
+
+            """
+            # Remove multiline comments (/* ... */)
+            sql = re.sub(r"/\*.*?\*/", "", sql, flags=re.DOTALL)
+            # Remove single-line comments (-- ...)
+            sql = re.sub(r"(?m)(^|;)\s*--.*?(\r\n|\r|\n)", r"\1", sql)
+            return sql
+
+        sql_content = remove_sql_comments(sql_content)
+
+        # Check for forbidden transaction statements
+        forbidden_statements = (
+            (
+                r"\bBEGIN\b\s*;",
+                "BEGIN; COMMIT; is not authroized in executed SQL since connections are handled by PUM.",
+            ),
+            (
+                r"\bCOMMIT\b\s*;",
+                "BEGIN; COMMIT; is not authroized in executed SQL since connections are handled by PUM.",
+            ),
+            (
+                r"SELECT +pg_catalog.set_config.*search_path.*;",
+                "Setting of search path is not authorized in executed SQL as it breaks PostGIS installation",
+            ),
+        )
+        for forbidden, message in forbidden_statements:
+            if re.search(forbidden, sql_content, re.IGNORECASE):
+                raise PumSqlError(f"SQL contains forbidden transaction statement: {message}")
+
+        def split_sql_statements(sql: str) -> list[str]:
+            """
+            Split SQL statements by semicolon, ignoring those inside single/double quotes, dollar-quoted blocks, and DO/BODY blocks.
+            Do NOT split on semicolons inside string literals (e.g. COMMENT ON ... IS '...;...'),
+            and do NOT split on semicolons inside dollar-quoted blocks (e.g. $$...;...$$, $BODY$...;...$BODY$),
+            but DO split on semicolons that are not inside a string, block, or quote, even if the statement contains a SQL comment with a semicolon.
+            """
+            # Step 1: Replace all dollar-quoted blocks with placeholders
+            body_blocks = []
+            block_pattern = (
+                r"(\$\$BODY\$\$.*?\$\$BODY\$\$"
+                r"|\$BODY\$.*?\$BODY\$"
+                r"|\$\$DO\$\$.*?\$\$DO\$\$"
+                r"|\$DO\$.*?\$DO\$"
+                r"|\$[A-Za-z0-9_]*\$.*?\$[A-Za-z0-9_]*\$"  # generic $tag$...$tag$
+                r"|\$\$.*?\$\$"  # generic $$...$$
+                r")"
+            )
+
+            def block_replacer(match):
+                body_blocks.append(match.group(0))
+                return f"__BLOCK_{len(body_blocks) - 1}__"
+
+            sql_wo_blocks = re.sub(
+                block_pattern, block_replacer, sql, flags=re.DOTALL | re.IGNORECASE
+            )
+
+            # Step 2: Split by semicolon, but only when not inside a string or comment
+            statements = []
+            current = []
+            in_single = False
+            in_double = False
+            in_line_comment = False
+            i = 0
+            while i < len(sql_wo_blocks):
+                c = sql_wo_blocks[i]
+                next2 = sql_wo_blocks[i : i + 2]
+                if in_line_comment:
+                    current.append(c)
+                    if c == "\n":
+                        in_line_comment = False
+                    i += 1
+                    continue
+                if not in_single and not in_double and next2 == "--":
+                    in_line_comment = True
+                    current.append("--")
+                    i += 2
+                    continue
+                if not in_double and c == "'":
+                    in_single = not in_single
+                    current.append(c)
+                    i += 1
+                    continue
+                if not in_single and c == '"':
+                    in_double = not in_double
+                    current.append(c)
+                    i += 1
+                    continue
+                if not in_single and not in_double and not in_line_comment and c == ";":
+                    statements.append("".join(current).strip())
+                    current = []
+                    i += 1
+                    continue
+                current.append(c)
+                i += 1
+            if current:
+                statements.append("".join(current).strip())
+
+            # Step 3: Restore dollar-quoted blocks
+            def restore_blocks(stmt):
+                for idx, block in enumerate(body_blocks):
+                    stmt = stmt.replace(f"__BLOCK_{idx}__", block)
+                return stmt
+
+            return [restore_blocks(stmt) for stmt in statements if stmt]
+
+        sql_code = split_sql_statements(sql_content)
+
+        # if we want to remove new lines from the SQL code, we need to handle comments starting with --
+        # and remove them before removing new lines
+        # sql_code = [re.sub(r"[\r\n]+", " ", stmt) for stmt in sql_code]
+        sql_code = [psycopg.sql.SQL(stmt) for stmt in sql_code]
+
+    return sql_code
+
+
+class SqlContent:
+    """Class to handle SQL content preparation and execution."""
+
+    def __init__(self, sql: str | psycopg.sql.SQL | Path) -> None:
+        """Initialize the SqlContent class.
+
+        Args:
+            sql: The SQL statement to execute or a path to a SQL file.
+
+        """
+        self.sql = sql
+
+    def validate(self, parameters: dict | None) -> bool:
+        """Validate the SQL content.
+        This is done by checking if the SQL content is not empty.
+
+        Args:
+            parameters: The parameters to pass to the SQL files.
+
+        Returns:
+            bool: True if valid, False otherwise.
+
+        """
+        if not self.sql:
+            raise PumSqlError("SQL content is empty.")
+        self._prepare_sql(parameters)
+        return True
+
+    def execute(
+        self,
+        connection: psycopg.Connection,
+        *,
+        parameters: dict | None = None,
+        commit: bool = False,
+    ) -> psycopg.Cursor:
+        """Execute a SQL statement with optional parameters.
+
+        Args:
+            connection: The database connection to execute the SQL statement.
+            parameters: Parameters to bind to the SQL statement. Defaults to ().
+            commit: Whether to commit the transaction. Defaults to False.
+
+        """
+        cursor = connection.cursor()
+
+        for sql_code in self._prepare_sql(parameters):
+            try:
+                statement = sql_code.as_string(connection)
+            except (psycopg.errors.SyntaxError, psycopg.errors.ProgrammingError) as e:
+                raise PumSqlError(
+                    f"SQL preparation failed for the following code: {statement} {e}"
+                ) from e
+            try:
+                logger.debug(f"Executing SQL statement: {statement}")
+                cursor.execute(statement)
+            except (psycopg.errors.SyntaxError, psycopg.errors.ProgrammingError) as e:
+                raise PumSqlError(
+                    f"SQL execution failed for the following code: {statement} {e}"
+                ) from e
+        if commit:
+            connection.commit()
+
+        return cursor
+
+    def _prepare_sql(self, parameters: dict | None) -> list[psycopg.sql.SQL]:
+        """Prepare SQL for execution.
+
+        Args:
+            sql: The SQL statement to execute or a path to a SQL file.
+            parameters: Parameters to bind to the SQL statement. Defaults to ().
+
+        Returns:
+            list: A list of prepared SQL statements.
+
+        Raises:
+            PumSqlError: If SQL preparation fails.
+
+        """
+        if isinstance(self.sql, Path):
+            logger.info(
+                f"Checking SQL from file: {self.sql} with parameters: {parameters}",
+            )
+            sql_code = sql_chunks_from_file(self.sql)
+        elif isinstance(self.sql, str):
+            sql_code = [psycopg.sql.SQL(self.sql)]
+        else:
+            sql_code = [self.sql]
+
+        def format_sql(
+            statement: psycopg.sql.SQL, parameters: dict | None = None
+        ) -> psycopg.sql.SQL:
+            for key, value in (parameters or {}).items():
+                if (
+                    not isinstance(value, psycopg.sql.Literal)
+                    and not isinstance(value, psycopg.sql.Identifier)
+                    and not isinstance(value, psycopg.sql.Composed)
+                ):
+                    raise PumSqlError(
+                        f"Invalid parameter type for key '{key}': {type(value)}. "
+                        "Parameters must be psycopg.sql.Literal or psycopg.sql.Identifier."
+                    )
+            try:
+                return statement.format(**parameters)
+            except TypeError:
+                # if parameters is None, we can ignore this error
+                return statement
+            except KeyError as e:
+                raise PumSqlError(
+                    f"SQL preparation failed for the following code: missing parameter: {statement} {e}"
+                ) from e
+
+        return [format_sql(statement, parameters) for statement in sql_code]

pum/upgrader.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python
+import logging
+
+import packaging
+import packaging.version
+import psycopg
+import copy
+
+from .pum_config import PumConfig
+from .exceptions import PumException
+from .schema_migrations import SchemaMigrations
+from .sql_content import SqlContent
+
+
+logger = logging.getLogger(__name__)
+
+
+class Upgrader:
+    """Class to handle the upgrade of a module.
+    This class is used to install a new instance or to upgrade an existing instance of a module.
+    It stores the info about the upgrade in a table on the database.
+    """
+
+    def __init__(
+        self,
+        config: PumConfig,
+        max_version: packaging.version.Version | str | None = None,
+    ) -> None:
+        """Initialize the Upgrader class.
+        This class is used to install a new instance or to upgrade an existing instance of a module.
+        Stores the info about the upgrade in a table on the database.
+        The table is created in the schema defined in the config file if it does not exist.
+
+        Args:
+            connection:
+                The database connection to use for the upgrade.
+            config:
+                The configuration object
+            max_version:
+                Maximum (including) version to run the deltas up to.
+
+        """
+        self.config = config
+        self.max_version = packaging.parse(max_version) if max_version else None
+        self.schema_migrations = SchemaMigrations(self.config)
+
+    def install(
+        self,
+        connection: psycopg.Connection = None,
+        *,
+        parameters: dict | None = None,
+        max_version: str | packaging.version.Version | None = None,
+        roles: bool = False,
+        grant: bool = False,
+        demo_data: str | None = None,
+        commit: bool = False,
+    ) -> None:
+        """Installs the given module
+        This will create the schema_migrations table if it does not exist.
+        The changelogs are applied in the order they are found in the directory.
+        It will also set the baseline version to the current version of the module.
+
+        Args:
+            connection:
+                The database connection to use for the upgrade.
+            parameters:
+                The parameters to pass for the migration.
+            max_version:
+                The maximum version to apply. If None, all versions are applied.
+            roles:
+                If True, roles will be created.
+            grant:
+                If True, permissions will be granted to the roles.
+            demo_data:
+                The name of the demo data to load. If None, no demo data is loaded.
+            commit:
+                If True, the changes will be committed to the database.
+        """
+        parameters_literals = self._prepare_parameters(parameters)
+
+        if demo_data and demo_data not in self.config.demo_data():
+            raise PumException(
+                f"Demo data '{demo_data}' not found in the configuration. Available demo data: {self.config.demo_data()}"
+            )
+
+        if self.schema_migrations.exists(connection):
+            msg = (
+                f"Schema migrations table {self.config.config.pum.migration_table_schema}.pum_migrations already exists. "
+                "This means that the module is already installed or the database is not empty. "
+                "Use upgrade() to upgrade the db or start with a clean db."
+            )
+            raise PumException(msg)
+        self.schema_migrations.create(connection, commit=False)
+
+        if roles or grant:
+            self.config.role_manager().create_roles(
+                connection=connection, grant=False, commit=False
+            )
+
+        for pre_hook in self.config.pre_hook_handlers():
+            pre_hook.execute(connection=connection, commit=False, parameters=parameters_literals)
+
+        last_changelog = None
+        for changelog in self.config.changelogs(max_version=max_version):
+            last_changelog = changelog
+            changelog_files = changelog.apply(
+                connection, commit=False, parameters=parameters_literals
+            )
+            changelog_files = [str(f) for f in changelog_files]
+            self.schema_migrations.set_baseline(
+                connection=connection,
+                version=changelog.version,
+                beta_testing=False,
+                commit=False,
+                changelog_files=changelog_files,
+                parameters=parameters,
+            )
+
+        for post_hook in self.config.post_hook_handlers():
+            post_hook.execute(connection=connection, commit=False, parameters=parameters_literals)
+
+        logger.info(
+            "Installed %s.pum_migrations table and applied changelogs up to version %s",
+            self.config.config.pum.migration_table_schema,
+            last_changelog.version,
+        )
+
+        if grant:
+            self.config.role_manager().grant_permissions(connection=connection, commit=False)
+
+        if commit:
+            connection.commit()
+            logger.info("Changes committed to the database.")
+
+    def install_demo_data(
+        self,
+        connection: psycopg.Connection,
+        name: str,
+        *,
+        parameters: dict | None = None,
+    ) -> None:
+        """Install demo data for the module.
+        Args:
+            connection: The database connection to use.
+            name: The name of the demo data to install.
+            parameters: The parameters to pass to the demo data SQL.
+        """
+        if name not in self.config.demo_data():
+            raise PumException(f"Demo data '{name}' not found in the configuration.")
+
+        parameters_literals = self._prepare_parameters(parameters)
+
+        demo_data_file = self.config.base_path / self.config.demo_data()[name]
+        logger.info("Installing demo data from %s", demo_data_file)
+
+        for pre_hook in self.config.pre_hook_handlers():
+            pre_hook.execute(connection=connection, commit=False, parameters=parameters_literals)
+
+        connection.commit()
+
+        SqlContent(sql=demo_data_file).execute(
+            connection=connection,
+            commit=False,
+            parameters=parameters_literals,
+        )
+
+        connection.commit()
+
+        for post_hook in self.config.post_hook_handlers():
+            post_hook.execute(connection=connection, commit=False, parameters=parameters_literals)
+
+        logger.info("Demo data '%s' installed successfully.", name)
+
+    @staticmethod
+    def _prepare_parameters(parameters: dict | None):
+        """
+        Prepares a dictionary of parameters for use in SQL queries by converting each value to a psycopg.sql.Literal.
+
+        Args:
+            parameters: A dictionary of parameters to be converted, or None.
+
+        Returns:
+            dict: A new dictionary with the same keys as `parameters`, where each value is wrapped in psycopg.sql.Literal.
+        """
+        parameters_literals = copy.deepcopy(parameters) if parameters else {}
+        for key, value in parameters_literals.items():
+            parameters_literals[key] = psycopg.sql.Literal(value)
+        return parameters_literals

pum-1.0.0.dist-info/METADATA

@@ -0,0 +1,61 @@
+Metadata-Version: 2.4
+Name: pum
+Version: 1.0.0
+Summary: Pum stands for "Postgres Upgrades Manager". It is a Database migration management tool very similar to flyway-db or Liquibase, based on metadata tables.
+Author-email: Denis Rouzaud <denis@opengis.ch>
+License-Expression: GPL-2.0-or-later
+Project-URL: homepage, https://opengisch.github.io/pum/
+Project-URL: documentation, https://opengisch.github.io/pum/
+Project-URL: repository, https://github.com/opengisch/pum/
+Project-URL: tracker, https://github.com/opengisch/pum/issues
+Keywords: postgres,database,versioning
+Classifier: Topic :: Database
+Classifier: Intended Audience :: System Administrators
+Classifier: Intended Audience :: Information Technology
+Classifier: Topic :: Software Development :: Version Control
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: packaging
+Requires-Dist: pydantic
+Requires-Dist: PyYAML
+Requires-Dist: psycopg[binary]
+Provides-Extra: dev
+Requires-Dist: black; extra == "dev"
+Requires-Dist: flake8-builtins; extra == "dev"
+Requires-Dist: flake8-isort; extra == "dev"
+Requires-Dist: flake8-print; extra == "dev"
+Requires-Dist: pre-commit; extra == "dev"
+Requires-Dist: nose2; extra == "dev"
+Dynamic: license-file
+
+# PostgreSQL Upgrades Manager (PUM)
+
+![logo](https://raw.githubusercontent.com/opengisch/pum/main/docs/docs/assets/images/pum.png)
+
+## New version
+
+This is the code of pum version 1.
+You can find version 0.x documentation at https://github.com/opengisch/pum/tree/old-v1
+
+## About
+
+PUM (PostgreSQL Upgrades Manager) is a robust database migration management tool designed to streamline the process of managing PostgreSQL database upgrades. Inspired by tools like FlywayDB and Liquibase, PUM leverages metadata tables to ensure seamless database versioning and migration.
+
+## Key Features
+
+- **Command-line and Python Integration**: Use PUM as a standalone CLI tool or integrate it into your Python project.
+- **Database Versioning**: Automatically manage database versioning with a metadata table.
+- **Changelog Management**: Apply and track SQL delta files for database upgrades.
+- **Migration Hooks**: Define custom hooks to execute additional SQL or Python code before or after migrations. This feature allows you to isolate data (table) code from application code (such as views and triggers), ensuring a clear separation of concerns and more maintainable database structures.
+
+## Why PUM?
+
+Managing database migrations in a Version Control System (VCS) can be challenging, especially for production databases. PUM simplifies this process by embedding version metadata directly into the database, enabling efficient tracking and application of migrations.
+
+PUM was developed to address challenges in the [TEKSI](https://github.com/TESKI) project, an open-source GIS for network management based on [QGIS](http://qgis.org/fr/site/).

pum-1.0.0.dist-info/RECORD

@@ -0,0 +1,22 @@
+pum/__init__.py,sha256=IG1g3LMuqSxlW8GC1-H5XdnPX3KF-Irq7MuMEbCYCUM,684
+pum/changelog.py,sha256=yDc5swmMd5gb2vCEAlenoq5gs-ZEGc4uXicBtiGxkOk,3692
+pum/checker.py,sha256=GT2v7793HP1g94dv0mL6CHtQfblQwAyeFHEWCy44lkc,14379
+pum/cli.py,sha256=GcxBiQM_S4CDG1v1Gy9xj0noaf-GdeJFi8jc2yfVjLU,13956
+pum/config_model.py,sha256=G8FsVGsYzMEQKrVIc3SltycbJvwmgfhu6uNduAL-D8E,5107
+pum/dumper.py,sha256=EJZ8T44JM0GKgdqw1ENOfhZ-RI89OQ4DNdoTZKtLdEw,3404
+pum/exceptions.py,sha256=HYgC0kLk6Gel7RtT9AjxHdtkmZ4BtjKdB5BHYL67LVs,1042
+pum/hook.py,sha256=L4Cnr34zrgPzxso9CdsUYWmtuOXRmFccQZ9Lp4IYCBM,9326
+pum/info.py,sha256=VSCUZJJ_ae-khKaudwbgqszZXBMKB_yskuQo5Mc1PgY,1024
+pum/parameter.py,sha256=e9f80kMZpART9laeImW_YECeTvwDyDSmZlTeJGvpS_8,2449
+pum/pum_config.py,sha256=WopogLoPEJkvuKEdpWq56YZaZc3mgK-pnII0TbYtIlQ,8454
+pum/role_manager.py,sha256=yr-fmytflGqANY3IZIpgJBoMOK98ynTWfemIBhAy79A,10131
+pum/schema_migrations.py,sha256=FiaqAbhFX7vd3Rk_R43kd7-QWfil-Q5587EU8xSLBkA,10504
+pum/sql_content.py,sha256=gwgvcdXOXxNz3RvLtL8Bqr5WO3KKq3sluhbj4OAEnQs,9756
+pum/upgrader.py,sha256=jvl6vmpgxGyYiw8rrWC_bDC7Zd4wHJqGLXCK8EMt9wY,7109
+pum/conf/pum_config_example.yaml,sha256=_nwV_7z6S_Se-mejh_My0JFLY-A0Q4nigeLGPZAfcqg,424
+pum-1.0.0.dist-info/licenses/LICENSE,sha256=2ylvL381vKOhdO-w6zkrOxe9lLNBhRQpo9_0EbHC_HM,18046
+pum-1.0.0.dist-info/METADATA,sha256=HVBChQBJ0xnjyYYxnYyU7EAhU4CLs0V8SIJhM09j9iE,3146
+pum-1.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pum-1.0.0.dist-info/entry_points.txt,sha256=U6dmxSpKs1Pe9vWiR29VPhJMDjrmZeJCSxvfLGR8BD4,36
+pum-1.0.0.dist-info/top_level.txt,sha256=ddiI4HLBhY6ql-NNm0Ez0JhoOHdWDIzrHeCdHmmagcc,4
+pum-1.0.0.dist-info/RECORD,,

pum-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

pum-1.0.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+pum = pum.cli:cli