infrahub-testcontainers 1.5.0b1__tar.gz → 1.5.1__tar.gz

{infrahub_testcontainers-1.5.0b1 → infrahub_testcontainers-1.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: infrahub-testcontainers
-Version: 1.5.0b1
+Version: 1.5.1
 Summary: Testcontainers instance for Infrahub to easily build integration tests
 License: Apache-2.0
 Author: OpsMill

{infrahub_testcontainers-1.5.0b1 → infrahub_testcontainers-1.5.1}/infrahub_testcontainers/container.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import os
 import shutil
 import time
@@ -72,7 +74,10 @@ class InfrahubDockerCompose(DockerCompose):
 
     @classmethod
     def init(
-        cls, directory: Path | None = None, version: str | None = None, deployment_type: str | None = None
+        cls,
+        directory: Path | None = None,
+        version: str | None = None,
+        deployment_type: str | None = None,
     ) -> Self:
         if not directory:
             directory = Path.cwd()
@@ -84,7 +89,11 @@ class InfrahubDockerCompose(DockerCompose):
         if version == "local" and infrahub_image_version:
             version = infrahub_image_version
 
-        compose = cls(project_name=cls.generate_project_name(), context=directory, deployment_type=deployment_type)
+        compose = cls(
+            project_name=cls.generate_project_name(),
+            context=directory,
+            deployment_type=deployment_type,
+        )
         compose.create_docker_file(directory=directory)
         compose.create_env_file(directory=directory, version=version)
 
@@ -152,6 +161,8 @@ class InfrahubDockerCompose(DockerCompose):
                     "INFRAHUB_TESTING_TASKMGR_BACKGROUND_SVC_REPLICAS": "1",
                     "PREFECT_MESSAGING_BROKER": "prefect_redis.messaging",
                     "PREFECT_MESSAGING_CACHE": "prefect_redis.messaging",
+                    "PREFECT_SERVER_EVENTS_CAUSAL_ORDERING": "prefect_redis.ordering",
+                    "PREFECT_SERVER_CONCURRENCY_LEASE_STORAGE": "prefect_redis.lease_storage",
                     "PREFECT__SERVER_WEBSERVER_ONLY": "true",
                     "PREFECT_API_DATABASE_MIGRATE_ON_START": "false",
                     "PREFECT_API_BLOCKS_REGISTER_ON_START": "false",
@@ -237,7 +248,12 @@ class InfrahubDockerCompose(DockerCompose):
             for service_name, service_data in INFRAHUB_SERVICES.items()
         }
 
-    def database_create_backup(self, backup_name: str = "neo4j_database.backup", dest_dir: Path | None = None) -> None:
+    def database_create_backup(
+        self,
+        backup_name: str = "neo4j_database.backup",
+        dest_dir: Path | None = None,
+        compress: bool = False,
+    ) -> None:
         assert self.use_neo4j_enterprise
 
         self.exec_in_container(
@@ -245,7 +261,7 @@ class InfrahubDockerCompose(DockerCompose):
                 "neo4j-admin",
                 "database",
                 "backup",
-                "--compress=false",
+                f"--compress={'true' if compress else 'false'}",
                 "--to-path",
                 str(self.internal_backup_dir),
             ],
@@ -279,7 +295,14 @@ class InfrahubDockerCompose(DockerCompose):
                 self.start_container(service_name=service_name)
 
             self.exec_in_container(
-                command=["cypher-shell", "-u", "neo4j", "-p", "admin", "STOP DATABASE neo4j;"],
+                command=[
+                    "cypher-shell",
+                    "-u",
+                    "neo4j",
+                    "-p",
+                    "admin",
+                    "STOP DATABASE neo4j;",
+                ],
                 service_name=service_name,
             )
 
@@ -364,7 +387,14 @@ class InfrahubDockerCompose(DockerCompose):
             time.sleep(10)
 
             self.exec_in_container(
-                command=["cypher-shell", "-u", "neo4j", "-p", "admin", "DROP DATABASE neo4j;"],
+                command=[
+                    "cypher-shell",
+                    "-u",
+                    "neo4j",
+                    "-p",
+                    "admin",
+                    "DROP DATABASE neo4j;",
+                ],
                 service_name=service_name,
             )
 
@@ -513,3 +543,111 @@ class InfrahubDockerCompose(DockerCompose):
                 )
             self.start()
             print("Database restored successfully")
+
+    def task_manager_create_backup(self, backup_name: str = "prefect.dump", dest_dir: Path | None = None) -> Path:
+        """Create a backup of the task manager PostgreSQL database using ``pg_dump``.
+
+        Args:
+            backup_name: Name of the archive file to create. Defaults to ``prefect.dump``.
+            dest_dir: Optional host directory where the backup should be copied after it is
+                produced. When omitted, the backup remains in ``external_backup_dir``.
+
+        Returns:
+            Path to the backup archive on the host filesystem.
+
+        Raises:
+            FileNotFoundError: If the pg_dump command completes but no archive is produced.
+        """
+
+        service_name = "task-manager-db"
+
+        try:
+            self.get_container(service_name=service_name)
+        except ContainerIsNotRunning:
+            self.start_container(service_name=service_name)
+
+        self.external_backup_dir.mkdir(parents=True, exist_ok=True)
+
+        internal_backup_path = self.internal_backup_dir / backup_name
+        dump_command = [
+            "pg_dump",
+            "--format=custom",
+            "--blobs",
+            "--no-owner",
+            "--no-privileges",
+            "--dbname=postgresql://postgres:postgres@localhost:5432/prefect",
+            f"--file={internal_backup_path}",
+        ]
+        self.exec_in_container(command=dump_command, service_name=service_name)
+
+        source_path = self.external_backup_dir / backup_name
+        if not source_path.exists():
+            raise FileNotFoundError(f"Backup file {source_path} was not created")
+
+        final_path = source_path
+        if dest_dir:
+            dest_dir.mkdir(parents=True, exist_ok=True)
+            if dest_dir.resolve() != self.external_backup_dir.resolve():
+                final_path = dest_dir / backup_name
+                shutil.copy(source_path, final_path)
+
+        return final_path
+
+    def task_manager_restore_backup(self, backup_file: Path) -> None:
+        """Restore the task manager PostgreSQL database from a ``pg_restore`` archive.
+
+        Args:
+            backup_file: Path to the backup archive on the host filesystem.
+
+        Raises:
+            FileNotFoundError: If the provided backup archive does not exist.
+        """
+
+        if not backup_file.exists():
+            raise FileNotFoundError(f"Backup file {backup_file} does not exist")
+
+        service_name = "task-manager-db"
+
+        try:
+            self.get_container(service_name=service_name)
+        except ContainerIsNotRunning:
+            self.start_container(service_name=service_name)
+
+        self.external_backup_dir.mkdir(parents=True, exist_ok=True)
+        target_path = self.external_backup_dir / backup_file.name
+        shutil.copy(backup_file, target_path)
+
+        admin_dsn = "postgresql://postgres:postgres@localhost:5432/postgres"
+        prefect_dsn = "postgresql://postgres:postgres@localhost:5432/prefect"
+        internal_backup_path = self.internal_backup_dir / backup_file.name
+
+        terminate_sessions_command = [
+            "psql",
+            f"--dbname={admin_dsn}",
+            "--command",
+            "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = 'prefect';",
+        ]
+        drop_database_command = [
+            "psql",
+            f"--dbname={admin_dsn}",
+            "--command",
+            "DROP DATABASE IF EXISTS prefect WITH (FORCE);",
+        ]
+        create_database_command = [
+            "psql",
+            f"--dbname={admin_dsn}",
+            "--command",
+            "CREATE DATABASE prefect OWNER postgres;",
+        ]
+        restore_command = [
+            "pg_restore",
+            "--no-owner",
+            "--role=postgres",
+            f"--dbname={prefect_dsn}",
+            str(internal_backup_path),
+        ]
+
+        self.exec_in_container(command=terminate_sessions_command, service_name=service_name)
+        self.exec_in_container(command=drop_database_command, service_name=service_name)
+        self.exec_in_container(command=create_database_command, service_name=service_name)
+        self.exec_in_container(command=restore_command, service_name=service_name)

{infrahub_testcontainers-1.5.0b1 → infrahub_testcontainers-1.5.1}/infrahub_testcontainers/docker-compose-cluster.test.yml

@@ -184,6 +184,8 @@ services:
 
       PREFECT_MESSAGING_BROKER:
       PREFECT_MESSAGING_CACHE:
+      PREFECT_SERVER_EVENTS_CAUSAL_ORDERING:
+      PREFECT_SERVER_CONCURRENCY_LEASE_STORAGE:
       PREFECT__SERVER_WEBSERVER_ONLY:
       PREFECT_API_DATABASE_MIGRATE_ON_START:
       PREFECT_API_BLOCKS_REGISTER_ON_START:
@@ -225,6 +227,8 @@ services:
       INFRAHUB_CACHE_ADDRESS: ${INFRAHUB_TESTING_CACHE_ADDRESS}
       PREFECT_MESSAGING_BROKER: prefect_redis.messaging
       PREFECT_MESSAGING_CACHE: prefect_redis.messaging
+      PREFECT_SERVER_EVENTS_CAUSAL_ORDERING: prefect_redis.ordering
+      PREFECT_SERVER_CONCURRENCY_LEASE_STORAGE: prefect_redis.lease_storage
       PREFECT_REDIS_MESSAGING_HOST: "${INFRAHUB_TESTING_CACHE_ADDRESS:-cache}"
       PREFECT_REDIS_MESSAGING_DB: "1"
       PREFECT_REDIS_MESSAGING_CONSUMER_MIN_IDLE_TIME: "30"
@@ -244,6 +248,7 @@ services:
       - POSTGRES_DB=prefect
     volumes:
       - workflow_db:/var/lib/postgresql/data
+      - "./${INFRAHUB_TESTING_LOCAL_DB_BACKUP_DIRECTORY}:${INFRAHUB_TESTING_INTERNAL_DB_BACKUP_DIRECTORY}"
     healthcheck:
       test: ["CMD-SHELL", "pg_isready"]
       interval: 10s

{infrahub_testcontainers-1.5.0b1 → infrahub_testcontainers-1.5.1}/infrahub_testcontainers/docker-compose.test.yml

@@ -95,6 +95,8 @@ services:
 
       PREFECT_MESSAGING_BROKER:
       PREFECT_MESSAGING_CACHE:
+      PREFECT_SERVER_EVENTS_CAUSAL_ORDERING:
+      PREFECT_SERVER_CONCURRENCY_LEASE_STORAGE:
       PREFECT__SERVER_WEBSERVER_ONLY:
       PREFECT_API_DATABASE_MIGRATE_ON_START:
       PREFECT_API_BLOCKS_REGISTER_ON_START:
@@ -136,6 +138,8 @@ services:
       INFRAHUB_CACHE_ADDRESS: ${INFRAHUB_TESTING_CACHE_ADDRESS}
       PREFECT_MESSAGING_BROKER: prefect_redis.messaging
       PREFECT_MESSAGING_CACHE: prefect_redis.messaging
+      PREFECT_SERVER_EVENTS_CAUSAL_ORDERING: prefect_redis.ordering
+      PREFECT_SERVER_CONCURRENCY_LEASE_STORAGE: prefect_redis.lease_storage
       PREFECT_REDIS_MESSAGING_HOST: "${INFRAHUB_TESTING_CACHE_ADDRESS:-cache}"
       PREFECT_REDIS_MESSAGING_DB: "1"
       PREFECT_REDIS_MESSAGING_CONSUMER_MIN_IDLE_TIME: "30"
@@ -155,6 +159,7 @@ services:
       - POSTGRES_DB=prefect
     volumes:
       - workflow_db:/var/lib/postgresql/data
+      - "./${INFRAHUB_TESTING_LOCAL_DB_BACKUP_DIRECTORY}:${INFRAHUB_TESTING_INTERNAL_DB_BACKUP_DIRECTORY}"
     healthcheck:
       test: ["CMD-SHELL", "pg_isready"]
       interval: 10s

{infrahub_testcontainers-1.5.0b1 → infrahub_testcontainers-1.5.1}/infrahub_testcontainers/helpers.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import os
 import subprocess  # noqa: S404
 import uuid
@@ -24,13 +26,21 @@ class TestInfrahubDocker:
         env["INFRAHUB_API_TOKEN"] = PROJECT_ENV_VARIABLES["INFRAHUB_TESTING_INITIAL_ADMIN_TOKEN"]
         env["INFRAHUB_MAX_CONCURRENT_EXECUTION"] = "1"
         result = subprocess.run(  # noqa: S602
-            f"infrahubctl run {script}", shell=True, capture_output=True, text=True, env=env, check=False
+            f"infrahubctl run {script}",
+            shell=True,
+            capture_output=True,
+            text=True,
+            env=env,
+            check=False,
         )
         return result.stdout
 
     @staticmethod
     def execute_command(
-        command: str, address: str, concurrent_execution: int = 10, pagination_size: int = 50
+        command: str,
+        address: str,
+        concurrent_execution: int = 10,
+        pagination_size: int = 50,
     ) -> subprocess.CompletedProcess[str]:
         env = os.environ.copy()
         env["INFRAHUB_ADDRESS"] = address
@@ -79,7 +89,9 @@ class TestInfrahubDocker:
         deployment_type: str | None,
     ) -> InfrahubDockerCompose:
         return InfrahubDockerCompose.init(
-            directory=tmp_directory, version=infrahub_version, deployment_type=deployment_type
+            directory=tmp_directory,
+            version=infrahub_version,
+            deployment_type=deployment_type,
         )
 
     @pytest.fixture(scope="class")
@@ -90,7 +102,10 @@ class TestInfrahubDocker:
             tests_failed_during_class = request.session.testsfailed - tests_failed_before_class
             if tests_failed_during_class > 0:
                 stdout, stderr = infrahub_compose.get_logs("infrahub-server", "task-worker")
-                warnings.warn(f"Container logs:\nStdout:\n{stdout}\nStderr:\n{stderr}", stacklevel=2)
+                warnings.warn(
+                    f"Container logs:\nStdout:\n{stdout}\nStderr:\n{stderr}",
+                    stacklevel=2,
+                )
             infrahub_compose.stop()
 
         request.addfinalizer(cleanup)

{infrahub_testcontainers-1.5.0b1 → infrahub_testcontainers-1.5.1}/infrahub_testcontainers/models.py

@@ -1,6 +1,8 @@
-from datetime import UTC, datetime
+from __future__ import annotations
+
+from datetime import datetime, timezone
 from enum import Enum
-from typing import Any
+from typing import Any, Union
 
 from pydantic import BaseModel, Field
 
@@ -15,24 +17,24 @@ class ContextUnit(str, Enum):
 class MeasurementDefinition(BaseModel):
     name: str
     description: str
-    dimensions: list[str] = Field(default_factory=dict)
+    dimensions: list[str] = Field(default_factory=list)
     unit: ContextUnit
 
 
 class InfrahubResultContext(BaseModel):
     name: str
-    value: int | float | str
+    value: Union[int, float, str]
     unit: ContextUnit
 
 
 class InfrahubActiveMeasurementItem(BaseModel):
     definition: MeasurementDefinition
-    start_time: datetime = Field(default_factory=lambda: datetime.now(UTC))
+    start_time: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
     context: dict[str, Any] = Field(default_factory=dict)
 
 
 class InfrahubMeasurementItem(BaseModel):
     name: str
-    value: int | float | str
+    value: Union[int, float, str]
     unit: ContextUnit
     context: dict[str, Any] = Field(default_factory=dict)

{infrahub_testcontainers-1.5.0b1 → infrahub_testcontainers-1.5.1}/infrahub_testcontainers/performance_test.py

@@ -1,6 +1,8 @@
+from __future__ import annotations
+
 import hashlib
 import json
-from datetime import UTC, datetime
+from datetime import datetime, timezone
 from types import TracebackType
 from typing import Any
 
@@ -35,7 +37,7 @@ class InfrahubPerformanceTest:
         self.env_vars = {}
         self.project_name = ""
         self.test_info = {}
-        self.start_time = datetime.now(UTC)
+        self.start_time = datetime.now(timezone.utc)
         self.end_time: datetime | None = None
         self.results_url = results_url
         self.scraper_endpoint = ""
@@ -57,7 +59,7 @@ class InfrahubPerformanceTest:
 
     def finalize(self, session: pytest.Session) -> None:
         if self.initialized:
-            self.end_time = datetime.now(UTC)
+            self.end_time = datetime.now(timezone.utc)
             self.extract_test_session_information(session)
             self.send_results()
 
@@ -129,7 +131,7 @@ class InfrahubPerformanceTest:
         if not exc_type and self.active_measurements:
             self.add_measurement(
                 definition=self.active_measurements.definition,
-                value=(datetime.now(UTC) - self.active_measurements.start_time).total_seconds() * 1000,
+                value=(datetime.now(timezone.utc) - self.active_measurements.start_time).total_seconds() * 1000,
                 context=self.active_measurements.context,
             )
 

{infrahub_testcontainers-1.5.0b1 → infrahub_testcontainers-1.5.1}/pyproject.toml

@@ -1,11 +1,11 @@
 [project]
 name = "infrahub-testcontainers"
-version = "1.5.0b1"
+version = "1.5.1"
 requires-python = ">=3.9"
 
 [tool.poetry]
 name = "infrahub-testcontainers"
-version = "1.5.0b1"
+version = "1.5.1"
 description = "Testcontainers instance for Infrahub to easily build integration tests"
 authors = ["OpsMill <info@opsmill.com>"]
 readme = "README.md"
@@ -43,3 +43,9 @@ prefect-client = "3.4.23"
 
 [tool.poetry.group.dev.dependencies]
 rich = "^13.9.4"
+pytest-cov = "7.0.0"
+pytest-xdist = "~3.4"
+
+[tool.ruff]
+line-length = 120
+target-version = "py39"