infrahub-server 1.5.0b0__py3-none-any.whl → 1.5.0b2__py3-none-any.whl

infrahub_testcontainers/container.py

@@ -29,7 +29,7 @@ INFRAHUB_SERVICES: dict[str, ContainerService] = {
 
 PROJECT_ENV_VARIABLES: dict[str, str] = {
     "MESSAGE_QUEUE_DOCKER_IMAGE": "rabbitmq:3.13.7-management",
-    "CACHE_DOCKER_IMAGE": "redis:7.2.4",
+    "CACHE_DOCKER_IMAGE": "redis:7.2.11",
     "INFRAHUB_TESTING_DOCKER_IMAGE": "registry.opsmill.io/opsmill/infrahub",
     "INFRAHUB_TESTING_DOCKER_ENTRYPOINT": f"gunicorn --config backend/infrahub/serve/gunicorn_config.py -w {os.environ.get('INFRAHUB_TESTING_WEB_CONCURRENCY', 4)} --logger-class infrahub.serve.log.GunicornLogger infrahub.server:app",  # noqa: E501
     "INFRAHUB_TESTING_IMAGE_VERSION": infrahub_version,
@@ -152,6 +152,8 @@ class InfrahubDockerCompose(DockerCompose):
                     "INFRAHUB_TESTING_TASKMGR_BACKGROUND_SVC_REPLICAS": "1",
                     "PREFECT_MESSAGING_BROKER": "prefect_redis.messaging",
                     "PREFECT_MESSAGING_CACHE": "prefect_redis.messaging",
+                    "PREFECT_SERVER_EVENTS_CAUSAL_ORDERING": "prefect_redis.ordering",
+                    "PREFECT_SERVER_CONCURRENCY_LEASE_STORAGE": "prefect_redis.lease_storage",
                     "PREFECT__SERVER_WEBSERVER_ONLY": "true",
                     "PREFECT_API_DATABASE_MIGRATE_ON_START": "false",
                     "PREFECT_API_BLOCKS_REGISTER_ON_START": "false",
@@ -237,7 +239,9 @@ class InfrahubDockerCompose(DockerCompose):
             for service_name, service_data in INFRAHUB_SERVICES.items()
         }
 
-    def database_create_backup(self, backup_name: str = "neo4j_database.backup", dest_dir: Path | None = None) -> None:
+    def database_create_backup(
+        self, backup_name: str = "neo4j_database.backup", dest_dir: Path | None = None, compress: bool = False
+    ) -> None:
         assert self.use_neo4j_enterprise
 
         self.exec_in_container(
@@ -245,7 +249,7 @@ class InfrahubDockerCompose(DockerCompose):
                 "neo4j-admin",
                 "database",
                 "backup",
-                "--compress=false",
+                f"--compress={'true' if compress else 'false'}",
                 "--to-path",
                 str(self.internal_backup_dir),
             ],
@@ -513,3 +517,111 @@ class InfrahubDockerCompose(DockerCompose):
                 )
             self.start()
             print("Database restored successfully")
+
+    def task_manager_create_backup(self, backup_name: str = "prefect.dump", dest_dir: Path | None = None) -> Path:
+        """Create a backup of the task manager PostgreSQL database using ``pg_dump``.
+
+        Args:
+            backup_name: Name of the archive file to create. Defaults to ``prefect.dump``.
+            dest_dir: Optional host directory where the backup should be copied after it is
+                produced. When omitted, the backup remains in ``external_backup_dir``.
+
+        Returns:
+            Path to the backup archive on the host filesystem.
+
+        Raises:
+            FileNotFoundError: If the pg_dump command completes but no archive is produced.
+        """
+
+        service_name = "task-manager-db"
+
+        try:
+            self.get_container(service_name=service_name)
+        except ContainerIsNotRunning:
+            self.start_container(service_name=service_name)
+
+        self.external_backup_dir.mkdir(parents=True, exist_ok=True)
+
+        internal_backup_path = self.internal_backup_dir / backup_name
+        dump_command = [
+            "pg_dump",
+            "--format=custom",
+            "--blobs",
+            "--no-owner",
+            "--no-privileges",
+            "--dbname=postgresql://postgres:postgres@localhost:5432/prefect",
+            f"--file={internal_backup_path}",
+        ]
+        self.exec_in_container(command=dump_command, service_name=service_name)
+
+        source_path = self.external_backup_dir / backup_name
+        if not source_path.exists():
+            raise FileNotFoundError(f"Backup file {source_path} was not created")
+
+        final_path = source_path
+        if dest_dir:
+            dest_dir.mkdir(parents=True, exist_ok=True)
+            if dest_dir.resolve() != self.external_backup_dir.resolve():
+                final_path = dest_dir / backup_name
+                shutil.copy(source_path, final_path)
+
+        return final_path
+
+    def task_manager_restore_backup(self, backup_file: Path) -> None:
+        """Restore the task manager PostgreSQL database from a ``pg_restore`` archive.
+
+        Args:
+            backup_file: Path to the backup archive on the host filesystem.
+
+        Raises:
+            FileNotFoundError: If the provided backup archive does not exist.
+        """
+
+        if not backup_file.exists():
+            raise FileNotFoundError(f"Backup file {backup_file} does not exist")
+
+        service_name = "task-manager-db"
+
+        try:
+            self.get_container(service_name=service_name)
+        except ContainerIsNotRunning:
+            self.start_container(service_name=service_name)
+
+        self.external_backup_dir.mkdir(parents=True, exist_ok=True)
+        target_path = self.external_backup_dir / backup_file.name
+        shutil.copy(backup_file, target_path)
+
+        admin_dsn = "postgresql://postgres:postgres@localhost:5432/postgres"
+        prefect_dsn = "postgresql://postgres:postgres@localhost:5432/prefect"
+        internal_backup_path = self.internal_backup_dir / backup_file.name
+
+        terminate_sessions_command = [
+            "psql",
+            f"--dbname={admin_dsn}",
+            "--command",
+            "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = 'prefect';",
+        ]
+        drop_database_command = [
+            "psql",
+            f"--dbname={admin_dsn}",
+            "--command",
+            "DROP DATABASE IF EXISTS prefect WITH (FORCE);",
+        ]
+        create_database_command = [
+            "psql",
+            f"--dbname={admin_dsn}",
+            "--command",
+            "CREATE DATABASE prefect OWNER postgres;",
+        ]
+        restore_command = [
+            "pg_restore",
+            "--no-owner",
+            "--role=postgres",
+            f"--dbname={prefect_dsn}",
+            str(internal_backup_path),
+        ]
+
+        self.exec_in_container(command=terminate_sessions_command, service_name=service_name)
+        self.exec_in_container(command=drop_database_command, service_name=service_name)
+        self.exec_in_container(command=create_database_command, service_name=service_name)
+        self.exec_in_container(command=restore_command, service_name=service_name)

infrahub_testcontainers/docker-compose-cluster.test.yml

@@ -37,7 +37,7 @@ services:
       - ${INFRAHUB_TESTING_MESSAGE_QUEUE_PORT:-0}:15692
 
   cache:
-    image: ${CACHE_DOCKER_IMAGE:-redis:7.2.4}
+    image: ${CACHE_DOCKER_IMAGE:-redis:7.2.11}
     restart: unless-stopped
     healthcheck:
       test: ["CMD-SHELL", "redis-cli ping | grep PONG"]
@@ -184,6 +184,8 @@ services:
 
       PREFECT_MESSAGING_BROKER:
       PREFECT_MESSAGING_CACHE:
+      PREFECT_SERVER_EVENTS_CAUSAL_ORDERING:
+      PREFECT_SERVER_CONCURRENCY_LEASE_STORAGE:
       PREFECT__SERVER_WEBSERVER_ONLY:
       PREFECT_API_DATABASE_MIGRATE_ON_START:
       PREFECT_API_BLOCKS_REGISTER_ON_START:
@@ -225,6 +227,8 @@ services:
       INFRAHUB_CACHE_ADDRESS: ${INFRAHUB_TESTING_CACHE_ADDRESS}
       PREFECT_MESSAGING_BROKER: prefect_redis.messaging
       PREFECT_MESSAGING_CACHE: prefect_redis.messaging
+      PREFECT_SERVER_EVENTS_CAUSAL_ORDERING: prefect_redis.ordering
+      PREFECT_SERVER_CONCURRENCY_LEASE_STORAGE: prefect_redis.lease_storage
       PREFECT_REDIS_MESSAGING_HOST: "${INFRAHUB_TESTING_CACHE_ADDRESS:-cache}"
       PREFECT_REDIS_MESSAGING_DB: "1"
       PREFECT_REDIS_MESSAGING_CONSUMER_MIN_IDLE_TIME: "30"
@@ -244,6 +248,7 @@ services:
       - POSTGRES_DB=prefect
     volumes:
       - workflow_db:/var/lib/postgresql/data
+      - "./${INFRAHUB_TESTING_LOCAL_DB_BACKUP_DIRECTORY}:${INFRAHUB_TESTING_INTERNAL_DB_BACKUP_DIRECTORY}"
     healthcheck:
       test: ["CMD-SHELL", "pg_isready"]
       interval: 10s

infrahub_testcontainers/docker-compose.test.yml

@@ -20,7 +20,7 @@ services:
       - ${INFRAHUB_TESTING_MESSAGE_QUEUE_PORT:-0}:15692
 
   cache:
-    image: ${CACHE_DOCKER_IMAGE:-redis:7.2.4}
+    image: ${CACHE_DOCKER_IMAGE:-redis:7.2.11}
     restart: unless-stopped
     healthcheck:
       test: ["CMD-SHELL", "redis-cli ping | grep PONG"]
@@ -95,6 +95,8 @@ services:
 
       PREFECT_MESSAGING_BROKER:
       PREFECT_MESSAGING_CACHE:
+      PREFECT_SERVER_EVENTS_CAUSAL_ORDERING:
+      PREFECT_SERVER_CONCURRENCY_LEASE_STORAGE:
       PREFECT__SERVER_WEBSERVER_ONLY:
       PREFECT_API_DATABASE_MIGRATE_ON_START:
       PREFECT_API_BLOCKS_REGISTER_ON_START:
@@ -136,6 +138,8 @@ services:
       INFRAHUB_CACHE_ADDRESS: ${INFRAHUB_TESTING_CACHE_ADDRESS}
       PREFECT_MESSAGING_BROKER: prefect_redis.messaging
       PREFECT_MESSAGING_CACHE: prefect_redis.messaging
+      PREFECT_SERVER_EVENTS_CAUSAL_ORDERING: prefect_redis.ordering
+      PREFECT_SERVER_CONCURRENCY_LEASE_STORAGE: prefect_redis.lease_storage
       PREFECT_REDIS_MESSAGING_HOST: "${INFRAHUB_TESTING_CACHE_ADDRESS:-cache}"
       PREFECT_REDIS_MESSAGING_DB: "1"
       PREFECT_REDIS_MESSAGING_CONSUMER_MIN_IDLE_TIME: "30"
@@ -155,6 +159,7 @@ services:
       - POSTGRES_DB=prefect
     volumes:
       - workflow_db:/var/lib/postgresql/data
+      - "./${INFRAHUB_TESTING_LOCAL_DB_BACKUP_DIRECTORY}:${INFRAHUB_TESTING_INTERNAL_DB_BACKUP_DIRECTORY}"
     healthcheck:
       test: ["CMD-SHELL", "pg_isready"]
       interval: 10s

infrahub/core/migrations/graph/m040_profile_attrs_in_db.py

@@ -1,166 +0,0 @@
-from __future__ import annotations
-
-from collections import defaultdict
-from typing import TYPE_CHECKING, Any
-
-from rich.console import Console
-from rich.progress import Progress
-
-from infrahub.core.branch.models import Branch
-from infrahub.core.initialization import initialization
-from infrahub.core.manager import NodeManager
-from infrahub.core.migrations.shared import MigrationResult
-from infrahub.core.query import Query, QueryType
-from infrahub.core.timestamp import Timestamp
-from infrahub.lock import initialize_lock
-from infrahub.log import get_logger
-from infrahub.profiles.node_applier import NodeProfilesApplier
-
-from ..shared import ArbitraryMigration
-
-if TYPE_CHECKING:
-    from infrahub.core.node import Node
-    from infrahub.database import InfrahubDatabase
-
-log = get_logger()
-
-
-class GetProfilesByBranchQuery(Query):
-    """
-    Get CoreProfile UUIDs by which branches they have attribute updates on
-    """
-
-    name = "get_profiles_by_branch"
-    type = QueryType.READ
-    insert_return = False
-
-    async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None:  # noqa: ARG002
-        query = """
-MATCH (profile:CoreProfile)-[:HAS_ATTRIBUTE]->(attr:Attribute)-[e:HAS_VALUE]->(:AttributeValue)
-WITH DISTINCT profile.uuid AS profile_uuid, e.branch AS branch
-RETURN profile_uuid, collect(branch) AS branches
-        """
-        self.add_to_query(query)
-        self.return_labels = ["profile_uuid", "branches"]
-
-    def get_profile_ids_by_branch(self) -> dict[str, set[str]]:
-        """Get dictionary of branch names to set of updated profile UUIDs"""
-        profiles_by_branch = defaultdict(set)
-        for result in self.get_results():
-            profile_uuid = result.get_as_type("profile_uuid", str)
-            branches = result.get_as_type("branches", list[str])
-            for branch in branches:
-                profiles_by_branch[branch].add(profile_uuid)
-        return profiles_by_branch
-
-
-class GetNodesWithProfileUpdatesByBranchQuery(Query):
-    """
-    Get Node UUIDs by which branches they have updated profiles on
-    """
-
-    name = "get_nodes_with_profile_updates_by_branch"
-    type = QueryType.READ
-    insert_return = False
-
-    async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None:  # noqa: ARG002
-        query = """
-MATCH (node:Node)-[e1:IS_RELATED]->(:Relationship {name: "node__profile"})
-WHERE NOT node:CoreProfile
-WITH DISTINCT node.uuid AS node_uuid, e1.branch AS branch
-RETURN node_uuid, collect(branch) AS branches
-        """
-        self.add_to_query(query)
-        self.return_labels = ["node_uuid", "branches"]
-
-    def get_node_ids_by_branch(self) -> dict[str, set[str]]:
-        """Get dictionary of branch names to set of updated node UUIDs"""
-        nodes_by_branch = defaultdict(set)
-        for result in self.get_results():
-            node_uuid = result.get_as_type("node_uuid", str)
-            branches = result.get_as_type("branches", list[str])
-            for branch in branches:
-                nodes_by_branch[branch].add(node_uuid)
-        return nodes_by_branch
-
-
-class Migration040(ArbitraryMigration):
-    """
-    Save profile attribute values on each node using the profile in the database
-    For any profile that has updates on a given branch (including default branch)
-    - run NodeProfilesApplier.apply_profiles on each node related to the profile on that branch
-    For any node that has an updated relationship to a profile on a given branch
-    - run NodeProfilesApplier.apply_profiles on the node on that branch
-    """
-
-    name: str = "040_profile_attrs_in_db"
-    minimum_version: int = 39
-
-    def __init__(self, *args: Any, **kwargs: Any) -> None:
-        super().__init__(*args, **kwargs)
-        self._appliers_by_branch: dict[str, NodeProfilesApplier] = {}
-
-    async def _get_profile_applier(self, db: InfrahubDatabase, branch_name: str) -> NodeProfilesApplier:
-        if branch_name not in self._appliers_by_branch:
-            branch = await Branch.get_by_name(db=db, name=branch_name)
-            self._appliers_by_branch[branch_name] = NodeProfilesApplier(db=db, branch=branch)
-        return self._appliers_by_branch[branch_name]
-
-    async def validate_migration(self, db: InfrahubDatabase) -> MigrationResult:  # noqa: ARG002
-        return MigrationResult()
-
-    async def execute(self, db: InfrahubDatabase) -> MigrationResult:
-        console = Console()
-        result = MigrationResult()
-        # load schemas from database into registry
-        initialize_lock()
-        await initialization(db=db)
-
-        console.print("Gathering profiles for each branch...", end="")
-        get_profiles_by_branch_query = await GetProfilesByBranchQuery.init(db=db)
-        await get_profiles_by_branch_query.execute(db=db)
-        profiles_ids_by_branch = get_profiles_by_branch_query.get_profile_ids_by_branch()
-
-        profiles_by_branch: dict[str, list[Node]] = {}
-        for branch_name, profile_ids in profiles_ids_by_branch.items():
-            profiles_map = await NodeManager.get_many(db=db, branch=branch_name, ids=list(profile_ids))
-            profiles_by_branch[branch_name] = list(profiles_map.values())
-        console.print("done")
-
-        node_ids_to_update_by_branch: dict[str, set[str]] = defaultdict(set)
-        total_size = sum(len(profiles) for profiles in profiles_by_branch.values())
-        with Progress() as progress:
-            gather_nodes_task = progress.add_task(
-                "Gathering affected objects for each profile on each branch...", total=total_size
-            )
-
-            for branch_name, profiles in profiles_by_branch.items():
-                for profile in profiles:
-                    node_relationship_manager = profile.get_relationship("related_nodes")
-                    node_peers = await node_relationship_manager.get_db_peers(db=db)
-                    node_ids_to_update_by_branch[branch_name].update({str(peer.peer_id) for peer in node_peers})
-                    progress.update(gather_nodes_task, advance=1)
-
-        console.print("Identifying nodes with profile updates by branch...", end="")
-        get_nodes_with_profile_updates_by_branch_query = await GetNodesWithProfileUpdatesByBranchQuery.init(db=db)
-        await get_nodes_with_profile_updates_by_branch_query.execute(db=db)
-        nodes_ids_by_branch = get_nodes_with_profile_updates_by_branch_query.get_node_ids_by_branch()
-        for branch_name, node_ids in nodes_ids_by_branch.items():
-            node_ids_to_update_by_branch[branch_name].update(node_ids)
-        console.print("done")
-
-        right_now = Timestamp()
-        total_size = sum(len(node_ids) for node_ids in node_ids_to_update_by_branch.values())
-        with Progress() as progress:
-            apply_task = progress.add_task("Applying profiles to nodes...", total=total_size)
-            for branch_name, node_ids in node_ids_to_update_by_branch.items():
-                applier = await self._get_profile_applier(db=db, branch_name=branch_name)
-                for node_id in node_ids:
-                    node = await NodeManager.get_one(db=db, branch=branch_name, id=node_id, at=right_now)
-                    if node:
-                        updated_field_names = await applier.apply_profiles(node=node)
-                        if updated_field_names:
-                            await node.save(db=db, fields=updated_field_names, at=right_now)
-                    progress.update(apply_task, advance=1)
-
-        return result