infrahub-testcontainers 1.2.11__tar.gz → 1.3.0__tar.gz

{infrahub_testcontainers-1.2.11 → infrahub_testcontainers-1.3.0}/PKG-INFO

@@ -1,12 +1,11 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: infrahub-testcontainers
-Version: 1.2.11
+Version: 1.3.0
 Summary: Testcontainers instance for Infrahub to easily build integration tests
-Home-page: https://opsmill.com
 License: Apache-2.0
 Author: OpsMill
 Author-email: info@opsmill.com
-Requires-Python: >=3.9,<4.0
+Requires-Python: >=3.9
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
@@ -16,10 +15,12 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: httpx (>=0.28.1,<0.29.0)
+Requires-Dist: prefect-client (==3.4.1)
 Requires-Dist: psutil
 Requires-Dist: pydantic (>=2.10.6,<3.0.0)
 Requires-Dist: pytest
 Requires-Dist: testcontainers (>=4.8,<4.9)
+Project-URL: Homepage, https://opsmill.com
 Project-URL: Repository, https://github.com/opsmill/infrahub
 Description-Content-Type: text/markdown
 

{infrahub_testcontainers-1.2.11 → infrahub_testcontainers-1.3.0}/infrahub_testcontainers/container.py

@@ -28,7 +28,6 @@ INFRAHUB_SERVICES: dict[str, ContainerService] = {
 }
 
 PROJECT_ENV_VARIABLES: dict[str, str] = {
-    "NEO4J_DOCKER_IMAGE": "neo4j:5.20.0-community",
     "MESSAGE_QUEUE_DOCKER_IMAGE": "rabbitmq:3.13.7-management",
     "CACHE_DOCKER_IMAGE": "redis:7.2.4",
     "INFRAHUB_TESTING_DOCKER_IMAGE": "registry.opsmill.io/opsmill/infrahub",
@@ -67,9 +66,12 @@ PROJECT_ENV_VARIABLES: dict[str, str] = {
 class InfrahubDockerCompose(DockerCompose):
     project_name: str | None = None
     env_vars: dict[str, str] = field(default_factory=dict)
+    deployment_type: str | None = None
 
     @classmethod
-    def init(cls, directory: Path | None = None, version: str | None = None) -> Self:
+    def init(
+        cls, directory: Path | None = None, version: str | None = None, deployment_type: str | None = None
+    ) -> Self:
         if not directory:
             directory = Path.cwd()
 
@@ -80,7 +82,7 @@ class InfrahubDockerCompose(DockerCompose):
         if version == "local" and infrahub_image_version:
             version = infrahub_image_version
 
-        compose = cls(project_name=cls.generate_project_name(), context=directory)
+        compose = cls(project_name=cls.generate_project_name(), context=directory, deployment_type=deployment_type)
         compose.create_docker_file(directory=directory)
         compose.create_env_file(directory=directory, version=version)
 
@@ -112,7 +114,10 @@ class InfrahubDockerCompose(DockerCompose):
 
     def create_docker_file(self, directory: Path) -> Path:
         current_directory = Path(__file__).resolve().parent
-        compose_file = current_directory / "docker-compose.test.yml"
+        compose_file_name = (
+            "docker-compose-cluster.test.yml" if self.deployment_type == "cluster" else "docker-compose.test.yml"
+        )
+        compose_file = current_directory / compose_file_name
 
         test_compose_file = directory / "docker-compose.yml"
         test_compose_file.write_bytes(compose_file.read_bytes())
@@ -161,7 +166,7 @@ class InfrahubDockerCompose(DockerCompose):
             cmd.extend(self.services)
         self._run_command(cmd=cmd)
 
-    def start_container(self, service_name: str) -> None:
+    def start_container(self, service_name: str | list[str]) -> None:
         """
         Starts a specific service of the docker compose environment.
 
@@ -171,7 +176,11 @@ class InfrahubDockerCompose(DockerCompose):
 
         # pull means running a separate command before starting
         if self.pull:
-            pull_cmd = [*base_cmd, "pull", service_name]
+            pull_cmd = [*base_cmd, "pull"]
+            if isinstance(service_name, list):
+                pull_cmd.extend(service_name)
+            else:
+                pull_cmd.append(service_name)
             self._run_command(cmd=pull_cmd)
 
         up_cmd = [*base_cmd, "up"]
@@ -186,7 +195,10 @@ class InfrahubDockerCompose(DockerCompose):
             # we run in detached mode instead of blocking
             up_cmd.append("--detach")
 
-        up_cmd.append(service_name)
+        if isinstance(service_name, list):
+            up_cmd.extend(service_name)
+        else:
+            up_cmd.append(service_name)
         self._run_command(cmd=up_cmd)
 
     # TODO would be good to the support for project_name upstream
@@ -234,7 +246,7 @@ class InfrahubDockerCompose(DockerCompose):
                 dest_dir / backup_name,
             )
 
-    def database_restore_backup(self, backup_file: Path) -> None:
+    def database_restore_backup(self, backup_file: Path) -> None:  # noqa: PLR0915
         assert self.use_neo4j_enterprise
 
         shutil.copy(
@@ -243,52 +255,35 @@ class InfrahubDockerCompose(DockerCompose):
         )
         service_name = "database"
 
-        # Ensure the database container is running otherwise start it
-        try:
-            self.get_container(service_name=service_name)
-        except ContainerIsNotRunning:
-            self.start_container(service_name=service_name)
-
-        self.exec_in_container(
-            command=["cypher-shell", "-u", "neo4j", "-p", "admin", "STOP DATABASE neo4j;"],
-            service_name=service_name,
-        )
+        if self.deployment_type != "cluster":  # noqa: PLR1702
+            try:
+                self.get_container(service_name=service_name)
+            except ContainerIsNotRunning:
+                self.start_container(service_name=service_name)
 
-        self.exec_in_container(
-            command=[
-                "neo4j-admin",
-                "database",
-                "restore",
-                "--overwrite-destination",
-                "--from-path",
-                str(self.internal_backup_dir / backup_file.name),
-            ],
-            service_name=service_name,
-        )
+            self.exec_in_container(
+                command=["cypher-shell", "-u", "neo4j", "-p", "admin", "STOP DATABASE neo4j;"],
+                service_name=service_name,
+            )
 
-        self.exec_in_container(
-            command=["chown", "-R", "neo4j:neo4j", "/data"],
-            service_name=service_name,
-        )
+            self.exec_in_container(
+                command=[
+                    "neo4j-admin",
+                    "database",
+                    "restore",
+                    "--overwrite-destination",
+                    "--from-path",
+                    str(self.internal_backup_dir / backup_file.name),
+                ],
+                service_name=service_name,
+            )
 
-        (restore_output, _, _) = self.exec_in_container(
-            command=[
-                "cypher-shell",
-                "--format",
-                "plain",
-                "-d",
-                "system",
-                "-u",
-                "neo4j",
-                "-p",
-                "admin",
-                "START DATABASE neo4j;",
-            ],
-            service_name=service_name,
-        )
+            self.exec_in_container(
+                command=["chown", "-R", "neo4j:neo4j", "/data"],
+                service_name=service_name,
+            )
 
-        for _ in range(3):
-            (stdout, _, _) = self.exec_in_container(
+            (restore_output, _, _) = self.exec_in_container(
                 command=[
                     "cypher-shell",
                     "--format",
@@ -299,26 +294,205 @@ class InfrahubDockerCompose(DockerCompose):
                     "neo4j",
                     "-p",
                     "admin",
-                    "SHOW DATABASES WHERE name = 'neo4j' AND currentStatus = 'online';",
+                    "START DATABASE neo4j;",
                 ],
                 service_name=service_name,
             )
-            if stdout:
-                break
-            time.sleep(5)
+
+            for _ in range(3):
+                (stdout, _, _) = self.exec_in_container(
+                    command=[
+                        "cypher-shell",
+                        "--format",
+                        "plain",
+                        "-d",
+                        "system",
+                        "-u",
+                        "neo4j",
+                        "-p",
+                        "admin",
+                        "SHOW DATABASES WHERE name = 'neo4j' AND currentStatus = 'online';",
+                    ],
+                    service_name=service_name,
+                )
+                if stdout:
+                    break
+                time.sleep(5)
+            else:
+                (debug_logs, _, _) = self.exec_in_container(
+                    command=["cat", "logs/debug.log"],
+                    service_name=service_name,
+                )
+                raise Exception(f"Failed to restore database:\n{restore_output}\nDebug logs:\n{debug_logs}")
+
+            old_services = self.services
+            self.services = ["infrahub-server", "task-worker"]
+            self.stop(down=False)
+            try:
+                self.start()
+            except Exception as exc:
+                stdout, stderr = self.get_logs()
+                raise Exception(f"Failed to start docker compose:\nStdout:\n{stdout}\nStderr:\n{stderr}") from exc
+            self.services = old_services
         else:
-            (debug_logs, _, _) = self.exec_in_container(
-                command=["cat", "logs/debug.log"],
+            print("Cluster mode detected")
+            try:
+                self.get_container(service_name=service_name)
+                self.get_container(service_name="database-core2")
+                self.get_container(service_name="database-core3")
+            except ContainerIsNotRunning:
+                self.start_container("database", "database-core2", "database-core3")
+
+            # Waiting for cluster to stabilize...
+            time.sleep(10)
+
+            self.exec_in_container(
+                command=["cypher-shell", "-u", "neo4j", "-p", "admin", "DROP DATABASE neo4j;"],
+                service_name=service_name,
+            )
+
+            self.exec_in_container(
+                command=["rm", "-rf", "/data/databases/neo4j"],
+                service_name=service_name,
+            )
+            self.exec_in_container(
+                command=["rm", "-rf", "/data/transactions/neo4j"],
+                service_name=service_name,
+            )
+
+            self.exec_in_container(
+                command=[
+                    "neo4j-admin",
+                    "database",
+                    "restore",
+                    "--from-path",
+                    str(self.internal_backup_dir / backup_file.name),
+                    "neo4j",
+                ],
                 service_name=service_name,
             )
-            raise Exception(f"Failed to restore database:\n{restore_output}\nDebug logs:\n{debug_logs}")
 
-        old_services = self.services
-        self.services = ["infrahub-server", "task-worker"]
-        self.stop(down=False)
-        try:
+            cmd = self.compose_command_property[:]
+            cmd += ["restart", "database"]
+            self._run_command(cmd=cmd)
+
+            main_node = service_name
+            cluster_nodes = ["database", "database-core2", "database-core3"]
+
+            for attempt in range(3):
+                try:
+                    (stdout, _, _) = self.exec_in_container(
+                        command=[
+                            "cypher-shell",
+                            "--format",
+                            "plain",
+                            "-d",
+                            "system",
+                            "-u",
+                            "neo4j",
+                            "-p",
+                            "admin",
+                            "SHOW DATABASES YIELD name, address, currentStatus WHERE name = 'system' RETURN address, currentStatus",
+                        ],
+                        service_name=main_node,
+                    )
+                except Exception:
+                    time.sleep(10)
+                    continue
+
+                raw_output = stdout
+                nodes_status = dict.fromkeys(cluster_nodes, False)
+                online_count = 0
+                total_entries = 0
+
+                try:
+                    for line_raw in stdout.splitlines():
+                        line = line_raw.strip()
+                        if not line or line.startswith("address"):
+                            continue
+
+                        total_entries += 1
+                        if "online" in line:
+                            online_count += 1
+                            for node in cluster_nodes:
+                                node_pattern = f'"{node}:'
+                                if node_pattern in line:
+                                    nodes_status[node] = True
+                                    break
+                    if all(nodes_status.values()) and online_count == len(cluster_nodes):
+                        break
+                except Exception as e:
+                    print(f"Error parsing database status on attempt {attempt + 1}: {e}")
+
+                print(f"Waiting for all nodes to be online. Current status: {nodes_status}")
+                time.sleep(5)
+            else:
+                debug_logs = {}
+                for node in cluster_nodes:
+                    try:
+                        (logs, _, _) = self.exec_in_container(
+                            command=["cat", "logs/debug.log"],
+                            service_name=node,
+                        )
+                        debug_logs[node] = logs
+                    except Exception as e:
+                        debug_logs[node] = f"Could not retrieve logs: {str(e)}"
+
+                debug_info = f"Raw output from SHOW DATABASES command:\n{raw_output}\n\n"
+                debug_info += f"Final node status: {nodes_status}\n\n"
+
+                status_str = ", ".join(
+                    [f"{node}: {'online' if status else 'offline'}" for node, status in nodes_status.items()]
+                )
+                logs_str = debug_info + "\n\n".join(
+                    [f"--- {node} logs ---\n{logs}" for node, logs in debug_logs.items()]
+                )
+
+                raise Exception(
+                    f"Failed to restore database cluster. Node status: {status_str}\nDebug logs:\n{logs_str}"
+                )
+
+            server_id = None
+            try:
+                stdout, _, _ = self.exec_in_container(
+                    command=[
+                        "cypher-shell",
+                        "--format",
+                        "plain",
+                        "-d",
+                        "system",
+                        "-u",
+                        "neo4j",
+                        "-p",
+                        "admin",
+                        'SHOW SERVERS YIELD name, address WHERE address = "database:7687" RETURN name;',
+                    ],
+                    service_name=service_name,
+                )
+
+                lines = stdout.splitlines()
+                for line_raw in lines:
+                    line = line_raw.strip()
+                    if not line or line == "name" or line.startswith("+"):
+                        continue
+                    server_id = line.strip('"')
+                    break
+            except Exception as e:
+                print(f"Error retrieving server ID with direct query: {e}")
+
+            if server_id:
+                self.exec_in_container(
+                    command=[
+                        "cypher-shell",
+                        "-d",
+                        "system",
+                        "-u",
+                        "neo4j",
+                        "-p",
+                        "admin",
+                        f"CREATE DATABASE neo4j TOPOLOGY 3 PRIMARIES OPTIONS {{ existingData: 'use', existingDataSeedInstance: '{server_id}' }};",
+                    ],
+                    service_name=service_name,
+                )
             self.start()
-        except Exception as exc:
-            stdout, stderr = self.get_logs()
-            raise Exception(f"Failed to start docker compose:\nStdout:\n{stdout}\nStderr:\n{stderr}") from exc
-        self.services = old_services
+            print("Database restored successfully")

infrahub_testcontainers-1.3.0/infrahub_testcontainers/docker-compose-cluster.test.yml

@@ -0,0 +1,321 @@
+---
+# yamllint disable rule:line-length
+# The following environment variables are part of the Infrahub configuration options.
+# For detailed information on these configuration options, please refer to the Infrahub documentation:
+# https://docs.infrahub.app/reference/configuration
+x-neo4j-config-common: &neo4j-config-common
+  NEO4J_AUTH: neo4j/admin
+  NEO4J_dbms_security_procedures_unrestricted: apoc.*
+  NEO4J_dbms_security_auth__minimum__password__length: 4
+  NEO4J_ACCEPT_LICENSE_AGREEMENT: 'yes'
+  NEO4J_server_backup_enabled: true
+  NEO4J_metrics_prometheus_enabled: true
+  NEO4J_server_metrics_filter: '*'
+  NEO4J_server_cluster_system__database__mode: PRIMARY
+  NEO4J_initial_server_mode__constraint: PRIMARY
+  NEO4J_dbms_cluster_discovery_endpoints: database:5000,database-core2:5000,database-core3:5000
+  NEO4J_initial_dbms_default__primaries__count: 3
+  NEO4J_dbms_memory_heap_initial__size: ${INFRAHUB_TESTING_DB_HEAP_INITIAL_SIZE}
+  NEO4J_dbms_memory_heap_max__size: ${INFRAHUB_TESTING_DB_HEAP_MAX_SIZE}
+  NEO4J_server_memory_pagecache_size: ${INFRAHUB_TESTING_DB_PAGECACHE_SIZE}
+
+
+services:
+  message-queue:
+    image: ${MESSAGE_QUEUE_DOCKER_IMAGE:-rabbitmq:3.13.7-management}
+    restart: unless-stopped
+    environment:
+      RABBITMQ_DEFAULT_USER: infrahub
+      RABBITMQ_DEFAULT_PASS: infrahub
+    healthcheck:
+      test: rabbitmq-diagnostics -q check_port_connectivity
+      interval: 5s
+      timeout: 30s
+      retries: 10
+      start_period: 3s
+    ports:
+      - ${INFRAHUB_TESTING_MESSAGE_QUEUE_PORT:-0}:15692
+
+  cache:
+    image: ${CACHE_DOCKER_IMAGE:-redis:7.2.4}
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD-SHELL", "redis-cli ping | grep PONG"]
+      interval: 5s
+      timeout: 5s
+      retries: 3
+
+  infrahub-server-lb:
+    image: haproxy:3.1-alpine
+    volumes:
+      - ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg
+    depends_on:
+      infrahub-server:
+        condition: service_started
+    healthcheck:
+      test: wget -O /dev/null http://127.0.0.1:8000/api/config || exit 1
+      interval: 5s
+      timeout: 5s
+      retries: 20
+      start_period: 10s
+    ports:
+      - ${INFRAHUB_TESTING_SERVER_PORT:-0}:8000
+
+  database:
+    deploy:
+      resources:
+        limits:
+          cpus: ${INFRAHUB_TESTING_DB_CPU_LIMIT}
+          memory: ${INFRAHUB_TESTING_DB_MEMORY_LIMIT}
+    image: "${DATABASE_DOCKER_IMAGE:-neo4j:5.20.0-enterprise}"
+    restart: unless-stopped
+    environment:
+      <<: *neo4j-config-common
+      NEO4J_metrics_prometheus_endpoint: 0.0.0.0:2004
+      NEO4J_server_backup_listen__address: 0.0.0.0:6362
+      NEO4J_server_discovery_advertised__address: database:5000
+      NEO4J_server_cluster_advertised__address: database:6000
+      NEO4J_server_cluster_raft_advertised__address: database:7000
+      NEO4J_server_bolt_advertised__address: database:7687
+      NEO4J_server_http_advertised__address: database:7474
+      NEO4J_server_https_advertised__address: database:7473
+    volumes:
+      - "database_data:/data"
+      - "database_logs:/logs"
+      - "./${INFRAHUB_TESTING_LOCAL_DB_BACKUP_DIRECTORY}:${INFRAHUB_TESTING_INTERNAL_DB_BACKUP_DIRECTORY}"
+    healthcheck:
+      test: wget http://localhost:7474 || exit 1
+      interval: 2s
+      timeout: 10s
+      retries: 20
+      start_period: 3s
+    ports:
+      - ${INFRAHUB_TESTING_DATABASE_PORT:-0}:6362
+      - ${INFRAHUB_TESTING_DATABASE_UI_PORT:-0}:7474
+
+  database-core2:
+    deploy:
+      resources:
+        limits:
+          cpus: ${INFRAHUB_TESTING_DB_CPU_LIMIT}
+          memory: ${INFRAHUB_TESTING_DB_MEMORY_LIMIT}
+    image: "${DATABASE_DOCKER_IMAGE:-neo4j:5.20.0-enterprise}"
+    environment:
+      <<: *neo4j-config-common
+      NEO4J_metrics_prometheus_endpoint: 0.0.0.0:2005
+      NEO4J_server_backup_listen__address: 0.0.0.0:6363
+      NEO4J_server_discovery_advertised__address: database-core2:5000
+      NEO4J_server_cluster_advertised__address: database-core2:6000
+      NEO4J_server_cluster_raft_advertised__address: database-core2:7000
+      NEO4J_server_bolt_advertised__address: database-core2:7687
+      NEO4J_server_http_advertised__address: database-core2:7474
+      NEO4J_server_https_advertised__address: database-core2:7473
+    volumes:
+      - "./plugins:/plugins"
+      - "database_data_core2:/data"
+      - "database_logs_core2:/logs"
+    healthcheck:
+      test: wget http://localhost:7474 || exit 1
+      interval: 5s
+      timeout: 10s
+      retries: 40
+      start_period: 30s
+    labels:
+      infrahub_role: "database"
+      com.github.run_id: "${GITHUB_RUN_ID:-unknown}"
+      com.github.job: "${JOB_NAME:-unknown}"
+    ports:
+      - "${INFRAHUB_TESTING_DATABASE_PORT:-0}:6363"
+
+  database-core3:
+    deploy:
+      resources:
+        limits:
+          cpus: ${INFRAHUB_TESTING_DB_CPU_LIMIT}
+          memory: ${INFRAHUB_TESTING_DB_MEMORY_LIMIT}
+    image: "${DATABASE_DOCKER_IMAGE:-neo4j:5.20.0-enterprise}"
+    environment:
+      <<: *neo4j-config-common
+      NEO4J_metrics_prometheus_endpoint: 0.0.0.0:2006
+      NEO4J_server_backup_listen__address: 0.0.0.0:6364
+      NEO4J_server_discovery_advertised__address: database-core3:5000
+      NEO4J_server_cluster_advertised__address: database-core3:6000
+      NEO4J_server_cluster_raft_advertised__address: database-core3:7000
+      NEO4J_server_bolt_advertised__address: database-core3:7687
+      NEO4J_server_http_advertised__address: database-core3:7474
+      NEO4J_server_https_advertised__address: database-core3:7473
+    volumes:
+      - "./plugins:/plugins"
+      - "database_data_core3:/data"
+      - "database_logs_core3:/logs"
+    healthcheck:
+      test: wget http://localhost:7474 || exit 1
+      interval: 5s
+      timeout: 10s
+      retries: 40
+      start_period: 30s
+    labels:
+      infrahub_role: "database"
+      com.github.run_id: "${GITHUB_RUN_ID:-unknown}"
+      com.github.job: "${JOB_NAME:-unknown}"
+    ports:
+      - "${INFRAHUB_TESTING_DATABASE_PORT:-0}:6364"
+
+  task-manager:
+    image: "${INFRAHUB_TESTING_DOCKER_IMAGE}:${INFRAHUB_TESTING_IMAGE_VERSION}"
+    command: uvicorn --host 0.0.0.0 --port 4200 --factory infrahub.prefect_server.app:create_infrahub_prefect
+    depends_on:
+      task-manager-db:
+        condition: service_healthy
+    environment:
+      PREFECT_UI_ENABLED: "${INFRAHUB_TESTING_PREFECT_UI_ENABLED}" # enabling UI requires permissions, run container as root to enable UI
+      PREFECT_API_DATABASE_CONNECTION_URL: postgresql+asyncpg://postgres:postgres@task-manager-db:5432/prefect
+    healthcheck:
+      test: /usr/local/bin/httpx http://localhost:4200/api/health || exit 1
+      interval: 5s
+      timeout: 5s
+      retries: 20
+      start_period: 10s
+    ports:
+      - ${INFRAHUB_TESTING_TASK_MANAGER_PORT:-0}:4200
+
+  task-manager-db:
+    image: "${POSTGRES_DOCKER_IMAGE:-postgres:16-alpine}"
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DB=prefect
+    volumes:
+      - workflow_db:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  infrahub-server:
+    deploy:
+      mode: replicated
+      replicas: ${INFRAHUB_TESTING_API_SERVER_COUNT}
+    image: "${INFRAHUB_TESTING_DOCKER_IMAGE}:${INFRAHUB_TESTING_IMAGE_VERSION}"
+    command: ${INFRAHUB_TESTING_DOCKER_ENTRYPOINT}
+    environment:
+      INFRAHUB_PRODUCTION: ${INFRAHUB_TESTING_PRODUCTION}
+      INFRAHUB_LOG_LEVEL: ${INFRAHUB_TESTING_LOG_LEVEL:-INFO}
+      INFRAHUB_BROKER_ADDRESS: ${INFRAHUB_TESTING_BROKER_ADDRESS}
+      INFRAHUB_CACHE_ADDRESS: ${INFRAHUB_TESTING_CACHE_ADDRESS}
+      INFRAHUB_DB_ADDRESS: ${INFRAHUB_TESTING_DB_ADDRESS}
+      INFRAHUB_DB_PROTOCOL: ${INFRAHUB_TESTING_DB_PROTOCOL:-neo4j}
+      INFRAHUB_WORKFLOW_ADDRESS: ${INFRAHUB_TESTING_WORKFLOW_ADDRESS}
+      INFRAHUB_WORKFLOW_DEFAULT_WORKER_TYPE: ${INFRAHUB_TESTING_WORKFLOW_DEFAULT_WORKER_TYPE}
+      INFRAHUB_INITIAL_ADMIN_TOKEN: ${INFRAHUB_TESTING_INITIAL_ADMIN_TOKEN}
+      INFRAHUB_INITIAL_AGENT_TOKEN: ${INFRAHUB_TESTING_INITIAL_AGENT_TOKEN}
+      INFRAHUB_SECURITY_SECRET_KEY: ${INFRAHUB_TESTING_SECURITY_SECRET_KEY}
+      PREFECT_API_URL: ${INFRAHUB_TESTING_PREFECT_API}
+      # Tracing
+      INFRAHUB_TRACE_ENABLE: ${INFRAHUB_TRACE_ENABLE:-false}
+      INFRAHUB_TRACE_EXPORTER_ENDPOINT:
+      INFRAHUB_TRACE_EXPORTER_PROTOCOL: ${INFRAHUB_TRACE_EXPORTER_PROTOCOL:-grpc}
+      INFRAHUB_TRACE_EXPORTER_TYPE: ${INFRAHUB_TRACE_EXPORTER_TYPE:-console}
+      INFRAHUB_TRACE_INSECURE: ${INFRAHUB_TRACE_INSECURE:-true}
+      OTEL_RESOURCE_ATTRIBUTES:
+    depends_on:
+      database:
+        condition: service_healthy
+      database-core2:
+        condition: service_healthy
+      database-core3:
+        condition: service_healthy
+      message-queue:
+        condition: service_healthy
+      cache:
+        condition: service_healthy
+      task-manager:
+        condition: service_healthy
+    volumes:
+      - "storage_data:/opt/infrahub/storage"
+    tty: true
+    healthcheck:
+      test: curl -s -f -o /dev/null http://localhost:8000/api/config || exit 1
+      interval: 5s
+      timeout: 5s
+      retries: 20
+      start_period: 10s
+
+  task-worker:
+    deploy:
+      mode: replicated
+      replicas: ${INFRAHUB_TESTING_TASK_WORKER_COUNT}
+    image: "${INFRAHUB_TESTING_DOCKER_IMAGE}:${INFRAHUB_TESTING_IMAGE_VERSION}"
+    command: prefect worker start --type ${INFRAHUB_TESTING_WORKFLOW_DEFAULT_WORKER_TYPE} --pool infrahub-worker --with-healthcheck
+    environment:
+      INFRAHUB_PRODUCTION: ${INFRAHUB_TESTING_PRODUCTION}
+      INFRAHUB_LOG_LEVEL: ${INFRAHUB_TESTING_LOG_LEVEL}
+      INFRAHUB_GIT_REPOSITORIES_DIRECTORY: ${INFRAHUB_TESTING_GIT_REPOSITORIES_DIRECTORY}
+      INFRAHUB_API_TOKEN: ${INFRAHUB_TESTING_INITIAL_AGENT_TOKEN}
+      INFRAHUB_SECURITY_SECRET_KEY: ${INFRAHUB_TESTING_SECURITY_SECRET_KEY}
+      INFRAHUB_ADDRESS: ${INFRAHUB_TESTING_ADDRESS}
+      INFRAHUB_INTERNAL_ADDRESS: ${INFRAHUB_TESTING_INTERNAL_ADDRESS}
+      INFRAHUB_BROKER_ADDRESS: ${INFRAHUB_TESTING_BROKER_ADDRESS}
+      INFRAHUB_CACHE_ADDRESS: ${INFRAHUB_TESTING_CACHE_ADDRESS}
+      INFRAHUB_DB_ADDRESS: ${INFRAHUB_TESTING_DB_ADDRESS:-database}
+      INFRAHUB_DB_PROTOCOL: ${INFRAHUB_TESTING_DB_PROTOCOL:-neo4j}
+      INFRAHUB_WORKFLOW_ADDRESS: ${INFRAHUB_TESTING_WORKFLOW_ADDRESS}
+      INFRAHUB_TIMEOUT: ${INFRAHUB_TESTING_TIMEOUT}
+      PREFECT_API_URL: ${INFRAHUB_TESTING_PREFECT_API}
+      # Tracing
+      INFRAHUB_TRACE_ENABLE: ${INFRAHUB_TRACE_ENABLE:-false}
+      INFRAHUB_TRACE_EXPORTER_ENDPOINT:
+      INFRAHUB_TRACE_EXPORTER_PROTOCOL: ${INFRAHUB_TRACE_EXPORTER_PROTOCOL:-grpc}
+      INFRAHUB_TRACE_EXPORTER_TYPE: ${INFRAHUB_TRACE_EXPORTER_TYPE:-console}
+      INFRAHUB_TRACE_INSECURE: ${INFRAHUB_TRACE_INSECURE:-true}
+      OTEL_RESOURCE_ATTRIBUTES:
+    depends_on:
+      - infrahub-server
+    volumes:
+      - "./${INFRAHUB_TESTING_LOCAL_REMOTE_GIT_DIRECTORY}:${INFRAHUB_TESTING_INTERNAL_REMOTE_GIT_DIRECTORY}"
+    tty: true
+
+  cadvisor:
+    image: "${CADVISOR_DOCKER_IMAGE:-gcr.io/cadvisor/cadvisor:v0.51.0}"
+    command:
+      - -disable_root_cgroup_stats=true
+      - -docker_only=true
+      - -store_container_labels=false
+      - -whitelisted_container_labels=com.docker.compose.project
+    privileged: true
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:ro
+      - /sys:/sys:ro
+      - /var/lib/docker:/var/lib/docker:ro
+      - /dev/disk/:/dev/disk:ro
+    ports:
+      - "${INFRAHUB_TESTING_CADVISOR_PORT:-0}:8080"
+
+  scraper:
+    image: "${SCRAPER_DOCKER_IMAGE:-victoriametrics/victoria-metrics:v1.110.0}"
+    volumes:
+      - vmdata:/victoria-metrics-data
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+    command:
+      - "--promscrape.config=/etc/prometheus/prometheus.yml"
+    ports:
+      - ${INFRAHUB_TESTING_SCRAPER_PORT:-0}:8428
+    healthcheck:
+      test: wget -qO- http://127.0.0.1:8428/-/healthy
+      start_period: 10s
+      interval: 5s
+      timeout: 5s
+      retries: 10
+
+volumes:
+  database_data:
+  database_logs:
+  database_data_core2:
+  database_logs_core2:
+  database_data_core3:
+  database_logs_core3:
+  storage_data:
+  workflow_db:
+  vmdata:

{infrahub_testcontainers-1.2.11 → infrahub_testcontainers-1.3.0}/infrahub_testcontainers/docker-compose.test.yml

@@ -50,7 +50,7 @@ services:
         limits:
           cpus: ${INFRAHUB_TESTING_DB_CPU_LIMIT:-0.0}
           memory: ${INFRAHUB_TESTING_DB_MEMORY_LIMIT:-0}
-    image: ${NEO4J_DOCKER_IMAGE:-neo4j:5.20.0-community}
+    image: ${NEO4J_DOCKER_IMAGE:-neo4j:2025.03.0-community}
     restart: unless-stopped
     environment:
       NEO4J_AUTH: neo4j/admin
@@ -59,6 +59,7 @@ services:
       NEO4J_ACCEPT_LICENSE_AGREEMENT: "yes"
       NEO4J_server_memory_heap_initial__size: ${INFRAHUB_TESTING_DB_HEAP_INITIAL_SIZE}
       NEO4J_server_memory_heap_max__size: ${INFRAHUB_TESTING_DB_HEAP_MAX_SIZE}
+      NEO4J_server_memory_pagecache_size: ${INFRAHUB_TESTING_DB_PAGECACHE_SIZE}
     volumes:
       - "database_data:/data"
       - "database_logs:/logs"

{infrahub_testcontainers-1.2.11 → infrahub_testcontainers-1.3.0}/infrahub_testcontainers/helpers.py

@@ -1,8 +1,11 @@
 import os
 import subprocess  # noqa: S404
+import uuid
+import warnings
 from pathlib import Path
 
 import pytest
+from prefect.client.orchestration import PrefectClient
 
 from infrahub_testcontainers import __version__ as infrahub_version
 
@@ -38,8 +41,8 @@ class TestInfrahubDocker:
 
     @pytest.fixture(scope="class")
     def tmp_directory(self, tmpdir_factory: pytest.TempdirFactory) -> Path:
-        directory = Path(str(tmpdir_factory.getbasetemp().strpath))
-        return directory
+        name = f"{self.__class__.__name__.lower()}_{uuid.uuid4().hex}"
+        return Path(str(tmpdir_factory.mktemp(name)))
 
     @pytest.fixture(scope="class")
     def remote_repos_dir(self, tmp_directory: Path) -> Path:
@@ -59,6 +62,10 @@ class TestInfrahubDocker:
     def default_branch(self) -> str:
         return "main"
 
+    @pytest.fixture(scope="class")
+    def deployment_type(self, request: pytest.FixtureRequest) -> str | None:
+        return request.config.getoption(name="infrahub_deployment_type", default=None)
+
     @pytest.fixture(scope="class")
     def infrahub_compose(
         self,
@@ -66,12 +73,21 @@ class TestInfrahubDocker:
         remote_repos_dir: Path,  # initialize repository before running docker compose to fix permissions issues # noqa: ARG002
         remote_backups_dir: Path,  # noqa: ARG002
         infrahub_version: str,
+        deployment_type: str | None,
     ) -> InfrahubDockerCompose:
-        return InfrahubDockerCompose.init(directory=tmp_directory, version=infrahub_version)
+        return InfrahubDockerCompose.init(
+            directory=tmp_directory, version=infrahub_version, deployment_type=deployment_type
+        )
 
     @pytest.fixture(scope="class")
     def infrahub_app(self, request: pytest.FixtureRequest, infrahub_compose: InfrahubDockerCompose) -> dict[str, int]:
+        tests_failed_before_class = request.session.testsfailed
+
         def cleanup() -> None:
+            tests_failed_during_class = request.session.testsfailed - tests_failed_before_class
+            if tests_failed_during_class > 0:
+                stdout, stderr = infrahub_compose.get_logs("infrahub-server", "task-worker")
+                warnings.warn(f"Container logs:\nStdout:\n{stdout}\nStderr:\n{stderr}", stacklevel=2)
             infrahub_compose.stop()
 
         request.addfinalizer(cleanup)
@@ -91,3 +107,7 @@ class TestInfrahubDocker:
     @pytest.fixture(scope="class")
     def task_manager_port(self, infrahub_app: dict[str, int]) -> int:
         return infrahub_app["task-manager"]
+
+    @pytest.fixture(scope="class")
+    def prefect_client(self, task_manager_port: int) -> PrefectClient:
+        return PrefectClient(api=f"http://localhost:{task_manager_port}/api/")

{infrahub_testcontainers-1.2.11 → infrahub_testcontainers-1.3.0}/infrahub_testcontainers/plugin.py

@@ -13,6 +13,15 @@ if TYPE_CHECKING:
 def pytest_addoption(parser: pytest.Parser) -> None:
     group = parser.getgroup("infrahub-performance-test")
 
+    group.addoption(
+        "--deployment-type",
+        action="store",
+        dest="infrahub_deployment_type",
+        default=None,
+        metavar="INFRAHUB_DEPLOYMENT_TYPE",
+        help="Type of deployment to use (default: None, options: cluster)",
+    )
+
     group.addoption(
         "--performance-result-address",
         action="store",

{infrahub_testcontainers-1.2.11 → infrahub_testcontainers-1.3.0}/pyproject.toml

@@ -1,11 +1,11 @@
 [project]
 name = "infrahub-testcontainers"
-version = "1.2.11"
+version =  "1.3.0"
 requires-python = ">=3.9"
 
 [tool.poetry]
 name = "infrahub-testcontainers"
-version = "1.2.11"
+version =  "1.3.0"
 description = "Testcontainers instance for Infrahub to easily build integration tests"
 authors = ["OpsMill <info@opsmill.com>"]
 readme = "README.md"
@@ -23,21 +23,23 @@ classifiers = [
     "Programming Language :: Python :: 3.13",
 ]
 
-include = ["infrahub_testcontainers/docker-compose.test.yml", "infrahub_testcontainers/haproxy.cfg", "infrahub_testcontainers/prometheus.yml"]
+include = ["infrahub_testcontainers/docker-compose.test.yml", "infrahub_testcontainers/docker-compose-cluster.test.yml", "infrahub_testcontainers/haproxy.cfg", "infrahub_testcontainers/prometheus.yml"]
+
 
 [tool.setuptools.package-data]
-infrahub_testcontainers = ["docker-compose.test.yml", "haproxy.cfg", "prometheus.yml"]
+infrahub_testcontainers = ["docker-compose.test.yml", "docker-compose-cluster.test.yml", "haproxy.cfg", "prometheus.yml"]
 
 [tool.poetry.plugins."pytest11"]
 "pytest-infrahub-performance-test" = "infrahub_testcontainers.plugin"
 
 [tool.poetry.dependencies]
-python = "^3.9"
+python = "^3.9, < 3.14"
 testcontainers = "~4.8"
 psutil = "*"
 pytest = "*"
 httpx = "^0.28.1"
 pydantic = "^2.10.6"
+prefect-client = "3.4.1"
 
 [tool.poetry.group.dev.dependencies]
 rich = "^13.9.4"