infrahub-server 1.3.0a0__py3-none-any.whl → 1.3.0b2__py3-none-any.whl

infrahub_testcontainers/container.py

@@ -66,9 +66,12 @@ PROJECT_ENV_VARIABLES: dict[str, str] = {
 class InfrahubDockerCompose(DockerCompose):
     project_name: str | None = None
     env_vars: dict[str, str] = field(default_factory=dict)
+    deployment_type: str | None = None
 
     @classmethod
-    def init(cls, directory: Path | None = None, version: str | None = None) -> Self:
+    def init(
+        cls, directory: Path | None = None, version: str | None = None, deployment_type: str | None = None
+    ) -> Self:
         if not directory:
             directory = Path.cwd()
 
@@ -79,7 +82,7 @@ class InfrahubDockerCompose(DockerCompose):
         if version == "local" and infrahub_image_version:
             version = infrahub_image_version
 
-        compose = cls(project_name=cls.generate_project_name(), context=directory)
+        compose = cls(project_name=cls.generate_project_name(), context=directory, deployment_type=deployment_type)
         compose.create_docker_file(directory=directory)
         compose.create_env_file(directory=directory, version=version)
 
@@ -111,7 +114,10 @@ class InfrahubDockerCompose(DockerCompose):
 
     def create_docker_file(self, directory: Path) -> Path:
         current_directory = Path(__file__).resolve().parent
-        compose_file = current_directory / "docker-compose.test.yml"
+        compose_file_name = (
+            "docker-compose-cluster.test.yml" if self.deployment_type == "cluster" else "docker-compose.test.yml"
+        )
+        compose_file = current_directory / compose_file_name
 
         test_compose_file = directory / "docker-compose.yml"
         test_compose_file.write_bytes(compose_file.read_bytes())
@@ -160,7 +166,7 @@ class InfrahubDockerCompose(DockerCompose):
             cmd.extend(self.services)
         self._run_command(cmd=cmd)
 
-    def start_container(self, service_name: str) -> None:
+    def start_container(self, service_name: str | list[str]) -> None:
         """
         Starts a specific service of the docker compose environment.
 
@@ -170,7 +176,11 @@ class InfrahubDockerCompose(DockerCompose):
 
         # pull means running a separate command before starting
         if self.pull:
-            pull_cmd = [*base_cmd, "pull", service_name]
+            pull_cmd = [*base_cmd, "pull"]
+            if isinstance(service_name, list):
+                pull_cmd.extend(service_name)
+            else:
+                pull_cmd.append(service_name)
             self._run_command(cmd=pull_cmd)
 
         up_cmd = [*base_cmd, "up"]
@@ -185,7 +195,10 @@ class InfrahubDockerCompose(DockerCompose):
             # we run in detached mode instead of blocking
             up_cmd.append("--detach")
 
-        up_cmd.append(service_name)
+        if isinstance(service_name, list):
+            up_cmd.extend(service_name)
+        else:
+            up_cmd.append(service_name)
         self._run_command(cmd=up_cmd)
 
     # TODO would be good to the support for project_name upstream
@@ -233,7 +246,7 @@ class InfrahubDockerCompose(DockerCompose):
                 dest_dir / backup_name,
             )
 
-    def database_restore_backup(self, backup_file: Path) -> None:
+    def database_restore_backup(self, backup_file: Path) -> None:  # noqa: PLR0915
         assert self.use_neo4j_enterprise
 
         shutil.copy(
@@ -242,52 +255,35 @@ class InfrahubDockerCompose(DockerCompose):
         )
         service_name = "database"
 
-        # Ensure the database container is running otherwise start it
-        try:
-            self.get_container(service_name=service_name)
-        except ContainerIsNotRunning:
-            self.start_container(service_name=service_name)
-
-        self.exec_in_container(
-            command=["cypher-shell", "-u", "neo4j", "-p", "admin", "STOP DATABASE neo4j;"],
-            service_name=service_name,
-        )
+        if self.deployment_type != "cluster":  # noqa: PLR1702
+            try:
+                self.get_container(service_name=service_name)
+            except ContainerIsNotRunning:
+                self.start_container(service_name=service_name)
 
-        self.exec_in_container(
-            command=[
-                "neo4j-admin",
-                "database",
-                "restore",
-                "--overwrite-destination",
-                "--from-path",
-                str(self.internal_backup_dir / backup_file.name),
-            ],
-            service_name=service_name,
-        )
+            self.exec_in_container(
+                command=["cypher-shell", "-u", "neo4j", "-p", "admin", "STOP DATABASE neo4j;"],
+                service_name=service_name,
+            )
 
-        self.exec_in_container(
-            command=["chown", "-R", "neo4j:neo4j", "/data"],
-            service_name=service_name,
-        )
+            self.exec_in_container(
+                command=[
+                    "neo4j-admin",
+                    "database",
+                    "restore",
+                    "--overwrite-destination",
+                    "--from-path",
+                    str(self.internal_backup_dir / backup_file.name),
+                ],
+                service_name=service_name,
+            )
 
-        (restore_output, _, _) = self.exec_in_container(
-            command=[
-                "cypher-shell",
-                "--format",
-                "plain",
-                "-d",
-                "system",
-                "-u",
-                "neo4j",
-                "-p",
-                "admin",
-                "START DATABASE neo4j;",
-            ],
-            service_name=service_name,
-        )
+            self.exec_in_container(
+                command=["chown", "-R", "neo4j:neo4j", "/data"],
+                service_name=service_name,
+            )
 
-        for _ in range(3):
-            (stdout, _, _) = self.exec_in_container(
+            (restore_output, _, _) = self.exec_in_container(
                 command=[
                     "cypher-shell",
                     "--format",
@@ -298,26 +294,205 @@ class InfrahubDockerCompose(DockerCompose):
                     "neo4j",
                     "-p",
                     "admin",
-                    "SHOW DATABASES WHERE name = 'neo4j' AND currentStatus = 'online';",
+                    "START DATABASE neo4j;",
                 ],
                 service_name=service_name,
             )
-            if stdout:
-                break
-            time.sleep(5)
+
+            for _ in range(3):
+                (stdout, _, _) = self.exec_in_container(
+                    command=[
+                        "cypher-shell",
+                        "--format",
+                        "plain",
+                        "-d",
+                        "system",
+                        "-u",
+                        "neo4j",
+                        "-p",
+                        "admin",
+                        "SHOW DATABASES WHERE name = 'neo4j' AND currentStatus = 'online';",
+                    ],
+                    service_name=service_name,
+                )
+                if stdout:
+                    break
+                time.sleep(5)
+            else:
+                (debug_logs, _, _) = self.exec_in_container(
+                    command=["cat", "logs/debug.log"],
+                    service_name=service_name,
+                )
+                raise Exception(f"Failed to restore database:\n{restore_output}\nDebug logs:\n{debug_logs}")
+
+            old_services = self.services
+            self.services = ["infrahub-server", "task-worker"]
+            self.stop(down=False)
+            try:
+                self.start()
+            except Exception as exc:
+                stdout, stderr = self.get_logs()
+                raise Exception(f"Failed to start docker compose:\nStdout:\n{stdout}\nStderr:\n{stderr}") from exc
+            self.services = old_services
         else:
-            (debug_logs, _, _) = self.exec_in_container(
-                command=["cat", "logs/debug.log"],
+            print("Cluster mode detected")
+            try:
+                self.get_container(service_name=service_name)
+                self.get_container(service_name="database-core2")
+                self.get_container(service_name="database-core3")
+            except ContainerIsNotRunning:
+                self.start_container("database", "database-core2", "database-core3")
+
+            # Waiting for cluster to stabilize...
+            time.sleep(10)
+
+            self.exec_in_container(
+                command=["cypher-shell", "-u", "neo4j", "-p", "admin", "DROP DATABASE neo4j;"],
+                service_name=service_name,
+            )
+
+            self.exec_in_container(
+                command=["rm", "-rf", "/data/databases/neo4j"],
+                service_name=service_name,
+            )
+            self.exec_in_container(
+                command=["rm", "-rf", "/data/transactions/neo4j"],
+                service_name=service_name,
+            )
+
+            self.exec_in_container(
+                command=[
+                    "neo4j-admin",
+                    "database",
+                    "restore",
+                    "--from-path",
+                    str(self.internal_backup_dir / backup_file.name),
+                    "neo4j",
+                ],
                 service_name=service_name,
             )
-            raise Exception(f"Failed to restore database:\n{restore_output}\nDebug logs:\n{debug_logs}")
 
-        old_services = self.services
-        self.services = ["infrahub-server", "task-worker"]
-        self.stop(down=False)
-        try:
+            cmd = self.compose_command_property[:]
+            cmd += ["restart", "database"]
+            self._run_command(cmd=cmd)
+
+            main_node = service_name
+            cluster_nodes = ["database", "database-core2", "database-core3"]
+
+            for attempt in range(3):
+                try:
+                    (stdout, _, _) = self.exec_in_container(
+                        command=[
+                            "cypher-shell",
+                            "--format",
+                            "plain",
+                            "-d",
+                            "system",
+                            "-u",
+                            "neo4j",
+                            "-p",
+                            "admin",
+                            "SHOW DATABASES YIELD name, address, currentStatus WHERE name = 'system' RETURN address, currentStatus",
+                        ],
+                        service_name=main_node,
+                    )
+                except Exception:
+                    time.sleep(10)
+                    continue
+
+                raw_output = stdout
+                nodes_status = dict.fromkeys(cluster_nodes, False)
+                online_count = 0
+                total_entries = 0
+
+                try:
+                    for line_raw in stdout.splitlines():
+                        line = line_raw.strip()
+                        if not line or line.startswith("address"):
+                            continue
+
+                        total_entries += 1
+                        if "online" in line:
+                            online_count += 1
+                            for node in cluster_nodes:
+                                node_pattern = f'"{node}:'
+                                if node_pattern in line:
+                                    nodes_status[node] = True
+                                    break
+                    if all(nodes_status.values()) and online_count == len(cluster_nodes):
+                        break
+                except Exception as e:
+                    print(f"Error parsing database status on attempt {attempt + 1}: {e}")
+
+                print(f"Waiting for all nodes to be online. Current status: {nodes_status}")
+                time.sleep(5)
+            else:
+                debug_logs = {}
+                for node in cluster_nodes:
+                    try:
+                        (logs, _, _) = self.exec_in_container(
+                            command=["cat", "logs/debug.log"],
+                            service_name=node,
+                        )
+                        debug_logs[node] = logs
+                    except Exception as e:
+                        debug_logs[node] = f"Could not retrieve logs: {str(e)}"
+
+                debug_info = f"Raw output from SHOW DATABASES command:\n{raw_output}\n\n"
+                debug_info += f"Final node status: {nodes_status}\n\n"
+
+                status_str = ", ".join(
+                    [f"{node}: {'online' if status else 'offline'}" for node, status in nodes_status.items()]
+                )
+                logs_str = debug_info + "\n\n".join(
+                    [f"--- {node} logs ---\n{logs}" for node, logs in debug_logs.items()]
+                )
+
+                raise Exception(
+                    f"Failed to restore database cluster. Node status: {status_str}\nDebug logs:\n{logs_str}"
+                )
+
+            server_id = None
+            try:
+                stdout, _, _ = self.exec_in_container(
+                    command=[
+                        "cypher-shell",
+                        "--format",
+                        "plain",
+                        "-d",
+                        "system",
+                        "-u",
+                        "neo4j",
+                        "-p",
+                        "admin",
+                        'SHOW SERVERS YIELD name, address WHERE address = "database:7687" RETURN name;',
+                    ],
+                    service_name=service_name,
+                )
+
+                lines = stdout.splitlines()
+                for line_raw in lines:
+                    line = line_raw.strip()
+                    if not line or line == "name" or line.startswith("+"):
+                        continue
+                    server_id = line.strip('"')
+                    break
+            except Exception as e:
+                print(f"Error retrieving server ID with direct query: {e}")
+
+            if server_id:
+                self.exec_in_container(
+                    command=[
+                        "cypher-shell",
+                        "-d",
+                        "system",
+                        "-u",
+                        "neo4j",
+                        "-p",
+                        "admin",
+                        f"CREATE DATABASE neo4j TOPOLOGY 3 PRIMARIES OPTIONS {{ existingData: 'use', existingDataSeedInstance: '{server_id}' }};",
+                    ],
+                    service_name=service_name,
+                )
             self.start()
-        except Exception as exc:
-            stdout, stderr = self.get_logs()
-            raise Exception(f"Failed to start docker compose:\nStdout:\n{stdout}\nStderr:\n{stderr}") from exc
-        self.services = old_services
+            print("Database restored successfully")

infrahub_testcontainers/docker-compose-cluster.test.yml

@@ -0,0 +1,321 @@
+---
+# yamllint disable rule:line-length
+# The following environment variables are part of the Infrahub configuration options.
+# For detailed information on these configuration options, please refer to the Infrahub documentation:
+# https://docs.infrahub.app/reference/configuration
+x-neo4j-config-common: &neo4j-config-common
+  NEO4J_AUTH: neo4j/admin
+  NEO4J_dbms_security_procedures_unrestricted: apoc.*
+  NEO4J_dbms_security_auth__minimum__password__length: 4
+  NEO4J_ACCEPT_LICENSE_AGREEMENT: 'yes'
+  NEO4J_server_backup_enabled: true
+  NEO4J_metrics_prometheus_enabled: true
+  NEO4J_server_metrics_filter: '*'
+  NEO4J_server_cluster_system__database__mode: PRIMARY
+  NEO4J_initial_server_mode__constraint: PRIMARY
+  NEO4J_dbms_cluster_discovery_endpoints: database:5000,database-core2:5000,database-core3:5000
+  NEO4J_initial_dbms_default__primaries__count: 3
+  NEO4J_dbms_memory_heap_initial__size: ${INFRAHUB_TESTING_DB_HEAP_INITIAL_SIZE}
+  NEO4J_dbms_memory_heap_max__size: ${INFRAHUB_TESTING_DB_HEAP_MAX_SIZE}
+  NEO4J_server_memory_pagecache_size: ${INFRAHUB_TESTING_DB_PAGECACHE_SIZE}
+
+
+services:
+  message-queue:
+    image: ${MESSAGE_QUEUE_DOCKER_IMAGE:-rabbitmq:3.13.7-management}
+    restart: unless-stopped
+    environment:
+      RABBITMQ_DEFAULT_USER: infrahub
+      RABBITMQ_DEFAULT_PASS: infrahub
+    healthcheck:
+      test: rabbitmq-diagnostics -q check_port_connectivity
+      interval: 5s
+      timeout: 30s
+      retries: 10
+      start_period: 3s
+    ports:
+      - ${INFRAHUB_TESTING_MESSAGE_QUEUE_PORT:-0}:15692
+
+  cache:
+    image: ${CACHE_DOCKER_IMAGE:-redis:7.2.4}
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD-SHELL", "redis-cli ping | grep PONG"]
+      interval: 5s
+      timeout: 5s
+      retries: 3
+
+  infrahub-server-lb:
+    image: haproxy:3.1-alpine
+    volumes:
+      - ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg
+    depends_on:
+      infrahub-server:
+        condition: service_started
+    healthcheck:
+      test: wget -O /dev/null http://127.0.0.1:8000/api/config || exit 1
+      interval: 5s
+      timeout: 5s
+      retries: 20
+      start_period: 10s
+    ports:
+      - ${INFRAHUB_TESTING_SERVER_PORT:-0}:8000
+
+  database:
+    deploy:
+      resources:
+        limits:
+          cpus: ${INFRAHUB_TESTING_DB_CPU_LIMIT}
+          memory: ${INFRAHUB_TESTING_DB_MEMORY_LIMIT}
+    image: "${DATABASE_DOCKER_IMAGE:-neo4j:5.20.0-enterprise}"
+    restart: unless-stopped
+    environment:
+      <<: *neo4j-config-common
+      NEO4J_metrics_prometheus_endpoint: 0.0.0.0:2004
+      NEO4J_server_backup_listen__address: 0.0.0.0:6362
+      NEO4J_server_discovery_advertised__address: database:5000
+      NEO4J_server_cluster_advertised__address: database:6000
+      NEO4J_server_cluster_raft_advertised__address: database:7000
+      NEO4J_server_bolt_advertised__address: database:7687
+      NEO4J_server_http_advertised__address: database:7474
+      NEO4J_server_https_advertised__address: database:7473
+    volumes:
+      - "database_data:/data"
+      - "database_logs:/logs"
+      - "./${INFRAHUB_TESTING_LOCAL_DB_BACKUP_DIRECTORY}:${INFRAHUB_TESTING_INTERNAL_DB_BACKUP_DIRECTORY}"
+    healthcheck:
+      test: wget http://localhost:7474 || exit 1
+      interval: 2s
+      timeout: 10s
+      retries: 20
+      start_period: 3s
+    ports:
+      - ${INFRAHUB_TESTING_DATABASE_PORT:-0}:6362
+      - ${INFRAHUB_TESTING_DATABASE_UI_PORT:-0}:7474
+
+  database-core2:
+    deploy:
+      resources:
+        limits:
+          cpus: ${INFRAHUB_TESTING_DB_CPU_LIMIT}
+          memory: ${INFRAHUB_TESTING_DB_MEMORY_LIMIT}
+    image: "${DATABASE_DOCKER_IMAGE:-neo4j:5.20.0-enterprise}"
+    environment:
+      <<: *neo4j-config-common
+      NEO4J_metrics_prometheus_endpoint: 0.0.0.0:2005
+      NEO4J_server_backup_listen__address: 0.0.0.0:6363
+      NEO4J_server_discovery_advertised__address: database-core2:5000
+      NEO4J_server_cluster_advertised__address: database-core2:6000
+      NEO4J_server_cluster_raft_advertised__address: database-core2:7000
+      NEO4J_server_bolt_advertised__address: database-core2:7687
+      NEO4J_server_http_advertised__address: database-core2:7474
+      NEO4J_server_https_advertised__address: database-core2:7473
+    volumes:
+      - "./plugins:/plugins"
+      - "database_data_core2:/data"
+      - "database_logs_core2:/logs"
+    healthcheck:
+      test: wget http://localhost:7474 || exit 1
+      interval: 5s
+      timeout: 10s
+      retries: 40
+      start_period: 30s
+    labels:
+      infrahub_role: "database"
+      com.github.run_id: "${GITHUB_RUN_ID:-unknown}"
+      com.github.job: "${JOB_NAME:-unknown}"
+    ports:
+      - "${INFRAHUB_TESTING_DATABASE_PORT:-0}:6363"
+
+  database-core3:
+    deploy:
+      resources:
+        limits:
+          cpus: ${INFRAHUB_TESTING_DB_CPU_LIMIT}
+          memory: ${INFRAHUB_TESTING_DB_MEMORY_LIMIT}
+    image: "${DATABASE_DOCKER_IMAGE:-neo4j:5.20.0-enterprise}"
+    environment:
+      <<: *neo4j-config-common
+      NEO4J_metrics_prometheus_endpoint: 0.0.0.0:2006
+      NEO4J_server_backup_listen__address: 0.0.0.0:6364
+      NEO4J_server_discovery_advertised__address: database-core3:5000
+      NEO4J_server_cluster_advertised__address: database-core3:6000
+      NEO4J_server_cluster_raft_advertised__address: database-core3:7000
+      NEO4J_server_bolt_advertised__address: database-core3:7687
+      NEO4J_server_http_advertised__address: database-core3:7474
+      NEO4J_server_https_advertised__address: database-core3:7473
+    volumes:
+      - "./plugins:/plugins"
+      - "database_data_core3:/data"
+      - "database_logs_core3:/logs"
+    healthcheck:
+      test: wget http://localhost:7474 || exit 1
+      interval: 5s
+      timeout: 10s
+      retries: 40
+      start_period: 30s
+    labels:
+      infrahub_role: "database"
+      com.github.run_id: "${GITHUB_RUN_ID:-unknown}"
+      com.github.job: "${JOB_NAME:-unknown}"
+    ports:
+      - "${INFRAHUB_TESTING_DATABASE_PORT:-0}:6364"
+
+  task-manager:
+    image: "${INFRAHUB_TESTING_DOCKER_IMAGE}:${INFRAHUB_TESTING_IMAGE_VERSION}"
+    command: uvicorn --host 0.0.0.0 --port 4200 --factory infrahub.prefect_server.app:create_infrahub_prefect
+    depends_on:
+      task-manager-db:
+        condition: service_healthy
+    environment:
+      PREFECT_UI_ENABLED: "${INFRAHUB_TESTING_PREFECT_UI_ENABLED}" # enabling UI requires permissions, run container as root to enable UI
+      PREFECT_API_DATABASE_CONNECTION_URL: postgresql+asyncpg://postgres:postgres@task-manager-db:5432/prefect
+    healthcheck:
+      test: /usr/local/bin/httpx http://localhost:4200/api/health || exit 1
+      interval: 5s
+      timeout: 5s
+      retries: 20
+      start_period: 10s
+    ports:
+      - ${INFRAHUB_TESTING_TASK_MANAGER_PORT:-0}:4200
+
+  task-manager-db:
+    image: "${POSTGRES_DOCKER_IMAGE:-postgres:16-alpine}"
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DB=prefect
+    volumes:
+      - workflow_db:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  infrahub-server:
+    deploy:
+      mode: replicated
+      replicas: ${INFRAHUB_TESTING_API_SERVER_COUNT}
+    image: "${INFRAHUB_TESTING_DOCKER_IMAGE}:${INFRAHUB_TESTING_IMAGE_VERSION}"
+    command: ${INFRAHUB_TESTING_DOCKER_ENTRYPOINT}
+    environment:
+      INFRAHUB_PRODUCTION: ${INFRAHUB_TESTING_PRODUCTION}
+      INFRAHUB_LOG_LEVEL: ${INFRAHUB_TESTING_LOG_LEVEL:-INFO}
+      INFRAHUB_BROKER_ADDRESS: ${INFRAHUB_TESTING_BROKER_ADDRESS}
+      INFRAHUB_CACHE_ADDRESS: ${INFRAHUB_TESTING_CACHE_ADDRESS}
+      INFRAHUB_DB_ADDRESS: ${INFRAHUB_TESTING_DB_ADDRESS}
+      INFRAHUB_DB_PROTOCOL: ${INFRAHUB_TESTING_DB_PROTOCOL:-neo4j}
+      INFRAHUB_WORKFLOW_ADDRESS: ${INFRAHUB_TESTING_WORKFLOW_ADDRESS}
+      INFRAHUB_WORKFLOW_DEFAULT_WORKER_TYPE: ${INFRAHUB_TESTING_WORKFLOW_DEFAULT_WORKER_TYPE}
+      INFRAHUB_INITIAL_ADMIN_TOKEN: ${INFRAHUB_TESTING_INITIAL_ADMIN_TOKEN}
+      INFRAHUB_INITIAL_AGENT_TOKEN: ${INFRAHUB_TESTING_INITIAL_AGENT_TOKEN}
+      INFRAHUB_SECURITY_SECRET_KEY: ${INFRAHUB_TESTING_SECURITY_SECRET_KEY}
+      PREFECT_API_URL: ${INFRAHUB_TESTING_PREFECT_API}
+      # Tracing
+      INFRAHUB_TRACE_ENABLE: ${INFRAHUB_TRACE_ENABLE:-false}
+      INFRAHUB_TRACE_EXPORTER_ENDPOINT:
+      INFRAHUB_TRACE_EXPORTER_PROTOCOL: ${INFRAHUB_TRACE_EXPORTER_PROTOCOL:-grpc}
+      INFRAHUB_TRACE_EXPORTER_TYPE: ${INFRAHUB_TRACE_EXPORTER_TYPE:-console}
+      INFRAHUB_TRACE_INSECURE: ${INFRAHUB_TRACE_INSECURE:-true}
+      OTEL_RESOURCE_ATTRIBUTES:
+    depends_on:
+      database:
+        condition: service_healthy
+      database-core2:
+        condition: service_healthy
+      database-core3:
+        condition: service_healthy
+      message-queue:
+        condition: service_healthy
+      cache:
+        condition: service_healthy
+      task-manager:
+        condition: service_healthy
+    volumes:
+      - "storage_data:/opt/infrahub/storage"
+    tty: true
+    healthcheck:
+      test: curl -s -f -o /dev/null http://localhost:8000/api/config || exit 1
+      interval: 5s
+      timeout: 5s
+      retries: 20
+      start_period: 10s
+
+  task-worker:
+    deploy:
+      mode: replicated
+      replicas: ${INFRAHUB_TESTING_TASK_WORKER_COUNT}
+    image: "${INFRAHUB_TESTING_DOCKER_IMAGE}:${INFRAHUB_TESTING_IMAGE_VERSION}"
+    command: prefect worker start --type ${INFRAHUB_TESTING_WORKFLOW_DEFAULT_WORKER_TYPE} --pool infrahub-worker --with-healthcheck
+    environment:
+      INFRAHUB_PRODUCTION: ${INFRAHUB_TESTING_PRODUCTION}
+      INFRAHUB_LOG_LEVEL: ${INFRAHUB_TESTING_LOG_LEVEL}
+      INFRAHUB_GIT_REPOSITORIES_DIRECTORY: ${INFRAHUB_TESTING_GIT_REPOSITORIES_DIRECTORY}
+      INFRAHUB_API_TOKEN: ${INFRAHUB_TESTING_INITIAL_AGENT_TOKEN}
+      INFRAHUB_SECURITY_SECRET_KEY: ${INFRAHUB_TESTING_SECURITY_SECRET_KEY}
+      INFRAHUB_ADDRESS: ${INFRAHUB_TESTING_ADDRESS}
+      INFRAHUB_INTERNAL_ADDRESS: ${INFRAHUB_TESTING_INTERNAL_ADDRESS}
+      INFRAHUB_BROKER_ADDRESS: ${INFRAHUB_TESTING_BROKER_ADDRESS}
+      INFRAHUB_CACHE_ADDRESS: ${INFRAHUB_TESTING_CACHE_ADDRESS}
+      INFRAHUB_DB_ADDRESS: ${INFRAHUB_TESTING_DB_ADDRESS:-database}
+      INFRAHUB_DB_PROTOCOL: ${INFRAHUB_TESTING_DB_PROTOCOL:-neo4j}
+      INFRAHUB_WORKFLOW_ADDRESS: ${INFRAHUB_TESTING_WORKFLOW_ADDRESS}
+      INFRAHUB_TIMEOUT: ${INFRAHUB_TESTING_TIMEOUT}
+      PREFECT_API_URL: ${INFRAHUB_TESTING_PREFECT_API}
+      # Tracing
+      INFRAHUB_TRACE_ENABLE: ${INFRAHUB_TRACE_ENABLE:-false}
+      INFRAHUB_TRACE_EXPORTER_ENDPOINT:
+      INFRAHUB_TRACE_EXPORTER_PROTOCOL: ${INFRAHUB_TRACE_EXPORTER_PROTOCOL:-grpc}
+      INFRAHUB_TRACE_EXPORTER_TYPE: ${INFRAHUB_TRACE_EXPORTER_TYPE:-console}
+      INFRAHUB_TRACE_INSECURE: ${INFRAHUB_TRACE_INSECURE:-true}
+      OTEL_RESOURCE_ATTRIBUTES:
+    depends_on:
+      - infrahub-server
+    volumes:
+      - "./${INFRAHUB_TESTING_LOCAL_REMOTE_GIT_DIRECTORY}:${INFRAHUB_TESTING_INTERNAL_REMOTE_GIT_DIRECTORY}"
+    tty: true
+
+  cadvisor:
+    image: "${CADVISOR_DOCKER_IMAGE:-gcr.io/cadvisor/cadvisor:v0.51.0}"
+    command:
+      - -disable_root_cgroup_stats=true
+      - -docker_only=true
+      - -store_container_labels=false
+      - -whitelisted_container_labels=com.docker.compose.project
+    privileged: true
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:ro
+      - /sys:/sys:ro
+      - /var/lib/docker:/var/lib/docker:ro
+      - /dev/disk/:/dev/disk:ro
+    ports:
+      - "${INFRAHUB_TESTING_CADVISOR_PORT:-0}:8080"
+
+  scraper:
+    image: "${SCRAPER_DOCKER_IMAGE:-victoriametrics/victoria-metrics:v1.110.0}"
+    volumes:
+      - vmdata:/victoria-metrics-data
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+    command:
+      - "--promscrape.config=/etc/prometheus/prometheus.yml"
+    ports:
+      - ${INFRAHUB_TESTING_SCRAPER_PORT:-0}:8428
+    healthcheck:
+      test: wget -qO- http://127.0.0.1:8428/-/healthy
+      start_period: 10s
+      interval: 5s
+      timeout: 5s
+      retries: 10
+
+volumes:
+  database_data:
+  database_logs:
+  database_data_core2:
+  database_logs_core2:
+  database_data_core3:
+  database_logs_core3:
+  storage_data:
+  workflow_db:
+  vmdata: