infrahub-server 1.3.7__py3-none-any.whl → 1.4.0__py3-none-any.whl

infrahub_testcontainers/container.py

@@ -59,6 +59,8 @@ PROJECT_ENV_VARIABLES: dict[str, str] = {
     "INFRAHUB_TESTING_PREFECT_UI_ENABLED": "true",
     "INFRAHUB_TESTING_DOCKER_PULL": "true",
     "INFRAHUB_TESTING_SCHEMA_STRICT_MODE": "true",
+    "INFRAHUB_TESTING_TASKMGR_API_WORKERS": "1",
+    "INFRAHUB_TESTING_TASKMGR_BACKGROUND_SVC_REPLICAS": "0",
 }
 
 
@@ -144,6 +146,21 @@ class InfrahubDockerCompose(DockerCompose):
                     "NEO4J_DOCKER_IMAGE": "neo4j:2025.03.0-enterprise",
                 }
             )
+        if os.environ.get("INFRAHUB_TESTING_TASKMGR_SCALEOUT"):
+            PROJECT_ENV_VARIABLES.update(
+                {
+                    "INFRAHUB_TESTING_TASKMGR_BACKGROUND_SVC_REPLICAS": "1",
+                    "PREFECT_MESSAGING_BROKER": "prefect_redis.messaging",
+                    "PREFECT_MESSAGING_CACHE": "prefect_redis.messaging",
+                    "PREFECT__SERVER_WEBSERVER_ONLY": "true",
+                    "PREFECT_API_DATABASE_MIGRATE_ON_START": "false",
+                    "PREFECT_API_BLOCKS_REGISTER_ON_START": "false",
+                    "PREFECT_SERVER_SERVICES_EVENT_LOGGER_ENABLED": "false",
+                    "PREFECT_SERVER_SERVICES_EVENT_PERSISTER_ENABLED": "false",
+                    "PREFECT_SERVER_SERVICES_TRIGGERS_ENABLED": "false",
+                    "PREFECT_SERVER_SERVICES_TASK_RUN_RECORDER_ENABLED": "false",
+                }
+            )
 
         with env_file.open(mode="w", encoding="utf-8") as file:
             for key, value in PROJECT_ENV_VARIABLES.items():

infrahub_testcontainers/docker-compose-cluster.test.yml

@@ -166,13 +166,33 @@ services:
 
   task-manager:
     image: "${INFRAHUB_TESTING_DOCKER_IMAGE}:${INFRAHUB_TESTING_IMAGE_VERSION}"
-    command: uvicorn --host 0.0.0.0 --port 4200 --factory infrahub.prefect_server.app:create_infrahub_prefect
+    command: "${PREFECT_SERVER_COMMAND:-gunicorn -k uvicorn.workers.UvicornWorker -b 0.0.0.0:4200 'infrahub.prefect_server.app:create_infrahub_prefect()'}"
     depends_on:
       task-manager-db:
         condition: service_healthy
+      cache:
+        condition: service_healthy
     environment:
       PREFECT_UI_ENABLED: "${INFRAHUB_TESTING_PREFECT_UI_ENABLED}" # enabling UI requires permissions, run container as root to enable UI
       PREFECT_API_DATABASE_CONNECTION_URL: postgresql+asyncpg://postgres:postgres@task-manager-db:5432/prefect
+
+      INFRAHUB_CACHE_ADDRESS: ${INFRAHUB_TESTING_CACHE_ADDRESS}
+      PREFECT_REDIS_MESSAGING_HOST: "${INFRAHUB_TESTING_CACHE_ADDRESS:-cache}"
+      PREFECT_REDIS_MESSAGING_DB: "1"
+      PREFECT_REDIS_MESSAGING_CONSUMER_MIN_IDLE_TIME: "30"
+      PREFECT_REDIS_MESSAGING_PUBLISHER_BATCH_SIZE: "1"
+
+      PREFECT_MESSAGING_BROKER:
+      PREFECT_MESSAGING_CACHE:
+      PREFECT__SERVER_WEBSERVER_ONLY:
+      PREFECT_API_DATABASE_MIGRATE_ON_START:
+      PREFECT_API_BLOCKS_REGISTER_ON_START:
+      PREFECT_SERVER_SERVICES_EVENT_LOGGER_ENABLED:
+      PREFECT_SERVER_SERVICES_EVENT_PERSISTER_ENABLED:
+      PREFECT_SERVER_SERVICES_TRIGGERS_ENABLED:
+      PREFECT_SERVER_SERVICES_TASK_RUN_RECORDER_ENABLED:
+
+      WEB_CONCURRENCY: ${INFRAHUB_TESTING_TASKMGR_API_WORKERS:-1}
     healthcheck:
       test: /usr/local/bin/httpx http://localhost:4200/api/health || exit 1
       interval: 5s
@@ -182,8 +202,42 @@ services:
     ports:
       - ${INFRAHUB_TESTING_TASK_MANAGER_PORT:-0}:4200
 
+  task-manager-background-svc:
+    deploy:
+      mode: replicated
+      replicas: ${INFRAHUB_TESTING_TASKMGR_BACKGROUND_SVC_REPLICAS:-0}
+    image: "${INFRAHUB_TESTING_DOCKER_IMAGE}:${INFRAHUB_TESTING_IMAGE_VERSION}"
+    command: prefect server services start
+    restart: unless-stopped
+    depends_on:
+      task-manager:
+        condition: service_healthy
+      task-manager-db:
+        condition: service_healthy
+      cache:
+        condition: service_healthy
+    environment:
+      PREFECT_UI_ENABLED: "${INFRAHUB_TESTING_PREFECT_UI_ENABLED}" # This might be required because triggers actions service depends on an in-memory API server
+      PREFECT_API_DATABASE_CONNECTION_URL: postgresql+asyncpg://postgres:postgres@task-manager-db:5432/prefect
+
+      PREFECT_API_DATABASE_MIGRATE_ON_START: "false"
+      PREFECT_API_BLOCKS_REGISTER_ON_START: "false"
+      INFRAHUB_CACHE_ADDRESS: ${INFRAHUB_TESTING_CACHE_ADDRESS}
+      PREFECT_MESSAGING_BROKER: prefect_redis.messaging
+      PREFECT_MESSAGING_CACHE: prefect_redis.messaging
+      PREFECT_REDIS_MESSAGING_HOST: "${INFRAHUB_TESTING_CACHE_ADDRESS:-cache}"
+      PREFECT_REDIS_MESSAGING_DB: "1"
+      PREFECT_REDIS_MESSAGING_CONSUMER_MIN_IDLE_TIME: "30"
+      PREFECT_REDIS_MESSAGING_PUBLISHER_BATCH_SIZE: "1"
+
+      PREFECT_SERVER_SERVICES_EVENT_LOGGER_ENABLED: "true"
+      PREFECT_SERVER_SERVICES_EVENT_PERSISTER_ENABLED: "true"
+      PREFECT_SERVER_SERVICES_TRIGGERS_ENABLED: "true"
+      PREFECT_SERVER_SERVICES_TASK_RUN_RECORDER_ENABLED: "true"
+
   task-manager-db:
     image: "${POSTGRES_DOCKER_IMAGE:-postgres:16-alpine}"
+    command: postgres -c 'max_connections=${INFRAHUB_TESTING_TASK_MANAGER_DB_MAX_CONNECTIONS:-100}'
     environment:
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=postgres
@@ -265,6 +319,7 @@ services:
       INFRAHUB_DB_PROTOCOL: ${INFRAHUB_TESTING_DB_PROTOCOL:-neo4j}
       INFRAHUB_WORKFLOW_ADDRESS: ${INFRAHUB_TESTING_WORKFLOW_ADDRESS}
       INFRAHUB_TIMEOUT: ${INFRAHUB_TESTING_TIMEOUT}
+      INFRAHUB_PAGINATION_SIZE: ${INFRAHUB_TESTING_PAGINATION_SIZE:-50}
       PREFECT_API_URL: ${INFRAHUB_TESTING_PREFECT_API}
       # Tracing
       INFRAHUB_TRACE_ENABLE: ${INFRAHUB_TRACE_ENABLE:-false}

infrahub_testcontainers/docker-compose.test.yml

@@ -77,13 +77,33 @@ services:
 
   task-manager:
     image: "${INFRAHUB_TESTING_DOCKER_IMAGE}:${INFRAHUB_TESTING_IMAGE_VERSION}"
-    command: uvicorn --host 0.0.0.0 --port 4200 --factory infrahub.prefect_server.app:create_infrahub_prefect
+    command: "${PREFECT_SERVER_COMMAND:-gunicorn -k uvicorn.workers.UvicornWorker -b 0.0.0.0:4200 'infrahub.prefect_server.app:create_infrahub_prefect()'}"
     depends_on:
       task-manager-db:
         condition: service_healthy
+      cache:
+        condition: service_healthy
     environment:
       PREFECT_UI_ENABLED: "${INFRAHUB_TESTING_PREFECT_UI_ENABLED}" # enabling UI requires permissions, run container as root to enable UI
       PREFECT_API_DATABASE_CONNECTION_URL: postgresql+asyncpg://postgres:postgres@task-manager-db:5432/prefect
+
+      INFRAHUB_CACHE_ADDRESS: ${INFRAHUB_TESTING_CACHE_ADDRESS}
+      PREFECT_REDIS_MESSAGING_HOST: "${INFRAHUB_TESTING_CACHE_ADDRESS:-cache}"
+      PREFECT_REDIS_MESSAGING_DB: "1"
+      PREFECT_REDIS_MESSAGING_CONSUMER_MIN_IDLE_TIME: "30"
+      PREFECT_REDIS_MESSAGING_PUBLISHER_BATCH_SIZE: "1"
+
+      PREFECT_MESSAGING_BROKER:
+      PREFECT_MESSAGING_CACHE:
+      PREFECT__SERVER_WEBSERVER_ONLY:
+      PREFECT_API_DATABASE_MIGRATE_ON_START:
+      PREFECT_API_BLOCKS_REGISTER_ON_START:
+      PREFECT_SERVER_SERVICES_EVENT_LOGGER_ENABLED:
+      PREFECT_SERVER_SERVICES_EVENT_PERSISTER_ENABLED:
+      PREFECT_SERVER_SERVICES_TRIGGERS_ENABLED:
+      PREFECT_SERVER_SERVICES_TASK_RUN_RECORDER_ENABLED:
+
+      WEB_CONCURRENCY: ${INFRAHUB_TESTING_TASKMGR_API_WORKERS:-1}
     healthcheck:
       test: /usr/local/bin/httpx http://localhost:4200/api/health || exit 1
       interval: 5s
@@ -93,8 +113,42 @@ services:
     ports:
       - ${INFRAHUB_TESTING_TASK_MANAGER_PORT:-0}:4200
 
+  task-manager-background-svc:
+    deploy:
+      mode: replicated
+      replicas: ${INFRAHUB_TESTING_TASKMGR_BACKGROUND_SVC_REPLICAS:-0}
+    image: "${INFRAHUB_TESTING_DOCKER_IMAGE}:${INFRAHUB_TESTING_IMAGE_VERSION}"
+    command: prefect server services start
+    restart: unless-stopped
+    depends_on:
+      task-manager:
+        condition: service_healthy
+      task-manager-db:
+        condition: service_healthy
+      cache:
+        condition: service_healthy
+    environment:
+      PREFECT_UI_ENABLED: "${INFRAHUB_TESTING_PREFECT_UI_ENABLED}" # This might be required because triggers actions service depends on an in-memory API server
+      PREFECT_API_DATABASE_CONNECTION_URL: postgresql+asyncpg://postgres:postgres@task-manager-db:5432/prefect
+
+      PREFECT_API_DATABASE_MIGRATE_ON_START: "false"
+      PREFECT_API_BLOCKS_REGISTER_ON_START: "false"
+      INFRAHUB_CACHE_ADDRESS: ${INFRAHUB_TESTING_CACHE_ADDRESS}
+      PREFECT_MESSAGING_BROKER: prefect_redis.messaging
+      PREFECT_MESSAGING_CACHE: prefect_redis.messaging
+      PREFECT_REDIS_MESSAGING_HOST: "${INFRAHUB_TESTING_CACHE_ADDRESS:-cache}"
+      PREFECT_REDIS_MESSAGING_DB: "1"
+      PREFECT_REDIS_MESSAGING_CONSUMER_MIN_IDLE_TIME: "30"
+      PREFECT_REDIS_MESSAGING_PUBLISHER_BATCH_SIZE: "1"
+
+      PREFECT_SERVER_SERVICES_EVENT_LOGGER_ENABLED: "true"
+      PREFECT_SERVER_SERVICES_EVENT_PERSISTER_ENABLED: "true"
+      PREFECT_SERVER_SERVICES_TRIGGERS_ENABLED: "true"
+      PREFECT_SERVER_SERVICES_TASK_RUN_RECORDER_ENABLED: "true"
+
   task-manager-db:
     image: "${POSTGRES_DOCKER_IMAGE:-postgres:16-alpine}"
+    command: postgres -c 'max_connections=${INFRAHUB_TESTING_TASK_MANAGER_DB_MAX_CONNECTIONS:-100}'
     environment:
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=postgres
@@ -172,6 +226,7 @@ services:
       INFRAHUB_DB_ADDRESS: ${INFRAHUB_TESTING_DB_ADDRESS:-database}
       INFRAHUB_WORKFLOW_ADDRESS: ${INFRAHUB_TESTING_WORKFLOW_ADDRESS}
       INFRAHUB_TIMEOUT: ${INFRAHUB_TESTING_TIMEOUT}
+      INFRAHUB_PAGINATION_SIZE: ${INFRAHUB_TESTING_PAGINATION_SIZE:-50}
       PREFECT_API_URL: ${INFRAHUB_TESTING_PREFECT_API}
       INFRAHUB_EXPERIMENTAL_VALUE_DB_INDEX:
       # Tracing

infrahub_testcontainers/helpers.py

@@ -29,11 +29,14 @@ class TestInfrahubDocker:
         return result.stdout
 
     @staticmethod
-    def execute_command(command: str, address: str, concurrent_execution: int = 10) -> subprocess.CompletedProcess[str]:
+    def execute_command(
+        command: str, address: str, concurrent_execution: int = 10, pagination_size: int = 50
+    ) -> subprocess.CompletedProcess[str]:
         env = os.environ.copy()
         env["INFRAHUB_ADDRESS"] = address
         env["INFRAHUB_API_TOKEN"] = PROJECT_ENV_VARIABLES["INFRAHUB_TESTING_INITIAL_ADMIN_TOKEN"]
         env["INFRAHUB_MAX_CONCURRENT_EXECUTION"] = f"{concurrent_execution}"
+        env["INFRAHUB_PAGINATION_SIZE"] = f"{pagination_size}"
         result = subprocess.run(  # noqa: S602
             command, shell=True, capture_output=True, text=True, env=env, check=False
         )