scriptgini 1.3.0__tar.gz → 1.4.0__tar.gz

{scriptgini-1.3.0 → scriptgini-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scriptgini
-Version: 1.3.0
+Version: 1.4.0
 Summary: Agentic AI system that converts functional test cases into automation test scripts.
 Author: ScriptGini Team
 License: Proprietary
@@ -16,7 +16,7 @@ Description-Content-Type: text/markdown
 
 > **Enterprise-grade Agentic AI system that converts functional test cases into high-quality, review-ready automation test scripts.**
 
-Current release: v1.3.0 (Sprint 3)
+Current release: v1.3.1 (Sprint 4 hardening increment)
 
 ---
 
@@ -441,9 +441,9 @@ The project follows an **enterprise-grade development roadmap** with 6 sprints c
 |--------|-------|--------|--------|
 | **Sprint 1** | IAM Core | 30-36pts | 🟡 Core delivered (auth hardening pending) |
 | **Sprint 2** | RBAC + Multi-Tenancy | 32-38pts | 🟡 Core delivered (RBAC hardening pending) |
-| **Sprint 3** | Durable Execution | 34-40pts | 🔲 Pending (Redis + Celery/Arq setup) |
-| **Sprint 4** | Security & Hardening | 30-36pts | 🔲 Pending (Container sandbox, audit logging) |
-| **Sprint 5** | Reporting & Analytics | 28-34pts | 🔲 Pending (Artifact storage, dashboards) |
+| **Sprint 3** | Durable Execution | 34-40pts | ✅ Completed (Redis + Celery queue foundation) |
+| **Sprint 4** | Security & Hardening | 30-36pts | 🟡 In progress (isolation boundary + breakout tests pending) |
+| **Sprint 5** | Reporting & Analytics | 28-34pts | ✅ Completed (Reports APIs, trends/flakiness, retention cleanup) |
 | **Sprint 6** | Advanced Features | 24-30pts | 🔲 Pending (Webhooks, defect sync, versioning) |
 
 ---

{scriptgini-1.3.0 → scriptgini-1.4.0}/README.md

@@ -2,7 +2,7 @@
 
 > **Enterprise-grade Agentic AI system that converts functional test cases into high-quality, review-ready automation test scripts.**
 
-Current release: v1.3.0 (Sprint 3)
+Current release: v1.3.1 (Sprint 4 hardening increment)
 
 ---
 
@@ -427,9 +427,9 @@ The project follows an **enterprise-grade development roadmap** with 6 sprints c
 |--------|-------|--------|--------|
 | **Sprint 1** | IAM Core | 30-36pts | 🟡 Core delivered (auth hardening pending) |
 | **Sprint 2** | RBAC + Multi-Tenancy | 32-38pts | 🟡 Core delivered (RBAC hardening pending) |
-| **Sprint 3** | Durable Execution | 34-40pts | 🔲 Pending (Redis + Celery/Arq setup) |
-| **Sprint 4** | Security & Hardening | 30-36pts | 🔲 Pending (Container sandbox, audit logging) |
-| **Sprint 5** | Reporting & Analytics | 28-34pts | 🔲 Pending (Artifact storage, dashboards) |
+| **Sprint 3** | Durable Execution | 34-40pts | ✅ Completed (Redis + Celery queue foundation) |
+| **Sprint 4** | Security & Hardening | 30-36pts | 🟡 In progress (isolation boundary + breakout tests pending) |
+| **Sprint 5** | Reporting & Analytics | 28-34pts | ✅ Completed (Reports APIs, trends/flakiness, retention cleanup) |
 | **Sprint 6** | Advanced Features | 24-30pts | 🔲 Pending (Webhooks, defect sync, versioning) |
 
 ---

scriptgini-1.4.0/app/__init__.py

@@ -0,0 +1,3 @@
+__version__ = "1.4.0"
+__api_version__ = "v1.4.0"
+

{scriptgini-1.3.0 → scriptgini-1.4.0}/app/celery_app.py

@@ -16,8 +16,8 @@ celery_app.conf.update(
     timezone="UTC",
     enable_utc=True,
     task_track_started=True,
-    task_time_limit=30 * 60,  # 30 minute hard time limit
-    task_soft_time_limit=25 * 60,  # 25 minute soft time limit
+    task_time_limit=settings.CELERY_TASK_HARD_TIMEOUT,
+    task_soft_time_limit=settings.CELERY_TASK_TIMEOUT,
     worker_prefetch_multiplier=4,
     worker_max_tasks_per_child=1000,
 )

{scriptgini-1.3.0 → scriptgini-1.4.0}/app/config.py

@@ -8,6 +8,11 @@ class Settings(BaseSettings):
     # App
     APP_NAME: str = "ScriptGini"
     DEBUG: bool = False
+    CORS_ALLOWED_ORIGINS: str = "http://localhost:3000,http://127.0.0.1:3000"
+    RATE_LIMIT_REQUESTS: int = 120
+    RATE_LIMIT_WINDOW_SECONDS: int = 60
+    RATE_LIMIT_EXEMPT_PATHS: str = "/health,/docs,/redoc,/openapi.json,/static"
+    SECURITY_AUDIT_LOG_ENABLED: bool = True
 
     # Database (PostgreSQL)
     # Format: postgresql://user:password@host:port/dbname
@@ -22,6 +27,7 @@ class Settings(BaseSettings):
     CELERY_BROKER_URL: str = "redis://localhost:6379/1"  # Different DB for task queue
     CELERY_RESULT_BACKEND: str = "redis://localhost:6379/2"  # Different DB for results
     CELERY_TASK_TIMEOUT: int = 600  # 10 minutes default task timeout
+    CELERY_TASK_HARD_TIMEOUT: int = 660
     CELERY_MAX_RETRIES: int = 3
     JWT_SECRET_KEY: str = "your-secret-key-change-in-production"
     JWT_ALGORITHM: str = "HS256"
@@ -34,6 +40,9 @@ class Settings(BaseSettings):
     SCRIPT_GENERATION_TIMEOUT_SECONDS: int = 180
     PLAYWRIGHT_RUN_HEADED: bool = True
     SCRIPT_EXECUTION_TIMEOUT_SECONDS: int = 300
+    ARTIFACT_RETENTION_DAYS: int = 30
+    EXECUTION_ENV_ALLOWED_KEYS: str = "PATH,SYSTEMROOT,TEMP,TMP,HOME,USERPROFILE,PYTHONPATH,PYTHONHOME,PYTHONIOENCODING"
+    EXECUTION_ENV_ALLOWED_PREFIXES: str = "PLAYWRIGHT_"
     SKIP_REVIEW_FOR_OLLAMA: bool = True
     USE_LLM_INTENT_ANALYSIS: bool = True
 
@@ -71,5 +80,25 @@ class Settings(BaseSettings):
     AWS_REGION_NAME: str = "us-east-1"
     BEDROCK_MODEL_ID: str = "anthropic.claude-3-5-sonnet-20241022-v2:0"
 
+    @staticmethod
+    def _split_csv(value: str) -> list[str]:
+        return [entry.strip() for entry in value.split(",") if entry.strip()]
+
+    @property
+    def cors_allowed_origins_list(self) -> list[str]:
+        return self._split_csv(self.CORS_ALLOWED_ORIGINS)
+
+    @property
+    def rate_limit_exempt_paths_list(self) -> list[str]:
+        return self._split_csv(self.RATE_LIMIT_EXEMPT_PATHS)
+
+    @property
+    def execution_env_allowed_keys_set(self) -> set[str]:
+        return set(self._split_csv(self.EXECUTION_ENV_ALLOWED_KEYS))
+
+    @property
+    def execution_env_allowed_prefixes_tuple(self) -> tuple[str, ...]:
+        return tuple(self._split_csv(self.EXECUTION_ENV_ALLOWED_PREFIXES))
+
 
 settings = Settings()

scriptgini-1.4.0/app/main.py

@@ -0,0 +1,192 @@
+import logging
+import threading
+import time
+from contextlib import asynccontextmanager
+from pathlib import Path
+from uuid import uuid4
+
+from fastapi import FastAPI, Request
+from fastapi.responses import JSONResponse
+from fastapi.responses import FileResponse
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.staticfiles import StaticFiles
+
+from app import __version__
+from app.config import settings
+from app.llm.provider import get_llm_diagnostics
+from app.routers import (
+    analytics,
+    api_key,
+    auth,
+    bulk_jobs,
+    demo,
+    execution,
+    organizations,
+    projects,
+    reports,
+    scripts,
+    test_cases,
+)
+
+logging.basicConfig(level=logging.DEBUG if settings.DEBUG else logging.INFO)
+logger = logging.getLogger(__name__)
+
+static_dir = Path(__file__).resolve().parent / "static"
+_RATE_LIMIT_BUCKETS: dict[str, tuple[int, float]] = {}
+_RATE_LIMIT_LOCK = threading.Lock()
+
+
+def _client_ip(request: Request) -> str:
+    forwarded_for = request.headers.get("x-forwarded-for", "")
+    if forwarded_for:
+        return forwarded_for.split(",")[0].strip()
+    return request.client.host if request.client else "unknown"
+
+
+def _is_exempt_path(path: str) -> bool:
+    for exempt in settings.rate_limit_exempt_paths_list:
+        if path == exempt or path.startswith(exempt):
+            return True
+    return False
+
+
+def _check_rate_limit(key: str, now: float) -> tuple[bool, int, int]:
+    window_seconds = max(1, settings.RATE_LIMIT_WINDOW_SECONDS)
+    max_requests = max(1, settings.RATE_LIMIT_REQUESTS)
+    window_start = now - window_seconds
+    with _RATE_LIMIT_LOCK:
+        current_count, reset_at = _RATE_LIMIT_BUCKETS.get(key, (0, now + window_seconds))
+        if reset_at <= now:
+            current_count = 0
+            reset_at = now + window_seconds
+        current_count += 1
+        _RATE_LIMIT_BUCKETS[key] = (current_count, reset_at)
+
+    remaining = max(0, max_requests - current_count)
+    retry_after = max(0, int(reset_at - now))
+    return current_count <= max_requests, remaining, retry_after
+
+
+def _security_audit(event: str, request: Request, status_code: int, request_id: str) -> None:
+    if not settings.SECURITY_AUDIT_LOG_ENABLED:
+        return
+    logger.info(
+        "security_audit event=%s request_id=%s method=%s path=%s status=%s ip=%s user_agent=%s",
+        event,
+        request_id,
+        request.method,
+        request.url.path,
+        status_code,
+        _client_ip(request),
+        request.headers.get("user-agent", "unknown"),
+    )
+
+
+@asynccontextmanager
+async def lifespan(_: FastAPI):
+    diagnostics = get_llm_diagnostics()
+    logger.info(
+        "Runtime LLM default: provider=%s model=%s api_key_env=%s api_key_present=%s api_key=%s",
+        diagnostics["provider"],
+        diagnostics["model"],
+        diagnostics["api_key_env"],
+        diagnostics["api_key_present"],
+        diagnostics["api_key_masked"],
+    )
+    yield
+
+app = FastAPI(
+    title=settings.APP_NAME,
+    description=(
+        "Enterprise-grade Agentic AI system that converts functional test cases "
+        "into high-quality automation scripts."
+    ),
+    version=__version__,
+    lifespan=lifespan,
+)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=settings.cors_allowed_origins_list,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.middleware("http")
+async def apply_security_controls(request: Request, call_next):
+    request_id = request.headers.get("X-Request-ID") or str(uuid4())
+    request.state.request_id = request_id
+    path = request.url.path
+    now = time.time()
+
+    if not _is_exempt_path(path):
+        key = f"{_client_ip(request)}:{path}"
+        allowed, remaining, retry_after = _check_rate_limit(key, now)
+        if not allowed:
+            _security_audit("rate_limit_block", request, 429, request_id)
+            return JSONResponse(
+                status_code=429,
+                content={"detail": "Rate limit exceeded", "request_id": request_id},
+                headers={
+                    "X-Request-ID": request_id,
+                    "X-RateLimit-Limit": str(max(1, settings.RATE_LIMIT_REQUESTS)),
+                    "X-RateLimit-Remaining": "0",
+                    "Retry-After": str(retry_after),
+                },
+            )
+
+    response = await call_next(request)
+    response.headers["X-Request-ID"] = request_id
+
+    if not _is_exempt_path(path):
+        response.headers["X-RateLimit-Limit"] = str(max(1, settings.RATE_LIMIT_REQUESTS))
+        response.headers["X-RateLimit-Remaining"] = str(response.headers.get("X-RateLimit-Remaining", ""))
+        if not response.headers["X-RateLimit-Remaining"]:
+            key = f"{_client_ip(request)}:{path}"
+            with _RATE_LIMIT_LOCK:
+                count, _ = _RATE_LIMIT_BUCKETS.get(key, (0, now))
+            response.headers["X-RateLimit-Remaining"] = str(max(0, max(1, settings.RATE_LIMIT_REQUESTS) - count))
+
+    if path.startswith("/api/v1/auth") or path.startswith("/api/v1/execution"):
+        if response.status_code >= 400 or request.method in {"POST", "PUT", "PATCH", "DELETE"}:
+            _security_audit("security_sensitive_request", request, response.status_code, request_id)
+
+    return response
+
+app.include_router(projects.router, prefix="/api/v1")
+app.include_router(test_cases.router, prefix="/api/v1")
+app.include_router(scripts.router, prefix="/api/v1")
+app.include_router(bulk_jobs.router, prefix="/api/v1")
+app.include_router(analytics.router, prefix="/api/v1")
+app.include_router(analytics.insights_router, prefix="/api/v1")
+app.include_router(demo.router, prefix="/api/v1")
+app.include_router(auth.router, prefix="/api/v1")
+app.include_router(api_key.router, prefix="/api/v1")
+app.include_router(organizations.router, prefix="/api/v1")
+app.include_router(execution.router, prefix="/api/v1")
+app.include_router(reports.router, prefix="/api/v1")
+app.mount("/static", StaticFiles(directory=static_dir), name="static")
+
+
+@app.get("/api/v1/runtime/llm", tags=["Runtime"])
+def runtime_llm():
+    default_diagnostics = get_llm_diagnostics()
+    return {
+        "default_provider": default_diagnostics["provider"],
+        "default_model": default_diagnostics["model"],
+        "provider_diagnostics": {
+            provider: get_llm_diagnostics(provider) for provider in ["openai", "openrouter", "gemini", "ollama", "bedrock"]
+        },
+    }
+
+
+@app.get("/", include_in_schema=False)
+def index():
+    return FileResponse(static_dir / "index.html")
+
+
+@app.get("/health", tags=["Health"])
+def health():
+    return {"status": "ok", "app": settings.APP_NAME}

scriptgini-1.4.0/app/routers/analytics.py

@@ -0,0 +1,198 @@
+from datetime import date, datetime, time, timezone
+
+from sqlalchemy import case, func
+from sqlalchemy.orm import Session
+from fastapi import APIRouter, Depends, HTTPException
+
+from app.database import get_db
+from app.models.project import Project
+from app.models.script_run import ScriptRun, ScriptRunStatus
+from app.models.test_case import TestCase
+from app.schemas.analytics import (
+    FlakinessItemResponse,
+    FlakinessResponse,
+    RecentFailureResponse,
+    RunAnalyticsResponse,
+    TrendPointResponse,
+    TrendsResponse,
+)
+
+router = APIRouter(prefix="/projects/{project_id}/analytics", tags=["Run Analytics"])
+insights_router = APIRouter(prefix="/analytics", tags=["Run Analytics"])
+
+
+@router.get("/runs", response_model=RunAnalyticsResponse)
+def get_run_analytics(project_id: int, db: Session = Depends(get_db)):
+    project = db.query(Project).filter(Project.id == project_id).first()
+    if not project:
+        raise HTTPException(status_code=404, detail="Project not found")
+
+    aggregate = (
+        db.query(
+            func.count(ScriptRun.id),
+            func.sum(case((ScriptRun.success.is_(True), 1), else_=0)),
+            func.sum(case((ScriptRun.status == ScriptRunStatus.failed, 1), else_=0)),
+            func.sum(case((ScriptRun.status == ScriptRunStatus.timed_out, 1), else_=0)),
+            func.avg(ScriptRun.duration_seconds),
+        )
+        .filter(ScriptRun.project_id == project_id)
+        .one()
+    )
+
+    total_runs = int(aggregate[0] or 0)
+    success_runs = int(aggregate[1] or 0)
+    failed_runs = int(aggregate[2] or 0)
+    timed_out_runs = int(aggregate[3] or 0)
+    avg_duration = float(aggregate[4] or 0.0)
+
+    failed_items = (
+        db.query(ScriptRun, TestCase.title)
+        .outerjoin(TestCase, TestCase.id == ScriptRun.test_case_id)
+        .filter(ScriptRun.project_id == project_id)
+        .filter(ScriptRun.success.is_(False))
+        .order_by(ScriptRun.created_at.desc(), ScriptRun.id.desc())
+        .limit(10)
+        .all()
+    )
+
+    recent_failures = [
+        RecentFailureResponse(
+            run_id=run.id,
+            script_id=run.script_id,
+            test_case_id=run.test_case_id,
+            test_case_title=tc_title,
+            exit_code=run.exit_code,
+            duration_seconds=run.duration_seconds,
+            stderr_excerpt=(run.stderr or "")[:240],
+            created_at=run.created_at,
+        )
+        for run, tc_title in failed_items
+    ]
+
+    success_rate = round((success_runs / total_runs) * 100, 2) if total_runs else 0.0
+
+    return RunAnalyticsResponse(
+        project_id=project_id,
+        total_runs=total_runs,
+        success_runs=success_runs,
+        failed_runs=failed_runs,
+        timed_out_runs=timed_out_runs,
+        success_rate=success_rate,
+        average_duration_seconds=round(avg_duration, 2),
+        recent_failures=recent_failures,
+    )
+
+
+def _resolve_datetime_range(start_date: date | None, end_date: date | None) -> tuple[datetime | None, datetime | None]:
+    start_dt = datetime.combine(start_date, time.min, tzinfo=timezone.utc) if start_date else None
+    end_dt = datetime.combine(end_date, time.max, tzinfo=timezone.utc) if end_date else None
+    return start_dt, end_dt
+
+
+@insights_router.get("/trends", response_model=TrendsResponse)
+def get_trends(
+    project_id: int | None = None,
+    start_date: date | None = None,
+    end_date: date | None = None,
+    db: Session = Depends(get_db),
+):
+    if project_id is not None:
+        project = db.query(Project).filter(Project.id == project_id).first()
+        if not project:
+            raise HTTPException(status_code=404, detail="Project not found")
+
+    start_dt, end_dt = _resolve_datetime_range(start_date, end_date)
+    bucket_expr = func.date(ScriptRun.created_at)
+    query = (
+        db.query(
+            bucket_expr.label("bucket"),
+            func.count(ScriptRun.id).label("total_runs"),
+            func.sum(case((ScriptRun.success.is_(True), 1), else_=0)).label("success_runs"),
+            func.avg(ScriptRun.duration_seconds).label("avg_duration"),
+        )
+        .group_by(bucket_expr)
+        .order_by(bucket_expr.asc())
+    )
+
+    if project_id is not None:
+        query = query.filter(ScriptRun.project_id == project_id)
+    if start_dt is not None:
+        query = query.filter(ScriptRun.created_at >= start_dt)
+    if end_dt is not None:
+        query = query.filter(ScriptRun.created_at <= end_dt)
+
+    points: list[TrendPointResponse] = []
+    for row in query.all():
+        total_runs = int(row.total_runs or 0)
+        success_runs = int(row.success_runs or 0)
+        success_rate = round((success_runs / total_runs) * 100, 2) if total_runs else 0.0
+        points.append(
+            TrendPointResponse(
+                bucket=str(row.bucket),
+                total_runs=total_runs,
+                success_rate=success_rate,
+                average_duration_seconds=round(float(row.avg_duration or 0.0), 2),
+            )
+        )
+
+    return TrendsResponse(project_id=project_id, start_date=start_dt, end_date=end_dt, points=points)
+
+
+@insights_router.get("/flakiness", response_model=FlakinessResponse)
+def get_flakiness(
+    project_id: int | None = None,
+    start_date: date | None = None,
+    end_date: date | None = None,
+    min_runs: int = 3,
+    db: Session = Depends(get_db),
+):
+    if project_id is not None:
+        project = db.query(Project).filter(Project.id == project_id).first()
+        if not project:
+            raise HTTPException(status_code=404, detail="Project not found")
+
+    min_runs = max(1, min_runs)
+    start_dt, end_dt = _resolve_datetime_range(start_date, end_date)
+
+    query = db.query(
+        ScriptRun.script_id.label("script_id"),
+        ScriptRun.test_case_id.label("test_case_id"),
+        func.count(ScriptRun.id).label("total_runs"),
+        func.sum(case((ScriptRun.success.is_(False), 1), else_=0)).label("failed_runs"),
+        func.max(case((ScriptRun.success.is_(False), ScriptRun.created_at), else_=None)).label("last_failure_at"),
+    ).group_by(ScriptRun.script_id, ScriptRun.test_case_id)
+
+    if project_id is not None:
+        query = query.filter(ScriptRun.project_id == project_id)
+    if start_dt is not None:
+        query = query.filter(ScriptRun.created_at >= start_dt)
+    if end_dt is not None:
+        query = query.filter(ScriptRun.created_at <= end_dt)
+
+    rows = query.having(func.count(ScriptRun.id) >= min_runs).all()
+    items: list[FlakinessItemResponse] = []
+    for row in rows:
+        total_runs = int(row.total_runs or 0)
+        failed_runs = int(row.failed_runs or 0)
+        flakiness_score = round((failed_runs / total_runs) * 100, 2) if total_runs else 0.0
+        confidence_score = round(min(1.0, total_runs / 10.0) * 100, 2)
+        items.append(
+            FlakinessItemResponse(
+                script_id=int(row.script_id),
+                test_case_id=int(row.test_case_id),
+                total_runs=total_runs,
+                failed_runs=failed_runs,
+                flakiness_score=flakiness_score,
+                confidence_score=confidence_score,
+                last_failure_at=row.last_failure_at,
+            )
+        )
+
+    items.sort(key=lambda item: (-item.flakiness_score, -item.confidence_score, -item.total_runs))
+    return FlakinessResponse(
+        project_id=project_id,
+        start_date=start_dt,
+        end_date=end_dt,
+        min_runs=min_runs,
+        items=items,
+    )

{scriptgini-1.3.0 → scriptgini-1.4.0}/app/routers/bulk_jobs.py

@@ -2,7 +2,7 @@ import logging
 import subprocess
 import sys
 
-from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, status
+from fastapi import APIRouter, Depends, HTTPException, status
 from sqlalchemy.orm import Session
 
 from app.database import get_db
@@ -13,6 +13,7 @@ from app.models.project import Project
 from app.models.script_run import ScriptRunStatus
 from app.models.test_case import TestCase
 from app.schemas.bulk_job import BulkGenerateRequest, BulkJobResponse, BulkRunRequest
+from app.tasks import process_bulk_execution_job, process_bulk_generation_job
 from app.routers.scripts import (
     _create_script_run,
     _get_project_or_404,
@@ -218,7 +219,6 @@ def _run_bulk_execution(job_id: int, request: BulkRunRequest):
 def bulk_generate_scripts(
     project_id: int,
     payload: BulkGenerateRequest,
-    background_tasks: BackgroundTasks,
     db: Session = Depends(get_db),
 ):
     _get_project_or_404(project_id, db)
@@ -238,7 +238,14 @@ def bulk_generate_scripts(
     _refresh_job_counts(job, db)
     db.commit()
 
-    background_tasks.add_task(_run_bulk_generation, job.id, payload)
+    try:
+        process_bulk_generation_job.delay(job.id, payload.model_dump())
+    except Exception as exc:
+        logger.exception("Failed to enqueue bulk generation job_id=%s", job.id)
+        job.status = BulkJobStatus.failed
+        db.commit()
+        raise HTTPException(status_code=503, detail="Failed to enqueue bulk generation") from exc
+
     return _serialize_bulk_job(job, db)
 
 
@@ -246,7 +253,6 @@ def bulk_generate_scripts(
 def bulk_run_scripts(
     project_id: int,
     payload: BulkRunRequest,
-    background_tasks: BackgroundTasks,
     db: Session = Depends(get_db),
 ):
     _get_project_or_404(project_id, db)
@@ -266,7 +272,14 @@ def bulk_run_scripts(
     _refresh_job_counts(job, db)
     db.commit()
 
-    background_tasks.add_task(_run_bulk_execution, job.id, payload)
+    try:
+        process_bulk_execution_job.delay(job.id, payload.model_dump())
+    except Exception as exc:
+        logger.exception("Failed to enqueue bulk run job_id=%s", job.id)
+        job.status = BulkJobStatus.failed
+        db.commit()
+        raise HTTPException(status_code=503, detail="Failed to enqueue bulk run") from exc
+
     return _serialize_bulk_job(job, db)