control-tower 6.0.0__py3-none-any.whl

control_tower/__init__.py

@@ -0,0 +1,3 @@
+from importlib.metadata import version
+
+__version__ = version("control-tower")

control_tower/backends/__init__.py

@@ -0,0 +1,31 @@
+"""Where an eval runs: in this process, on the EC2 fleet, or on Hawk."""
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from inspect_ai.log import EvalLog
+
+    from control_tower.eval.launch.options import EvalCliInputs
+
+
+class Backend(ABC):
+    """Where `run_task` runs an eval, given its completed task args."""
+
+    def resolvable(self, name: str) -> bool:
+        """Whether the task set, policy, protocol or monitor ``name`` is looked up before the run; the rest are looked
+        up where the task is built."""
+        return True
+
+    @abstractmethod
+    def run(
+        self,
+        task_name: str,
+        task_args: dict[str, Any],
+        eval_inputs: "EvalCliInputs",
+        *,
+        upload: bool,
+        metadata: dict[str, Any],
+    ) -> "EvalLog | None":
+        """Run ``task_name`` with ``task_args``, ``metadata`` joining its eval metadata, and upload the log when
+        ``upload``; return it, or None when it stays where it ran."""

control_tower/backends/fleet/__init__.py

@@ -0,0 +1 @@
+"""Fleet mode: distributed evaluation on EC2 workers."""

control_tower/backends/fleet/ami.py

@@ -0,0 +1,45 @@
+"""Worker AMI resolution for fleet mode: the pre-baked AMI named in SSM ``/linuxbench/worker-ami-id``."""
+
+import boto3
+import botocore.exceptions
+
+SSM_AMI_PARAM = "/linuxbench/worker-ami-id"
+
+
+def get_worker_ami_id(region: str = "us-east-1", requested: str | None = None) -> str:
+    """Return the AMI to use for fleet workers, refusing one that cannot boot now.
+
+    ``requested`` is an AMI id, or the SSM parameter naming one; the default is ``SSM_AMI_PARAM``.
+    """
+    source = requested or SSM_AMI_PARAM
+    ami_id = source
+    if source.startswith("/"):
+        ssm = boto3.client("ssm", region_name=region)
+        try:
+            ami_id = ssm.get_parameter(Name=source)["Parameter"]["Value"]
+        except ssm.exceptions.ParameterNotFound as e:
+            raise RuntimeError(
+                f"No worker AMI: SSM parameter {source} does not exist in {region}. "
+                f"Build one with scripts/build_worker_ami.sh --region {region}."
+            ) from e
+    label = ami_id if ami_id == source else f"{ami_id} (from {source})"
+    ec2 = boto3.client("ec2", region_name=region)
+    try:
+        images = ec2.describe_images(ImageIds=[ami_id])["Images"]
+    except botocore.exceptions.ClientError as e:
+        if not e.response["Error"]["Code"].startswith("InvalidAMIID"):
+            raise
+        raise RuntimeError(
+            f"Worker AMI {label} not found in {region}: "
+            f"{e.response['Error']['Message']}"
+        ) from e
+    if not images:
+        raise RuntimeError(f"Worker AMI {label} not found in {region}.")
+    image = images[0]
+    if image["State"] != "available":
+        reason = image.get("StateReason", {}).get("Message", "")
+        raise RuntimeError(
+            f"Worker AMI {label} in {region} is {image['State']}"
+            f"{f' ({reason})' if reason else ''}, not available; no workers launched."
+        )
+    return ami_id

control_tower/backends/fleet/aws_identity.py

@@ -0,0 +1,9 @@
+"""AWS caller identity lookup for resource tagging."""
+
+import boto3
+
+
+def caller_identity_name(region: str) -> str:
+    """Get the caller's identity name (IAM username or SSO session name) for resource tagging."""
+    sts = boto3.client("sts", region_name=region)
+    return sts.get_caller_identity()["Arn"].rsplit("/", 1)[-1]

control_tower/backends/fleet/bootstrap.sh

@@ -0,0 +1,349 @@
+#!/bin/bash
+# Fleet worker bootstrap script — runs as EC2 user-data (root).
+#
+# Template variables (replaced by controller at launch time):
+#   {s3_bucket}       — S3 bucket for fleet results and code artifact
+#   {s3_prefix}       — S3 prefix for this fleet run (fleet_id/)
+#   {code_s3_key}     — S3 key of the code tarball
+#   {rates_s3_key}    — S3 key of the driver's rate table (empty when pricing is off)
+#   {worker_setup_s3_key} — S3 key of the run's worker setup script (empty when none)
+#   {region}          — AWS region
+#   {work_queue_url}  — SQS work queue URL
+#   {done_queue_url}  — SQS done queue URL
+#   {concurrency}     — Concurrent eval tasks per worker
+#   {idle_timeout}    — Seconds idle before worker self-terminates
+#   {fleet_secrets_prefix} — SSM path holding this fleet's per-instance
+#                            SecureStrings (JSON of API keys, GH_TOKEN, Docker
+#                            auth, env-required vars), one per instance id
+#   {setting_archives} — space-separated `<s3 key>=<dir>` pairs: the controller's
+#                        setting checkouts, one archive of each repo's files and
+#                        one of its shallow .git, each extracted into
+#                        /opt/control-tower/<dir>
+#   {env_secret_packages} — space-separated setting packages whose `<setting>.env` text
+#                           sits in "<instance secrets param>.env.<package>"
+#   {docker_org_name_ssm_param} — shared SSM param for Docker org name
+#   {docker_org_pat_ssm_param}  — shared SSM param for Docker PAT
+#
+# What the pre-baked AMI provides (see scripts/build_worker_ami.sh):
+#   - Docker + compose plugin, with the address pool below already in daemon.json
+#   - NVIDIA driver + container toolkit, which a GPU worker needs to pass its GPU check
+#   - uv installed for both ubuntu and root users
+#   - Pre-pulled Docker images for all environments and shared services
+#   - Python wheel cache at /opt/uv-cache
+
+set -euo pipefail
+export HOME=/root
+exec > /var/log/lb-worker-bootstrap.log 2>&1
+
+# Upload bootstrap log to S3 on exit (success or failure) for debugging
+IMDS_TOKEN=$(curl -sX PUT http://169.254.169.254/latest/api/token -H "X-aws-ec2-metadata-token-ttl-seconds: 21600" || true)
+INSTANCE_ID=$(curl -s -H "X-aws-ec2-metadata-token: $IMDS_TOKEN" http://169.254.169.254/latest/meta-data/instance-id || echo "unknown")
+on_exit() {{
+    local rc=$?
+    aws s3 cp /var/log/lb-worker-bootstrap.log \
+        's3://{s3_bucket}/{s3_prefix}logs/'$INSTANCE_ID'-bootstrap.log' \
+        --region {region} || true
+    if [ "$rc" -ne 0 ]; then
+        detail=$(tail -n 5 /var/log/lb-worker-bootstrap.log | python3 -c 'import json,sys; print(json.dumps(sys.stdin.read()))')
+        aws sqs send-message --queue-url '{done_queue_url}' --region {region} \
+            --message-body '{{"type": "bootstrap_failed", "instance_id": "'"$INSTANCE_ID"'", "detail": '"$detail"'}}' || true
+        shutdown -h now
+    fi
+}}
+trap on_exit EXIT
+
+SECONDS=0
+echo "=== Control Tower Fleet Worker Bootstrap ==="
+echo "Instance: $INSTANCE_ID"
+echo "Started at $(date)"
+
+resolve_ssm_param() {{
+    local param_name="$1"
+    local with_decryption="${{2:-false}}"
+    if [ -z "$param_name" ]; then
+        return 0
+    fi
+    if [ "$with_decryption" = "true" ]; then
+        aws ssm get-parameter \
+            --name "$param_name" \
+            --with-decryption \
+            --query 'Parameter.Value' \
+            --output text \
+            --region {region} || true
+    else
+        aws ssm get-parameter \
+            --name "$param_name" \
+            --query 'Parameter.Value' \
+            --output text \
+            --region {region} || true
+    fi
+}}
+
+configure_docker_auth() {{
+    local org_name="${{DOCKERHUB_USERNAME:-}}"
+    local org_pat="${{DOCKERHUB_TOKEN:-}}"
+    local org_name_param='{docker_org_name_ssm_param}'
+    local org_pat_param='{docker_org_pat_ssm_param}'
+
+    if [ -z "$org_name" ]; then
+        org_name=$(resolve_ssm_param "$org_name_param")
+    fi
+    if [ -z "$org_pat" ]; then
+        org_pat=$(resolve_ssm_param "$org_pat_param" true)
+    fi
+
+    if [ -z "$org_name" ] && [ -z "$org_pat" ]; then
+        echo "FATAL: No Docker registry auth configured (need DOCKERHUB_USERNAME/DOCKERHUB_TOKEN in the launching shell or the shared SSM params)"
+        exit 1
+    fi
+    if [ -z "$org_name" ] || [ -z "$org_pat" ]; then
+        echo "FATAL: Incomplete Docker registry auth configuration"
+        exit 1
+    fi
+
+    if ! printf '%s\n' "$org_pat" | docker login -u "$org_name" --password-stdin >/dev/null; then
+        echo "FATAL: Docker registry login failed for $org_name"
+        exit 1
+    fi
+    unset org_pat
+    echo "  [+${{SECONDS}}s] Docker registry login configured for $org_name"
+}}
+
+# --- inotify limit ------------------------------------------------------
+# The kernel's 128 inotify instances per user are shared by every container; file-watching dev servers exhaust them at fleet concurrency.
+sysctl -w fs.inotify.max_user_instances=8192 >/dev/null
+echo "  [+${{SECONDS}}s] inotify max_user_instances=$(cat /proc/sys/fs/inotify/max_user_instances)"
+
+# --- Docker daemon config: the AMI's address pool -----------------------
+# Docker's ~30-network default pool exhausts at concurrency; 10.224/11 as /24s gives 8192 networks clear of the VPC's 172.31/16.
+if ! python3 -c '
+import json, sys
+config = json.load(open("/etc/docker/daemon.json"))
+sys.exit(config.get("default-address-pools") != [{{"base": "10.224.0.0/11", "size": 24}}])
+'; then
+    echo "FATAL: /etc/docker/daemon.json lacks the 10.224.0.0/11 /24 default address pool. Fleet workers boot only from the worker AMI built by scripts/build_worker_ami.sh, which bakes it."
+    exit 1
+fi
+
+
+# --- Docker -----------------------------------------------------------
+docker version --format json >/dev/null 2>&1 || {{ echo "FATAL: Docker is not running. Fleet workers boot only from the worker AMI built by scripts/build_worker_ami.sh."; exit 1; }}
+echo "  [+${{SECONDS}}s] Docker ready"
+
+# --- Secrets (per-instance SSM SecureString; user-data carries the path only) ---
+# The controller writes this parameter right after RunInstances and tags it
+# with this instance's ARN; the instance role reads only parameters tagged
+# with its own ARN, so the fetch is retried until the write and tag land.
+if [ "$INSTANCE_ID" = unknown ]; then
+    echo "FATAL: IMDS did not return an instance id, so this instance's secrets parameter cannot be located"
+    exit 1
+fi
+FLEET_SECRETS_PARAM='{fleet_secrets_prefix}'/"$INSTANCE_ID"
+SECRETS_DEADLINE=$((SECONDS + 900))
+SECRETS_NEXT_LOG=$SECONDS
+until secrets=$(AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws ssm get-parameter \
+        --name "$FLEET_SECRETS_PARAM" --with-decryption \
+        --query 'Parameter.Value' --output text --region {region} 2>/tmp/secrets-fetch.err); do
+    if [ "$SECONDS" -ge "$SECRETS_DEADLINE" ]; then
+        echo "FATAL: cannot read $FLEET_SECRETS_PARAM after 15 minutes: $(cat /tmp/secrets-fetch.err)"
+        echo "  (does the instance role allow ssm:GetParameter where the InstanceArn tag is this instance, and did the controller tag the parameter?)"
+        exit 1
+    fi
+    if [ "$SECONDS" -ge "$SECRETS_NEXT_LOG" ]; then
+        echo "  [+${{SECONDS}}s] waiting for $FLEET_SECRETS_PARAM: $(tail -n 1 /tmp/secrets-fetch.err)"
+        SECRETS_NEXT_LOG=$((SECONDS + 60))
+    fi
+    sleep 5
+done
+exports=$(printf '%s' "$secrets" | python3 -c '
+import json, shlex, sys
+for name, value in json.load(sys.stdin).items():
+    print("export " + name + "=" + shlex.quote(value))
+')
+eval "$exports"
+unset secrets exports
+echo "  [+${{SECONDS}}s] Secrets fetched from SSM"
+
+# --- Docker registry auth (early — fail fast before expensive steps) ----
+configure_docker_auth
+unset DOCKERHUB_TOKEN DOCKERHUB_USERNAME
+
+PCI_VENDORS=$(cat /sys/bus/pci/devices/*/vendor)
+if grep -qx 0x10de <<< "$PCI_VENDORS"; then
+    docker run --rm --gpus all ubuntu:22.04 nvidia-smi -L
+fi
+
+export PATH="/root/.local/bin:$PATH"
+
+# --- Code download ----------------------------------------------------
+rm -rf /opt/control-tower && mkdir -p /opt/control-tower
+cd /opt/control-tower
+aws s3 cp 's3://{s3_bucket}/{code_s3_key}' code.tar.gz --region {region}
+tar xzf code.tar.gz
+rm code.tar.gz
+echo "  [+${{SECONDS}}s] Code downloaded and extracted"
+
+# --- SSH known hosts (needed for git+ssh deps in uv sync) ----------------
+mkdir -p ~/.ssh
+ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null
+chmod 600 ~/.ssh/known_hosts
+echo "  [+${{SECONDS}}s] GitHub SSH host key added to known_hosts"
+
+# --- GitHub credentials (needed BEFORE uv sync for private git+https deps) ---
+# uv shells out to plain git, which ignores GH_TOKEN — only gh reads it
+# natively. A credential helper scoped to https://github.com supplies it
+# when git asks; deliberately NOT a url.insteadOf rewrite, which would
+# mutate every GitHub URL on the worker.
+git config --global credential.'https://github.com'.helper \
+    '!f() {{ echo "username=x-access-token"; echo "password=${{GH_TOKEN}}"; }}; f'
+echo "  [+${{SECONDS}}s] GitHub HTTPS credential helper configured"
+
+# --- Python deps -------------------------------------------------------
+export UV_CACHE_DIR=/opt/uv-cache
+export UV_PYTHON_INSTALL_DIR=/opt/uv-python
+# git archive strips .git, so setuptools-scm can't detect version. Fake it.
+export SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0
+if [ -f uv.lock ]; then
+    uv sync --frozen
+else
+    echo "  No uv.lock in code artifact; resolving dependencies from pyproject.toml"
+    uv sync
+fi
+echo "  [+${{SECONDS}}s] Python deps installed"
+# Defeat AMI shadowing: a pre-baked control_tower elsewhere in the worker
+# Python's sys.path can win priority over the bundle's editable install,
+# so the worker silently runs OLDER code even though uv sync reports success.
+# (Verified via sentinel string round-trip: stale install wins by default.)
+# Three defense layers:
+#   1. Re-install editable into the venv explicitly.
+#   2. Purge any other control_tower sitting in the venv's site-packages.
+#   3. Prepend /opt/control-tower/src to PYTHONPATH so the bundle is the FIRST
+#      sys.path entry no matter what.
+uv pip install --no-deps -e .
+VENV_SITE=$(uv run python -c "import sysconfig; print(sysconfig.get_paths()['purelib'])")
+echo "  [+${{SECONDS}}s] venv site-packages = $VENV_SITE"
+# Remove any non-editable control_tower copy — but only when the bundle
+# ships its own (out-of-tree consumers rely on the pip-installed one).
+if [ -d /opt/control-tower/src/control_tower ] && [ -d "$VENV_SITE/control_tower" ] && [ ! -L "$VENV_SITE/control_tower" ]; then
+    echo "  [+${{SECONDS}}s] Removing non-editable control_tower from $VENV_SITE"
+    rm -rf "$VENV_SITE/control_tower"
+fi
+# On PYTHONPATH, src/control_tower.egg-info shadows the venv's dist-info, and Inspect then registers tasks, scorers and metrics without the control_tower/ prefix.
+rm -rf /opt/control-tower/src/control_tower.egg-info
+export PYTHONPATH="/opt/control-tower/src${{PYTHONPATH:+:$PYTHONPATH}}"
+echo "  [+${{SECONDS}}s] PYTHONPATH=$PYTHONPATH"
+# Verify resolution
+RESOLVED=$(uv run python -c "import control_tower; print(control_tower.__file__)")
+echo "  [+${{SECONDS}}s] control_tower resolves to: $RESOLVED"
+
+export CT_FLEET_WORKER_USER='{worker_user}'
+if [ "$CT_FLEET_WORKER_USER" != root ]; then
+    worker_home="$(getent passwd "$CT_FLEET_WORKER_USER" | cut -d: -f6)"
+    install -d -o "$CT_FLEET_WORKER_USER" -m 700 "$worker_home/.docker"
+    install -o "$CT_FLEET_WORKER_USER" -m 600 /root/.docker/config.json "$worker_home/.docker/config.json"
+    usermod -aG docker "$CT_FLEET_WORKER_USER"
+    runuser -u "$CT_FLEET_WORKER_USER" -- env HOME="$worker_home" docker info >/dev/null || {{ echo "FATAL: $CT_FLEET_WORKER_USER cannot reach the Docker daemon"; exit 1; }}
+fi
+
+# --- Settings: the controller's checkouts of each shipped repo ---
+SETTING_ARCHIVES='{setting_archives}'
+for archive in $SETTING_ARCHIVES; do
+    mkdir -p "/opt/control-tower/${{archive#*=}}"
+    aws s3 cp "s3://{s3_bucket}/${{archive%%=*}}" - --region {region} \
+        | tar xz -C "/opt/control-tower/${{archive#*=}}"
+done
+echo "  [+${{SECONDS}}s] Settings extracted: $(ls .settings 2>/dev/null | tr '\n' ' ')"
+
+# --- Setting values: each shipped setting's `<setting>.env`, beside its checkout as on the controller ---
+ENV_SECRET_PACKAGES='{env_secret_packages}'
+for package in $ENV_SECRET_PACKAGES; do
+    attempt=0
+    until package_values=$(AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws ssm get-parameter \
+            --name "$FLEET_SECRETS_PARAM.env.$package" --with-decryption \
+            --query 'Parameter.Value' --output text --region {region} 2>/tmp/secrets-fetch.err); do
+        attempt=$((attempt + 1))
+        if [ "$attempt" -ge 5 ]; then
+            echo "FATAL: cannot read $FLEET_SECRETS_PARAM.env.$package: $(cat /tmp/secrets-fetch.err)"
+            exit 1
+        fi
+        sleep $((attempt * 5))
+    done
+    install -o "$CT_FLEET_WORKER_USER" -m 600 /dev/null ".settings/$package.env"
+    printf '%s\n' "$package_values" > ".settings/$package.env"
+done
+unset package_values
+
+# --- Setting package venvs: the env processes CT spawns run from them ---
+for package_dir in .settings/*/; do
+    [ -f "$package_dir/pyproject.toml" ] || continue
+    sync_flags=()
+    [ -f "$package_dir/uv.lock" ] && sync_flags=(--frozen)
+    if ! uv sync --project "$package_dir" --no-dev "${{sync_flags[@]}}" 2>&1; then
+        echo "  [+${{SECONDS}}s] FATAL: uv sync failed for setting package $package_dir."
+        echo "  Workers cannot start its env process without its venv; check the"
+        echo "  package's pyproject.toml and uv.lock as shipped."
+        exit 1
+    fi
+    echo "  [+${{SECONDS}}s] Setting package venv ready: $package_dir"
+done
+
+# --- Setting plugins: each shipped setting's ct_plugin/ joins CT's venv, as `ct settings pull` does ---
+for plugin_dir in .settings/*/ct_plugin/; do
+    [ -f "$plugin_dir/pyproject.toml" ] || continue
+    if ! uv pip install --python .venv/bin/python -e "$plugin_dir" 2>&1; then
+        echo "  [+${{SECONDS}}s] FATAL: installing setting plugin $plugin_dir failed."
+        exit 1
+    fi
+    echo "  [+${{SECONDS}}s] Setting plugin installed: $plugin_dir"
+done
+
+# --- Worker setup (--ec2-arg worker_setup): e.g. an external sandbox's host runtime ---
+# It sees the venv at /opt/control-tower/.venv; /opt/control-tower-worker-setup.env is sourced after it.
+WORKER_SETUP_S3_KEY='{worker_setup_s3_key}'
+if [ -n "$WORKER_SETUP_S3_KEY" ]; then
+    aws s3 cp "s3://{s3_bucket}/$WORKER_SETUP_S3_KEY" /opt/control-tower-worker-setup.sh --region {region}
+    echo "  [+${{SECONDS}}s] Running worker setup..."
+    if ! bash -e /opt/control-tower-worker-setup.sh; then
+        echo "  [+${{SECONDS}}s] FATAL: worker setup failed"
+        exit 1
+    fi
+    if [ -f /opt/control-tower-worker-setup.env ]; then
+        # shellcheck disable=SC1091
+        source /opt/control-tower-worker-setup.env
+    fi
+    echo "  [+${{SECONDS}}s] Worker setup finished"
+fi
+
+
+# --- Model rates: the driver's table, so workers price as the driver does ---
+RATES_S3_KEY='{rates_s3_key}'
+if [ -n "$RATES_S3_KEY" ]; then
+    aws s3 cp "s3://{s3_bucket}/$RATES_S3_KEY" /opt/control-tower/model_rates.json --region {region}
+    export CONTROL_TOWER_MODEL_RATES=/opt/control-tower/model_rates.json
+else
+    export CONTROL_TOWER_PRICING=off
+fi
+
+# --- Worker -------------------------------------------------------------
+echo "=== Starting worker at +${{SECONDS}}s ==="
+export CONTROL_TOWER_IMAGES=pull
+export INSPECT_LOG_LEVEL='{inspect_log_level}'
+if [ "$CT_FLEET_WORKER_USER" != root ]; then
+    mkdir -p /opt/uv-python
+    chown -R "$CT_FLEET_WORKER_USER": /opt/control-tower /opt/uv-cache /opt/uv-python
+    if [ ! "$(command -v uv)" -ef /usr/local/bin/uv ]; then
+        install -m 755 "$(command -v uv)" /usr/local/bin/uv
+    fi
+    echo "$CT_FLEET_WORKER_USER ALL=(ALL) NOPASSWD: /usr/sbin/shutdown" > /etc/sudoers.d/fleet-worker
+    chmod 440 /etc/sudoers.d/fleet-worker
+fi
+runuser -u "$CT_FLEET_WORKER_USER" -- env HOME="$(getent passwd "$CT_FLEET_WORKER_USER" | cut -d: -f6)" PATH="$PATH" \
+    uv run python -m control_tower.backends.fleet.worker_cli \
+    --queue-url '{work_queue_url}' \
+    --done-queue-url '{done_queue_url}' \
+    --s3-bucket '{s3_bucket}' \
+    --concurrency {concurrency} \
+    --idle-timeout {idle_timeout} \
+    --region {region}
+
+echo "=== Worker finished at $(date) ==="

control_tower/backends/fleet/config.py

@@ -0,0 +1,40 @@
+"""Fleet run options."""
+
+from typing import ClassVar
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+
+class Ec2Config(BaseModel):
+    """The fleet ``--ec2`` launches.
+
+    Every field is settable as ``--ec2-arg key=value``, so a new knob is a field here and never a
+    new CLI flag.
+    """
+
+    model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid")
+
+    instance_type: str = "c7i.4xlarge"
+    instance_type_fallbacks: list[str] = []
+    region: str = "us-east-1"
+    max_workers: int = 24
+    # Workers start at 10 and adapt within [1, max] from observed rate-limit pressure.
+    concurrency_per_worker: int = Field(default=32, ge=1)
+    # Print the cost estimate and exit without launching.
+    estimate: bool = False
+    # Skip the confirmation wait.
+    auto_confirm: bool = False
+    # Force a fresh fleet, skipping auto-resume.
+    new_fleet: bool = False
+    spot: bool = False
+    ami: str | None = None
+    worker_user: str = "root"
+    # Local script each worker runs as root before taking jobs, e.g. to install an external sandbox's host side.
+    worker_setup: str | None = None
+
+    @field_validator("instance_type_fallbacks", mode="before")
+    @classmethod
+    def _split_instance_type_fallbacks(cls, value: object) -> object:
+        if isinstance(value, str):
+            return [t.strip() for t in value.split(",") if t.strip()]
+        return value