tribunal-kit 9.2.3 → 9.2.4

package/.agent/skills/hf-cloud-sagemaker-production-defaults/scripts/teardown.py

@@ -0,0 +1,246 @@
+#!/usr/bin/env python3
+"""Tear down a SageMaker endpoint and its associated resources. Cross-platform.
+
+Deletes in safe order: alarms -> autoscaling -> inference components ->
+endpoint (stops billing) -> endpoint config -> model.
+
+Handles both deployment shapes:
+  - model-based endpoints (deploy.py, deploy_async.py): the variant carries the
+    model, autoscaling targets endpoint/<name>/variant/AllTraffic
+  - inference-component endpoints (deploy_ic.py, scale-to-zero): the component
+    carries the model, autoscaling targets inference-component/<name>
+
+Does NOT delete: IAM role, data capture S3 objects, SNS topic, model artifacts.
+Idempotent — missing resources are skipped, not errors.
+
+Calls the `aws` CLI from the current shell, so it inherits the same AWS context
+(profile, region, SSO session) — no Bash/WSL context-sharing problem on Windows.
+
+Usage:
+    python teardown.py <endpoint-name> [<region>]
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+import sys
+import time
+
+VARIANT_DIM = "sagemaker:variant:DesiredInstanceCount"
+IC_DIM = "sagemaker:inference-component:DesiredCopyCount"
+
+
+def log(msg: str) -> None:
+    print(f"[teardown] {msg}", file=sys.stderr, flush=True)
+
+
+def aws_bin() -> str:
+    exe = shutil.which("aws")
+    if not exe:
+        log("ERROR: the 'aws' CLI was not found on PATH. Install AWS CLI v2.")
+        sys.exit(2)
+    return exe
+
+
+def run_aws(args: list[str]) -> subprocess.CompletedProcess:
+    return subprocess.run([aws_bin(), *args], capture_output=True, text=True)
+
+
+def json_out(proc: subprocess.CompletedProcess, default):
+    if proc.returncode != 0:
+        return default
+    try:
+        return json.loads(proc.stdout) or default
+    except json.JSONDecodeError:
+        return default
+
+
+def resolve_region(arg_region: str | None) -> str:
+    """Region from arg, then env, then the active profile's config."""
+    if arg_region:
+        return arg_region
+    for var in ("AWS_REGION", "AWS_DEFAULT_REGION"):
+        if os.environ.get(var):
+            return os.environ[var]
+    proc = run_aws(["configure", "get", "region"])
+    return proc.stdout.strip() if proc.returncode == 0 else ""
+
+
+def delete_alarms(prefix: str, reg: list[str]) -> None:
+    alarms = json_out(
+        run_aws(["cloudwatch", "describe-alarms", "--alarm-name-prefix", prefix,
+                 "--query", "MetricAlarms[*].AlarmName", "--output", "json", *reg]),
+        [],
+    )
+    if alarms:
+        run_aws(["cloudwatch", "delete-alarms", "--alarm-names", *alarms, *reg])
+        log(f"Deleted alarms: {' '.join(alarms)}")
+
+
+def delete_scaling_stack(resource_id: str, dimension: str, reg: list[str]) -> None:
+    """Delete every scaling policy on a target, then deregister the target."""
+    policies = json_out(
+        run_aws(["application-autoscaling", "describe-scaling-policies",
+                 "--service-namespace", "sagemaker", "--resource-id", resource_id,
+                 "--query", "ScalingPolicies[*].PolicyName", "--output", "json", *reg]),
+        [],
+    )
+    for policy in policies:
+        run_aws(["application-autoscaling", "delete-scaling-policy",
+                 "--service-namespace", "sagemaker", "--resource-id", resource_id,
+                 "--scalable-dimension", dimension, "--policy-name", policy, *reg])
+        log(f"Deleted autoscaling policy: {policy}")
+
+    targets = json_out(
+        run_aws(["application-autoscaling", "describe-scalable-targets",
+                 "--service-namespace", "sagemaker", "--resource-ids", resource_id,
+                 "--query", "ScalableTargets[*].ResourceId", "--output", "json", *reg]),
+        [],
+    )
+    if resource_id in targets:
+        run_aws(["application-autoscaling", "deregister-scalable-target",
+                 "--service-namespace", "sagemaker", "--resource-id", resource_id,
+                 "--scalable-dimension", dimension, *reg])
+        log(f"Deregistered scalable target: {resource_id}")
+
+
+def list_inference_components(endpoint_name: str, reg: list[str]) -> list[str]:
+    return json_out(
+        run_aws(["sagemaker", "list-inference-components",
+                 "--endpoint-name-equals", endpoint_name,
+                 "--query", "InferenceComponents[*].InferenceComponentName",
+                 "--output", "json", *reg]),
+        [],
+    )
+
+
+def component_model_name(ic_name: str, reg: list[str]) -> str:
+    desc = json_out(
+        run_aws(["sagemaker", "describe-inference-component",
+                 "--inference-component-name", ic_name, *reg]),
+        {},
+    )
+    return desc.get("Specification", {}).get("ModelName", "")
+
+
+def delete_inference_components(ic_names: list[str], reg: list[str],
+                                timeout_seconds: int = 900) -> None:
+    """Delete components and wait until they are gone.
+
+    Retries the delete on every poll, because SageMaker refuses it in two
+    transient states seen in practice:
+      - CREATE_IN_PROGRESS (container still starting)
+      - UPDATE_RC_IN_PROGRESS (a scaling action is changing the copy count)
+    An adapter component must also go before its base component, and the
+    ordering falls out of the same retry loop.
+    """
+    pending = list(ic_names)
+    deadline = time.time() + timeout_seconds
+    announced: set[str] = set()
+    while pending and time.time() < deadline:
+        for ic_name in pending:
+            if ic_name not in announced:
+                log(f"Deleting inference component: {ic_name}")
+                announced.add(ic_name)
+            run_aws(["sagemaker", "delete-inference-component",
+                     "--inference-component-name", ic_name, *reg])
+        still_there = [
+            ic_name for ic_name in pending
+            if run_aws(["sagemaker", "describe-inference-component",
+                        "--inference-component-name", ic_name, *reg]).returncode == 0
+        ]
+        pending = still_there
+        if pending:
+            time.sleep(15)
+
+    if pending:
+        log(f"WARNING: components still present after {timeout_seconds}s: {' '.join(pending)}")
+    else:
+        log("All inference components deleted")
+
+
+def main() -> int:
+    if len(sys.argv) < 2:
+        log(f"Usage: {os.path.basename(sys.argv[0])} <endpoint-name> [<region>]")
+        return 64
+
+    endpoint_name = sys.argv[1]
+    region = resolve_region(sys.argv[2] if len(sys.argv) > 2 else None)
+    if not region:
+        log("ERROR: no AWS region. Pass region as 2nd arg or set AWS_REGION.")
+        return 1
+
+    reg = ["--region", region]
+    log(f"Tearing down endpoint: {endpoint_name} in {region}")
+
+    # Discover what's attached to this endpoint.
+    config_name = ""
+    model_names: list[str] = []
+    desc = run_aws(["sagemaker", "describe-endpoint", "--endpoint-name", endpoint_name, *reg])
+    endpoint_exists = desc.returncode == 0
+    if endpoint_exists:
+        config_name = json_out(desc, {}).get("EndpointConfigName", "")
+    else:
+        log("Endpoint not found — checking for orphan resources anyway")
+
+    if config_name:
+        cfg = json_out(
+            run_aws(["sagemaker", "describe-endpoint-config",
+                     "--endpoint-config-name", config_name, *reg]),
+            {},
+        )
+        # Model-based endpoints name the model on the variant. IC-based ones do not.
+        for variant in cfg.get("ProductionVariants", []):
+            if variant.get("ModelName"):
+                model_names.append(variant["ModelName"])
+
+    ic_names = list_inference_components(endpoint_name, reg)
+    if ic_names:
+        log(f"Inference components on this endpoint: {' '.join(ic_names)}")
+        for ic_name in ic_names:
+            model = component_model_name(ic_name, reg)
+            if model and model not in model_names:
+                model_names.append(model)
+
+    # Alarms — discovered by name prefix. deploy.py / deploy_async.py name them
+    # "<endpoint>-*"; deploy_ic.py names the wake alarm "<component>-*".
+    delete_alarms(f"{endpoint_name}-", reg)
+    for ic_name in ic_names:
+        delete_alarms(f"{ic_name}-", reg)
+
+    # Autoscaling — variant target (model-based) and component targets (IC-based).
+    delete_scaling_stack(f"endpoint/{endpoint_name}/variant/AllTraffic", VARIANT_DIM, reg)
+    for ic_name in ic_names:
+        delete_scaling_stack(f"inference-component/{ic_name}", IC_DIM, reg)
+
+    # Inference components must go before the endpoint.
+    if ic_names:
+        delete_inference_components(ic_names, reg)
+
+    # Endpoint (stops billing)
+    if endpoint_exists:
+        run_aws(["sagemaker", "delete-endpoint", "--endpoint-name", endpoint_name, *reg])
+        log(f"Deleted endpoint: {endpoint_name} (billing stopped)")
+
+    # Endpoint config
+    if config_name and run_aws(
+        ["sagemaker", "describe-endpoint-config", "--endpoint-config-name", config_name, *reg]
+    ).returncode == 0:
+        run_aws(["sagemaker", "delete-endpoint-config", "--endpoint-config-name", config_name, *reg])
+        log(f"Deleted endpoint config: {config_name}")
+
+    # Models
+    for model_name in model_names:
+        if run_aws(["sagemaker", "describe-model", "--model-name", model_name, *reg]).returncode == 0:
+            run_aws(["sagemaker", "delete-model", "--model-name", model_name, *reg])
+            log(f"Deleted model: {model_name}")
+
+    log("Teardown complete. Data capture S3 objects (if any) NOT deleted — manage separately.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/.agent/skills/hf-cloud-serving-image-selection/SKILL.md

@@ -0,0 +1,220 @@
+---
+name: hf-cloud-serving-image-selection
+description: 'Pick the right serving container for a SageMaker model deployment and find its current image URI. Use this skill whenever about to deploy a model to a SageMaker endpoint and an image URI needs to be chosen — including when the user says "deploy this LLM", "host this HuggingFace model", "serve this fine-tuned model", "deploy this embedding model", "host a reranker", "serve a sentence-transformers model", or when about to hardcode any container URI in deployment code. HuggingFace-curated Deep Learning Containers are ALWAYS preferred: HuggingFace vLLM (LLMs and generative rerankers), HuggingFace vLLM-Omni (multimodal), TEI (embeddings/cross-encoder rerankers), HF Inference Toolkit (other transformers). Generic images (AWS vLLM, DJL-LMI, SGLang) are used only when no HuggingFace image is compatible — never merely because they carry a newer version. Never hardcode a container URI from memory and never default to TGI. Prevents stale-image failures and wrong-region URIs.'
+---
+
+# Serving Image Selection
+
+The serving container is the single thing most likely to break a SageMaker deployment that "looked correct on paper". Wrong container, stale tag, or the wrong AMI — all produce the same opaque `Failed to pass health check` error.
+
+## Rule zero: HuggingFace images always win
+
+When both a HuggingFace-curated family (`huggingface-vllm`, `huggingface-vllm-omni`, `huggingface-sglang`, `tei`, `huggingface-pytorch-inference`) and a generic family (`vllm`, `vllm-omni`, `sglang`, `djl-inference`) can serve the model, **the HuggingFace one is mandatory, not preferred**. The only valid reasons to use a generic image:
+
+1. **Verified incompatibility** — the model needs an architecture/modality/feature no available HuggingFace tag supports, confirmed against the catalog (not assumed).
+2. **No HuggingFace tag exists in the target region** and mirroring is not an option.
+3. **The HuggingFace image is in "Known-broken images"** below.
+
+A **newer version number on the generic repo is not a reason**. The AWS `vllm` repo often publishes a higher vLLM version than `huggingface-vllm`; an older-but-compatible `huggingface-vllm` tag still wins. "Latest vLLM" is not a requirement anyone stated — compatibility with the model is. If you fall back, record in the deployment log which of the three reasons applied.
+
+## Where image URIs come from
+
+**Primary source: AWS's official Deep Learning Containers catalog.**
+
+URL: https://aws.github.io/deep-learning-containers/reference/available_images/
+
+This page is AWS-maintained and lists every image family with example URIs, tags, CUDA versions, Python versions, and platform (SageMaker vs EC2/ECS/EKS). When picking a URI for a deployment, **read it from this page directly** — copy the example URL, substitute `<region>` with the user's region, and pass it to `deploy.py --image-uri`.
+
+The example URLs use `763104351884` as the account ID for most regions. A few regions use different accounts (e.g. `eu-south-1` uses `692866216735`). Check the [Region Availability page](https://aws.github.io/deep-learning-containers/reference/region_availability/) when in doubt.
+
+**Exception: none currently.** Every image family used by this workflow is now on the AWS catalog page (TEI was added in late 2026). If you encounter a new family that isn't there, mirror it via `mirror_image.py` and pass the resulting URI directly.
+
+## Quick decision
+
+| Model | Container family | How to get the URI |
+|---|---|---|
+| HuggingFace text-generation LLM (Llama, Qwen, Mistral, etc.) | **HuggingFace vLLM** | AWS catalog → "HuggingFace vLLM Inference" (ECR repo `huggingface-vllm`) |
+| Same as above, multimodal | **HuggingFace vLLM-Omni** | AWS catalog → "HuggingFace vLLM-Omni Inference" (ECR repo `huggingface-vllm-omni`) |
+| HuggingFace embeddings | TEI | AWS catalog → "HuggingFace Text Embeddings Inference" |
+| Encoder / cross-encoder rerankers (BERT-family `*ForSequenceClassification`) | TEI | Same as embeddings |
+| **Generative rerankers** (causal-LM, e.g. Qwen3-Reranker) | HuggingFace vLLM | Same as text-generation LLMs — **not TEI**, see "Rerankers: TEI or vLLM?" |
+| Text-to-image / diffusion (Stable Diffusion, FLUX) | DJL Inference | AWS catalog → "DJL Inference" — **not** HF Inference Toolkit, see "Known-broken images" |
+| HuggingFace classifiers, NER, QA, summarization | HF Inference Toolkit (CPU) | AWS catalog → "HuggingFace PyTorch Inference"; GPU tags currently broken — see "Known-broken images" |
+| User specifically wants SGLang | HuggingFace SGLang | AWS catalog → "HuggingFace SGLang Inference" |
+| No compatible `huggingface-vllm` tag (verified incompatibility or region gap — see "Rule zero") | vLLM (AWS) | AWS catalog → "vLLM" section — fallback only, never for version freshness |
+| User specifically wants DJL-LMI | DJL Inference | AWS catalog → "DJL Inference" |
+| Amazon Nova | SageMaker JumpStart | Use JumpStart, not raw endpoint creation |
+| Custom inference code | BYOC | User provides URI |
+
+**HuggingFace-curated DLCs are mandatory when one is compatible (see "Rule zero").** `huggingface-vllm` is layered directly on the AWS vLLM DLC — **identical `SM_VLLM_*` env contract and the same cu130 AMI rule** — and adds current `transformers`, current `huggingface_hub` + `hf_xet` (avoids the XET-CDN 403 download failures older images hit), and HF performance defaults. It is also what SageMaker SDK v3 auto-routes to. The AWS `vllm` image is a compatibility escape hatch only; it usually shows a higher vLLM version than `huggingface-vllm`, and that is not a reason to pick it.
+
+**Do not use TGI.** Text Generation Inference is archived. Models released after the archive (Qwen3 most famously) fail ping health checks on TGI. Use vLLM instead. (The SageMaker SDK v3 agrees: since [PR #5960](https://github.com/aws/sagemaker-python-sdk/pull/5960), June 2026, its `ModelBuilder` auto-routes `text-generation` to the HuggingFace vLLM DLC and multimodal tasks to HuggingFace vLLM-Omni.)
+
+Full reasoning for each family in `references/model-to-image.md`.
+
+## Rerankers: TEI or vLLM?
+
+"Reranker" covers two very different architectures, and picking wrong wastes a full endpoint-creation cycle (~20 min) before TEI rejects the model:
+
+- **Encoder cross-encoders** (BAAI/bge-reranker-*, mixedbread, most `sentence-transformers` rerankers) — BERT-family models with a classification head. `config.json` has `architectures: [..ForSequenceClassification]` on a TEI-supported encoder type. → **TEI**.
+- **Generative rerankers** (Qwen/Qwen3-Reranker-*, and similar causal-LM judges) — decoder LLMs that score relevance via the logprob of a yes/no token. `config.json` has `architectures: [..ForCausalLM]`. → **HuggingFace vLLM**, deployed exactly like a text-generation LLM. TEI will load the architecture then reject the `classifier` model type (Qwen3 support in TEI is *embeddings-only*). Invocation pattern (raw completions API, `max_tokens=1`, logprobs scoring) is in `hf-cloud-sagemaker-production-defaults`.
+
+**Preflight before creating any resources** — one HTTP GET settles it:
+
+```bash
+curl -s https://huggingface.co/<model-id>/raw/main/config.json
+# "architectures": ["Qwen3ForCausalLM"]              → vLLM
+# "architectures": ["XLMRobertaForSequenceClassification"] → TEI
+```
+
+For TEI also confirm the *(architecture, task)* pair: an architecture appearing in TEI's supported list means embeddings support, not necessarily classification/reranking support.
+
+Heads-up: SageMaker SDK v3 (PR #5960) routes the `text-ranking` task to TEI **unconditionally** — correct for cross-encoders, wrong for generative rerankers. Don't treat the SDK's routing as evidence that TEI can serve a given reranker.
+
+## Workflow
+
+For every family: **read the URI from the AWS catalog page**.
+
+1. Open https://aws.github.io/deep-learning-containers/reference/available_images/
+2. Find the section for the right family (e.g. "HuggingFace vLLM Inference" for HuggingFace LLMs, "HuggingFace Text Embeddings Inference" for embeddings)
+3. Pick the newest row marked `SageMaker` for the platform column — newest **within that family**. Do not switch to another family's section because it lists a higher engine version (see "Rule zero")
+4. Substitute `<region>` with the user's region (from `hf-cloud-aws-context-discovery`)
+5. For vLLM: also check the AMI requirement (see "vLLM AMI requirement" below)
+6. Pass the URI to `deploy.py --image-uri` (real-time) or `deploy_async.py --image-uri` (async)
+
+### TEI: pick the right variant
+
+The TEI catalog row lists two URIs — GPU (`tei` repo) and CPU (`tei-cpu` repo). Pick based on the instance type:
+
+- `ml.g*`, `ml.p*`, `ml.inf*` → GPU variant
+- `ml.c*`, `ml.m*`, `ml.t*` → CPU variant
+
+Mixing them fails: CPU image on a GPU instance wastes hardware, GPU image on a CPU instance fails to start.
+
+**Note on the TEI account ID**: the catalog page shows `683313688378` as the example account, but TEI is published from a different account namespace than the main AWS DLCs and the per-region account IDs vary. If `683313688378.dkr.ecr.<region>.amazonaws.com/tei:...` returns an ECR pull error for a region other than us-east-1, check the [Region Availability page](https://aws.github.io/deep-learning-containers/reference/region_availability/) for the correct account ID for that region.
+
+## vLLM AMI requirement
+
+vLLM DLC images with **CUDA 13 or higher** (current default: `cu130`) require setting `InferenceAmiVersion=al2-ami-sagemaker-inference-gpu-3-1` on the ProductionVariant. This applies equally to `huggingface-vllm` and `huggingface-vllm-omni` (layered on the same cu130 base) and to the AWS `vllm` repo. Without it the container dies on startup with no CloudWatch logs ever created. The failure looks identical to many other things (account-level issues, quota, networking) and routinely sends people down wrong diagnostic paths.
+
+Lookup table:
+
+| Tag contains | InferenceAmiVersion to pass |
+|---|---|
+| `cu130` (or higher) | `al2-ami-sagemaker-inference-gpu-3-1` |
+| `cu129` or lower | (omit the flag; default AMI works) |
+
+Rule of thumb: if the vLLM tag you picked contains `cu130` or later, pass `--inference-ami-version al2-ami-sagemaker-inference-gpu-3-1` to `deploy.py`. If a future CUDA version (cu140+) needs a different AMI, add a row to the table when AWS publishes the new image.
+
+This is a vLLM-specific concern. TEI and HF Inference Toolkit images don't need an AMI override.
+
+## Configuring the vLLM DLCs (HuggingFace vLLM and AWS vLLM)
+
+Both images share the same contract: configuration as environment variables on the SageMaker model definition, `SM_VLLM_*` mapped to vLLM CLI flags. The `huggingface-vllm` entrypoint additionally auto-detects the model when `SM_VLLM_MODEL` is unset — from `/opt/ml/model` if S3 artifacts are mounted, else from `HF_MODEL_ID`. For production, point `SM_VLLM_MODEL` at `/opt/ml/model`; loading directly from the Hub at runtime is the exception.
+
+### Required for every HuggingFace LLM deployment
+
+| Env var | Purpose | Notes |
+|---|---|---|
+| `SM_VLLM_MODEL` | HF model ID (e.g. `Qwen/Qwen3-0.6B`) or `/opt/ml/model` if loading from S3 | — |
+| `SM_VLLM_HOST` | **Must be `0.0.0.0`** | Otherwise vLLM binds localhost only, ping fails, container dies before logs. Top cause of mystery failures with this image. |
+| `SM_VLLM_TRUST_REMOTE_CODE` | Whether model artifacts may execute custom Python code | **Default `false`.** Set `true` only for a specific architecture that cannot load without its repository code, after reviewing and pinning those artifacts. Enabling it allows model-supplied code to run inside the container. |
+| `HUGGING_FACE_HUB_TOKEN` | HF token | Required for gated models. |
+
+Stage production model artifacts in an account-controlled S3 bucket, pass their URI as `--model-s3-uri`, and use `SM_VLLM_MODEL=/opt/ml/model`. This avoids downloading or executing repository code at runtime. Use a Hub model ID only when runtime Hub access is explicitly required; keep `SM_VLLM_TRUST_REMOTE_CODE=false` unless that architecture requires reviewed custom code.
+
+### Tuning (optional)
+
+| Env var | Purpose |
+|---|---|
+| `SM_VLLM_MAX_MODEL_LEN` | Max sequence length — set this; defaults can be wrong for fine-tunes |
+| `SM_VLLM_GPU_MEMORY_UTILIZATION` | Float 0.0–1.0, ~0.9 reasonable |
+| `SM_VLLM_TENSOR_PARALLEL_SIZE` | GPU count for multi-GPU instances |
+| `SM_VLLM_DTYPE` | `auto`, `bfloat16`, `float16` |
+
+Any vLLM CLI flag works — uppercase, replace dashes with underscores, prepend `SM_VLLM_`.
+
+## Configuring TEI
+
+Simpler env contract than vLLM:
+
+| Env var | Purpose | Required |
+|---|---|---|
+| `HF_MODEL_ID` | HF model ID (e.g. `BAAI/bge-large-en-v1.5`) or `/opt/ml/model` | Yes |
+| `HF_TOKEN` | HF auth token | Only for gated models |
+| `MAX_BATCH_TOKENS` | Max tokens per batch (default 16384) | No |
+| `MAX_CLIENT_BATCH_SIZE` | Max requests per client batch (default 32) | No |
+
+No host-binding to configure, no trust-remote-code flag. The architectures TEI supports (BERT, CamemBERT, RoBERTa, XLM-RoBERTa, NomicBert, JinaBert, JinaCodeBert, Mistral, Qwen2/3, Gemma2/3, ModernBert) are baked into the image.
+
+## CUDA / instance compatibility
+
+Critical and easy to get wrong:
+
+| CUDA in image tag | Default AMI | With `al2-ami-sagemaker-inference-gpu-3-1` |
+|---|---|---|
+| cu124 / cu128 | g5, g6, p5 all work | (not needed) |
+| cu129 | g6, p5; g5 fails (driver mismatch → CannotStartContainerError) | expected to fix g5 (unverified) |
+| cu130+ | fails everywhere — AMI flag is mandatory | g5, g6, p5 all work (cu130-on-g5 verified June 2026) |
+
+The driver comes from the host AMI, not the instance family — so passing the gpu-3-1 AMI (which vLLM cu130 images require anyway) also makes `ml.g5.*` viable for cu129+ images.
+
+## VPC / NAT gateway problem
+
+SageMaker endpoints inside a VPC **without** a NAT gateway can't pull from `public.ecr.aws`. The deployment fails with an image-pull error that doesn't mention "VPC" or "egress".
+
+For images on AWS's regional ECR (everything in the catalog): SageMaker reaches them through built-in routing, no NAT needed. Use the regional URI pattern (`<account>.dkr.ecr.<region>.amazonaws.com/...`), not the `public.ecr.aws/...` pattern.
+
+For images requiring `public.ecr.aws` access (less common): mirror to a private ECR repo in your account with `scripts/mirror_image.py` (cross-platform; needs Docker + the `aws` CLI). Run it from the shell where the AWS CLI works.
+
+```bash
+# macOS / Linux
+PRIVATE_URI=$(python3 scripts/mirror_image.py \
+    public.ecr.aws/deep-learning-containers/vllm:<tag> \
+    vllm-mirror)
+```
+
+```powershell
+# Windows (PowerShell) — capture stdout into a variable
+$PRIVATE_URI = python scripts\mirror_image.py `
+    public.ecr.aws/deep-learning-containers/vllm:<tag> vllm-mirror
+```
+
+## When the catalog page won't render, is stale, or is wrong
+
+**The page won't render / fetch returns junk**: the catalog page is JavaScript-heavy and some fetch tools get an empty shell. Fallbacks, in order:
+
+1. **The catalog's source data on GitHub** — the page is generated from one YAML file per version, listing exact tags, CUDA, and Python versions. List a family's files, then fetch the newest:
+   ```bash
+   curl -s https://api.github.com/repos/aws/deep-learning-containers/contents/docs/src/data/huggingface-vllm
+   curl -s https://raw.githubusercontent.com/aws/deep-learning-containers/main/docs/src/data/huggingface-vllm/0.21.0-gpu-sagemaker.yml
+   ```
+   Directory names match ECR repos (`huggingface-vllm`, `huggingface-vllm-omni`, `huggingface-tei`, `vllm`, `djl-inference`, ...).
+2. **Query ECR directly** for current tags in the target region (works with credentials that can read the DLC registry; if it returns AccessDenied, use the YAML files):
+   ```bash
+   aws ecr describe-images --registry-id 763104351884 --repository-name huggingface-vllm \
+       --region <region> --query 'sort_by(imageDetails,&imagePushedAt)[-5:].imageTags' --output json
+   ```
+3. [Release notes on the DLC GitHub repo](https://github.com/aws/deep-learning-containers/releases).
+
+**A tag was just released and isn't on the page yet**: rare; AWS updates the page on each release. Check the release notes above.
+
+**An architecture you need isn't supported by the listed image yet**: for TEI specifically, you can mirror the upstream image from GHCR (`ghcr.io/huggingface/text-embeddings-inference:<version>`) into private ECR and pass the resulting URI directly to `deploy.py --image-uri`. Same `mirror_image.py` script.
+
+## Known-broken images (last checked July 2026)
+
+| Image | Defect | Use instead |
+|---|---|---|
+| `huggingface-pytorch-inference` **GPU** tags — all recent ones tested (PT 2.3–2.6, cu121/cu124, transformers 4.48–5.5.3) | `ImportError: libtorch_cuda.so: undefined symbol: ncclCommResume` at `import torch`. The NCCL bundled in the image is older than what torch links against — a packaging defect *inside the container*, on g5 **and** g6, regardless of AMI, model, or inference code. The MMS Java front-end keeps answering `/ping`, so the endpoint can reach InService while the Python worker crash-loops and serves nothing. | DJL Inference (bundles its own complete CUDA/NCCL stack) or BYOC. CPU tags are unaffected. |
+
+Re-check when AWS publishes new `huggingface-pytorch-inference` GPU tags — remove the row once a fixed image is confirmed.
+
+**General fallback rule**: when an HF DLC fails with CUDA/NCCL linker errors, switch to DJL Inference rather than iterating over sibling tags — the defect class is per-repo, not per-tag (three different tags were tried for the case above; all broken).
+
+**Related HF Hub gotcha**: older DLCs can fail model download with `403 Forbidden` from HF's XET CDN (their bundled `huggingface_hub` predates XET auth). Set `HF_HUB_ENABLE_HF_TRANSFER=0` to force the standard download path, or pre-stage weights in S3.
+
+## Hub download time at first boot
+
+Loading the model from HF Hub happens *inside the container after the endpoint starts* — expect **5–15+ minutes** before InService even for small models, longer for multi-GB ones. A slow first boot is not a failure; don't tear down or re-diagnose before the deploy script's 30-minute wait expires.
+
+For production or repeated deployments, pre-stage the weights in S3 and pass `--model-s3-uri` to `deploy.py` (the model then loads from `/opt/ml/model`) — faster, immune to Hub rate limits/outages, and no `HUGGING_FACE_HUB_TOKEN` needed at runtime.

package/.agent/skills/hf-cloud-serving-image-selection/references/model-to-image.md

@@ -0,0 +1,111 @@
+# Model Family → Serving Container Decision Table
+
+Consult this **before** writing deployment code that hardcodes an image URI.
+
+The canonical source for AWS Deep Learning Container image URIs is:
+**https://aws.github.io/deep-learning-containers/reference/available_images/**
+
+This page is AWS-maintained and lists every published image family with example URIs, tags, CUDA versions, and platform (SageMaker vs EC2/ECS/EKS). Read URIs from there directly — substitute `<region>` with the user's region and pass to `deploy.py --image-uri`. The doc below explains *which* family to pick for which use case; the URI itself comes from the AWS page.
+
+## Decision summary
+
+| Model family | Container | Source |
+|---|---|---|
+| HuggingFace text-generation LLMs (Llama, Qwen, Mistral, Mixtral, DeepSeek, Phi, Gemma, GPT-OSS, etc.) | **HuggingFace vLLM DLC** | AWS catalog → "HuggingFace vLLM Inference" |
+| Same as above, multimodal (vision-language, audio, any-to-any) | **HuggingFace vLLM-Omni DLC** | AWS catalog → "HuggingFace vLLM-Omni Inference" |
+| Same family, when NO `huggingface-vllm` tag is compatible (verified) or the region has none | AWS vLLM DLC (fallback — never for version freshness) | AWS catalog → "vLLM" |
+| Same family, alternative serving stack | DJL-LMI | AWS catalog → "DJL Inference" |
+| HuggingFace embeddings + encoder cross-encoder rerankers | **TEI DLC** | AWS catalog → "HuggingFace Text Embeddings Inference" |
+| Generative rerankers (causal-LM, e.g. Qwen3-Reranker) | **HuggingFace vLLM DLC** | Same as text-generation LLMs — see "Generative rerankers" below |
+| HuggingFace classifiers, NER, QA, summarization | HF Inference Toolkit (CPU; GPU tags broken as of July 2026 — see SKILL.md "Known-broken images") | AWS catalog → "HuggingFace PyTorch Inference" |
+| HuggingFace-curated SGLang build | HuggingFace SGLang | AWS catalog → "HuggingFace SGLang Inference" |
+| SGLang (without HF wrapper) | SGLang | AWS catalog → "SGLang" |
+| Amazon Nova (Lite, Micro, Pro) | SageMaker JumpStart | Use JumpStart deployment, not raw endpoint creation |
+| Stable Diffusion / diffusion / image generation | **DJL Inference** | AWS catalog → "DJL Inference" — not the HF Inference Toolkit (GPU tags broken, see below); StabilityAI DLC exists but is stale |
+| Inferentia / Trainium hardware | NeuronX variants | AWS catalog → search for "NeuronX" |
+| Custom inference code | BYOC | User provides URI |
+
+## Why vLLM, not TGI
+
+Text Generation Inference (TGI) was the long-standing default for HuggingFace LLMs. As of late 2025 / early 2026, **TGI is archived** — no more major updates. Models released after the archive (Qwen3 most famously) fail health checks on TGI. Use vLLM instead.
+
+The SageMaker SDK v2 helper `get_huggingface_llm_image_uri` returned TGI URIs; the v3 SDK removed it entirely. Since [sagemaker-python-sdk PR #5960](https://github.com/aws/sagemaker-python-sdk/pull/5960) (merged June 2026), v3's `ModelBuilder` auto-routes HuggingFace models by task: `text-generation` → HuggingFace vLLM DLC, multimodal tasks (`image-text-to-text`, `any-to-any`, `audio-text-to-text`) → vLLM-Omni, `feature-extraction` / `sentence-similarity` / `text-ranking` → TEI, SGLang opt-in only. Our decision table matches, with one deliberate divergence: the SDK sends **all** `text-ranking` models to TEI, which fails for generative rerankers (see below). Either way, don't use TGI for new deployments.
+
+## Generative rerankers → vLLM, not TEI
+
+Two reranker architectures exist:
+
+- **Encoder cross-encoders** (BAAI/bge-reranker-*, mixedbread rerankers): BERT-family + classification head, `config.json` `architectures` ends in `ForSequenceClassification`. TEI serves these natively via its `/rerank` route.
+- **Generative rerankers** (Qwen/Qwen3-Reranker-0.6B/4B/8B and similar): decoder-only causal LMs that emit a yes/no judgment token; relevance = P(yes)/(P(yes)+P(no)) from the logprobs. `config.json` `architectures` ends in `ForCausalLM`. These need **vLLM** — use the HuggingFace vLLM DLC, same as any text-generation LLM. TEI recognizes the Qwen3 architecture, starts loading it, then rejects the `classifier` model type — Qwen3 support in TEI is embeddings-only. That rejection costs a full endpoint-creation cycle (~20 min observed), so preflight with the model's `config.json` (`curl -s https://huggingface.co/<id>/raw/main/config.json`) before creating anything.
+
+Deploy generative rerankers exactly like a vLLM LLM deployment (`SM_VLLM_*` env vars, AMI rule applies). The invocation pattern — raw completions API, `max_tokens=1`, logprobs scoring — is documented in `hf-cloud-sagemaker-production-defaults`.
+
+## HuggingFace vLLM DLC (mandatory default for LLMs)
+
+This is the default for **every** vLLM deployment, not a suggestion. The AWS vLLM repo further down is strictly a compatibility escape hatch.
+
+URI pattern (from AWS catalog → "HuggingFace vLLM Inference"):
+```
+763104351884.dkr.ecr.<region>.amazonaws.com/huggingface-vllm:<version>-transformers<tv>-gpu-py<py>-cu<cuda>-ubuntu22.04
+```
+
+Example: `763104351884.dkr.ecr.us-east-1.amazonaws.com/huggingface-vllm:0.21.0-transformers5.8.1-gpu-py312-cu130-ubuntu22.04` (0.22.1 in the release pipeline as of July 2026 — read the newest tag from the catalog).
+
+The image is built directly on the AWS vLLM DLC, so everything below about the AWS image applies to it too — same `SM_VLLM_*` env contract, same cu130 AMI requirement. On top of the base it carries current `transformers`, current `huggingface_hub` + `hf_xet` (current-generation Hub downloads; older images 403 against the XET CDN), ffmpeg for multimodal preprocessing, and HF performance defaults (expandable-segments allocator, LMCache CPU KV-offload). Model selection: `SM_VLLM_MODEL` as usual; if unset, the entrypoint auto-detects `/opt/ml/model` or falls back to `HF_MODEL_ID`.
+
+For multimodal models (vision-language, audio, any-to-any) use the sibling `huggingface-vllm-omni` repo (catalog → "HuggingFace vLLM-Omni Inference") — same rules.
+
+## AWS vLLM DLC (fallback only)
+
+Use **only** when no `huggingface-vllm` tag is compatible with the model (a verified architecture/feature gap, not an assumed one) or the target region has no `huggingface-vllm` tag at all. This repo usually lists a **newer vLLM version** than `huggingface-vllm` — that alone is never a reason to pick it. If you are reading this section because the version number here is higher, go back to `huggingface-vllm`. URI pattern (from AWS catalog):
+```
+763104351884.dkr.ecr.<region>.amazonaws.com/vllm:<version>-gpu-py<py>-cu<cuda>-ubuntu22.04-sagemaker
+```
+
+Example: `763104351884.dkr.ecr.eu-west-1.amazonaws.com/vllm:0.23.0-gpu-py312-cu130-ubuntu22.04-sagemaker`
+
+**vLLM AMI requirement (both repos)**: images with `cu130` or higher require setting `InferenceAmiVersion=al2-ami-sagemaker-inference-gpu-3-1` on the ProductionVariant. Without it the container dies on startup with no CloudWatch logs created. See the "vLLM AMI requirement" lookup table in the SKILL.md to map a tag to the right AMI version.
+
+For environment variable configuration of the vLLM DLCs, see the SKILL.md.
+
+## TEI DLC
+
+Listed on the AWS catalog under "HuggingFace Text Embeddings Inference". The catalog row uses account ID `683313688378` (different from the main `763104351884` used by most other DLCs). TEI is published from its own account namespace and the per-region account IDs vary — if the canonical `683313688378` returns an ECR pull error for a non-us-east-1 region, check the [Region Availability page](https://aws.github.io/deep-learning-containers/reference/region_availability/) for the correct mapping.
+
+Two variants:
+
+- Repo `tei` — GPU build
+- Repo `tei-cpu` — CPU build
+
+Pick by instance type:
+- `ml.g*`, `ml.p*`, `ml.inf*` → `tei` (GPU)
+- `ml.c*`, `ml.m*`, `ml.t*` → `tei-cpu` (CPU)
+
+CPU embeddings are dramatically cheaper than GPU and often fast enough — `ml.c6i.2xlarge` (~$0.20/hr) is a common starting point. GPU is needed for large embedding models (>1B params) or sustained high throughput.
+
+### Supported architectures and staleness
+
+TEI bakes architecture support into the image. The current upstream version supports BERT, CamemBERT, RoBERTa, XLM-RoBERTa, NomicBert, JinaBert, JinaCodeBert, Mistral, Qwen2/3, Gemma2/3, ModernBert. The AWS-published DLC sometimes lags upstream by months — if a recent architecture isn't supported, mirror the upstream image from `ghcr.io/huggingface/text-embeddings-inference:<version>` using `scripts/mirror_image.py` and pass the result to `deploy.py --image-uri` directly.
+
+**Support is per (architecture, task), not per architecture.** Qwen3 appearing in the list means Qwen3 *embeddings*; TEI 1.8.x rejects Qwen3 with a classification head ("`classifier` model type is not supported"). Before picking TEI for a reranker, check the model's `config.json`: `architectures` ending in `ForSequenceClassification` on a BERT-family encoder is TEI territory; `ForCausalLM` means it's a generative reranker → vLLM (see above).
+
+For environment variable configuration of TEI, see the SKILL.md.
+
+## DJL Inference
+
+Listed on the AWS catalog under "DJL Inference", published from the main `763104351884` account. Two reasons to reach for it:
+
+1. **Diffusion / text-to-image models** — the recommended container for Stable Diffusion-class workloads (usually behind an async endpoint; see `hf-cloud-sagemaker-production-defaults`).
+2. **Fallback when an HF DLC fails with CUDA/NCCL errors** — DJL images bundle their own complete CUDA/NCCL stack and don't depend on host libraries, so they sidestep the packaging-defect class entirely.
+
+Configuration goes in a `serving.properties` file packaged alongside the model artifacts (or the equivalent `OPTION_*` env vars) rather than `SM_VLLM_*`/`HF_MODEL_ID` — check the [DJL Serving docs](https://docs.djl.ai/master/docs/serving/index.html) for the engine matching your model type rather than guessing keys.
+
+## HF Inference Toolkit: GPU tags broken (NCCL packaging defect)
+
+As of July 2026, every recently-tested GPU tag of `huggingface-pytorch-inference` (PT 2.3–2.6, cu121/cu124, transformers 4.48–5.5.3) fails at `import torch` with:
+
+```
+ImportError: libtorch_cuda.so: undefined symbol: ncclCommResume
+```
+
+torch in the image links NCCL 2.19+ symbols but the image ships an older NCCL. Verified on g5 **and** g6 — the defect is inside the container, so changing instance type, AMI, model, or inference code does not help, and neither does trying sibling tags. Worse, the MMS Java server keeps answering `/ping` while the Python worker crash-loops, so the endpoint can reach InService and silently serve nothing (async requests queue forever). Use DJL Inference or BYOC for GPU workloads until AWS ships fixed images; CPU tags are unaffected.