tribunal-kit 9.2.3 → 9.2.5

package/.agents/skills/hf-cloud-serving-image-selection/scripts/mirror_image.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+"""Mirror an ECR Public image to a private ECR repo. Cross-platform.
+
+Needed when SageMaker runs in a VPC without NAT gateway: SageMaker can't pull
+from public.ecr.aws in that case, so the image must be in private ECR.
+
+Requires: docker daemon, aws CLI with ecr/ecr-public permissions.
+Idempotent: skips push if the tag already exists in the private repo.
+
+Runs `aws`/`docker` from the current shell, inheriting its AWS context — no
+Bash/WSL context-sharing problem on Windows.
+
+Usage:
+    python mirror_image.py <public-image-uri> <private-repo-name> [<tag-override>]
+
+Prints the resulting private URI to stdout.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+import sys
+
+
+def log(msg: str) -> None:
+    print(f"[mirror_image] {msg}", file=sys.stderr, flush=True)
+
+
+def require(cmd: str) -> str:
+    exe = shutil.which(cmd)
+    if not exe:
+        log(f"ERROR: {cmd} not installed or not on PATH")
+        sys.exit(1)
+    return exe
+
+
+def run(args: list[str], **kwargs) -> subprocess.CompletedProcess:
+    return subprocess.run(args, capture_output=True, text=True, **kwargs)
+
+
+def resolve_region() -> str:
+    for var in ("AWS_REGION", "AWS_DEFAULT_REGION"):
+        if os.environ.get(var):
+            return os.environ[var]
+    proc = run([shutil.which("aws"), "configure", "get", "region"])
+    return proc.stdout.strip() if proc.returncode == 0 else ""
+
+
+def main() -> int:
+    if len(sys.argv) < 3:
+        log(f"Usage: {os.path.basename(sys.argv[0])} <public-image-uri> <private-repo-name> [<tag-override>]")
+        return 64
+
+    public_uri = sys.argv[1]
+    private_repo = sys.argv[2]
+    tag_override = sys.argv[3] if len(sys.argv) > 3 else ""
+
+    aws = require("aws")
+    docker = require("docker")
+    if run([docker, "info"]).returncode != 0:
+        log("ERROR: docker daemon not running")
+        return 1
+
+    # Extract tag
+    if tag_override:
+        tag = tag_override
+    elif ":" in public_uri.rsplit("/", 1)[-1]:
+        tag = public_uri.rsplit(":", 1)[-1]
+    else:
+        log("ERROR: public URI has no tag — refusing implicit ':latest'")
+        return 1
+
+    acct = run([aws, "sts", "get-caller-identity", "--query", "Account", "--output", "text"])
+    if acct.returncode != 0:
+        log("ERROR: 'aws sts get-caller-identity' failed. Configure AWS credentials.")
+        return 1
+    account_id = acct.stdout.strip()
+
+    region = resolve_region()
+    if not region:
+        log("ERROR: no AWS region. Set AWS_REGION or configure profile.")
+        return 1
+
+    registry = f"{account_id}.dkr.ecr.{region}.amazonaws.com"
+    private_uri = f"{registry}/{private_repo}:{tag}"
+
+    log(f"Public : {public_uri}")
+    log(f"Private: {private_uri}")
+
+    reg = ["--region", region]
+
+    # Create private repo if missing
+    if run([aws, "ecr", "describe-repositories", "--repository-names", private_repo, *reg]).returncode != 0:
+        log(f"Creating private ECR repo: {private_repo}")
+        run(
+            [aws, "ecr", "create-repository", "--repository-name", private_repo,
+             "--image-scanning-configuration", "scanOnPush=true", *reg]
+        )
+
+    # Skip if tag already exists in private
+    if run(
+        [aws, "ecr", "describe-images", "--repository-name", private_repo,
+         "--image-ids", f"imageTag={tag}", *reg]
+    ).returncode == 0:
+        log(f"Tag '{tag}' already in private ECR — skipping pull/push")
+        print(private_uri)
+        return 0
+
+    # Auth. ECR Public auth always uses us-east-1.
+    def docker_login(password: str, registry_host: str) -> bool:
+        login = subprocess.run(
+            [docker, "login", "--username", "AWS", "--password-stdin", registry_host],
+            input=password, capture_output=True, text=True,
+        )
+        return login.returncode == 0
+
+    pub_pw = run([aws, "ecr-public", "get-login-password", "--region", "us-east-1"])
+    if pub_pw.returncode != 0 or not docker_login(pub_pw.stdout, "public.ecr.aws"):
+        log("ERROR: failed to authenticate to public.ecr.aws")
+        return 1
+
+    priv_pw = run([aws, "ecr", "get-login-password", *reg])
+    if priv_pw.returncode != 0 or not docker_login(priv_pw.stdout, registry):
+        log(f"ERROR: failed to authenticate to {registry}")
+        return 1
+
+    log(f"Pulling {public_uri} (may take several minutes)...")
+    if subprocess.run([docker, "pull", public_uri]).returncode != 0:
+        log("ERROR: docker pull failed")
+        return 1
+    subprocess.run([docker, "tag", public_uri, private_uri])
+    log("Pushing to private ECR...")
+    if subprocess.run([docker, "push", private_uri]).returncode != 0:
+        log("ERROR: docker push failed")
+        return 1
+
+    log("Done.")
+    print(private_uri)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/.agents/skills/hf-mem/SKILL.md

@@ -0,0 +1,79 @@
+---
+name: hf-mem
+description: Hugging Face CLI to estimate the required memory to load Safetensors or GGUF model weights for inference from the Hugging Face Hub
+---
+
+`hf_mem` estimates the required memory for inference, including model weights and an optional KV cache, for Safetensors and GGUF for models on the Hugging Face Hub using HTTP Range requests i.e., without downloading or loading any weights locally.
+
+## When to use?
+
+- User asks how much VRAM or memory a model needs to run
+- User wants to know if a model fits on their GPU or a given instance
+- User references a Hugging Face model ID or URL and asks about inference requirements
+
+## What are the requirements?
+
+- `uv` installed (for `uvx`)
+- `HF_TOKEN` env var or `--hf-token` flag (for gated or private models only)
+
+## How to run?
+
+Run with `--model-id` pointing to the Hugging Face Hub repository which will check that it either contains Safetensors (via `model.safetensors`, `model.safetensors.index.json` if sharded, or `model_index.json` for Diffusers) or GGUF model weights within.
+
+```bash
+uvx hf-mem --model-id <model-id> --json-output
+```
+
+If the repository contains GGUF model weights in multiple precisions / quantizations, the estimations will be on a per-file basis, whereas for inference you won't load all of those but rather only a single precision. This being said, for GGUF you might as well need to provide `--gguf-file` to target the specific file (or path if sharded) you want to run.
+
+```bash
+uvx hf-mem --model-id <model-id> --gguf-file <file-or-path> --json-output
+```
+
+Additionally, `hf-mem` comes with an `--experimental` flag that will also calculate the KV cache memory requirements too, useful for large-language models, meaning it applies to LLMs (`...ForCausalLM`), VLMs (`...ForConditionalGeneration`), and GGUF models.
+
+As per the context window, it will be read from the default or overridden with `--max-model-len` a la vLLM. And, same goes for the KV cache precision, which will default to the model precision unless manually set via `--kv-cache-dtype` a la vLLM too.
+
+For Safetensors use as:
+
+```bash
+uvx hf-mem --model-id <model-id> --experimental [--max-model-len N] [--batch-size N] [--kv-cache-dtype auto|bfloat16|fp8|fp8_ds_mla|fp8_e4m3|fp8_e5m2|fp8_inc] --json-output
+```
+
+And, for GGUF use as:
+
+```bash
+uvx hf-mem --model-id <model-id> --gguf-file <file-or-path> --experimental [--max-model-len N] [--batch-size N] [--kv-cache-dtype auto|F32|F16|Q4_0|Q4_1|Q5_0|Q5_1|Q8_0|Q8_1|Q2_K|Q3_K|Q4_K|Q5_K|Q6_K|Q8_K|IQ2_XXS|IQ2_XS|IQ3_XXS|IQ1_S|IQ4_NL|IQ3_S|IQ2_S|IQ4_XS|I8|I16|I32|I64|F64|IQ1_M|BF16|TQ1_0|TQ2_0|MXFP4] --json-output
+```
+
+## Examples
+
+For Transformers with Safetensors weights:
+
+```bash
+uvx hf-mem --model-id MiniMaxAI/MiniMax-M2 --json-output
+```
+
+For Diffusers with Safetensors weights:
+
+```bash
+uvx hf-mem --model-id Qwen/Qwen-Image --json-output
+```
+
+For Sentence Transformers with Safetensors weights:
+
+```bash
+uvx hf-mem --model-id google/embeddinggemma-300m --json-output
+```
+
+With `--experimental` to include the KV cache estimation for LLMs and VLMs:
+
+```bash
+uvx hf-mem --model-id mistralai/Mistral-7B-v0.1 --experimental --json-output
+```
+
+And, for LLMs or VLMs with GGUF weights:
+
+```bash
+uvx hf-mem --model-id unsloth/Qwen3.5-397B-A17B-GGUF --gguf-file Q4_K_M --experimental --json-output
+```

package/.agents/skills/huggingface-best/SKILL.md

@@ -0,0 +1,134 @@
+---
+name: huggingface-best
+description: >-
+  Use when the user asks about finding the best, top, or recommended model for a task,
+  wants to know what AI model to use, or wants to compare models by benchmark scores.
+  Triggers on: "best model for X", "what model should I use for", "top models for [task]",
+  "which model runs on my laptop/machine/device", "recommend a model for", "what LLM should
+  I use for", "compare models for", "what's state of the art for", or any question about
+  choosing an AI model for a specific use case. Always use this skill when the user wants
+  model recommendations or comparisons, even if they don't explicitly mention HuggingFace
+  or benchmarks.
+---
+
+# HuggingFace Best Model Finder
+
+Finds the best models for a task by querying official HF benchmark leaderboards, enriching
+results with model size data, filtering for what fits on the user's device, and returning a
+comparison table with benchmark scores.
+
+---
+
+## Step 1: Parse the request
+
+Extract from the user's message:
+- **Task**: what they want the model to do (coding, math/reasoning, chat, OCR, RAG/retrieval, speech recognition, image classification, multimodal, agents, etc.)
+- **Device**: hardware constraints (MacBook M-series 8/16/32/64GB unified memory, RTX GPU with VRAM amount, CPU-only, cloud/no constraint, etc.)
+
+If device is not mentioned, skip filtering entirely and return the highest-performing models regardless of size. If the task is genuinely ambiguous, ask one clarifying question.
+
+### Device → max parameter budget
+
+When a device is specified, extract its available memory (unified RAM for Apple Silicon, VRAM for discrete GPUs) and apply:
+
+- **fp16 max params (B)** ≈ memory (GB) ÷ 2
+- **Q4 max params (B)** ≈ memory (GB) × 2
+
+Examples: 16GB → 8B fp16 / 32B Q4 — 24GB VRAM → 12B fp16 / 48B Q4 — 8GB → 4B fp16 / 16B Q4
+
+---
+
+## Step 2: Find relevant benchmark datasets
+
+Fetch the full list of official HF benchmarks:
+
+```bash
+curl -s -H "Authorization: Bearer $(cat ~/.cache/huggingface/token)" \
+  "https://huggingface.co/api/datasets?filter=benchmark:official&limit=500" | jq '[.[] | {id, tags, description}]'
+```
+
+Read the returned list and select the datasets most relevant to the user's task — match on dataset id, tags, and description. Use your judgment; don't limit yourself to 2-3. Aim for comprehensive coverage: if 5 benchmarks clearly cover the task, use all 5.
+
+---
+
+## Step 3: Fetch top models from leaderboards
+
+For each selected benchmark dataset:
+
+```bash
+curl -s -H "Authorization: Bearer $(cat ~/.cache/huggingface/token)" \
+  "https://huggingface.co/api/datasets/<namespace>/<repo>/leaderboard" | jq '[.[:15] | .[] | {rank, modelId, value, verified}]'
+```
+
+Collect model IDs and scores across all benchmarks. If a leaderboard returns an error (404, 401, etc.), skip it and note it in the output.
+
+---
+
+## Step 4: Enrich with model metadata
+
+For the top 10-15 candidate model IDs, get model infos.
+
+```bash
+# REST API
+curl -s -H "Authorization: Bearer $(cat ~/.cache/huggingface/token)" \
+  "https://huggingface.co/api/models/org/model1" | jq '{safetensors, tags, cardData}'
+
+# CLI (hf-cli)
+hf models info org/model1 --json | jq '{safetensors, tags, cardData}'
+```
+
+Extract from each response:
+- **Parameters**: `safetensors.total` → convert to B (e.g., 7_241_748_480 → "7.2B")
+- **License**: from model card tags (look for `license:apache-2.0`, `license:mit`, etc.)
+- If `safetensors` is absent, parse size from the model name (look for "7b", "8b", "13b", "70b", "72b", etc.)
+
+---
+
+## Step 5: Filter and rank
+
+**If a device was specified:**
+1. Remove models exceeding the fp16 parameter budget for the device
+2. Flag models that fit only with Q4 quantization (multiply budget by ~4 for Q4 capacity)
+3. If a highly-ranked model is slightly over budget, keep it with a "needs Q4" note — don't silently drop it
+
+**If no device was mentioned:** skip all size filtering — just rank by benchmark score.
+
+Then: rank by benchmark score (descending), keep top 5-8 models.
+
+Include proprietary models (GPT-4, Claude, Gemini) if they appear on leaderboards, but flag them as "API only / not self-hostable". If the user explicitly asked for local/open models only, exclude them.
+
+---
+
+## Step 6: Output
+
+### Comparison table
+
+```markdown
+| # | Model | Params | [Benchmark 1] | [Benchmark 2] | License | On device |
+|---|-------|--------|--------------|--------------|---------|-----------|
+| ⭐1 | [org/name](https://huggingface.co/org/name) | 7B | 85.2% | — | Apache 2.0 | Yes (fp16) |
+| 2 | [org/name](https://huggingface.co/org/name) | 13B | 83.1% | 71.5% | MIT | Q4 only |
+| 3 | [org/name](https://huggingface.co/org/name) | 70B | 90.0% | 81.0% | Llama | Too large |
+```
+
+- Link model names to `https://huggingface.co/<model_id>`
+- Use `—` for benchmarks where the model wasn't evaluated
+- Star the top recommended pick with ⭐
+- "On device" values: `Yes (fp16)`, `Q4 only`, `Too large`, `API only`
+
+### Follow-up
+
+After presenting the table, ask the user: "Would you like to run **[top recommended model]**?"
+
+If they say yes, ask whether they'd prefer to:
+- **Run locally** — ask about their device if not already known, then give appropriate setup instructions
+- **Run on HF Jobs** — point them to the HF Jobs guide: https://huggingface.co/docs/huggingface_hub/en/guides/jobs
+
+---
+
+## Error handling
+
+- **Leaderboard not found**: skip, note "leaderboard unavailable" in output
+- **Model missing from hub_repo_details**: fall back to parsing size from model name
+- **No benchmarks found for task**: use the curated fallback table above, or try `hub_repo_search` with `filters=["<task>"]` sorted by `trendingScore`
+- **All leaderboards fail**: fall back to `hub_repo_search` for popular models tagged with the task, note that results are by popularity rather than benchmark score

package/.agents/skills/huggingface-community-evals/SKILL.md

@@ -0,0 +1,207 @@
+---
+name: huggingface-community-evals
+description: Run evaluations for Hugging Face Hub models using inspect-ai and lighteval on local hardware. Use for backend selection, local GPU evals, and choosing between vLLM / Transformers / accelerate. Not for HF Jobs orchestration, model-card PRs, .eval_results publication, or community-evals automation.
+---
+
+# Overview
+
+This skill is for **running evaluations against models on the Hugging Face Hub on local hardware**.
+
+It covers:
+- `inspect-ai` with local inference
+- `lighteval` with local inference
+- choosing between `vllm`, Hugging Face Transformers, and `accelerate`
+- smoke tests, task selection, and backend fallback strategy
+
+It does **not** cover:
+- Hugging Face Jobs orchestration
+- model-card or `model-index` edits
+- README table extraction
+- Artificial Analysis imports
+- `.eval_results` generation or publishing
+- PR creation or community-evals automation
+
+If the user wants to **run the same eval remotely on Hugging Face Jobs**, hand off to the `hugging-face-jobs` skill and pass it one of the local scripts in this skill.
+
+If the user wants to **publish results into the community evals workflow**, stop after generating the evaluation run and hand off that publishing step to `~/code/community-evals`.
+
+> All paths below are relative to the directory containing this `SKILL.md`.
+
+# When To Use Which Script
+
+| Use case | Script |
+|---|---|
+| Local `inspect-ai` eval on a Hub model via inference providers | `scripts/inspect_eval_uv.py` |
+| Local GPU eval with `inspect-ai` using `vllm` or Transformers | `scripts/inspect_vllm_uv.py` |
+| Local GPU eval with `lighteval` using `vllm` or `accelerate` | `scripts/lighteval_vllm_uv.py` |
+| Extra command patterns | `examples/USAGE_EXAMPLES.md` |
+
+# Prerequisites
+
+- Prefer `uv run` for local execution.
+- Set `HF_TOKEN` for gated/private models.
+- For local GPU runs, verify GPU access before starting:
+
+```bash
+uv --version
+printenv HF_TOKEN >/dev/null
+nvidia-smi
+```
+
+If `nvidia-smi` is unavailable, either:
+- use `scripts/inspect_eval_uv.py` for lighter provider-backed evaluation, or
+- hand off to the `hugging-face-jobs` skill if the user wants remote compute.
+
+# Core Workflow
+
+1. Choose the evaluation framework.
+   - Use `inspect-ai` when you want explicit task control and inspect-native flows.
+   - Use `lighteval` when the benchmark is naturally expressed as a lighteval task string, especially leaderboard-style tasks.
+2. Choose the inference backend.
+   - Prefer `vllm` for throughput on supported architectures.
+   - Use Hugging Face Transformers (`--backend hf`) or `accelerate` as compatibility fallbacks.
+3. Start with a smoke test.
+   - `inspect-ai`: add `--limit 10` or similar.
+   - `lighteval`: add `--max-samples 10`.
+4. Scale up only after the smoke test passes.
+5. If the user wants remote execution, hand off to `hugging-face-jobs` with the same script + args.
+
+# Quick Start
+
+## Option A: inspect-ai with local inference providers path
+
+Best when the model is already supported by Hugging Face Inference Providers and you want the lowest local setup overhead.
+
+```bash
+uv run scripts/inspect_eval_uv.py \
+  --model meta-llama/Llama-3.2-1B \
+  --task mmlu \
+  --limit 20
+```
+
+Use this path when:
+- you want a quick local smoke test
+- you do not need direct GPU control
+- the task already exists in `inspect-evals`
+
+## Option B: inspect-ai on Local GPU
+
+Best when you need to load the Hub model directly, use `vllm`, or fall back to Transformers for unsupported architectures.
+
+Local GPU:
+
+```bash
+uv run scripts/inspect_vllm_uv.py \
+  --model meta-llama/Llama-3.2-1B \
+  --task gsm8k \
+  --limit 20
+```
+
+Transformers fallback:
+
+```bash
+uv run scripts/inspect_vllm_uv.py \
+  --model microsoft/phi-2 \
+  --task mmlu \
+  --backend hf \
+  --trust-remote-code \
+  --limit 20
+```
+
+## Option C: lighteval on Local GPU
+
+Best when the task is naturally expressed as a `lighteval` task string, especially Open LLM Leaderboard style benchmarks.
+
+Local GPU:
+
+```bash
+uv run scripts/lighteval_vllm_uv.py \
+  --model meta-llama/Llama-3.2-3B-Instruct \
+  --tasks "leaderboard|mmlu|5,leaderboard|gsm8k|5" \
+  --max-samples 20 \
+  --use-chat-template
+```
+
+`accelerate` fallback:
+
+```bash
+uv run scripts/lighteval_vllm_uv.py \
+  --model microsoft/phi-2 \
+  --tasks "leaderboard|mmlu|5" \
+  --backend accelerate \
+  --trust-remote-code \
+  --max-samples 20
+```
+
+# Remote Execution Boundary
+
+This skill intentionally stops at **local execution and backend selection**.
+
+If the user wants to:
+- run these scripts on Hugging Face Jobs
+- pick remote hardware
+- pass secrets to remote jobs
+- schedule recurring runs
+- inspect / cancel / monitor jobs
+
+then switch to the **`hugging-face-jobs`** skill and pass it one of these scripts plus the chosen arguments.
+
+# Task Selection
+
+`inspect-ai` examples:
+- `mmlu`
+- `gsm8k`
+- `hellaswag`
+- `arc_challenge`
+- `truthfulqa`
+- `winogrande`
+- `humaneval`
+
+`lighteval` task strings use `suite|task|num_fewshot`:
+- `leaderboard|mmlu|5`
+- `leaderboard|gsm8k|5`
+- `leaderboard|arc_challenge|25`
+- `lighteval|hellaswag|0`
+
+Multiple `lighteval` tasks can be comma-separated in `--tasks`.
+
+# Backend Selection
+
+- Prefer `inspect_vllm_uv.py --backend vllm` for fast GPU inference on supported architectures.
+- Use `inspect_vllm_uv.py --backend hf` when `vllm` does not support the model.
+- Prefer `lighteval_vllm_uv.py --backend vllm` for throughput on supported models.
+- Use `lighteval_vllm_uv.py --backend accelerate` as the compatibility fallback.
+- Use `inspect_eval_uv.py` when Inference Providers already cover the model and you do not need direct GPU control.
+
+# Hardware Guidance
+
+| Model size | Suggested local hardware |
+|---|---|
+| `< 3B` | consumer GPU / Apple Silicon / small dev GPU |
+| `3B - 13B` | stronger local GPU |
+| `13B+` | high-memory local GPU or hand off to `hugging-face-jobs` |
+
+For smoke tests, prefer cheaper local runs plus `--limit` or `--max-samples`.
+
+# Troubleshooting
+
+- CUDA or vLLM OOM:
+  - reduce `--batch-size`
+  - reduce `--gpu-memory-utilization`
+  - switch to a smaller model for the smoke test
+  - if necessary, hand off to `hugging-face-jobs`
+- Model unsupported by `vllm`:
+  - switch to `--backend hf` for `inspect-ai`
+  - switch to `--backend accelerate` for `lighteval`
+- Gated/private repo access fails:
+  - verify `HF_TOKEN`
+- Custom model code required:
+  - add `--trust-remote-code`
+
+# Examples
+
+See:
+- `examples/USAGE_EXAMPLES.md` for local command patterns
+- `scripts/inspect_eval_uv.py`
+- `scripts/inspect_vllm_uv.py`
+- `scripts/lighteval_vllm_uv.py`

package/.agents/skills/huggingface-community-evals/examples/.env.example

@@ -0,0 +1,3 @@
+# Hugging Face Token (required for gated/private models)
+# Get your token at: https://huggingface.co/settings/tokens
+HF_TOKEN=hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxx

package/.agents/skills/huggingface-community-evals/examples/USAGE_EXAMPLES.md

@@ -0,0 +1,101 @@
+# Usage Examples
+
+This document provides practical examples for **running evaluations locally** against Hugging Face Hub models.
+
+## What this skill covers
+
+- `inspect-ai` local runs
+- `inspect-ai` with `vllm` or Transformers backends
+- `lighteval` local runs with `vllm` or `accelerate`
+- smoke tests and backend fallback patterns
+
+## What this skill does NOT cover
+
+- `model-index`
+- `.eval_results`
+- community eval publication workflows
+- model-card PR creation
+- Hugging Face Jobs orchestration
+
+If you want to run these same scripts remotely, use the `hugging-face-jobs` skill and pass one of the scripts in `scripts/`.
+
+## Setup
+
+```bash
+cd skills/hugging-face-evaluation
+export HF_TOKEN=hf_xxx
+uv --version
+```
+
+For local GPU runs:
+
+```bash
+nvidia-smi
+```
+
+## inspect-ai examples
+
+### Quick smoke test
+
+```bash
+uv run scripts/inspect_eval_uv.py \
+  --model meta-llama/Llama-3.2-1B \
+  --task mmlu \
+  --limit 10
+```
+
+### Local GPU with vLLM
+
+```bash
+uv run scripts/inspect_vllm_uv.py \
+  --model meta-llama/Llama-3.2-8B-Instruct \
+  --task gsm8k \
+  --limit 20
+```
+
+### Transformers fallback
+
+```bash
+uv run scripts/inspect_vllm_uv.py \
+  --model microsoft/phi-2 \
+  --task mmlu \
+  --backend hf \
+  --trust-remote-code \
+  --limit 20
+```
+
+## lighteval examples
+
+### Single task
+
+```bash
+uv run scripts/lighteval_vllm_uv.py \
+  --model meta-llama/Llama-3.2-3B-Instruct \
+  --tasks "leaderboard|mmlu|5" \
+  --max-samples 20
+```
+
+### Multiple tasks
+
+```bash
+uv run scripts/lighteval_vllm_uv.py \
+  --model meta-llama/Llama-3.2-3B-Instruct \
+  --tasks "leaderboard|mmlu|5,leaderboard|gsm8k|5" \
+  --max-samples 20 \
+  --use-chat-template
+```
+
+### accelerate fallback
+
+```bash
+uv run scripts/lighteval_vllm_uv.py \
+  --model microsoft/phi-2 \
+  --tasks "leaderboard|mmlu|5" \
+  --backend accelerate \
+  --trust-remote-code \
+  --max-samples 20
+```
+
+## Hand-off to Hugging Face Jobs
+
+When local hardware is not enough, switch to the `hugging-face-jobs` skill and run one of these scripts remotely. Keep the script path and args; move the orchestration there.