tribunal-kit 9.2.1 → 9.2.4

package/skills/huggingface-community-evals/scripts/inspect_eval_uv.py

@@ -0,0 +1,104 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "inspect-ai>=0.3.0",
+#     "inspect-evals",
+#     "openai",
+# ]
+# ///
+
+"""
+Entry point script for running inspect-ai evaluations against Hugging Face inference providers.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import subprocess
+import sys
+from pathlib import Path
+from typing import Optional
+
+
+def _inspect_evals_tasks_root() -> Optional[Path]:
+    """Return the installed inspect_evals package path if available."""
+    try:
+        import inspect_evals
+
+        return Path(inspect_evals.__file__).parent
+    except Exception:
+        return None
+
+
+def _normalize_task(task: str) -> str:
+    """Allow lighteval-style `suite|task|shots` strings by keeping the task name."""
+    if "|" in task:
+        parts = task.split("|")
+        if len(parts) >= 2 and parts[1]:
+            return parts[1]
+    return task
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Inspect-ai job runner")
+    parser.add_argument("--model", required=True, help="Model ID on Hugging Face Hub")
+    parser.add_argument("--task", required=True, help="inspect-ai task to execute")
+    parser.add_argument("--limit", type=int, default=None, help="Limit number of samples to evaluate")
+    parser.add_argument(
+        "--tasks-root",
+        default=None,
+        help="Optional path to inspect task files. Defaults to the installed inspect_evals package.",
+    )
+    parser.add_argument(
+        "--sandbox",
+        default="local",
+        help="Sandbox backend to use (default: local for HF jobs without Docker).",
+    )
+    args = parser.parse_args()
+
+    # Ensure downstream libraries can read the token passed as a secret
+    hf_token = os.getenv("HF_TOKEN")
+    if hf_token:
+        os.environ.setdefault("HUGGING_FACE_HUB_TOKEN", hf_token)
+        os.environ.setdefault("HF_HUB_TOKEN", hf_token)
+
+    task = _normalize_task(args.task)
+    tasks_root = Path(args.tasks_root) if args.tasks_root else _inspect_evals_tasks_root()
+    if tasks_root and not tasks_root.exists():
+        tasks_root = None
+
+    cmd = [
+        "inspect",
+        "eval",
+        task,
+        "--model",
+        f"hf-inference-providers/{args.model}",
+        "--log-level",
+        "info",
+        # Reduce batch size to avoid OOM errors (default is 32)
+        "--max-connections",
+        "1",
+        # Set a small positive temperature (HF doesn't allow temperature=0)
+        "--temperature",
+        "0.001",
+    ]
+
+    if args.sandbox:
+        cmd.extend(["--sandbox", args.sandbox])
+
+    if args.limit:
+        cmd.extend(["--limit", str(args.limit)])
+
+    try:
+        subprocess.run(cmd, check=True, cwd=tasks_root)
+        print("Evaluation complete.")
+    except subprocess.CalledProcessError as exc:
+        location = f" (cwd={tasks_root})" if tasks_root else ""
+        print(f"Evaluation failed with exit code {exc.returncode}{location}", file=sys.stderr)
+        raise
+
+
+if __name__ == "__main__":
+    main()
+

package/skills/huggingface-community-evals/scripts/inspect_vllm_uv.py

@@ -0,0 +1,306 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "inspect-ai>=0.3.0",
+#     "inspect-evals",
+#     "vllm>=0.4.0",
+#     "torch>=2.0.0",
+#     "transformers>=4.40.0",
+# ]
+# ///
+
+"""
+Entry point script for running inspect-ai evaluations with vLLM or HuggingFace Transformers backend.
+
+This script runs evaluations on custom HuggingFace models using local GPU inference,
+separate from inference provider scripts (which use external APIs).
+
+Usage (standalone):
+    uv run scripts/inspect_vllm_uv.py --model "meta-llama/Llama-3.2-1B" --task "mmlu"
+
+Model backends:
+    - vllm: Fast inference with vLLM (recommended for large models)
+    - hf: HuggingFace Transformers backend (broader model compatibility)
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import subprocess
+import sys
+from typing import Optional
+
+
+def setup_environment() -> None:
+    """Configure environment variables for HuggingFace authentication."""
+    hf_token = os.getenv("HF_TOKEN")
+    if hf_token:
+        os.environ.setdefault("HUGGING_FACE_HUB_TOKEN", hf_token)
+        os.environ.setdefault("HF_HUB_TOKEN", hf_token)
+
+
+def run_inspect_vllm(
+    model_id: str,
+    task: str,
+    limit: Optional[int] = None,
+    max_connections: int = 4,
+    temperature: float = 0.0,
+    tensor_parallel_size: int = 1,
+    gpu_memory_utilization: float = 0.8,
+    dtype: str = "auto",
+    trust_remote_code: bool = False,
+    log_level: str = "info",
+) -> None:
+    """
+    Run inspect-ai evaluation with vLLM backend.
+
+    Args:
+        model_id: HuggingFace model ID
+        task: inspect-ai task to execute (e.g., "mmlu", "gsm8k")
+        limit: Limit number of samples to evaluate
+        max_connections: Maximum concurrent connections
+        temperature: Sampling temperature
+        tensor_parallel_size: Number of GPUs for tensor parallelism
+        gpu_memory_utilization: GPU memory fraction
+        dtype: Data type (auto, float16, bfloat16)
+        trust_remote_code: Allow remote code execution
+        log_level: Logging level
+    """
+    setup_environment()
+
+    model_spec = f"vllm/{model_id}"
+    cmd = [
+        "inspect",
+        "eval",
+        task,
+        "--model",
+        model_spec,
+        "--log-level",
+        log_level,
+        "--max-connections",
+        str(max_connections),
+    ]
+
+    # vLLM supports temperature=0 unlike HF inference providers
+    cmd.extend(["--temperature", str(temperature)])
+
+    # Older inspect-ai CLI versions do not support --model-args; rely on defaults
+    # and let vLLM choose sensible settings for small models.
+    if tensor_parallel_size != 1:
+        cmd.extend(["--tensor-parallel-size", str(tensor_parallel_size)])
+    if gpu_memory_utilization != 0.8:
+        cmd.extend(["--gpu-memory-utilization", str(gpu_memory_utilization)])
+    if dtype != "auto":
+        cmd.extend(["--dtype", dtype])
+    if trust_remote_code:
+        cmd.append("--trust-remote-code")
+
+    if limit:
+        cmd.extend(["--limit", str(limit)])
+
+    print(f"Running: {' '.join(cmd)}")
+
+    try:
+        subprocess.run(cmd, check=True)
+        print("Evaluation complete.")
+    except subprocess.CalledProcessError as exc:
+        print(f"Evaluation failed with exit code {exc.returncode}", file=sys.stderr)
+        sys.exit(exc.returncode)
+
+
+def run_inspect_hf(
+    model_id: str,
+    task: str,
+    limit: Optional[int] = None,
+    max_connections: int = 1,
+    temperature: float = 0.001,
+    device: str = "auto",
+    dtype: str = "auto",
+    trust_remote_code: bool = False,
+    log_level: str = "info",
+) -> None:
+    """
+    Run inspect-ai evaluation with HuggingFace Transformers backend.
+
+    Use this when vLLM doesn't support the model architecture.
+
+    Args:
+        model_id: HuggingFace model ID
+        task: inspect-ai task to execute
+        limit: Limit number of samples
+        max_connections: Maximum concurrent connections (keep low for memory)
+        temperature: Sampling temperature
+        device: Device to use (auto, cuda, cpu)
+        dtype: Data type
+        trust_remote_code: Allow remote code execution
+        log_level: Logging level
+    """
+    setup_environment()
+
+    model_spec = f"hf/{model_id}"
+
+    cmd = [
+        "inspect",
+        "eval",
+        task,
+        "--model",
+        model_spec,
+        "--log-level",
+        log_level,
+        "--max-connections",
+        str(max_connections),
+        "--temperature",
+        str(temperature),
+    ]
+
+    if device != "auto":
+        cmd.extend(["--device", device])
+    if dtype != "auto":
+        cmd.extend(["--dtype", dtype])
+    if trust_remote_code:
+        cmd.append("--trust-remote-code")
+
+    if limit:
+        cmd.extend(["--limit", str(limit)])
+
+    print(f"Running: {' '.join(cmd)}")
+
+    try:
+        subprocess.run(cmd, check=True)
+        print("Evaluation complete.")
+    except subprocess.CalledProcessError as exc:
+        print(f"Evaluation failed with exit code {exc.returncode}", file=sys.stderr)
+        sys.exit(exc.returncode)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Run inspect-ai evaluations with vLLM or HuggingFace Transformers on custom models",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Run MMLU with vLLM backend
+  uv run scripts/inspect_vllm_uv.py --model meta-llama/Llama-3.2-1B --task mmlu
+
+  # Run with HuggingFace Transformers backend
+  uv run scripts/inspect_vllm_uv.py --model meta-llama/Llama-3.2-1B --task mmlu --backend hf
+
+  # Run with limited samples for testing
+  uv run scripts/inspect_vllm_uv.py --model meta-llama/Llama-3.2-1B --task mmlu --limit 10
+
+  # Run on multiple GPUs with tensor parallelism
+  uv run scripts/inspect_vllm_uv.py --model meta-llama/Llama-3.2-70B --task mmlu --tensor-parallel-size 4
+
+Available tasks (from inspect-evals):
+  - mmlu: Massive Multitask Language Understanding
+  - gsm8k: Grade School Math
+  - hellaswag: Common sense reasoning
+  - arc_challenge: AI2 Reasoning Challenge
+  - truthfulqa: TruthfulQA benchmark
+  - winogrande: Winograd Schema Challenge
+  - humaneval: Code generation (HumanEval)
+
+        """,
+    )
+
+    parser.add_argument(
+        "--model",
+        required=True,
+        help="HuggingFace model ID (e.g., meta-llama/Llama-3.2-1B)",
+    )
+    parser.add_argument(
+        "--task",
+        required=True,
+        help="inspect-ai task to execute (e.g., mmlu, gsm8k)",
+    )
+    parser.add_argument(
+        "--backend",
+        choices=["vllm", "hf"],
+        default="vllm",
+        help="Model backend (default: vllm)",
+    )
+    parser.add_argument(
+        "--limit",
+        type=int,
+        default=None,
+        help="Limit number of samples to evaluate",
+    )
+    parser.add_argument(
+        "--max-connections",
+        type=int,
+        default=None,
+        help="Maximum concurrent connections (default: 4 for vllm, 1 for hf)",
+    )
+    parser.add_argument(
+        "--temperature",
+        type=float,
+        default=None,
+        help="Sampling temperature (default: 0.0 for vllm, 0.001 for hf)",
+    )
+    parser.add_argument(
+        "--tensor-parallel-size",
+        type=int,
+        default=1,
+        help="Number of GPUs for tensor parallelism (vLLM only, default: 1)",
+    )
+    parser.add_argument(
+        "--gpu-memory-utilization",
+        type=float,
+        default=0.8,
+        help="GPU memory fraction to use (vLLM only, default: 0.8)",
+    )
+    parser.add_argument(
+        "--dtype",
+        default="auto",
+        choices=["auto", "float16", "bfloat16", "float32"],
+        help="Data type for model weights (default: auto)",
+    )
+    parser.add_argument(
+        "--device",
+        default="auto",
+        help="Device for HF backend (auto, cuda, cpu)",
+    )
+    parser.add_argument(
+        "--trust-remote-code",
+        action="store_true",
+        help="Allow executing remote code from model repository",
+    )
+    parser.add_argument(
+        "--log-level",
+        default="info",
+        choices=["debug", "info", "warning", "error"],
+        help="Logging level (default: info)",
+    )
+
+    args = parser.parse_args()
+
+    if args.backend == "vllm":
+        run_inspect_vllm(
+            model_id=args.model,
+            task=args.task,
+            limit=args.limit,
+            max_connections=args.max_connections or 4,
+            temperature=args.temperature if args.temperature is not None else 0.0,
+            tensor_parallel_size=args.tensor_parallel_size,
+            gpu_memory_utilization=args.gpu_memory_utilization,
+            dtype=args.dtype,
+            trust_remote_code=args.trust_remote_code,
+            log_level=args.log_level,
+        )
+    else:
+        run_inspect_hf(
+            model_id=args.model,
+            task=args.task,
+            limit=args.limit,
+            max_connections=args.max_connections or 1,
+            temperature=args.temperature if args.temperature is not None else 0.001,
+            device=args.device,
+            dtype=args.dtype,
+            trust_remote_code=args.trust_remote_code,
+            log_level=args.log_level,
+        )
+
+
+if __name__ == "__main__":
+    main()

package/skills/huggingface-community-evals/scripts/lighteval_vllm_uv.py

@@ -0,0 +1,297 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "lighteval[accelerate,vllm]>=0.6.0",
+#     "torch>=2.0.0",
+#     "transformers>=4.40.0",
+#     "accelerate>=0.30.0",
+#     "vllm>=0.4.0",
+# ]
+# ///
+
+"""
+Entry point script for running lighteval evaluations with local GPU backends.
+
+This script runs evaluations using vLLM or accelerate on custom HuggingFace models.
+It is separate from inference provider scripts and evaluates models directly on local hardware.
+
+Usage (standalone):
+    uv run scripts/lighteval_vllm_uv.py --model "meta-llama/Llama-3.2-1B" --tasks "leaderboard|mmlu|5"
+
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import subprocess
+import sys
+from typing import Optional
+
+
+def setup_environment() -> None:
+    """Configure environment variables for HuggingFace authentication."""
+    hf_token = os.getenv("HF_TOKEN")
+    if hf_token:
+        os.environ.setdefault("HUGGING_FACE_HUB_TOKEN", hf_token)
+        os.environ.setdefault("HF_HUB_TOKEN", hf_token)
+
+
+def run_lighteval_vllm(
+    model_id: str,
+    tasks: str,
+    output_dir: Optional[str] = None,
+    max_samples: Optional[int] = None,
+    batch_size: int = 1,
+    tensor_parallel_size: int = 1,
+    gpu_memory_utilization: float = 0.8,
+    dtype: str = "auto",
+    trust_remote_code: bool = False,
+    use_chat_template: bool = False,
+    system_prompt: Optional[str] = None,
+) -> None:
+    """
+    Run lighteval with vLLM backend for efficient GPU inference.
+
+    Args:
+        model_id: HuggingFace model ID (e.g., "meta-llama/Llama-3.2-1B")
+        tasks: Task specification (e.g., "leaderboard|mmlu|5" or "lighteval|hellaswag|0")
+        output_dir: Directory for evaluation results
+        max_samples: Limit number of samples per task
+        batch_size: Batch size for evaluation
+        tensor_parallel_size: Number of GPUs for tensor parallelism
+        gpu_memory_utilization: GPU memory fraction to use (0.0-1.0)
+        dtype: Data type for model weights (auto, float16, bfloat16)
+        trust_remote_code: Allow executing remote code from model repo
+        use_chat_template: Apply chat template for conversational models
+        system_prompt: System prompt for chat models
+    """
+    setup_environment()
+
+    # Build lighteval vllm command
+    cmd = [
+        "lighteval",
+        "vllm",
+        model_id,
+        tasks,
+        "--batch-size", str(batch_size),
+        "--tensor-parallel-size", str(tensor_parallel_size),
+        "--gpu-memory-utilization", str(gpu_memory_utilization),
+        "--dtype", dtype,
+    ]
+
+    if output_dir:
+        cmd.extend(["--output-dir", output_dir])
+
+    if max_samples:
+        cmd.extend(["--max-samples", str(max_samples)])
+
+    if trust_remote_code:
+        cmd.append("--trust-remote-code")
+
+    if use_chat_template:
+        cmd.append("--use-chat-template")
+
+    if system_prompt:
+        cmd.extend(["--system-prompt", system_prompt])
+
+    print(f"Running: {' '.join(cmd)}")
+
+    try:
+        subprocess.run(cmd, check=True)
+        print("Evaluation complete.")
+    except subprocess.CalledProcessError as exc:
+        print(f"Evaluation failed with exit code {exc.returncode}", file=sys.stderr)
+        sys.exit(exc.returncode)
+
+
+def run_lighteval_accelerate(
+    model_id: str,
+    tasks: str,
+    output_dir: Optional[str] = None,
+    max_samples: Optional[int] = None,
+    batch_size: int = 1,
+    dtype: str = "bfloat16",
+    trust_remote_code: bool = False,
+    use_chat_template: bool = False,
+    system_prompt: Optional[str] = None,
+) -> None:
+    """
+    Run lighteval with accelerate backend for multi-GPU distributed inference.
+
+    Use this backend when vLLM is not available or for models not supported by vLLM.
+
+    Args:
+        model_id: HuggingFace model ID
+        tasks: Task specification
+        output_dir: Directory for evaluation results
+        max_samples: Limit number of samples per task
+        batch_size: Batch size for evaluation
+        dtype: Data type for model weights
+        trust_remote_code: Allow executing remote code
+        use_chat_template: Apply chat template
+        system_prompt: System prompt for chat models
+    """
+    setup_environment()
+
+    # Build lighteval accelerate command
+    cmd = [
+        "lighteval",
+        "accelerate",
+        model_id,
+        tasks,
+        "--batch-size", str(batch_size),
+        "--dtype", dtype,
+    ]
+
+    if output_dir:
+        cmd.extend(["--output-dir", output_dir])
+
+    if max_samples:
+        cmd.extend(["--max-samples", str(max_samples)])
+
+    if trust_remote_code:
+        cmd.append("--trust-remote-code")
+
+    if use_chat_template:
+        cmd.append("--use-chat-template")
+
+    if system_prompt:
+        cmd.extend(["--system-prompt", system_prompt])
+
+    print(f"Running: {' '.join(cmd)}")
+
+    try:
+        subprocess.run(cmd, check=True)
+        print("Evaluation complete.")
+    except subprocess.CalledProcessError as exc:
+        print(f"Evaluation failed with exit code {exc.returncode}", file=sys.stderr)
+        sys.exit(exc.returncode)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Run lighteval evaluations with vLLM or accelerate backend on custom HuggingFace models",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Run MMLU evaluation with vLLM
+  uv run scripts/lighteval_vllm_uv.py --model meta-llama/Llama-3.2-1B --tasks "leaderboard|mmlu|5"
+
+  # Run with accelerate backend instead of vLLM
+  uv run scripts/lighteval_vllm_uv.py --model meta-llama/Llama-3.2-1B --tasks "leaderboard|mmlu|5" --backend accelerate
+
+  # Run with chat template for instruction-tuned models
+  uv run scripts/lighteval_vllm_uv.py --model meta-llama/Llama-3.2-1B-Instruct --tasks "leaderboard|mmlu|5" --use-chat-template
+
+  # Run with limited samples for testing
+  uv run scripts/lighteval_vllm_uv.py --model meta-llama/Llama-3.2-1B --tasks "leaderboard|mmlu|5" --max-samples 10
+
+Task format:
+  Tasks use the format: "suite|task|num_fewshot"
+  - leaderboard|mmlu|5 (MMLU with 5-shot)
+  - lighteval|hellaswag|0 (HellaSwag zero-shot)
+  - leaderboard|gsm8k|5 (GSM8K with 5-shot)
+  - Multiple tasks: "leaderboard|mmlu|5,leaderboard|gsm8k|5"
+        """,
+    )
+
+    parser.add_argument(
+        "--model",
+        required=True,
+        help="HuggingFace model ID (e.g., meta-llama/Llama-3.2-1B)",
+    )
+    parser.add_argument(
+        "--tasks",
+        required=True,
+        help="Task specification (e.g., 'leaderboard|mmlu|5')",
+    )
+    parser.add_argument(
+        "--backend",
+        choices=["vllm", "accelerate"],
+        default="vllm",
+        help="Inference backend to use (default: vllm)",
+    )
+    parser.add_argument(
+        "--output-dir",
+        default=None,
+        help="Directory for evaluation results",
+    )
+    parser.add_argument(
+        "--max-samples",
+        type=int,
+        default=None,
+        help="Limit number of samples per task (useful for testing)",
+    )
+    parser.add_argument(
+        "--batch-size",
+        type=int,
+        default=1,
+        help="Batch size for evaluation (default: 1)",
+    )
+    parser.add_argument(
+        "--tensor-parallel-size",
+        type=int,
+        default=1,
+        help="Number of GPUs for tensor parallelism (vLLM only, default: 1)",
+    )
+    parser.add_argument(
+        "--gpu-memory-utilization",
+        type=float,
+        default=0.8,
+        help="GPU memory fraction to use (vLLM only, default: 0.8)",
+    )
+    parser.add_argument(
+        "--dtype",
+        default="auto",
+        choices=["auto", "float16", "bfloat16", "float32"],
+        help="Data type for model weights (default: auto)",
+    )
+    parser.add_argument(
+        "--trust-remote-code",
+        action="store_true",
+        help="Allow executing remote code from model repository",
+    )
+    parser.add_argument(
+        "--use-chat-template",
+        action="store_true",
+        help="Apply chat template for instruction-tuned/chat models",
+    )
+    parser.add_argument(
+        "--system-prompt",
+        default=None,
+        help="System prompt for chat models",
+    )
+
+    args = parser.parse_args()
+
+    if args.backend == "vllm":
+        run_lighteval_vllm(
+            model_id=args.model,
+            tasks=args.tasks,
+            output_dir=args.output_dir,
+            max_samples=args.max_samples,
+            batch_size=args.batch_size,
+            tensor_parallel_size=args.tensor_parallel_size,
+            gpu_memory_utilization=args.gpu_memory_utilization,
+            dtype=args.dtype,
+            trust_remote_code=args.trust_remote_code,
+            use_chat_template=args.use_chat_template,
+            system_prompt=args.system_prompt,
+        )
+    else:
+        run_lighteval_accelerate(
+            model_id=args.model,
+            tasks=args.tasks,
+            output_dir=args.output_dir,
+            max_samples=args.max_samples,
+            batch_size=args.batch_size,
+            dtype=args.dtype if args.dtype != "auto" else "bfloat16",
+            trust_remote_code=args.trust_remote_code,
+            use_chat_template=args.use_chat_template,
+            system_prompt=args.system_prompt,
+        )
+
+
+if __name__ == "__main__":
+    main()