crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/model_deployment_registry.py

@@ -4,7 +4,7 @@ from dataclasses import dataclass
 import cattrs
 import yaml
 
-from helm.common.hierarchical_logger import hlog
+from helm.common.hierarchical_logger import hlog, hwarn
 from helm.common.object_spec import ObjectSpec
 from helm.benchmark.model_metadata_registry import (
     ModelMetadata,
@@ -104,9 +104,7 @@ def register_model_deployment(model_deployment: ModelDeployment) -> None:
     try:
         model_metadata = get_model_metadata(model_name)
     except ValueError:
-        hlog(
-            f"WARNING: Could not find model metadata for model {model_name} of model deployment {model_deployment.name}"
-        )
+        hwarn(f"Could not find model metadata for model {model_name} of model deployment {model_deployment.name}")
         model_metadata = get_unknown_model_metadata(model_name)
         register_model_metadata(model_metadata)
     deployment_names: List[str] = model_metadata.deployment_names or [model_metadata.name]
@@ -130,7 +128,7 @@ def get_model_deployment(name: str, warn_deprecated: bool = False) -> ModelDeplo
         raise ValueError(f"Model deployment {name} not found")
     deployment: ModelDeployment = DEPLOYMENT_NAME_TO_MODEL_DEPLOYMENT[name]
     if deployment.deprecated and warn_deprecated:
-        hlog(f"WARNING: DEPLOYMENT Model deployment {name} is deprecated")
+        hwarn(f"DEPLOYMENT Model deployment {name} is deprecated")
     return deployment
 
 
@@ -148,3 +146,69 @@ def get_model_names_with_tokenizer(tokenizer_name: str) -> List[str]:
         deployment for deployment in ALL_MODEL_DEPLOYMENTS if deployment.tokenizer_name == tokenizer_name
     ]
     return [deployment.model_name or deployment.name for deployment in deployments]
+
+
+def get_default_model_deployment_for_model(
+    model_name: str, warn_arg_deprecated: bool = False, ignore_deprecated: bool = False
+) -> Optional[str]:
+    """Returns a valid model deployment name corresponding to the given model arg.
+    This is used as a backwards compatibility layer for model names that are now moved to model deployments.
+    Example: "anthropic/claude-v1.3" => "anthropic/claude-v1.3"
+    Example: "meta/llama-7b" => "together/llama-7b"
+
+    The process to find a model deployment name is as follows:
+    1. If there is at least one deployment for the model, use the last one that is available.
+    2. If there are no deployments for the model, returns None.
+
+    This function will also try to find a model deployment name that is not deprecated.
+    If there are no non-deprecated deployments, it will return the last deployment (even if it's deprecated).
+    If ignore_deprecated is True, this function will return None if the model deployment is deprecated.
+
+    If warn_arg_deprecated is True, this function will print a warning if the model deployment name is not the same
+    as the model arg. This is to remind the user that the model name is deprecated and should be replaced with
+    the model deployment name (in their config).
+
+    Args:
+        model_arg: The model arg to convert to a model deployment name.
+        warn_arg_deprecated: Whether to print a warning if the model deployment name is not the same as the model arg.
+        ignore_deprecated: Whether to return None if the model deployment is deprecated.
+    """
+
+    # If there is at least one deployment for the model, use the last one that is available.
+    available_deployments: List[ModelDeployment] = [
+        deployment for deployment in ALL_MODEL_DEPLOYMENTS if deployment.model_name == model_name
+    ]
+    if len(available_deployments) > 0:
+        available_deployment_names: List[str] = [deployment.name for deployment in available_deployments]
+        if warn_arg_deprecated:
+            hwarn("Model name is deprecated. Please use the model deployment name instead.")
+            hlog(f"Available model deployments for model {model_name}: {available_deployment_names}")
+
+        # Additionally, if there is a non-deprecated deployment, use it.
+        non_deprecated_deployments: List[ModelDeployment] = [
+            deployment for deployment in available_deployments if not deployment.deprecated
+        ]
+        if len(non_deprecated_deployments) > 0:
+            chosen_deployment = non_deprecated_deployments[-1]
+        # There are no non-deprecated deployments, so there are two options:
+        # 1. If we can return an empty string, return it. (no model deployment is available)
+        # 2. If we can't return an empty string, return the last deployment (even if it's deprecated).
+        elif ignore_deprecated:
+            return None
+        elif len(available_deployments) > 0:
+            chosen_deployment = available_deployments[-1]
+            if warn_arg_deprecated:
+                hwarn(f"All model deployments for model {model_name} are deprecated.")
+        else:
+            return None
+        if warn_arg_deprecated:
+            hlog(
+                f"Choosing {chosen_deployment.name} (the last one) as "
+                f"the default model deployment for model {model_name}"
+            )
+            hlog("If you want to use a different model deployment, please specify it explicitly.")
+        return chosen_deployment.name
+
+    # Some models are added but have no deployments yet.
+    # In this case, we return None.
+    return None

helm/benchmark/model_metadata_registry.py

@@ -22,11 +22,18 @@ CHATML_MODEL_TAG: str = "CHATML_MODEL_TAG"
 # OpenAI Chat format
 OPENAI_CHATGPT_MODEL_TAG: str = "OPENAI_CHATGPT_MODEL_TAG"
 
+# For NOVA models
+NOVA_MODEL_TAG: str = "NOVA_MODEL_TAG"
+
 # For Anthropic models
 ANTHROPIC_CLAUDE_1_MODEL_TAG: str = "ANTHROPIC_CLAUDE_1_MODEL_TAG"
 ANTHROPIC_CLAUDE_2_MODEL_TAG: str = "ANTHROPIC_CLAUDE_2_MODEL_TAG"
+ANTHROPIC_CLAUDE_3_MODEL_TAG: str = "ANTHROPIC_CLAUDE_3_MODEL_TAG"
 
 GOOGLE_PALM_2_MODEL_TAG: str = "GOOGLE_PALM_2_MODEL_TAG"
+GOOGLE_GEMINI_MODEL_TAG: str = "GOOGLE_GEMINI_MODEL_TAG"
+GOOGLE_GEMINI_PRO_VISION_V1_TAG: str = "GOOGLE_GEMINI_PRO_VISION_V1_TAG"
+GOOGLE_GEMMA_INSTRUCT_MODEL_TAG: str = "GOOGLE_GEMMA_INSTRUCT_MODEL_TAG"
 
 # Models which emit garbage tokens when temperature=0.
 BUGGY_TEMP_0_TAG: str = "BUGGY_TEMP_0_TAG"
@@ -46,12 +53,38 @@ NLG_PREFIX_TAG: str = "NLG_PREFIX_TAG"
 # Some models can follow instructions.
 INSTRUCTION_FOLLOWING_MODEL_TAG: str = "INSTRUCTION_FOLLOWING_MODEL_TAG"
 
+# For text-to-image models
+TEXT_TO_IMAGE_MODEL_TAG: str = "TEXT_TO_IMAGE_MODEL_TAG"
+
 # For Vision-langauge models (VLMs)
 VISION_LANGUAGE_MODEL_TAG: str = "VISION_LANGUAGE_MODEL_TAG"
-
+# IDEFICS require a special prompt format (see `IDEFICSInstructRunExpander`)
+IDEFICS_INSTRUCT_MODEL_TAG: str = "IDEFICS_INSTRUCT_MODEL_TAG"
+IDEFICS_MODEL_TAG: str = "IDEFICS_MODEL_TAG"
+# Llava should use a special prompt format (see `LlavaRunExpander`)
+LLAVA_MODEL_TAG: str = "LLAVA_MODEL_TAG"
+# OpenFlamingo has a special prompt format (see `OpenFlamingoRunExpander`)
+OPEN_FLAMINGO_MODEL_TAG: str = "OPEN_FLAMINGO_MODEL_TAG"
+# Some VLMs do not support multiple images in the prompt
+LIMITED_FUNCTIONALITY_VLM_TAG: str = "LIMITED_FUNCTIONALITY_VLM_TAG"
+FULL_FUNCTIONALITY_VLM_TAG: str = "FULL_FUNCTIONALITY_VLM_TAG"
+
+# For Audio-langauge models (AudioLMs)
+AUDIO_LANGUAGE_MODEL_TAG: str = "AUDIO_LANGUAGE_MODEL_TAG"
+
+# Deprecated models that are no longer available.
+# These are usually closed API models that have been permanently removed
+DEPRECATED_MODEL_TAG: str = "DEPRECATED_MODEL_TAG"
+
+# Unsupported models.
+# These are models that we have chosen not to support because they are
+# private, stale, non-notable, or difficult to implement.
+UNSUPPORTED_MODEL_TAG: str = "UNSUPPORTED_MODEL_TAG"
 
 # Frozen is set to false as the model_deployment_registry.py file
 # might populate the deployment_names field.
+
+
 @dataclass(frozen=False)
 class ModelMetadata:
     name: str
@@ -138,7 +171,10 @@ def register_model_metadata(model_metadata: ModelMetadata) -> None:
 def get_model_metadata(model_name: str) -> ModelMetadata:
     """Return the `ModelMetadata` for the model name."""
     if model_name not in MODEL_NAME_TO_MODEL_METADATA:
-        raise ValueError(f"No model with name: {model_name}")
+        raise ValueError(
+            f"No model metadata for model name: {model_name} - "
+            "did you remember to add this model to model_metadata.yaml?"
+        )
 
     return MODEL_NAME_TO_MODEL_METADATA[model_name]
 
@@ -153,6 +189,11 @@ def get_model_names_with_tag(tag: str) -> List[str]:
     return [model.name for model in ALL_MODELS_METADATA if tag in model.tags]
 
 
+def model_has_tag(model_name: str, tag: str) -> bool:
+    """Return True if the model has the given tag. False otherwise."""
+    return tag in get_model_metadata(model_name).tags
+
+
 def get_all_text_models() -> List[str]:
     """Return all model names of text models."""
     return get_model_names_with_tag(TEXT_MODEL_TAG)
@@ -168,6 +209,21 @@ def get_all_instruction_following_models() -> List[str]:
     return get_model_names_with_tag(INSTRUCTION_FOLLOWING_MODEL_TAG)
 
 
+def is_text_to_image_model(model_name: str) -> bool:
+    """Returns True if the model is a text-to-image model. False otherwise."""
+    return model_has_tag(model_name, TEXT_TO_IMAGE_MODEL_TAG)
+
+
+def is_vlm(model_name: str) -> bool:
+    """Returns True if the model is a vision-language model (VLM). False otherwise."""
+    return model_has_tag(model_name, VISION_LANGUAGE_MODEL_TAG)
+
+
+def is_audiolm(model_name: str) -> bool:
+    """Returns True if the model is a audio-language model (AudioLM). False otherwise."""
+    return model_has_tag(model_name, AUDIO_LANGUAGE_MODEL_TAG)
+
+
 def get_unknown_model_metadata(helm_model_name: str) -> ModelMetadata:
     """Return placeholder ModelMetadata for an unknown model."""
     return ModelMetadata(

helm/benchmark/multi_gpu_runner.py

@@ -0,0 +1,133 @@
+import signal
+import threading
+import traceback
+from typing import List
+import os
+import time
+import torch
+import torch.multiprocessing as multiprocessing
+from concurrent.futures import ProcessPoolExecutor as Pool
+from tqdm import tqdm
+
+from helm.benchmark.config_registry import (
+    register_configs_from_directory,
+    register_builtin_configs_from_helm_package,
+)
+from helm.benchmark.executor import ExecutionSpec
+from helm.benchmark.runner import Runner, RunSpec, RunnerError
+from helm.common.hierarchical_logger import hlog, htrack_block
+from helm.benchmark.runner_config_registry import RUNNER_CONFIG
+
+_MAX_CONCURRENT_WORKERS_ENV_NAME = "HELM_MAX_CONCURRENT_WORKERS"
+
+
+# From
+# https://stackoverflow.com/questions/71300294/how-to-terminate-pythons-processpoolexecutor-when-parent-process-dies
+def start_thread_to_terminate_when_parent_process_dies(ppid):
+    pid = os.getpid()
+
+    def f():
+        while True:
+            try:
+                os.kill(ppid, 0)
+            except OSError:
+                os.kill(pid, signal.SIGTERM)
+            time.sleep(1)
+
+    thread = threading.Thread(target=f, daemon=True)
+    thread.start()
+
+
+def initialize_worker(gpu_id: int):
+    hlog(f"Worker {gpu_id} initializing")
+
+    # Wait for 0.1 seconds to ensure all workers are initialized with different CUDA_VISIBLE_DEVICES
+    time.sleep(0.1)
+
+    # Pin GPU to worker process
+    os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu_id)
+
+    # Necessary for code_metrics in humaneval to work properly
+    multiprocessing.set_start_method("fork", force=True)
+
+
+class MultiGPURunner(Runner):
+    """Runner that runs the entire benchmark on multiple GPUs.
+
+    This is a thin wrapper around `Runner` that runs the entire benchmark on
+    multiple GPUs using `multiprocessing`.
+
+    Note that this runner will load multiple models into memory at the same
+    time if your running configuration specifies that, similar to the `Runner`
+    class. `SlurmRunner` on the other hand will load at most one model on a
+    GPU"""
+
+    def __init__(
+        self,
+        execution_spec: ExecutionSpec,
+        output_path: str,
+        suite: str,
+        skip_instances: bool,
+        cache_instances: bool,
+        cache_instances_only: bool,
+        skip_completed_runs: bool,
+        exit_on_error: bool,
+    ):
+        super().__init__(
+            execution_spec=execution_spec,
+            output_path=output_path,
+            suite=suite,
+            skip_instances=skip_instances,
+            cache_instances=cache_instances,
+            cache_instances_only=cache_instances_only,
+            skip_completed_runs=skip_completed_runs,
+            exit_on_error=exit_on_error,
+        )
+        # Configure max concurrent worker jobs from the environment variable.
+        env_max_concurrent_workers = os.getenv(_MAX_CONCURRENT_WORKERS_ENV_NAME)
+        self.max_concurrent_workers = (
+            int(env_max_concurrent_workers)
+            if env_max_concurrent_workers
+            else (
+                RUNNER_CONFIG.helm_max_concurrent_workers
+                if RUNNER_CONFIG.helm_max_concurrent_workers > 0
+                else torch.cuda.device_count()
+            )
+        )
+
+    def safe_run_one(self, run_spec: RunSpec):
+        register_builtin_configs_from_helm_package()
+        if self.executor.execution_spec.local_path is not None:
+            register_configs_from_directory(self.executor.execution_spec.local_path)
+
+        try:
+            with htrack_block(f"Running {run_spec.name}"):
+                self.run_one(run_spec)
+        except Exception as e:
+            hlog(f"Error when running {run_spec.name}:\n{traceback.format_exc()}")
+            return e
+
+    def run_all(self, run_specs: List[RunSpec]):
+        """Run the entire benchmark on multiple GPU"""
+
+        # Set the start method to forkserver to avoid issues with CUDA.
+        multiprocessing.set_start_method("forkserver")
+
+        with Pool(
+            max_workers=self.max_concurrent_workers,
+            initializer=start_thread_to_terminate_when_parent_process_dies,
+            initargs=(os.getpid(),),
+        ) as pool:
+            # Pin GPUs to each worker process
+            pool.map(initialize_worker, [i for i in range(self.max_concurrent_workers)])
+
+            # Run all queued tasks
+            error_msgs = list(tqdm(pool.map(self.safe_run_one, run_specs), total=len(run_specs), disable=None))
+
+        # Raise exception for failed runs, if any.
+        failed_run_names = [
+            run_spec.name for error_msg, run_spec in zip(error_msgs, run_specs) if error_msg is not None
+        ]
+        if failed_run_names:
+            failed_runs_str = ", ".join([f'"{run_name}"' for run_name in failed_run_names])
+            raise RunnerError(f"Failed runs: [{failed_runs_str}]")

helm/benchmark/presentation/contamination.py

@@ -4,7 +4,7 @@ import dacite
 import importlib_resources as resources
 import yaml
 
-from helm.common.hierarchical_logger import htrack, hlog
+from helm.common.hierarchical_logger import htrack, hlog, hwarn
 from helm.benchmark.model_metadata_registry import MODEL_NAME_TO_MODEL_METADATA
 from helm.benchmark.presentation.schema import Schema
 
@@ -71,10 +71,10 @@ def validate_contamination(contamination: Contamination, schema: Schema):
     for point in contamination.points:
         for model in point.models:
             if model not in MODEL_NAME_TO_MODEL_METADATA:
-                hlog(f"WARNING: model {model} not defined in schema")
+                hwarn(f"model {model} not defined in schema")
         for group in point.groups:
             if group not in schema.name_to_run_group:
-                hlog(f"WARNING: group {group} not defined in schema")
+                hwarn(f"group {group} not defined in schema")
 
 
 def read_contamination():

helm/benchmark/presentation/create_plots.py

@@ -1,4 +1,7 @@
-# mypy: check_untyped_defs = False
+# type: ignore
+# flake8: noqa
+# fmt: off
+
 import argparse
 from collections import defaultdict
 from dataclasses import dataclass
@@ -10,10 +13,10 @@ from typing import List, Dict, Optional, Any, Callable, Union, Mapping, Tuple, S
 import numpy as np
 from scipy.stats import pearsonr
 
-from helm.common.hierarchical_logger import hlog
+from helm.benchmark.config_registry import register_builtin_configs_from_helm_package
+from helm.common.hierarchical_logger import hlog, setup_default_logging
 from helm.common.optional_dependencies import handle_module_not_found_error
-from helm.benchmark.presentation.schema import read_schema, SCHEMA_CLASSIC_YAML_FILENAME
-from helm.benchmark.presentation.summarize import AGGREGATE_WIN_RATE_COLUMN
+from helm.benchmark.model_metadata_registry import MODEL_NAME_TO_MODEL_METADATA
 
 try:
     import colorcet
@@ -38,6 +41,7 @@ metric_group_to_label = {
     "Efficiency": f"Inference time (s) {DOWN_ARROW}",
 }
 all_metric_groups = list(metric_group_to_label.keys())
+AGGREGATE_WIN_RATE_COLUMN = 1
 
 
 @dataclass
@@ -133,9 +137,6 @@ class Plotter:
         self.plot_format = plot_format
         self._tables_cache: Dict[str, Dict[str, Table]] = {}
 
-        schema = read_schema(SCHEMA_CLASSIC_YAML_FILENAME)
-        self.model_metadata = {model_field.display_name: model_field for model_field in schema.models}
-
     def get_group_tables(self, group_name: str) -> Dict[str, Table]:
         """Reads and parses group tables. Uses _tables_cache to avoid reprocessing the same table multiple times."""
         if group_name in self._tables_cache:
@@ -338,14 +339,14 @@ class Plotter:
 
         def get_model_release_date(model_name: str) -> Optional[date]:
             """Maps a model name to the month of model release."""
-            release_date = self.model_metadata[model_name].release_date
+            release_date = MODEL_NAME_TO_MODEL_METADATA[model_name].release_date
             if release_date is None:
                 return None
             return release_date.replace(day=1)
 
         def get_model_size(model_name: str) -> Optional[int]:
             """Maps a model name to the number of parameters, rounding to the nearest leading digit."""
-            size = self.model_metadata[model_name].num_parameters
+            size = MODEL_NAME_TO_MODEL_METADATA[model_name].num_parameters
             if size is None:
                 return None
             grain = 10 ** (len(str(size)) - 1)
@@ -401,7 +402,9 @@ class Plotter:
 
         for i, access_level in enumerate(access_levels):
             model_indices: List[int] = [
-                idx for idx, model in enumerate(table.adapters) if self.model_metadata[model].access == access_level
+                idx
+                for idx, model in enumerate(table.adapters)
+                if MODEL_NAME_TO_MODEL_METADATA[model].access == access_level
             ]
             best_model_index = model_indices[table.mean_win_rates[model_indices].argmax()]
 
@@ -600,6 +603,17 @@ class Plotter:
         self.create_constrast_set_plots()
 
 
+def create_plots(args):
+    register_builtin_configs_from_helm_package()
+    base_path = os.path.join(args.output_path, "runs", args.suite)
+    if not os.path.exists(os.path.join(base_path, "groups")):
+        hlog(f"ERROR: Could not find `groups` directory under {base_path}. Did you run `summarize.py` first?")
+        return
+    save_path = os.path.join(base_path, "plots")
+    plotter = Plotter(base_path=base_path, save_path=save_path, plot_format=args.plot_format)
+    plotter.create_all_plots()
+
+
 def main():
     """
     This script creates the plots used in the HELM paper (https://arxiv.org/abs/2211.09110).
@@ -607,17 +621,34 @@ def main():
     the top-level command `helm-create-plots`.
     """
     parser = argparse.ArgumentParser()
-    parser.add_argument("-o", "--output-path", type=str, help="Path to benchmarking output", default="benchmark_output")
-    parser.add_argument("--suite", type=str, help="Name of the suite that we are plotting", required=True)
-    parser.add_argument("--plot-format", help="Format for saving plots", default="png", choices=["png", "pdf"])
+    parser.add_argument(
+        "-o",
+        "--output-path",
+        type=str,
+        help="Path to benchmarking output",
+        default="benchmark_output",
+    )
+    parser.add_argument(
+        "--suite",
+        type=str,
+        help="Name of the suite that we are plotting",
+        required=True,
+    )
+    parser.add_argument(
+        "--plot-format",
+        help="Format for saving plots",
+        default="png",
+        choices=["png", "pdf"],
+    )
+    parser.add_argument(
+        "--log-config",
+        type=str,
+        default=None,
+        help="PATH to a YAML file to customize logging",
+    )
     args = parser.parse_args()
-    base_path = os.path.join(args.output_path, "runs", args.suite)
-    if not os.path.exists(os.path.join(base_path, "groups")):
-        hlog(f"ERROR: Could not find `groups` directory under {base_path}. Did you run `summarize.py` first?")
-        return
-    save_path = os.path.join(base_path, "plots")
-    plotter = Plotter(base_path=base_path, save_path=save_path, plot_format=args.plot_format)
-    plotter.create_all_plots()
+    setup_default_logging(args.log_config)
+    create_plots(args)
 
 
 if __name__ == "__main__":

helm/benchmark/presentation/run_display.py

@@ -1,9 +1,10 @@
 from collections import OrderedDict, defaultdict
 from dataclasses import dataclass
 import os
-from typing import Dict, Iterable, List, Optional, Set, Tuple
+import re
+from typing import Dict, Iterable, List, Optional, Set, Tuple, Any
 
-from helm.benchmark.adaptation.adapters.adapter_factory import (
+from helm.benchmark.adaptation.adapter_spec import (
     ADAPT_MULTIPLE_CHOICE_SEPARATE_METHODS,
     ADAPT_MULTIPLE_CHOICE_SEPARATE_CALIBRATED,
 )
@@ -12,11 +13,13 @@ from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.adaptation.scenario_state import ScenarioState
 from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
 from helm.benchmark.metrics.metric import PerInstanceStats
+from helm.common.multimodal_request_utils import gather_generated_image_locations
 from helm.benchmark.presentation.schema import Schema
-from helm.benchmark.runner import RunSpec
+from helm.benchmark.run_spec import RunSpec
 from helm.benchmark.scenarios.scenario import Instance
 from helm.common.general import write
 from helm.common.hierarchical_logger import hlog, htrack
+from helm.common.images_utils import encode_base64
 from helm.common.request import Request
 from helm.common.codec import from_json, to_json
 
@@ -43,6 +46,9 @@ class DisplayPrediction:
     truncated_predicted_text: Optional[str]
     """The truncated prediction text, if truncation is required by the Adapter method."""
 
+    base64_images: Optional[List[str]]
+    """Images in base64."""
+
     mapped_output: Optional[str]
     """The mapped output, if an output mapping exists and the prediction can be mapped"""
 
@@ -52,6 +58,11 @@ class DisplayPrediction:
     stats: Dict[str, float]
     """Statistics computed from the predicted output"""
 
+    annotations: Optional[Dict[str, Any]]
+
+    thinking_text: Optional[str]
+    """Thinking text from thinking models."""
+
 
 @dataclass(frozen=True)
 class DisplayRequest:
@@ -73,7 +84,7 @@ class DisplayRequest:
     """The actual Request to display in the web frontend.
 
     There can be multiple requests per trial. The displayed request should be the
-    most relevant request e.g. the request for the chosen cohice for multiple choice questions."""
+    most relevant request e.g. the request for the chosen choice for multiple choice questions."""
 
 
 def _read_scenario_state(scenario_state_path: str) -> ScenarioState:
@@ -102,8 +113,7 @@ def _truncate_predicted_text(
             tokens = request_state.result.completions[0].tokens
             if tokens:
                 first_token = tokens[0]
-                if not first_token.top_logprobs:
-                    prefix = first_token.text
+                prefix = first_token.text
     if prefix:
         predicted_text = predicted_text
         prefix = prefix
@@ -126,7 +136,7 @@ def _get_metric_names_for_group(run_group_name: str, schema: Schema) -> Set[str]
         if metric_group is None:
             continue
         for metric_name_matcher in metric_group.metrics:
-            if metric_name_matcher.perturbation_name:
+            if metric_name_matcher.perturbation_name and metric_name_matcher.perturbation_name != "__all__":
                 continue
             result.add(metric_name_matcher.substitute(run_group.environment).name)
     return result
@@ -253,12 +263,38 @@ def write_run_display_json(run_path: str, run_spec: RunSpec, schema: Schema, ski
             if request_state.result is not None and request_state.result.completions
             else ""
         )
-        mapped_output = (
-            request_state.output_mapping.get(predicted_text.strip()) if request_state.output_mapping else None
+        mapped_output: Optional[str] = None
+        if request_state.output_mapping is not None:
+            output_to_map = predicted_text.strip()
+            if run_spec.adapter_spec.output_mapping_pattern:
+                match = re.search(run_spec.adapter_spec.output_mapping_pattern, output_to_map)
+                if not match:
+                    output_to_map = ""
+                elif match.groups():
+                    output_to_map = match.group(0)
+                else:
+                    output_to_map = match.string
+            mapped_output = request_state.output_mapping.get(output_to_map)
+        instance_id_to_instance[(request_state.instance.id, request_state.instance.perturbation)] = (
+            request_state.instance
+        )
+
+        if request_state.result.completions[0].multimodal_content:
+            additional_prediction: str = request_state.result.completions[0].multimodal_content.text
+            if additional_prediction:
+                predicted_text = f"{additional_prediction} {predicted_text}"
+
+        # Process images and include if they exist
+        images: List[str] = [
+            encode_base64(image_location)
+            for image_location in gather_generated_image_locations(request_state.result)
+            if os.path.exists(image_location)
+        ]
+
+        thinking_text: Optional[str] = (
+            request_state.result.completions[0].thinking.text if request_state.result.completions[0].thinking else None
         )
-        instance_id_to_instance[
-            (request_state.instance.id, request_state.instance.perturbation)
-        ] = request_state.instance
+
         predictions.append(
             DisplayPrediction(
                 instance_id=request_state.instance.id,
@@ -266,9 +302,12 @@ def write_run_display_json(run_path: str, run_spec: RunSpec, schema: Schema, ski
                 train_trial_index=request_state.train_trial_index,
                 predicted_text=predicted_text,
                 truncated_predicted_text=_truncate_predicted_text(predicted_text, request_state, run_spec.adapter_spec),
+                base64_images=images,
                 mapped_output=mapped_output,
                 reference_index=request_state.reference_index,
                 stats=trial_stats,
+                annotations=request_state.annotations,
+                thinking_text=thinking_text,
             )
         )
         requests.append(

helm/benchmark/presentation/run_entry.py

@@ -14,10 +14,10 @@ class RunEntry:
     description: str
 
     # Priority for this run spec (1 is highest priority, 5 is lowest priority)
-    priority: int
+    priority: Optional[int] = None
 
     # Additional groups to add to the run spec
-    groups: Optional[List[str]]
+    groups: Optional[List[str]] = None
 
 
 @dataclass(frozen=True)