crfm-helm 0.5.0__py3-none-any.whl → 0.5.2__py3-none-any.whl

helm/benchmark/metrics/prometheus_vision_critique_metrics.py

@@ -0,0 +1,185 @@
+from typing import Dict, List, Optional
+import re
+
+from tqdm import tqdm
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.adaptation.scenario_state import ScenarioState
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.metrics.metric import MetricInterface, MetricResult, PerInstanceStats, add_context
+from helm.benchmark.metrics.metric_name import MetricContext, MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat, merge_stat
+from helm.common.critique_request import CritiqueTaskTemplate, CritiqueQuestionTemplate, CritiqueRequest, QuestionType
+from helm.common.hierarchical_logger import hlog
+from helm.common.request import RequestResult, GeneratedOutput
+from helm.common.media_object import MultimediaObject, IMAGE_TYPE, MediaObject, TEXT_TYPE
+
+
+class PrometheusVisionCritiqueMetric(MetricInterface):
+    """
+    We compute the same metrics from the Prometheus-Vision: Vision-Language Model as a Judge for
+    Fine-Grained Evaluation paper:
+    https://arxiv.org/pdf/2401.06591.pdf
+
+    In this paper, the output of a Vision-Language Model named Prometheus-Vision is used to evaluate
+    the quality of the output of other Vision-Language Models to be evaluated.
+    """
+
+    # We can add more evaluation aspects here
+    METRIC_NAME: str = "prometheus_vision"
+    METRIC_PROMPT: str = """A chat between a curious human and an artificial intelligence assistant. \
+        The assistant gives helpful, detailed, and polite answers to the human's questions.\nUSER:<image>\
+###Task Description:
+An instruction (might include an Input inside it), a response to evaluate, a reference answer that gets a score of 5, \
+    image and a score rubric representing an evaluation criterion is given.
+1. Write a detailed feedback that assesses the quality of the response strictly based on the given score rubric, not \
+    evaluating in general.
+2. After writing a feedback, write a score that is an integer between 1 and 5. You should refer to the score rubric.
+3. The output format should look as follows: Feedback: (write a feedback for criteria) [RESULT] (an integer number \
+    between 1 and 5)
+4. Please do not generate any other opening, closing, and explanations.
+
+###The instruction to evaluate:
+{{orig_instruction}}
+
+###Response to evaluate:
+{{orig_response}}
+
+###Reference Answer (Score 5):
+{{orig_reference_answer}}
+
+###Score Rubrics:
+[{{orig_criteria}}]
+Score 1: {{orig_score1_description}}
+Score 2: {{orig_score2_description}}
+Score 3: {{orig_score3_description}}
+Score 4: {{orig_score4_description}}
+Score 5: {{orig_score5_description}}
+
+###Feedback:
+ASSISTANT:
+"""
+
+    def __init__(self, num_respondents: int, max_tokens: int):
+        self._num_respondents = num_respondents
+        self._max_tokens = max_tokens
+
+    def __repr__(self) -> str:
+        return "PrometheusVisionCritiqueMetric()"
+
+    def _extract_score_from_prometheus_vision_output(self, evaluator_response: str):
+        evaluator_response = evaluator_response.split("ASSISTANT:")[1]
+        re_match = re.search(r"\s*([1-5])", evaluator_response)
+        if re_match is None:
+            hlog(f"Error parsing answer: {evaluator_response}. Skipping question (and so the respondent entirely)")
+            return None
+        return int(re_match.group(1))
+
+    def evaluate(
+        self,
+        scenario_state: ScenarioState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+        parallelism: int,
+    ) -> MetricResult:
+        request_states: List[RequestState] = scenario_state.request_states
+
+        all_stats: Dict[MetricName, Stat] = {}
+        per_instance_stats: List[PerInstanceStats] = []
+        for request_state in tqdm(request_states):
+            context = MetricContext.from_instance(request_state.instance)
+            stats_without_context = self.evaluate_generation(
+                scenario_state.adapter_spec,
+                request_state,
+                metric_service,
+                eval_cache_path,
+            )
+            stats = [add_context(stat_without_context, context) for stat_without_context in stats_without_context]
+            for stat in stats:
+                merge_stat(all_stats, stat)
+            assert request_state.instance.id is not None
+            per_instance_stats.append(
+                PerInstanceStats(
+                    instance_id=request_state.instance.id,
+                    perturbation=request_state.instance.perturbation,
+                    train_trial_index=request_state.train_trial_index,
+                    stats=stats,
+                )
+            )
+        return MetricResult(aggregated_stats=list(all_stats.values()), per_instance_stats=per_instance_stats)
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        input_content = request_state.request
+        # Predicted outputs and their prometheus vision scores
+        assert request_state.result is not None
+        request_result: RequestResult = request_state.result
+        # Get input image and generated response for evaluation
+        assert input_content.multimodal_prompt is not None
+        completions: List[GeneratedOutput] = request_result.completions
+        generated_text: str = completions[0].text
+        input_media: MultimediaObject = input_content.multimodal_prompt
+        ref_text: str = request_state.instance.references[0].output.text
+        image_objects: List[MediaObject] = [
+            item for item in input_media.media_objects if item.is_type(IMAGE_TYPE) and item.location
+        ]
+        input_text: Optional[str] = [item for item in input_media.media_objects if item.is_type(TEXT_TYPE)][0].text
+
+        template = CritiqueTaskTemplate(
+            name="vhelm_prometheus_vision",
+            instructions=self.METRIC_PROMPT,
+            num_respondents=self._num_respondents,
+            max_tokens=self._max_tokens,
+            questions=[
+                CritiqueQuestionTemplate(
+                    name=self.METRIC_NAME,
+                    question_type=QuestionType.FREE_RESPONSE,
+                    text="",
+                    options=[],
+                    media_object=image_objects[0],  # we only take the first image as input
+                )
+            ],
+        )
+        request = CritiqueRequest(
+            template=template,
+            fields={
+                "orig_instruction": input_text if input_text is not None else "",
+                "orig_response": generated_text,
+                "orig_reference_answer": ref_text,
+                "orig_criteria": "similarity between the reponse and the reference.",
+                "orig_score1_description": "The model's responses do not follow the instructions provided.",
+                "orig_score2_description": "The resulting response follows the instructions, but the answer \
+                    is completely wrong relative to the reference answer.",
+                "orig_score3_description": "The resulting response follows the instructions, but the answer is \
+                    partially wrong relative to the reference answer.",
+                "orig_score4_description": "The resulting response follows the instructions, the overall answer \
+                    is relatively perfect with only a very few errors.",
+                "orig_score5_description": "The overall answer is completely correct compared to the reference \
+                    answer, and conforms to the instructions provided.",
+            },
+        )
+        # send to critique request
+        result = metric_service.make_critique_request(request)
+        if not result or not result.responses:
+            # Skip computing metrics if there aren't any responses yet
+            hlog("Waiting for responses to be generated.")
+            return []
+
+        stats: Dict[str, Stat] = {}
+        for question in template.questions:
+            stats[question.name] = Stat(MetricName(question.name))
+
+        for response in result.responses:
+            for answer_name, answer in response.answers.items():
+                assert isinstance(answer, str)
+                answer_value: float
+                answer_value = self._extract_score_from_prometheus_vision_output(answer)
+                stats[answer_name].add(answer_value)
+
+        return list(stats.values())

helm/benchmark/metrics/reka_vibe_critique_metrics.py

@@ -0,0 +1,158 @@
+from typing import Dict, List, Optional
+import re
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.adaptation.scenario_state import ScenarioState
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.metrics.metric import MetricInterface, MetricResult, PerInstanceStats, add_context
+from helm.benchmark.metrics.metric_name import MetricContext, MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat, merge_stat
+from helm.common.critique_request import CritiqueTaskTemplate, CritiqueQuestionTemplate, CritiqueRequest, QuestionType
+from helm.common.hierarchical_logger import hlog
+from helm.common.request import RequestResult, GeneratedOutput
+from helm.common.media_object import MultimediaObject, IMAGE_TYPE, TEXT_TYPE, MediaObject
+
+
+class RekaVibeCritiqueMetric(MetricInterface):
+    """
+    Critique evaluation for evaluating the correctness of generated response given the image and
+    reference by Reka-vibe-eval.
+    """
+
+    # We can add more evaluation aspects here
+    VIBE_EVAL_NAME: str = "reka_vibe"
+    REKA_VIBE_PROMPT_WITH_IMAGE: str = """\
+[Question]
+{{prompt}}
+
+[Assistant Response]
+{{generation}}
+
+[Ground Truth Response]
+{{reference}}
+
+[System]
+Rate whether the assistant response correctly matches the ground truth, in regards to the image above.
+The rating should be 1-5, where 1 is incorrect and 5 is correct.
+Your response should be in the format:
+Short Explanation: (explanation in only one sentence)
+Rating: (int)"""
+
+    def __init__(self, num_respondents: int, max_tokens: int):
+        self._num_respondents = num_respondents
+        self._max_tokens = max_tokens
+
+    def __repr__(self) -> str:
+        return "RekaVibeCritiqueMetric()"
+
+    def _extract_score_from_reka_output(self, evaluator_response: str):
+        """
+        Extract the score from the evaluator response. Refer to the official Vibe-Eval implementation:
+        https://github.com/reka-ai/reka-vibe-eval/blob/3852d4712da172a7b85dddeffc4f9c3482a6f4c9/evaluate.py#L159-#L164
+        """
+        re_match = re.search(r"Rating:\s*([1-5])", evaluator_response)
+        if re_match is None:
+            hlog(f"Error parsing answer: {evaluator_response}. Skipping question (and so the respondent entirely)")
+            return None
+        return int(re_match.group(1))
+
+    def evaluate(
+        self,
+        scenario_state: ScenarioState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+        parallelism: int,
+    ) -> MetricResult:
+        request_states: List[RequestState] = scenario_state.request_states
+
+        all_stats: Dict[MetricName, Stat] = {}
+        per_instance_stats: List[PerInstanceStats] = []
+        for request_state in request_states:
+            context = MetricContext.from_instance(request_state.instance)
+            stats_without_context = self.evaluate_generation(
+                scenario_state.adapter_spec,
+                request_state,
+                metric_service,
+                eval_cache_path,
+            )
+            stats = [add_context(stat_without_context, context) for stat_without_context in stats_without_context]
+            for stat in stats:
+                merge_stat(all_stats, stat)
+            assert request_state.instance.id is not None
+            per_instance_stats.append(
+                PerInstanceStats(
+                    instance_id=request_state.instance.id,
+                    perturbation=request_state.instance.perturbation,
+                    train_trial_index=request_state.train_trial_index,
+                    stats=stats,
+                )
+            )
+        return MetricResult(aggregated_stats=list(all_stats.values()), per_instance_stats=per_instance_stats)
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        input_content = request_state.request
+        # Predicted outputs and their originality scores
+        assert request_state.result is not None
+        request_result: RequestResult = request_state.result
+        # Get input image and generated response for the originality evaluation
+        assert input_content.multimodal_prompt is not None
+        completions: List[GeneratedOutput] = request_result.completions
+        generated_text: str = completions[0].text
+        input_media: MultimediaObject = input_content.multimodal_prompt
+        ref_text: str = request_state.instance.references[0].output.text
+
+        image_objects: List[MediaObject] = [
+            item for item in input_media.media_objects if item.is_type(IMAGE_TYPE) and item.location
+        ]
+        input_text: Optional[str] = [item for item in input_media.media_objects if item.is_type(TEXT_TYPE)][0].text
+
+        template = CritiqueTaskTemplate(
+            name="vhelm_vibe_eval",
+            instructions=self.REKA_VIBE_PROMPT_WITH_IMAGE,
+            num_respondents=self._num_respondents,
+            max_tokens=self._max_tokens,
+            questions=[
+                CritiqueQuestionTemplate(
+                    name=self.VIBE_EVAL_NAME,
+                    question_type=QuestionType.FREE_RESPONSE,
+                    text="",
+                    options=[],
+                    media_object=image_objects[0],  # we only take the first image as input
+                )
+            ],
+        )
+
+        request = CritiqueRequest(
+            template=template,
+            fields={
+                "prompt": input_text if input_text is not None else "",
+                "generation": generated_text,
+                "reference": ref_text,
+            },
+        )
+
+        # send to critique request
+        result = metric_service.make_critique_request(request)
+        if not result or not result.responses:
+            # Skip computing metrics if there aren't any responses yet
+            hlog("Waiting for responses to be generated.")
+            return []
+        stats: Dict[str, Stat] = {}
+        for question in template.questions:
+            stats[question.name] = Stat(MetricName(question.name))
+
+        for response in result.responses:
+            for answer_name, answer in response.answers.items():
+                assert isinstance(answer, str)
+                answer_value: float
+                answer_value = self._extract_score_from_reka_output(answer)
+                stats[answer_name].add(answer_value)
+
+        return list(stats.values())

helm/benchmark/metrics/unitxt_metrics.py

@@ -50,16 +50,20 @@ class UnitxtMetric(MetricInterface):
             for metric_name, metric_score in instance_results.items():
                 if metric_name == "score" or metric_name == "score_name":
                     continue
-                instance_stats.append(
-                    Stat(
-                        MetricName(
-                            name=metric_name,
-                            split=instance.split,
-                            sub_split=instance.sub_split,
-                            perturbation=instance.perturbation,
-                        )
-                    ).add(metric_score)
+                stat = Stat(
+                    MetricName(
+                        name=metric_name,
+                        split=instance.split,
+                        sub_split=instance.sub_split,
+                        perturbation=instance.perturbation,
+                    )
                 )
+                if isinstance(metric_score, list):
+                    for metric_score_element in metric_score:
+                        stat = stat.add(metric_score_element)
+                else:
+                    stat = stat.add(metric_score)
+                instance_stats.append(stat)
             assert instance.id
             per_instance_stats.append(
                 PerInstanceStats(
@@ -77,5 +81,11 @@ class UnitxtMetric(MetricInterface):
             for metric_name, metric_score in global_results.items():
                 if metric_name == "score" or metric_name == "score_name":
                     continue
-                aggregated_stats.append(Stat(MetricName(name=metric_name)).add(metric_score))
+                stat = Stat(MetricName(name=metric_name))
+                if isinstance(metric_score, list):
+                    for metric_score_element in metric_score:
+                        stat = stat.add(metric_score_element)
+                else:
+                    stat = stat.add(metric_score)
+                aggregated_stats.append(stat)
         return MetricResult(aggregated_stats=aggregated_stats, per_instance_stats=per_instance_stats)

helm/benchmark/metrics/vision_language/emd_utils.py

@@ -280,6 +280,10 @@ def compute_emd_recursive(
     assert max_num_patches > 0
     assert 0 < weight_most_frequent_color <= 1
 
+    # Convert the images to RGB first. Some images have 4 channels (RGBA)
+    img1_PIL = img1_PIL.convert("RGB")
+    img2_PIL = img2_PIL.convert("RGB")
+
     # Resize the images so that there are not too many patches
     # Try to maintain the aspect ratio and resize to a multiple of the patch size
     num_patches = math.ceil(img1_PIL.size[0] / patch_size[0]) * math.ceil(img1_PIL.size[1] / patch_size[1])

helm/benchmark/metrics/vision_language/image_metrics.py

@@ -28,7 +28,7 @@ from helm.benchmark.metrics.vision_language.image_utils import (
     pixel_similarity,
     sift_similarity,
 )
-from helm.benchmark.metrics.vision_language.emd_utils import compute_emd_recursive
+from helm.benchmark.metrics.vision_language.emd_utils import compute_emd_recursive, get_most_frequent_color, to_gray
 
 try:
     from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity
@@ -78,7 +78,8 @@ class AnnotatedImageMetrics(Metric):
 
     # Metric names
     COMPILE_METRIC: str = "compilation_success"
-    EARTH_MOVER_SIMILARITY: str = "earth_mover_similarity"
+    EARTH_MOVER_SIMILARITY = "earth_mover_similarity"
+    BLOCK_EMD: str = "block_emd"
     PIXEL_SIMILARITY: str = "pixel_similarity"
     SIFT_SIMILARITY: str = "sift_similarity"
     LPIPS_SIMILARITY: str = "lpips_similarity"
@@ -106,7 +107,10 @@ class AnnotatedImageMetrics(Metric):
         metrics: List[AnnotatedMetric] = [
             AnnotatedMetric(self.PIXEL_SIMILARITY, pixel_similarity, "image_np_gray"),
             AnnotatedMetric(self.SIFT_SIMILARITY, sift_similarity, "image_np"),
-            AnnotatedMetric(self.EARTH_MOVER_SIMILARITY, self.compute_emd_similarity_recursive, "image_PIL"),
+            AnnotatedMetric(self.BLOCK_EMD, self.compute_block_emd_raw, "image_PIL"),  # Raw block-EMD
+            AnnotatedMetric(
+                self.EARTH_MOVER_SIMILARITY, self.ems, "image_PIL"
+            ),  # Normalized block-EMD against black/white
             AnnotatedMetric(self.LPIPS_SIMILARITY, self.lpips_similarity, "image_PIL"),
             AnnotatedMetric(self.FID_SIMILARITY, self.fid_similarity, "image_PIL"),
             AnnotatedMetric(self.SSIM_SIMILARITY, self.compute_ssim, "image_np_gray"),
@@ -384,9 +388,15 @@ class AnnotatedImageMetrics(Metric):
         features1 = self._get_inception_features(img1_tensor)
         features2 = self._get_inception_features(img2_tensor)
 
-        fid_score = self._calculate_fid(features1, features2)
-        normalize_fid: float = np.exp(-fid_score * self.NORMALIZE_FID_FACTOR)
-        return normalize_fid
+        # TODO: Justify the value of the constant here or remove this code to only keep the cosine similarity.
+        # fid_score = self._calculate_fid(features1, features2)
+        # normalize_fid: float = np.exp(-fid_score * self.NORMALIZE_FID_FACTOR)
+        # return normalize_fid
+
+        # Use the cosine similarity between the features as a proxy for FID
+        # Return a score between 0 and 1, where 1 is the most similar
+        score = 0.5 * (1 + np.dot(features1[0], features2[0]) / (np.linalg.norm(features1) * np.linalg.norm(features2)))
+        return score
 
     def compute_ssim(self, generated_image: np.ndarray, reference_image: np.ndarray) -> float:
         """Compute the Structural Similarity Index (SSIM) between the generated and reference images."""
@@ -407,7 +417,7 @@ class AnnotatedImageMetrics(Metric):
         result = _edit_similarity(completion_tokens, truncated_reference_tokens)
         return result
 
-    def compute_emd_similarity_recursive(
+    def ems(
         self,
         pred_image: Image.Image,
         ref_image: Image.Image,
@@ -417,18 +427,31 @@ class AnnotatedImageMetrics(Metric):
         weight_most_frequent_color: float = 0.001,
         use_tqdm: bool = False,
     ):
-        emd_value = compute_emd_recursive(
-            pred_image,
-            ref_image,
-            threshold_most_frequent_color,
-            patch_size,
-            max_num_patches,
-            weight_most_frequent_color,
-            use_tqdm,
-        )
+        """Same as compute_emd_similarity_recursive EXCEPT that
+        the normalization is against an image of the median color.
+        """
 
-        def do_it():
-            constant_image = Image.new("RGB", ref_image.size, (255, 255, 255))  # default color is white
+        def compute_numerator():
+            return self.compute_block_emd_raw_wrapper(
+                pred_image,
+                ref_image,
+                threshold_most_frequent_color,
+                patch_size,
+                max_num_patches,
+                weight_most_frequent_color,
+                use_tqdm,
+            )
+
+        def compute_denominator():
+            ref_img_np = np.array(ref_image)
+            (rgb_most_frequent_color, _) = get_most_frequent_color(ref_img_np)
+            grayscale_most_frequent_color = to_gray(rgb_most_frequent_color)[0]
+
+            # Most frequent color as base
+            if grayscale_most_frequent_color < 127:
+                constant_image = Image.new("RGB", ref_image.size, (255, 255, 255))  # Make it white
+            else:
+                constant_image = Image.new("RGB", ref_image.size, (0, 0, 0))  # Make it black
             value = compute_emd_recursive(
                 constant_image,
                 ref_image,
@@ -442,9 +465,69 @@ class AnnotatedImageMetrics(Metric):
 
         hash_dict = {
             "reference_image": str(AnnotatedImageMetrics.HASH_FUNC(ref_image, hash_size=self.HASH_LENGTH)),
+            "generated_image": str(AnnotatedImageMetrics.HASH_FUNC(pred_image, hash_size=self.HASH_LENGTH)),
+        }
+        cache_key_numerator = {"metric_name": f"intermediate_{self.BLOCK_EMD}", **hash_dict}
+        cache_key_denominator = {"metric_name": "intermediate_ems_extreme_denominator", **hash_dict}
+
+        assert self._cache is not None
+        emd_raw, _ = self._cache.get(cache_key_numerator, compute_numerator)
+        emd_base, _ = self._cache.get(cache_key_denominator, compute_denominator)
+
+        return 1.0 - emd_raw["value"] / emd_base["value"]
+
+    def compute_block_emd_raw(
+        self,
+        pred_image: Image.Image,
+        ref_image: Image.Image,
+        threshold_most_frequent_color: float = 0.5,
+        patch_size: Tuple[int, int] = (8, 8),
+        max_num_patches: int = 100,
+        weight_most_frequent_color: float = 0.001,
+        use_tqdm: bool = False,
+    ):
+        def compute():
+            return self.compute_block_emd_raw_wrapper(
+                pred_image,
+                ref_image,
+                threshold_most_frequent_color,
+                patch_size,
+                max_num_patches,
+                weight_most_frequent_color,
+                use_tqdm,
+            )
+
+        hash_dict = {
+            "reference_image": str(AnnotatedImageMetrics.HASH_FUNC(ref_image, hash_size=self.HASH_LENGTH)),
+            "generated_image": str(AnnotatedImageMetrics.HASH_FUNC(pred_image, hash_size=self.HASH_LENGTH)),
         }
-        cache_key = {"metric_name": f"intermediate_{self.EARTH_MOVER_SIMILARITY}", **hash_dict}
+        cache_key = {"metric_name": f"intermediate_{self.BLOCK_EMD}", **hash_dict}
         assert self._cache is not None
-        response_metric, _ = self._cache.get(cache_key, do_it)
+        emd_raw, _ = self._cache.get(cache_key, compute)
 
-        return 1.0 - emd_value / response_metric["value"]
+        return emd_raw["value"]
+
+    def compute_block_emd_raw_wrapper(
+        self,
+        pred_image: Image.Image,
+        ref_image: Image.Image,
+        threshold_most_frequent_color: float = 0.5,
+        patch_size: Tuple[int, int] = (8, 8),
+        max_num_patches: int = 100,
+        weight_most_frequent_color: float = 0.001,
+        use_tqdm: bool = False,
+    ):
+        """Computes the block Earth Moving Distance (EMD). This attempts to
+        speed up EMD for images with huge areas by considering
+        movement/transformation of blocks of pixels.
+        """
+        emd_value = compute_emd_recursive(
+            pred_image,
+            ref_image,
+            threshold_most_frequent_color,
+            patch_size,
+            max_num_patches,
+            weight_most_frequent_color,
+            use_tqdm,
+        )
+        return {"value": emd_value}

helm/benchmark/model_metadata_registry.py

@@ -32,6 +32,7 @@ ANTHROPIC_CLAUDE_3_MODEL_TAG: str = "ANTHROPIC_CLAUDE_3_MODEL_TAG"
 
 GOOGLE_PALM_2_MODEL_TAG: str = "GOOGLE_PALM_2_MODEL_TAG"
 GOOGLE_GEMINI_MODEL_TAG: str = "GOOGLE_GEMINI_MODEL_TAG"
+GOOGLE_GEMINI_PRO_VISION_V1_TAG: str = "GOOGLE_GEMINI_PRO_VISION_V1_TAG"
 GOOGLE_GEMMA_INSTRUCT_MODEL_TAG: str = "GOOGLE_GEMMA_INSTRUCT_MODEL_TAG"
 
 # Models which emit garbage tokens when temperature=0.
@@ -159,7 +160,10 @@ def register_model_metadata(model_metadata: ModelMetadata) -> None:
 def get_model_metadata(model_name: str) -> ModelMetadata:
     """Return the `ModelMetadata` for the model name."""
     if model_name not in MODEL_NAME_TO_MODEL_METADATA:
-        raise ValueError(f"No model with name: {model_name}")
+        raise ValueError(
+            f"No model metadata for model name: {model_name} - "
+            "did you remember to add this model to model_metadata.yaml?"
+        )
 
     return MODEL_NAME_TO_MODEL_METADATA[model_name]