crfm-helm 0.5.1__py3-none-any.whl → 0.5.2__py3-none-any.whl

helm/benchmark/metrics/prometheus_vision_critique_metrics.py

@@ -0,0 +1,185 @@
+from typing import Dict, List, Optional
+import re
+
+from tqdm import tqdm
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.adaptation.scenario_state import ScenarioState
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.metrics.metric import MetricInterface, MetricResult, PerInstanceStats, add_context
+from helm.benchmark.metrics.metric_name import MetricContext, MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat, merge_stat
+from helm.common.critique_request import CritiqueTaskTemplate, CritiqueQuestionTemplate, CritiqueRequest, QuestionType
+from helm.common.hierarchical_logger import hlog
+from helm.common.request import RequestResult, GeneratedOutput
+from helm.common.media_object import MultimediaObject, IMAGE_TYPE, MediaObject, TEXT_TYPE
+
+
+class PrometheusVisionCritiqueMetric(MetricInterface):
+    """
+    We compute the same metrics from the Prometheus-Vision: Vision-Language Model as a Judge for
+    Fine-Grained Evaluation paper:
+    https://arxiv.org/pdf/2401.06591.pdf
+
+    In this paper, the output of a Vision-Language Model named Prometheus-Vision is used to evaluate
+    the quality of the output of other Vision-Language Models to be evaluated.
+    """
+
+    # We can add more evaluation aspects here
+    METRIC_NAME: str = "prometheus_vision"
+    METRIC_PROMPT: str = """A chat between a curious human and an artificial intelligence assistant. \
+        The assistant gives helpful, detailed, and polite answers to the human's questions.\nUSER:<image>\
+###Task Description:
+An instruction (might include an Input inside it), a response to evaluate, a reference answer that gets a score of 5, \
+    image and a score rubric representing an evaluation criterion is given.
+1. Write a detailed feedback that assesses the quality of the response strictly based on the given score rubric, not \
+    evaluating in general.
+2. After writing a feedback, write a score that is an integer between 1 and 5. You should refer to the score rubric.
+3. The output format should look as follows: Feedback: (write a feedback for criteria) [RESULT] (an integer number \
+    between 1 and 5)
+4. Please do not generate any other opening, closing, and explanations.
+
+###The instruction to evaluate:
+{{orig_instruction}}
+
+###Response to evaluate:
+{{orig_response}}
+
+###Reference Answer (Score 5):
+{{orig_reference_answer}}
+
+###Score Rubrics:
+[{{orig_criteria}}]
+Score 1: {{orig_score1_description}}
+Score 2: {{orig_score2_description}}
+Score 3: {{orig_score3_description}}
+Score 4: {{orig_score4_description}}
+Score 5: {{orig_score5_description}}
+
+###Feedback:
+ASSISTANT:
+"""
+
+    def __init__(self, num_respondents: int, max_tokens: int):
+        self._num_respondents = num_respondents
+        self._max_tokens = max_tokens
+
+    def __repr__(self) -> str:
+        return "PrometheusVisionCritiqueMetric()"
+
+    def _extract_score_from_prometheus_vision_output(self, evaluator_response: str):
+        evaluator_response = evaluator_response.split("ASSISTANT:")[1]
+        re_match = re.search(r"\s*([1-5])", evaluator_response)
+        if re_match is None:
+            hlog(f"Error parsing answer: {evaluator_response}. Skipping question (and so the respondent entirely)")
+            return None
+        return int(re_match.group(1))
+
+    def evaluate(
+        self,
+        scenario_state: ScenarioState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+        parallelism: int,
+    ) -> MetricResult:
+        request_states: List[RequestState] = scenario_state.request_states
+
+        all_stats: Dict[MetricName, Stat] = {}
+        per_instance_stats: List[PerInstanceStats] = []
+        for request_state in tqdm(request_states):
+            context = MetricContext.from_instance(request_state.instance)
+            stats_without_context = self.evaluate_generation(
+                scenario_state.adapter_spec,
+                request_state,
+                metric_service,
+                eval_cache_path,
+            )
+            stats = [add_context(stat_without_context, context) for stat_without_context in stats_without_context]
+            for stat in stats:
+                merge_stat(all_stats, stat)
+            assert request_state.instance.id is not None
+            per_instance_stats.append(
+                PerInstanceStats(
+                    instance_id=request_state.instance.id,
+                    perturbation=request_state.instance.perturbation,
+                    train_trial_index=request_state.train_trial_index,
+                    stats=stats,
+                )
+            )
+        return MetricResult(aggregated_stats=list(all_stats.values()), per_instance_stats=per_instance_stats)
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        input_content = request_state.request
+        # Predicted outputs and their prometheus vision scores
+        assert request_state.result is not None
+        request_result: RequestResult = request_state.result
+        # Get input image and generated response for evaluation
+        assert input_content.multimodal_prompt is not None
+        completions: List[GeneratedOutput] = request_result.completions
+        generated_text: str = completions[0].text
+        input_media: MultimediaObject = input_content.multimodal_prompt
+        ref_text: str = request_state.instance.references[0].output.text
+        image_objects: List[MediaObject] = [
+            item for item in input_media.media_objects if item.is_type(IMAGE_TYPE) and item.location
+        ]
+        input_text: Optional[str] = [item for item in input_media.media_objects if item.is_type(TEXT_TYPE)][0].text
+
+        template = CritiqueTaskTemplate(
+            name="vhelm_prometheus_vision",
+            instructions=self.METRIC_PROMPT,
+            num_respondents=self._num_respondents,
+            max_tokens=self._max_tokens,
+            questions=[
+                CritiqueQuestionTemplate(
+                    name=self.METRIC_NAME,
+                    question_type=QuestionType.FREE_RESPONSE,
+                    text="",
+                    options=[],
+                    media_object=image_objects[0],  # we only take the first image as input
+                )
+            ],
+        )
+        request = CritiqueRequest(
+            template=template,
+            fields={
+                "orig_instruction": input_text if input_text is not None else "",
+                "orig_response": generated_text,
+                "orig_reference_answer": ref_text,
+                "orig_criteria": "similarity between the reponse and the reference.",
+                "orig_score1_description": "The model's responses do not follow the instructions provided.",
+                "orig_score2_description": "The resulting response follows the instructions, but the answer \
+                    is completely wrong relative to the reference answer.",
+                "orig_score3_description": "The resulting response follows the instructions, but the answer is \
+                    partially wrong relative to the reference answer.",
+                "orig_score4_description": "The resulting response follows the instructions, the overall answer \
+                    is relatively perfect with only a very few errors.",
+                "orig_score5_description": "The overall answer is completely correct compared to the reference \
+                    answer, and conforms to the instructions provided.",
+            },
+        )
+        # send to critique request
+        result = metric_service.make_critique_request(request)
+        if not result or not result.responses:
+            # Skip computing metrics if there aren't any responses yet
+            hlog("Waiting for responses to be generated.")
+            return []
+
+        stats: Dict[str, Stat] = {}
+        for question in template.questions:
+            stats[question.name] = Stat(MetricName(question.name))
+
+        for response in result.responses:
+            for answer_name, answer in response.answers.items():
+                assert isinstance(answer, str)
+                answer_value: float
+                answer_value = self._extract_score_from_prometheus_vision_output(answer)
+                stats[answer_name].add(answer_value)
+
+        return list(stats.values())

helm/benchmark/metrics/reka_vibe_critique_metrics.py

@@ -0,0 +1,158 @@
+from typing import Dict, List, Optional
+import re
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.adaptation.scenario_state import ScenarioState
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.metrics.metric import MetricInterface, MetricResult, PerInstanceStats, add_context
+from helm.benchmark.metrics.metric_name import MetricContext, MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat, merge_stat
+from helm.common.critique_request import CritiqueTaskTemplate, CritiqueQuestionTemplate, CritiqueRequest, QuestionType
+from helm.common.hierarchical_logger import hlog
+from helm.common.request import RequestResult, GeneratedOutput
+from helm.common.media_object import MultimediaObject, IMAGE_TYPE, TEXT_TYPE, MediaObject
+
+
+class RekaVibeCritiqueMetric(MetricInterface):
+    """
+    Critique evaluation for evaluating the correctness of generated response given the image and
+    reference by Reka-vibe-eval.
+    """
+
+    # We can add more evaluation aspects here
+    VIBE_EVAL_NAME: str = "reka_vibe"
+    REKA_VIBE_PROMPT_WITH_IMAGE: str = """\
+[Question]
+{{prompt}}
+
+[Assistant Response]
+{{generation}}
+
+[Ground Truth Response]
+{{reference}}
+
+[System]
+Rate whether the assistant response correctly matches the ground truth, in regards to the image above.
+The rating should be 1-5, where 1 is incorrect and 5 is correct.
+Your response should be in the format:
+Short Explanation: (explanation in only one sentence)
+Rating: (int)"""
+
+    def __init__(self, num_respondents: int, max_tokens: int):
+        self._num_respondents = num_respondents
+        self._max_tokens = max_tokens
+
+    def __repr__(self) -> str:
+        return "RekaVibeCritiqueMetric()"
+
+    def _extract_score_from_reka_output(self, evaluator_response: str):
+        """
+        Extract the score from the evaluator response. Refer to the official Vibe-Eval implementation:
+        https://github.com/reka-ai/reka-vibe-eval/blob/3852d4712da172a7b85dddeffc4f9c3482a6f4c9/evaluate.py#L159-#L164
+        """
+        re_match = re.search(r"Rating:\s*([1-5])", evaluator_response)
+        if re_match is None:
+            hlog(f"Error parsing answer: {evaluator_response}. Skipping question (and so the respondent entirely)")
+            return None
+        return int(re_match.group(1))
+
+    def evaluate(
+        self,
+        scenario_state: ScenarioState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+        parallelism: int,
+    ) -> MetricResult:
+        request_states: List[RequestState] = scenario_state.request_states
+
+        all_stats: Dict[MetricName, Stat] = {}
+        per_instance_stats: List[PerInstanceStats] = []
+        for request_state in request_states:
+            context = MetricContext.from_instance(request_state.instance)
+            stats_without_context = self.evaluate_generation(
+                scenario_state.adapter_spec,
+                request_state,
+                metric_service,
+                eval_cache_path,
+            )
+            stats = [add_context(stat_without_context, context) for stat_without_context in stats_without_context]
+            for stat in stats:
+                merge_stat(all_stats, stat)
+            assert request_state.instance.id is not None
+            per_instance_stats.append(
+                PerInstanceStats(
+                    instance_id=request_state.instance.id,
+                    perturbation=request_state.instance.perturbation,
+                    train_trial_index=request_state.train_trial_index,
+                    stats=stats,
+                )
+            )
+        return MetricResult(aggregated_stats=list(all_stats.values()), per_instance_stats=per_instance_stats)
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        input_content = request_state.request
+        # Predicted outputs and their originality scores
+        assert request_state.result is not None
+        request_result: RequestResult = request_state.result
+        # Get input image and generated response for the originality evaluation
+        assert input_content.multimodal_prompt is not None
+        completions: List[GeneratedOutput] = request_result.completions
+        generated_text: str = completions[0].text
+        input_media: MultimediaObject = input_content.multimodal_prompt
+        ref_text: str = request_state.instance.references[0].output.text
+
+        image_objects: List[MediaObject] = [
+            item for item in input_media.media_objects if item.is_type(IMAGE_TYPE) and item.location
+        ]
+        input_text: Optional[str] = [item for item in input_media.media_objects if item.is_type(TEXT_TYPE)][0].text
+
+        template = CritiqueTaskTemplate(
+            name="vhelm_vibe_eval",
+            instructions=self.REKA_VIBE_PROMPT_WITH_IMAGE,
+            num_respondents=self._num_respondents,
+            max_tokens=self._max_tokens,
+            questions=[
+                CritiqueQuestionTemplate(
+                    name=self.VIBE_EVAL_NAME,
+                    question_type=QuestionType.FREE_RESPONSE,
+                    text="",
+                    options=[],
+                    media_object=image_objects[0],  # we only take the first image as input
+                )
+            ],
+        )
+
+        request = CritiqueRequest(
+            template=template,
+            fields={
+                "prompt": input_text if input_text is not None else "",
+                "generation": generated_text,
+                "reference": ref_text,
+            },
+        )
+
+        # send to critique request
+        result = metric_service.make_critique_request(request)
+        if not result or not result.responses:
+            # Skip computing metrics if there aren't any responses yet
+            hlog("Waiting for responses to be generated.")
+            return []
+        stats: Dict[str, Stat] = {}
+        for question in template.questions:
+            stats[question.name] = Stat(MetricName(question.name))
+
+        for response in result.responses:
+            for answer_name, answer in response.answers.items():
+                assert isinstance(answer, str)
+                answer_value: float
+                answer_value = self._extract_score_from_reka_output(answer)
+                stats[answer_name].add(answer_value)
+
+        return list(stats.values())

helm/benchmark/metrics/unitxt_metrics.py

@@ -50,16 +50,20 @@ class UnitxtMetric(MetricInterface):
             for metric_name, metric_score in instance_results.items():
                 if metric_name == "score" or metric_name == "score_name":
                     continue
-                instance_stats.append(
-                    Stat(
-                        MetricName(
-                            name=metric_name,
-                            split=instance.split,
-                            sub_split=instance.sub_split,
-                            perturbation=instance.perturbation,
-                        )
-                    ).add(metric_score)
+                stat = Stat(
+                    MetricName(
+                        name=metric_name,
+                        split=instance.split,
+                        sub_split=instance.sub_split,
+                        perturbation=instance.perturbation,
+                    )
                 )
+                if isinstance(metric_score, list):
+                    for metric_score_element in metric_score:
+                        stat = stat.add(metric_score_element)
+                else:
+                    stat = stat.add(metric_score)
+                instance_stats.append(stat)
             assert instance.id
             per_instance_stats.append(
                 PerInstanceStats(
@@ -77,5 +81,11 @@ class UnitxtMetric(MetricInterface):
             for metric_name, metric_score in global_results.items():
                 if metric_name == "score" or metric_name == "score_name":
                     continue
-                aggregated_stats.append(Stat(MetricName(name=metric_name)).add(metric_score))
+                stat = Stat(MetricName(name=metric_name))
+                if isinstance(metric_score, list):
+                    for metric_score_element in metric_score:
+                        stat = stat.add(metric_score_element)
+                else:
+                    stat = stat.add(metric_score)
+                aggregated_stats.append(stat)
         return MetricResult(aggregated_stats=aggregated_stats, per_instance_stats=per_instance_stats)

helm/benchmark/metrics/vision_language/emd_utils.py

@@ -280,6 +280,10 @@ def compute_emd_recursive(
     assert max_num_patches > 0
     assert 0 < weight_most_frequent_color <= 1
 
+    # Convert the images to RGB first. Some images have 4 channels (RGBA)
+    img1_PIL = img1_PIL.convert("RGB")
+    img2_PIL = img2_PIL.convert("RGB")
+
     # Resize the images so that there are not too many patches
     # Try to maintain the aspect ratio and resize to a multiple of the patch size
     num_patches = math.ceil(img1_PIL.size[0] / patch_size[0]) * math.ceil(img1_PIL.size[1] / patch_size[1])

helm/benchmark/metrics/vision_language/image_metrics.py

@@ -28,7 +28,7 @@ from helm.benchmark.metrics.vision_language.image_utils import (
     pixel_similarity,
     sift_similarity,
 )
-from helm.benchmark.metrics.vision_language.emd_utils import compute_emd_recursive, get_most_frequent_color
+from helm.benchmark.metrics.vision_language.emd_utils import compute_emd_recursive, get_most_frequent_color, to_gray
 
 try:
     from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity
@@ -78,9 +78,8 @@ class AnnotatedImageMetrics(Metric):
 
     # Metric names
     COMPILE_METRIC: str = "compilation_success"
-    BLOCK_EARTH_MOVER_SIMILARITY_NORM1: str = "block_emd_similarity_white"
-    BLOCK_EARTH_MOVER_SIMILARITY_NORM2: str = "block_emd_similarity_median_color"
-    BLOCK_EARTH_MOVER_SIMILARITY: str = "block_emd_similarity"
+    EARTH_MOVER_SIMILARITY = "earth_mover_similarity"
+    BLOCK_EMD: str = "block_emd"
     PIXEL_SIMILARITY: str = "pixel_similarity"
     SIFT_SIMILARITY: str = "sift_similarity"
     LPIPS_SIMILARITY: str = "lpips_similarity"
@@ -108,12 +107,10 @@ class AnnotatedImageMetrics(Metric):
         metrics: List[AnnotatedMetric] = [
             AnnotatedMetric(self.PIXEL_SIMILARITY, pixel_similarity, "image_np_gray"),
             AnnotatedMetric(self.SIFT_SIMILARITY, sift_similarity, "image_np"),
-            # Raw block EMD
-            AnnotatedMetric(self.BLOCK_EARTH_MOVER_SIMILARITY, self.compute_block_emd_raw, "image_PIL"),
-            # Normalized block EMD against white
-            AnnotatedMetric(self.BLOCK_EARTH_MOVER_SIMILARITY_NORM1, self.compute_block_emd_white, "image_PIL"),
-            # Normalized block EMD against median
-            AnnotatedMetric(self.BLOCK_EARTH_MOVER_SIMILARITY_NORM2, self.compute_block_emd_median, "image_PIL"),
+            AnnotatedMetric(self.BLOCK_EMD, self.compute_block_emd_raw, "image_PIL"),  # Raw block-EMD
+            AnnotatedMetric(
+                self.EARTH_MOVER_SIMILARITY, self.ems, "image_PIL"
+            ),  # Normalized block-EMD against black/white
             AnnotatedMetric(self.LPIPS_SIMILARITY, self.lpips_similarity, "image_PIL"),
             AnnotatedMetric(self.FID_SIMILARITY, self.fid_similarity, "image_PIL"),
             AnnotatedMetric(self.SSIM_SIMILARITY, self.compute_ssim, "image_np_gray"),
@@ -391,9 +388,15 @@ class AnnotatedImageMetrics(Metric):
         features1 = self._get_inception_features(img1_tensor)
         features2 = self._get_inception_features(img2_tensor)
 
-        fid_score = self._calculate_fid(features1, features2)
-        normalize_fid: float = np.exp(-fid_score * self.NORMALIZE_FID_FACTOR)
-        return normalize_fid
+        # TODO: Justify the value of the constant here or remove this code to only keep the cosine similarity.
+        # fid_score = self._calculate_fid(features1, features2)
+        # normalize_fid: float = np.exp(-fid_score * self.NORMALIZE_FID_FACTOR)
+        # return normalize_fid
+
+        # Use the cosine similarity between the features as a proxy for FID
+        # Return a score between 0 and 1, where 1 is the most similar
+        score = 0.5 * (1 + np.dot(features1[0], features2[0]) / (np.linalg.norm(features1) * np.linalg.norm(features2)))
+        return score
 
     def compute_ssim(self, generated_image: np.ndarray, reference_image: np.ndarray) -> float:
         """Compute the Structural Similarity Index (SSIM) between the generated and reference images."""
@@ -414,58 +417,7 @@ class AnnotatedImageMetrics(Metric):
         result = _edit_similarity(completion_tokens, truncated_reference_tokens)
         return result
 
-    def compute_block_emd_white(
-        self,
-        pred_image: Image.Image,
-        ref_image: Image.Image,
-        threshold_most_frequent_color: float = 0.5,
-        patch_size: Tuple[int, int] = (8, 8),
-        max_num_patches: int = 100,
-        weight_most_frequent_color: float = 0.001,
-        use_tqdm: bool = False,
-    ):
-        """Computes the block Earth Moving Distance (EMD). This attempts to
-        speed up EMD for images with huge areas by considering movement/transformatio
-        of blocks of pixels. The score is normalized against EMD against white images
-        """
-
-        def compute_numerator():
-            return self.compute_block_emd_raw_wrapper(
-                pred_image,
-                ref_image,
-                threshold_most_frequent_color,
-                patch_size,
-                max_num_patches,
-                weight_most_frequent_color,
-                use_tqdm,
-            )
-
-        def compute_denominator():
-            constant_image = Image.new("RGB", ref_image.size, (255, 255, 255))  # default color is white
-            value = compute_emd_recursive(
-                constant_image,
-                ref_image,
-                threshold_most_frequent_color,
-                patch_size,
-                max_num_patches,
-                weight_most_frequent_color,
-                use_tqdm,
-            )
-            return {"value": value}
-
-        hash_dict = {
-            "reference_image": str(AnnotatedImageMetrics.HASH_FUNC(ref_image, hash_size=self.HASH_LENGTH)),
-        }
-        cache_key_numerator = {"metric_name": f"intermediate_{self.BLOCK_EARTH_MOVER_SIMILARITY}", **hash_dict}
-        cache_key_denominator = {"metric_name": f"intermediate_{self.BLOCK_EARTH_MOVER_SIMILARITY_NORM1}", **hash_dict}
-
-        assert self._cache is not None
-        emd_raw, _ = self._cache.get(cache_key_numerator, compute_numerator)
-        emd_base, _ = self._cache.get(cache_key_denominator, compute_denominator)
-
-        return 1.0 - emd_raw["value"] / emd_base["value"]
-
-    def compute_block_emd_median(
+    def ems(
         self,
         pred_image: Image.Image,
         ref_image: Image.Image,
@@ -493,9 +445,13 @@ class AnnotatedImageMetrics(Metric):
         def compute_denominator():
             ref_img_np = np.array(ref_image)
             (rgb_most_frequent_color, _) = get_most_frequent_color(ref_img_np)
+            grayscale_most_frequent_color = to_gray(rgb_most_frequent_color)[0]
 
             # Most frequent color as base
-            constant_image = Image.new("RGB", ref_image.size, tuple(rgb_most_frequent_color))  # type: ignore
+            if grayscale_most_frequent_color < 127:
+                constant_image = Image.new("RGB", ref_image.size, (255, 255, 255))  # Make it white
+            else:
+                constant_image = Image.new("RGB", ref_image.size, (0, 0, 0))  # Make it black
             value = compute_emd_recursive(
                 constant_image,
                 ref_image,
@@ -509,9 +465,10 @@ class AnnotatedImageMetrics(Metric):
 
         hash_dict = {
             "reference_image": str(AnnotatedImageMetrics.HASH_FUNC(ref_image, hash_size=self.HASH_LENGTH)),
+            "generated_image": str(AnnotatedImageMetrics.HASH_FUNC(pred_image, hash_size=self.HASH_LENGTH)),
         }
-        cache_key_numerator = {"metric_name": f"intermediate_{self.BLOCK_EARTH_MOVER_SIMILARITY}", **hash_dict}
-        cache_key_denominator = {"metric_name": f"intermediate_{self.BLOCK_EARTH_MOVER_SIMILARITY_NORM2}", **hash_dict}
+        cache_key_numerator = {"metric_name": f"intermediate_{self.BLOCK_EMD}", **hash_dict}
+        cache_key_denominator = {"metric_name": "intermediate_ems_extreme_denominator", **hash_dict}
 
         assert self._cache is not None
         emd_raw, _ = self._cache.get(cache_key_numerator, compute_numerator)
@@ -542,8 +499,9 @@ class AnnotatedImageMetrics(Metric):
 
         hash_dict = {
             "reference_image": str(AnnotatedImageMetrics.HASH_FUNC(ref_image, hash_size=self.HASH_LENGTH)),
+            "generated_image": str(AnnotatedImageMetrics.HASH_FUNC(pred_image, hash_size=self.HASH_LENGTH)),
         }
-        cache_key = {"metric_name": f"intermediate_{self.BLOCK_EARTH_MOVER_SIMILARITY}", **hash_dict}
+        cache_key = {"metric_name": f"intermediate_{self.BLOCK_EMD}", **hash_dict}
         assert self._cache is not None
         emd_raw, _ = self._cache.get(cache_key, compute)
 
@@ -560,8 +518,8 @@ class AnnotatedImageMetrics(Metric):
         use_tqdm: bool = False,
     ):
         """Computes the block Earth Moving Distance (EMD). This attempts to
-        speed up EMD for images with huge areas by considering movement/transformatio
-        of blocks of pixels. The score is normalized against EMD against white images
+        speed up EMD for images with huge areas by considering
+        movement/transformation of blocks of pixels.
         """
         emd_value = compute_emd_recursive(
             pred_image,

helm/benchmark/presentation/schema.py

@@ -1,6 +1,9 @@
+import ast
+import dataclasses
 from dataclasses import dataclass, field
 from typing import List, Optional, Dict
 import dacite
+from inspect import cleandoc
 import mako.template
 import yaml
 import importlib_resources as resources
@@ -17,6 +20,11 @@ SCHEMA_YAML_PACKAGE: str = "helm.benchmark.static"
 SCHEMA_CLASSIC_YAML_FILENAME: str = "schema_classic.yaml"
 
 
+_ADAPTER_SPEC_PACKAGE = "helm.benchmark.adaptation"
+_ADAPTER_SPEC_FILENAME = "adapter_spec.py"
+_ADAPTER_SPEC_CLASS_NAME = "AdapterSpec"
+
+
 @dataclass(frozen=True)
 class Field:
     """
@@ -198,9 +206,6 @@ class RunGroup(Field):
 class Schema:
     """Specifies information about what to display on the frontend."""
 
-    # Adapter fields (e.g., temperature)
-    adapter: List[Field]
-
     # Information about each field
     metrics: List[Field]
 
@@ -213,6 +218,11 @@ class Schema:
     # Group the scenarios
     run_groups: List[RunGroup]
 
+    # Adapter fields (e.g., temperature)
+    # Automatically populated from the docstrings in the AdapterSpec class definition.
+    # Should not be specified in the user's YAML file.
+    adapter: Optional[List[Field]] = None
+
     def __post_init__(self):
         self.name_to_metric = {metric.name: metric for metric in self.metrics}
         self.name_to_perturbation = {perturbation.name: perturbation for perturbation in self.perturbations}
@@ -220,6 +230,43 @@ class Schema:
         self.name_to_run_group = {run_group.name: run_group for run_group in self.run_groups}
 
 
+def get_adapter_fields() -> List[Field]:
+    """Generate the adapter fields from the docstrings in the AdapterSpec class definition."""
+    # Unfortunately there is no standard library support for getting docstrings of class fields,
+    # so we have to do the parsing outselves. Fortunately, the parsing is quite straightforward.
+    adapter_spec_path = resources.files(_ADAPTER_SPEC_PACKAGE).joinpath(_ADAPTER_SPEC_FILENAME)
+    with open(adapter_spec_path, "r") as f:
+        contents = f.read()
+    module_node = ast.parse(contents)
+    adapter_spec_node = [
+        node
+        for node in ast.iter_child_nodes(module_node)
+        if isinstance(node, ast.ClassDef) and node.name == _ADAPTER_SPEC_CLASS_NAME
+    ][0]
+    metadata_fields: List[Field] = []
+    field_name: str = ""
+    for node in ast.iter_child_nodes(adapter_spec_node):
+        if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
+            # This node is a field definition.
+            # Save the name of the field for later.
+            field_name = node.target.id
+        else:
+            # If this is a docstring that immediately follows a field definition,
+            # output an adapter field with the name set to  the field definition and
+            # the description set to the docstring.
+            if (
+                field_name
+                and isinstance(node, ast.Expr)
+                and isinstance(node.value, ast.Constant)
+                and isinstance(node.value.value, str)
+            ):
+                description = cleandoc(node.value.value).replace("\n", " ")
+                metadata_fields.append(Field(name=field_name, description=description))
+            field_name = ""
+
+    return metadata_fields
+
+
 def get_default_schema_path() -> str:
     return resources.files(SCHEMA_YAML_PACKAGE).joinpath(SCHEMA_CLASSIC_YAML_FILENAME)
 
@@ -229,4 +276,7 @@ def read_schema(schema_path: str) -> Schema:
     hlog(f"Reading schema file {schema_path}...")
     with open(schema_path, "r") as f:
         raw = yaml.safe_load(f)
-    return dacite.from_dict(Schema, raw)
+    schema = dacite.from_dict(Schema, raw)
+    if schema.adapter:
+        hlog(f"WARNING: The `adapter` field is deprecated and should be removed from schema file {schema_path}")
+    return dataclasses.replace(schema, adapter=get_adapter_fields())

helm/benchmark/presentation/test_schema.py

@@ -0,0 +1,11 @@
+from helm.benchmark.presentation.schema import get_adapter_fields
+
+
+def test_get_adapter_fields() -> None:
+    adapter_fields = get_adapter_fields()
+    assert adapter_fields
+    assert adapter_fields[0].name == "method"
+    assert (
+        adapter_fields[0].description
+        == "The high-level strategy for converting instances into a prompt for the language model."
+    )