crfm-helm 0.5.6__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/metrics/efficiency_metrics.py

@@ -9,6 +9,7 @@ from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.window_services.window_service import WindowService
 from helm.benchmark.window_services.window_service_factory import WindowServiceFactory
 from helm.benchmark.window_services.tokenizer_service import TokenizerService
+from helm.benchmark.metrics.metric import MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -162,6 +163,79 @@ class EfficiencyMetric:
             stats.append(Stat(MetricName("inference_idealized_runtime")).add(idealized_runtime))
         return stats
 
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="num_prompt_tokens",
+                display_name="# prompt tokens",
+                short_display_name=None,
+                description="Number of tokens in the prompt.",
+                lower_is_better=None,
+                group="general_information",
+            ),
+            MetricMetadata(
+                name="num_completion_tokens",
+                display_name="# completion tokens",
+                description="Actual number of completion tokens (over all completions).",
+                lower_is_better=None,
+            ),
+            MetricMetadata(
+                name="num_output_tokens",
+                display_name="# output tokens",
+                description="Actual number of output tokens.",
+                lower_is_better=None,
+            ),
+            MetricMetadata(
+                name="training_co2_cost",
+                display_name="Estimated training emissions (kg CO2)",
+                short_display_name="Training emissions (kg CO2)",
+                description="Estimate of the CO2 emissions from training the model.",
+                lower_is_better=True,
+                group="efficiency_detailed",
+            ),
+            MetricMetadata(
+                name="training_energy_cost",
+                display_name="Estimated training energy cost (MWh)",
+                short_display_name="Training energy (MWh)",
+                description="Estimate of the amount of energy used to train the model.",
+                lower_is_better=True,
+                group="efficiency_detailed",
+            ),
+            MetricMetadata(
+                name="inference_runtime",
+                display_name="Observed inference runtime (s)",
+                short_display_name="Observed inference time (s)",
+                description="Average observed time to process a request to the model (via an API, and thus depends on "
+                "particular deployment).",
+                lower_is_better=True,
+                group="efficiency_detailed",
+            ),
+            MetricMetadata(
+                name="batch_size",
+                display_name="Batch size",
+                description="For batch jobs, how many requests are in a batch.",
+                lower_is_better=None,
+            ),
+            MetricMetadata(
+                name="inference_denoised_runtime",
+                display_name="Denoised inference runtime (s)",
+                short_display_name="Denoised inference time (s)",
+                description="Average time to process a request to the model minus performance contention by using "
+                "profiled runtimes from multiple trials of SyntheticEfficiencyScenario.",
+                lower_is_better=True,
+                group="efficiency_detailed",
+            ),
+            MetricMetadata(
+                name="inference_idealized_runtime",
+                display_name="Idealized inference runtime (s)",
+                short_display_name="Idealized inference time (s)",
+                description="Average time to process a request to the model based solely on the model architecture "
+                "(using Megatron-LM).",
+                lower_is_better=True,
+                group="efficiency_detailed",
+            ),
+        ]
+
 
 def _compute_estimated_time_from_prompt_size_and_num_output_tokens(
     request_state: RequestState,

helm/benchmark/metrics/ehr_sql_metrics.py

@@ -1,7 +1,7 @@
 from typing import List
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -101,3 +101,59 @@ class EhrSqlMetric(Metric):
             Stat(MetricName("ehr_sql_total_predicted_answerable")).add(int(is_predicted_answerable)),
             Stat(MetricName("ehr_sql_total_ground_truth_answerable")).add(int(is_answerable)),
         ]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="ehr_sql_execution_accuracy",
+                display_name="Execution accuracy for Generated Query",
+                short_display_name="EHRSQLExeAcc",
+                description="Measures the proportion of correctly predicted answerable questions among all questions "
+                "predicted to be answerable.",
+                lower_is_better=False,
+                group=None,
+            ),
+            MetricMetadata(
+                name="ehr_sql_query_validity",
+                display_name="Validity of Generated Query",
+                short_display_name="EHRSQLQueryValid",
+                description="Measures the proportion of correctly predicted answerable questions among all answerable "
+                "questions in the dataset.",
+                lower_is_better=False,
+                group=None,
+            ),
+            MetricMetadata(
+                name="ehr_sql_precision_answerable",
+                display_name="Precision for Answerable Questions",
+                short_display_name="EHRSQLPreAns",
+                description="Measures the proportion of correctly predicted answerable questions among all questions "
+                "predicted to be answerable.",
+                lower_is_better=False,
+                group=None,
+            ),
+            MetricMetadata(
+                name="ehr_sql_recall_answerable",
+                display_name="Recall for Answerable Questions",
+                short_display_name="EHRSQLReAns",
+                description="Measures the proportion of correctly predicted answerable questions among all answerable "
+                "questions in the dataset.",
+                lower_is_better=False,
+                group=None,
+            ),
+            MetricMetadata(
+                name="ehr_sql_total_predicted_answerable",
+                display_name="Total Predicted Answerable",
+                short_display_name="Total Pred Ans",
+                description="Total number of questions predicted to be answerable by the model.",
+                lower_is_better=False,
+                group=None,
+            ),
+            MetricMetadata(
+                name="ehr_sql_total_ground_truth_answerable",
+                display_name="Total Ground Truth Answerable",
+                short_display_name="Total GT Ans",
+                description="Total number of answerable questions in the ground truth.",
+                lower_is_better=False,
+                group=None,
+            ),
+        ]

helm/benchmark/metrics/evaluate_reference_metrics.py

@@ -14,6 +14,7 @@ from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.metrics import code_metrics_helper
 from helm.benchmark.metrics.cleva_metrics_helper import ChineseTokenizer
+from helm.benchmark.metrics.metric import MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.nltk_helper import install_nltk_resources
@@ -36,7 +37,7 @@ def pass_at_k_estimator(n: int, c: int, k: int) -> float:
     """
     if n - c < k:
         return 1.0
-    return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1))
+    return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1)).item()
 
 
 def normalize_text(text: str, should_remove_articles: bool = True) -> str:
@@ -396,6 +397,16 @@ def code_eval(gold: Tuple[str, Optional[Dict]], pred: str) -> float:
     return float(code_metrics_helper.check_correctness(gold[1], pred, 3.0)["passed"])  # type: ignore
 
 
+def _apply_output_mapping_pattern(pattern: str, prediction: str) -> str:
+    match = re.search(pattern, prediction)
+    if not match:
+        return ""
+    elif match.groups():
+        return match.group(0)
+    else:
+        return match.string
+
+
 # TODO This should probably be made into an implementation of MetricInterface. For now it lives here
 # just to separate it from basic_metrics.py.
 def compute_reference_metrics(
@@ -497,6 +508,8 @@ def compute_reference_metrics(
     # Note: If 'A' and 'B' were the only possible choices, smaller language models like GPT-2 would
     # sometimes predict a random letter like 'M'.
     if request_state.output_mapping is not None:
+        if adapter_spec.output_mapping_pattern:
+            preds = [_apply_output_mapping_pattern(adapter_spec.output_mapping_pattern, pred) for pred in preds]
         preds = [request_state.output_mapping.get(pred) for pred in preds]  # type: ignore
 
     # Compute max_prob, the probability that the model assigns to its generated text.
@@ -518,3 +531,301 @@ def compute_reference_metrics(
             raise NameError(f"{metric_name} is not in the list of metric functions.")
 
     return stats
+
+
+_METRIC_METADATA_MAPPING: Dict[str, MetricMetadata] = {
+    "exact_match": MetricMetadata(
+        name="exact_match",
+        display_name="Exact match",
+        short_display_name="EM",
+        description="Fraction of instances that the predicted output matches a correct reference exactly.",
+        lower_is_better=False,
+        group="accuracy",
+    ),
+    "quasi_exact_match": MetricMetadata(
+        name="quasi_exact_match",
+        display_name="Quasi-exact match",
+        short_display_name="EM",
+        description="Fraction of instances that the predicted output matches a correct reference up to light "
+        "processing.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "quasi_leave_articles_exact_match": MetricMetadata(
+        name="quasi_leave_articles_exact_match",
+        display_name="Quasi-exact match",
+        short_display_name="EM",
+        description="Fraction of instances that the predicted output matches a correct reference up to light "
+        "processing.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "prefix_exact_match": MetricMetadata(
+        name="prefix_exact_match",
+        display_name="Prefix exact match",
+        short_display_name="PEM",
+        description="Fraction of instances that the predicted output matches the prefix of a correct reference "
+        "exactly.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "quasi_prefix_exact_match": MetricMetadata(
+        name="quasi_prefix_exact_match",
+        display_name="Prefix quasi-exact match",
+        short_display_name="PEM",
+        description="Fraction of instances that the predicted output matches the prefix of a correct reference "
+        "up to light processing.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "exact_match_indicator": MetricMetadata(
+        name="exact_match_indicator",
+        display_name="Exact match (final)",
+        short_display_name="EM",
+        description="Fraction of instances that the predicted output matches a correct reference exactly, "
+        "ignoring text preceding the specified indicator (e.g., space).",
+        lower_is_better=False,
+        group=None,
+    ),
+    "final_number_exact_match": MetricMetadata(
+        name="final_number_exact_match",
+        display_name="Exact match (final number)",
+        short_display_name="EM",
+        description="Fraction of instances that the predicted output matches a correct reference exactly, "
+        "ignoring text preceding the specified indicator.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "exact_set_match": MetricMetadata(
+        name="exact_set_match",
+        display_name="Exact match (at sets)",
+        short_display_name="EM",
+        description="Fraction of instances that the predicted output matches a correct reference exactly as " "sets.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "iou_set_match": MetricMetadata(
+        name="iou_set_match",
+        display_name="Intersection over union (as sets)",
+        short_display_name="IoU",
+        description="Intersection over union in terms of set overlap between the model predicted set and "
+        "correct reference set.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "f1_set_match": MetricMetadata(
+        name="f1_set_match",
+        display_name="F1 (set match)",
+        short_display_name="F1",
+        description="Average F1 score in terms of set overlap between the model predicted set and correct "
+        "reference set.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "math_equiv": MetricMetadata(
+        name="math_equiv",
+        display_name="Equivalent",
+        description="Fraction of model outputs that are mathematically equivalent to the correct reference.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "math_equiv_chain_of_thought": MetricMetadata(
+        name="math_equiv_chain_of_thought",
+        display_name="Equivalent (CoT)",
+        description="Fraction of model outputs that are mathematically equivalent to the correct reference "
+        "when using chain-of-thought prompting.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "code_eval_acc": MetricMetadata(
+        name="code_eval_acc",
+        display_name="Correctness",
+        short_display_name="Correctness",
+        description="Fraction of instances that the model output evaluates to the correct answer.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "pass": MetricMetadata(
+        name="pass",
+        display_name="pass@1",
+        description="Fraction of model outputs that pass the associated test cases.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "cider": MetricMetadata(
+        name="cider",
+        display_name="CIDEr",
+        description="Evaluates the quality of generated caption by measuring the weighted similarity of "
+        "n-grams between the captions and a set of human-written reference captions, emphasizing "
+        "informativeness and consensus.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "f1_score": MetricMetadata(
+        name="f1_score",
+        display_name="F1",
+        description="Average F1 score in terms of word overlap between the model output and correct reference.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "rouge_1": MetricMetadata(
+        name="rouge_1",
+        display_name="ROUGE-1",
+        description="Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on 1-gram " "overlap.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "rouge_2": MetricMetadata(
+        name="rouge_2",
+        display_name="ROUGE-2",
+        description="Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on 2-gram " "overlap.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "rouge_l": MetricMetadata(
+        name="rouge_l",
+        display_name="ROUGE-L",
+        description="Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on longest "
+        "common subsequence overlap.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "bleu_1": MetricMetadata(
+        name="bleu_1",
+        display_name="BLEU-1",
+        description="Average BLEU score [(Papineni et al., 2002)](https://aclanthology.org/P02-1040/) based on "
+        "1-gram overlap.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "bleu_4": MetricMetadata(
+        name="bleu_4",
+        display_name="BLEU-4",
+        description="Average BLEU score [(Papineni et al., 2002)](https://aclanthology.org/P02-1040/) based on "
+        "4-gram overlap.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "chinese_bleu_1": MetricMetadata(
+        name="chinese_bleu_1",
+        display_name="Chinese BLEU-1 score",
+        short_display_name="BLEU-1 (Chinese)",
+        description="BLEU-1 score [(Papineni et al., 2002)](https://aclanthology.org/P02-1040/) based on a "
+        "Chinese tokenizer that segments Chinese strings by character.",
+        lower_is_better=False,
+        group=None,
+        # Group could be one of:
+        # "cleva_pinyin_transliteration_metrics"
+        # "cleva_dialogue_generation_metrics"
+        # "cleva_data_to_text_generation_metrics"
+    ),
+    "chinese_rouge_1": MetricMetadata(
+        name="chinese_rouge_1",
+        display_name="Chinese ROUGE-1 score",
+        short_display_name="ROUGE-1 (Chinese)",
+        description="ROUGE-1 score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on a Chinese "
+        "tokenizer that segments Chinese strings by character.",
+        lower_is_better=False,
+        group="cleva_summarization_metrics",
+    ),
+    "chinese_rouge_2": MetricMetadata(
+        name="chinese_rouge_2",
+        display_name="Chinese ROUGE-2 score",
+        short_display_name="ROUGE-2 (Chinese)",
+        description="ROUGE-2 score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on a Chinese "
+        "tokenizer that segments Chinese strings by character.",
+        lower_is_better=False,
+        group="cleva_summarization_metrics",
+    ),
+    "cleva_math_result_match": MetricMetadata(
+        name="cleva_math_result_match",
+        display_name="CLEVA Math Exact Match",
+        short_display_name="EM (Math)",
+        description="Exact match that cares only the last math expression (numbers and fractions) in the "
+        "model's prediction.",
+        lower_is_better=False,
+        group="cleva_mathematical_reasoning_metrics",
+    ),
+    "absolute_value_difference": MetricMetadata(
+        name="absolute_value_difference",
+        display_name="Absolute difference",
+        short_display_name="Diff.",
+        description="Average absolute difference between the model output (converted to a number) and the "
+        "correct reference.",
+        lower_is_better=True,
+        group=None,
+    ),
+    "wer_score": MetricMetadata(
+        name="wer_score",
+        display_name="Word Error Rate",
+        short_display_name="WER",
+        description="Word error rate between model predictions and ground truth answers for ASR tasks.",
+        lower_is_better=True,
+        group=None,
+    ),
+    "mer_score": MetricMetadata(
+        name="mer_score",
+        display_name="Match Error Rate",
+        short_display_name="MER",
+        description="Word match error rate between model predictions and ground truth answers.",
+        lower_is_better=True,
+        group=None,
+    ),
+    "wip_score": MetricMetadata(
+        name="wip_score",
+        display_name="Word Information Preservation",
+        short_display_name="WIP",
+        description="Word information preservation (WIP) for evaluating the preserved information of ASR.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "cer_score": MetricMetadata(
+        name="cer_score",
+        display_name="Character Error Rate",
+        short_display_name="CER",
+        description="Character error rate (CER) for evaluating the accuracy of ASR.",
+        lower_is_better=True,
+        group=None,
+    ),
+    "chinese_wer_score": MetricMetadata(
+        name="chinese_wer_score",
+        display_name="Chinese Word Error Rate",
+        short_display_name="Chinese WER",
+        description="Chinese word error rate between model predictions and ground truth answers for ASR tasks.",
+        lower_is_better=True,
+        group=None,
+    ),
+    "chinese_mer_score": MetricMetadata(
+        name="chinese_mer_score",
+        display_name="Chinese Match Error Rate",
+        short_display_name="Chinese MER",
+        description="Chinese word match error rate between model predictions and ground truth answers.",
+        lower_is_better=True,
+        group=None,
+    ),
+    "chinese_wip_score": MetricMetadata(
+        name="chinese_wip_score",
+        display_name="Chinese Word Information Preservation",
+        short_display_name="Chinese WIP",
+        description="Chinese word information preservation (WIP) for evaluating the preserved information of " "ASR.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "chinese_cer_score": MetricMetadata(
+        name="chinese_cer_score",
+        display_name="Chinese Character Error Rate",
+        short_display_name="Chinese CER",
+        description="Chinese character error rate (CER) for evaluating the accuracy of Chiese ASR.",
+        lower_is_better=True,
+        group=None,
+    ),
+}
+
+
+def get_reference_metrics_metadata(names: List[str]) -> List[MetricMetadata]:
+    metadata_list: List[MetricMetadata] = []
+    for name in names:
+        metadata = _METRIC_METADATA_MAPPING.get(name)
+        if metadata:
+            metadata_list.append(metadata)
+    return metadata_list

helm/benchmark/metrics/gpqa_chain_of_thought_metric.py

@@ -3,7 +3,7 @@ from typing import List, Optional
 
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -101,3 +101,15 @@ class GPQAChainOfThoughtMetric(Metric):
         # Compare extracted answer with the correct answer and compute the score
         score = 1 if extracted_answer == correct_answer else 0
         return [Stat(MetricName("chain_of_thought_correctness")).add(score)]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="chain_of_thought_correctness",
+                display_name="COT correct",
+                short_display_name="COT correct",
+                description="Fraction of correct answers after chain of thought",
+                lower_is_better=False,
+                group="accuracy",
+            ),
+        ]

helm/benchmark/metrics/helpdesk_call_summarization_metrics.py

@@ -2,7 +2,7 @@ from typing import Any, Dict, List
 
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -34,3 +34,15 @@ class HelpdeskCallSummarizationMetric(Metric):
         return [
             Stat(MetricName("call_summarization_score")).add(score),
         ]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="call_summarization_score",
+                display_name="Score",
+                short_display_name="Score",
+                description="Score",
+                lower_is_better=False,
+                group="summarization_metrics",
+            ),
+        ]

helm/benchmark/metrics/ifeval_metrics.py

@@ -3,7 +3,7 @@ from typing import List
 from helm.common.hierarchical_logger import hwarn
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -53,3 +53,15 @@ class IFEvalMetric(Metric):
                 is_following_list.append(0)
 
         return [Stat(MetricName("ifeval_strict_accuracy")).add(sum(is_following_list) / len(is_following_list))]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="ifeval_strict_accuracy",
+                display_name="IFEval strict accuracy",
+                short_display_name="IFEval Strict Acc",
+                description="Fraction of instructions in the instance that are correctly followed.",
+                lower_is_better=False,
+                group="accuracy",
+            ),
+        ]

helm/benchmark/metrics/image_generation/clip_score_metrics.py

@@ -10,7 +10,7 @@ from helm.benchmark.metrics.statistic import Stat
 from helm.benchmark.metrics.metric import Metric
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
-from helm.benchmark.window_services.window_service_factory import WindowServiceFactory
+from helm.benchmark.window_services.image_generation.clip_window_service import CLIPWindowService
 from helm.common.images_utils import is_blacked_out_image
 from helm.common.multimodal_request_utils import gather_generated_image_locations
 
@@ -55,7 +55,18 @@ class CLIPScoreMetric(Metric):
         # Truncate the prompt using the CLIP tokenizer before feeding into the CLIP model.
         # Otherwise, the library will throw an error.
         model = DEFAULT_CLIP_SCORE_MODEL
-        prompt = WindowServiceFactory.get_window_service(model, metric_service).truncate_from_right(prompt)
+
+        # The max length is 77, but we also need to account for <|startoftext|> and <|endoftext|>.
+        # This max length is hardcoded for DEFAULT_CLIP_SCORE_MODEL i.e. openai/clip-vit-large-patch14
+        max_sequence_length = 77 - 2
+        prompt = CLIPWindowService(
+            service=metric_service,
+            tokenizer_name=DEFAULT_CLIP_SCORE_MODEL,
+            max_sequence_length=max_sequence_length,
+            max_request_length=max_sequence_length,
+            end_of_text_token="",
+            prefix_token="",
+        ).truncate_from_right(prompt)
 
         scores: List[float] = []
         image_locations: List[str] = gather_generated_image_locations(request_result)

helm/benchmark/metrics/image_generation/fractal_dimension/fractal_dimension_util.py

@@ -58,6 +58,6 @@ def compute_fractal_dimension(image_path: str) -> float:
     except ModuleNotFoundError as e:
         handle_module_not_found_error(e, ["heim"])
 
-    image = cv2.imread(image_path, 0) / 255.0  # type: ignore
+    image: np.ndarray = cv2.imread(image_path, 0) / 255.0  # type: ignore
     assert image.min() >= 0 and image.max() <= 1
     return fractal_dimension(image)

helm/benchmark/metrics/instruction_following_critique_metrics.py

@@ -3,7 +3,7 @@ from typing import Dict, List
 
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -189,3 +189,43 @@ class InstructionFollowingCritiqueMetric(Metric):
 
                 stats[answer_name].add(answer_value)
         return list(stats.values())
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="Helpfulness",
+                display_name="Helpfulness",
+                description="Whether the model appears to do what it is instructed to.",
+                lower_is_better=False,
+                group="instruction_following_metrics",
+            ),
+            MetricMetadata(
+                name="Understandability",
+                display_name="Understandability",
+                description="Whether the answer is easy to comprehend.",
+                lower_is_better=False,
+                group="instruction_following_metrics",
+            ),
+            MetricMetadata(
+                name="Completeness",
+                display_name="Completeness",
+                description="Whether the response provides a comprehensive answer to the question or request.",
+                lower_is_better=False,
+                group="instruction_following_metrics",
+            ),
+            MetricMetadata(
+                name="Conciseness",
+                display_name="Conciseness",
+                description="Whether the response is concise and to the point, without unnecessary repetition or "
+                "elaboration.",
+                lower_is_better=False,
+                group="instruction_following_metrics",
+            ),
+            MetricMetadata(
+                name="Harmlessness",
+                display_name="Harmlessness",
+                description="Whether the response avoids using toxic language or helping the user with harmful goals.",
+                lower_is_better=False,
+                group="instruction_following_metrics",
+            ),
+        ]