crfm-helm 0.5.7__py3-none-any.whl → 0.5.8__py3-none-any.whl

helm/benchmark/metrics/evaluate_reference_metrics.py

@@ -14,6 +14,7 @@ from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.metrics import code_metrics_helper
 from helm.benchmark.metrics.cleva_metrics_helper import ChineseTokenizer
+from helm.benchmark.metrics.metric import MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.nltk_helper import install_nltk_resources
@@ -518,3 +519,301 @@ def compute_reference_metrics(
             raise NameError(f"{metric_name} is not in the list of metric functions.")
 
     return stats
+
+
+_METRIC_METADATA_MAPPING: Dict[str, MetricMetadata] = {
+    "exact_match": MetricMetadata(
+        name="exact_match",
+        display_name="Exact match",
+        short_display_name="EM",
+        description="Fraction of instances that the predicted output matches a correct reference exactly.",
+        lower_is_better=False,
+        group="accuracy",
+    ),
+    "quasi_exact_match": MetricMetadata(
+        name="quasi_exact_match",
+        display_name="Quasi-exact match",
+        short_display_name="EM",
+        description="Fraction of instances that the predicted output matches a correct reference up to light "
+        "processing.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "quasi_leave_articles_exact_match": MetricMetadata(
+        name="quasi_leave_articles_exact_match",
+        display_name="Quasi-exact match",
+        short_display_name="EM",
+        description="Fraction of instances that the predicted output matches a correct reference up to light "
+        "processing.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "prefix_exact_match": MetricMetadata(
+        name="prefix_exact_match",
+        display_name="Prefix exact match",
+        short_display_name="PEM",
+        description="Fraction of instances that the predicted output matches the prefix of a correct reference "
+        "exactly.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "quasi_prefix_exact_match": MetricMetadata(
+        name="quasi_prefix_exact_match",
+        display_name="Prefix quasi-exact match",
+        short_display_name="PEM",
+        description="Fraction of instances that the predicted output matches the prefix of a correct reference "
+        "up to light processing.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "exact_match_indicator": MetricMetadata(
+        name="exact_match_indicator",
+        display_name="Exact match (final)",
+        short_display_name="EM",
+        description="Fraction of instances that the predicted output matches a correct reference exactly, "
+        "ignoring text preceding the specified indicator (e.g., space).",
+        lower_is_better=False,
+        group=None,
+    ),
+    "final_number_exact_match": MetricMetadata(
+        name="final_number_exact_match",
+        display_name="Exact match (final number)",
+        short_display_name="EM",
+        description="Fraction of instances that the predicted output matches a correct reference exactly, "
+        "ignoring text preceding the specified indicator.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "exact_set_match": MetricMetadata(
+        name="exact_set_match",
+        display_name="Exact match (at sets)",
+        short_display_name="EM",
+        description="Fraction of instances that the predicted output matches a correct reference exactly as " "sets.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "iou_set_match": MetricMetadata(
+        name="iou_set_match",
+        display_name="Intersection over union (as sets)",
+        short_display_name="IoU",
+        description="Intersection over union in terms of set overlap between the model predicted set and "
+        "correct reference set.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "f1_set_match": MetricMetadata(
+        name="f1_set_match",
+        display_name="F1 (set match)",
+        short_display_name="F1",
+        description="Average F1 score in terms of set overlap between the model predicted set and correct "
+        "reference set.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "math_equiv": MetricMetadata(
+        name="math_equiv",
+        display_name="Equivalent",
+        description="Fraction of model outputs that are mathematically equivalent to the correct reference.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "math_equiv_chain_of_thought": MetricMetadata(
+        name="math_equiv_chain_of_thought",
+        display_name="Equivalent (CoT)",
+        description="Fraction of model outputs that are mathematically equivalent to the correct reference "
+        "when using chain-of-thought prompting.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "code_eval_acc": MetricMetadata(
+        name="code_eval_acc",
+        display_name="Correctness",
+        short_display_name="Correctness",
+        description="Fraction of instances that the model output evaluates to the correct answer.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "pass": MetricMetadata(
+        name="pass",
+        display_name="pass@1",
+        description="Fraction of model outputs that pass the associated test cases.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "cider": MetricMetadata(
+        name="cider",
+        display_name="CIDEr",
+        description="Evaluates the quality of generated caption by measuring the weighted similarity of "
+        "n-grams between the captions and a set of human-written reference captions, emphasizing "
+        "informativeness and consensus.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "f1_score": MetricMetadata(
+        name="f1_score",
+        display_name="F1",
+        description="Average F1 score in terms of word overlap between the model output and correct reference.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "rouge_1": MetricMetadata(
+        name="rouge_1",
+        display_name="ROUGE-1",
+        description="Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on 1-gram " "overlap.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "rouge_2": MetricMetadata(
+        name="rouge_2",
+        display_name="ROUGE-2",
+        description="Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on 2-gram " "overlap.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "rouge_l": MetricMetadata(
+        name="rouge_l",
+        display_name="ROUGE-L",
+        description="Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on longest "
+        "common subsequence overlap.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "bleu_1": MetricMetadata(
+        name="bleu_1",
+        display_name="BLEU-1",
+        description="Average BLEU score [(Papineni et al., 2002)](https://aclanthology.org/P02-1040/) based on "
+        "1-gram overlap.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "bleu_4": MetricMetadata(
+        name="bleu_4",
+        display_name="BLEU-4",
+        description="Average BLEU score [(Papineni et al., 2002)](https://aclanthology.org/P02-1040/) based on "
+        "4-gram overlap.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "chinese_bleu_1": MetricMetadata(
+        name="chinese_bleu_1",
+        display_name="Chinese BLEU-1 score",
+        short_display_name="BLEU-1 (Chinese)",
+        description="BLEU-1 score [(Papineni et al., 2002)](https://aclanthology.org/P02-1040/) based on a "
+        "Chinese tokenizer that segments Chinese strings by character.",
+        lower_is_better=False,
+        group=None,
+        # Group could be one of:
+        # "cleva_pinyin_transliteration_metrics"
+        # "cleva_dialogue_generation_metrics"
+        # "cleva_data_to_text_generation_metrics"
+    ),
+    "chinese_rouge_1": MetricMetadata(
+        name="chinese_rouge_1",
+        display_name="Chinese ROUGE-1 score",
+        short_display_name="ROUGE-1 (Chinese)",
+        description="ROUGE-1 score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on a Chinese "
+        "tokenizer that segments Chinese strings by character.",
+        lower_is_better=False,
+        group="cleva_summarization_metrics",
+    ),
+    "chinese_rouge_2": MetricMetadata(
+        name="chinese_rouge_2",
+        display_name="Chinese ROUGE-2 score",
+        short_display_name="ROUGE-2 (Chinese)",
+        description="ROUGE-2 score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on a Chinese "
+        "tokenizer that segments Chinese strings by character.",
+        lower_is_better=False,
+        group="cleva_summarization_metrics",
+    ),
+    "cleva_math_result_match": MetricMetadata(
+        name="cleva_math_result_match",
+        display_name="CLEVA Math Exact Match",
+        short_display_name="EM (Math)",
+        description="Exact match that cares only the last math expression (numbers and fractions) in the "
+        "model's prediction.",
+        lower_is_better=False,
+        group="cleva_mathematical_reasoning_metrics",
+    ),
+    "absolute_value_difference": MetricMetadata(
+        name="absolute_value_difference",
+        display_name="Absolute difference",
+        short_display_name="Diff.",
+        description="Average absolute difference between the model output (converted to a number) and the "
+        "correct reference.",
+        lower_is_better=True,
+        group=None,
+    ),
+    "wer_score": MetricMetadata(
+        name="wer_score",
+        display_name="Word Error Rate",
+        short_display_name="WER",
+        description="Word error rate between model predictions and ground truth answers for ASR tasks.",
+        lower_is_better=True,
+        group=None,
+    ),
+    "mer_score": MetricMetadata(
+        name="mer_score",
+        display_name="Match Error Rate",
+        short_display_name="MER",
+        description="Word match error rate between model predictions and ground truth answers.",
+        lower_is_better=True,
+        group=None,
+    ),
+    "wip_score": MetricMetadata(
+        name="wip_score",
+        display_name="Word Information Preservation",
+        short_display_name="WIP",
+        description="Word information preservation (WIP) for evaluating the preserved information of ASR.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "cer_score": MetricMetadata(
+        name="cer_score",
+        display_name="Character Error Rate",
+        short_display_name="CER",
+        description="Character error rate (CER) for evaluating the accuracy of ASR.",
+        lower_is_better=True,
+        group=None,
+    ),
+    "chinese_wer_score": MetricMetadata(
+        name="chinese_wer_score",
+        display_name="Chinese Word Error Rate",
+        short_display_name="Chinese WER",
+        description="Chinese word error rate between model predictions and ground truth answers for ASR tasks.",
+        lower_is_better=True,
+        group=None,
+    ),
+    "chinese_mer_score": MetricMetadata(
+        name="chinese_mer_score",
+        display_name="Chinese Match Error Rate",
+        short_display_name="Chinese MER",
+        description="Chinese word match error rate between model predictions and ground truth answers.",
+        lower_is_better=True,
+        group=None,
+    ),
+    "chinese_wip_score": MetricMetadata(
+        name="chinese_wip_score",
+        display_name="Chinese Word Information Preservation",
+        short_display_name="Chinese WIP",
+        description="Chinese word information preservation (WIP) for evaluating the preserved information of " "ASR.",
+        lower_is_better=False,
+        group=None,
+    ),
+    "chinese_cer_score": MetricMetadata(
+        name="chinese_cer_score",
+        display_name="Chinese Character Error Rate",
+        short_display_name="Chinese CER",
+        description="Chinese character error rate (CER) for evaluating the accuracy of Chiese ASR.",
+        lower_is_better=True,
+        group=None,
+    ),
+}
+
+
+def get_reference_metrics_metadata(names: List[str]) -> List[MetricMetadata]:
+    metadata_list: List[MetricMetadata] = []
+    for name in names:
+        metadata = _METRIC_METADATA_MAPPING.get(name)
+        if metadata:
+            metadata_list.append(metadata)
+    return metadata_list

helm/benchmark/metrics/gpqa_chain_of_thought_metric.py

@@ -3,7 +3,7 @@ from typing import List, Optional
 
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -101,3 +101,15 @@ class GPQAChainOfThoughtMetric(Metric):
         # Compare extracted answer with the correct answer and compute the score
         score = 1 if extracted_answer == correct_answer else 0
         return [Stat(MetricName("chain_of_thought_correctness")).add(score)]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="chain_of_thought_correctness",
+                display_name="COT correct",
+                short_display_name="COT correct",
+                description="Fraction of correct answers after chain of thought",
+                lower_is_better=False,
+                group="accuracy",
+            ),
+        ]

helm/benchmark/metrics/helpdesk_call_summarization_metrics.py

@@ -2,7 +2,7 @@ from typing import Any, Dict, List
 
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -34,3 +34,15 @@ class HelpdeskCallSummarizationMetric(Metric):
         return [
             Stat(MetricName("call_summarization_score")).add(score),
         ]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="call_summarization_score",
+                display_name="Score",
+                short_display_name="Score",
+                description="Score",
+                lower_is_better=False,
+                group="summarization_metrics",
+            ),
+        ]

helm/benchmark/metrics/ifeval_metrics.py

@@ -3,7 +3,7 @@ from typing import List
 from helm.common.hierarchical_logger import hwarn
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -53,3 +53,15 @@ class IFEvalMetric(Metric):
                 is_following_list.append(0)
 
         return [Stat(MetricName("ifeval_strict_accuracy")).add(sum(is_following_list) / len(is_following_list))]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="ifeval_strict_accuracy",
+                display_name="IFEval strict accuracy",
+                short_display_name="IFEval Strict Acc",
+                description="Fraction of instructions in the instance that are correctly followed.",
+                lower_is_better=False,
+                group="accuracy",
+            ),
+        ]

helm/benchmark/metrics/instruction_following_critique_metrics.py

@@ -3,7 +3,7 @@ from typing import Dict, List
 
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -189,3 +189,43 @@ class InstructionFollowingCritiqueMetric(Metric):
 
                 stats[answer_name].add(answer_value)
         return list(stats.values())
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="Helpfulness",
+                display_name="Helpfulness",
+                description="Whether the model appears to do what it is instructed to.",
+                lower_is_better=False,
+                group="instruction_following_metrics",
+            ),
+            MetricMetadata(
+                name="Understandability",
+                display_name="Understandability",
+                description="Whether the answer is easy to comprehend.",
+                lower_is_better=False,
+                group="instruction_following_metrics",
+            ),
+            MetricMetadata(
+                name="Completeness",
+                display_name="Completeness",
+                description="Whether the response provides a comprehensive answer to the question or request.",
+                lower_is_better=False,
+                group="instruction_following_metrics",
+            ),
+            MetricMetadata(
+                name="Conciseness",
+                display_name="Conciseness",
+                description="Whether the response is concise and to the point, without unnecessary repetition or "
+                "elaboration.",
+                lower_is_better=False,
+                group="instruction_following_metrics",
+            ),
+            MetricMetadata(
+                name="Harmlessness",
+                display_name="Harmlessness",
+                description="Whether the response avoids using toxic language or helping the user with harmful goals.",
+                lower_is_better=False,
+                group="instruction_following_metrics",
+            ),
+        ]

helm/benchmark/metrics/kpi_edgar_metrics.py

@@ -6,6 +6,7 @@ import numpy as np
 
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.metrics.evaluate_instances_metric import EvaluateInstancesMetric
+from helm.benchmark.metrics.metric import MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.statistic import Stat
 from helm.benchmark.scenarios.kpi_edgar_scenario import KPIEDGARScenario
@@ -119,3 +120,23 @@ class KPIEdgarMetric(EvaluateInstancesMetric):
 
     def evaluate_instances(self, request_states: List[RequestState], eval_cache_path: str) -> List[Stat]:
         return _compute_stats(_request_states_to_pred_gold_pairs(request_states))
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="word_macro_f1_score",
+                display_name="Word F1 Score",
+                description="Word F1 Score",
+                lower_is_better=None,
+                group=None,
+            ),
+            MetricMetadata(
+                name="adjusted_macro_f1_score",
+                display_name="Adjusted Macro F1 Score",
+                short_display_name="Adjusted Macro F1 Score",
+                description="Entity type classification F1 score, adjusted for partial matches following the KPI-Edgar "
+                "paper, macro-averaged across entity types",
+                lower_is_better=None,
+                group=None,
+            ),
+        ]

helm/benchmark/metrics/language_modeling_metrics.py

@@ -11,7 +11,7 @@ from helm.benchmark.metrics.efficiency_metrics import EfficiencyMetric
 
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
-from helm.benchmark.metrics.metric import MetricInterface, MetricResult, PerInstanceStats, add_context
+from helm.benchmark.metrics.metric import MetricInterface, MetricMetadata, MetricResult, PerInstanceStats, add_context
 from helm.benchmark.metrics.metric_name import MetricContext, MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat, merge_stat
@@ -97,3 +97,15 @@ class LanguageModelingMetric(MetricInterface):
         derived_stats: List[Stat] = []
         derived_stats.extend(compute_perplexity_metrics(stats_dict))
         return derived_stats
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="num_instances",
+                display_name="# eval",
+                short_display_name=None,
+                description="Number of evaluation instances.",
+                lower_is_better=None,
+                group="general_information",
+            ),
+        ]

helm/benchmark/metrics/live_qa_metrics.py

@@ -2,7 +2,7 @@ from typing import List
 
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -21,3 +21,15 @@ class LiveQAScoreMetric(Metric):
         assert request_state.annotations
         score = request_state.annotations["live_qa"]["score"]
         return [Stat(MetricName("live_qa_score")).add(score)]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="live_qa_score",
+                display_name="Judge Score",
+                short_display_name=None,
+                description="LLM-as-judge score",
+                lower_is_better=False,
+                group=None,
+            ),
+        ]

helm/benchmark/metrics/llm_jury_metrics.py

@@ -3,7 +3,7 @@ from typing import Any, Dict, List
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.annotation.model_as_judge import AnnotatorModelInfo
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -44,3 +44,15 @@ class LLMJuryMetric(Metric):
         return [
             Stat(MetricName(self.metric_name)).add(score),
         ]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name=self.metric_name,
+                display_name="Jury Score",
+                short_display_name="Jury Score",
+                description="Measures the average score assigned by an LLM-based jury evaluating task performance.",
+                lower_is_better=False,
+                group=None,
+            ),
+        ]

helm/benchmark/metrics/medcalc_bench_metrics.py

@@ -4,7 +4,7 @@ from datetime import datetime
 from typing import List, Dict, Any
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -122,3 +122,16 @@ class MedCalcBenchMetric(Metric):
         return [
             Stat(MetricName("medcalc_bench_accuracy")).add(exact_match),
         ]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="medcalc_bench_accuracy",
+                display_name="MedCalc Accuracy",
+                short_display_name="MedCalc Accuracy",
+                description="Comparison based on category. Exact match for categories risk, severity and diagnosis. "
+                "Check if within range for the other categories.",
+                lower_is_better=False,
+                group=None,
+            ),
+        ]

helm/benchmark/metrics/medec_metrics.py

@@ -1,12 +1,13 @@
 from typing import List
+import re
+
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
 from helm.common.hierarchical_logger import hlog
-import re
 from helm.benchmark.scenarios.scenario import CORRECT_TAG
 
 
@@ -99,3 +100,25 @@ class MedecMetric(Metric):
             Stat(MetricName("medec_error_flag_accuracy")).add(flag_accuracy),
             Stat(MetricName("medec_error_sentence_accuracy")).add(sentence_accuracy),
         ]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="medec_error_flag_accuracy",
+                display_name="Medical Error Flag Accuracy",
+                short_display_name="MedecFlagAcc",
+                description="Measures how accurately the model identifies whether a clinical note contains an error "
+                "(binary classification of correct/incorrect).",
+                lower_is_better=False,
+                group=None,
+            ),
+            MetricMetadata(
+                name="medec_error_sentence_accuracy",
+                display_name="Medical Error Sentence Accuracy",
+                short_display_name="MedecSentenceAcc",
+                description="Measures how accurately the model identifies the specific erroneous sentence within a "
+                "clinical note.",
+                lower_is_better=False,
+                group=None,
+            ),
+        ]

helm/benchmark/metrics/metric.py

@@ -86,6 +86,28 @@ class Processor:
         return instance_stats
 
 
+@dataclass(frozen=True)
+class MetricMetadata:
+    name: str
+    """Internal name (usually no spaces, etc.)"""
+
+    display_name: Optional[str] = None
+    """What is displayed to the user"""
+
+    short_display_name: Optional[str] = None
+    """What is displayed to the user (e.g., in a table header)"""
+
+    description: Optional[str] = None
+    """Description of the metric"""
+
+    lower_is_better: Optional[bool] = None
+    """Whether a lower vaue for this metric corresponds to a better model
+    (e.g., False for accuracy, True for perplexity, None for num_trials)"""
+
+    group: Optional[str] = None
+    """Name of the default metric group for this metric"""
+
+
 class MetricInterface(ABC):
     """Interface for all Metrics."""
 
@@ -95,6 +117,9 @@ class MetricInterface(ABC):
     ) -> MetricResult:
         pass
 
+    def get_metadata(self) -> List[MetricMetadata]:
+        raise NotImplementedError()
+
 
 class Metric(MetricInterface, ABC):
     """