crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/metrics/mimiciv_billing_code_metrics.py

@@ -0,0 +1,127 @@
+from typing import List
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+from helm.common.hierarchical_logger import hlog
+import re
+from sklearn.metrics import precision_score, recall_score, f1_score
+from sklearn.preprocessing import MultiLabelBinarizer
+
+
+class MIMICIVBillingCodeMetric(Metric):
+    """
+    Metric for evaluating the MIMIC Billing Code dataset, assessing the model's ability to match the
+    reference ICD codes. Handles cases where raw prediction output contains additional text.
+
+    Calculates:
+    1. Precision: proportion of correctly predicted ICD codes among all predicted codes
+    2. Recall: proportion of correctly predicted ICD codes among all reference codes
+    3. F1 score: harmonic mean of precision and recall
+
+    ICD codes format: letter followed by 1-3 digits, optional period, optional additional digits
+    Examples: "J18.9", "J45.909", "J47.1", "J96.01"
+    """
+
+    def extract_icd_codes(self, text: str) -> List[str]:
+        """Extract ICD codes from text, handling markdown and standardizing format."""
+        if not text:
+            return []
+
+        # Remove markdown bold formatting
+        cleaned_text = re.sub(r"\*\*", "", text)
+        # Match ICD code pattern with optional period and trailing digits
+        pattern = r"\b[A-Z]\d{1,3}(?:\.\d{1,4})?\.?\b"
+        codes = re.findall(pattern, cleaned_text)
+        # Standardize by removing trailing periods
+        return [code.rstrip(".").strip() for code in codes]
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        """
+        Evaluate a single generation against reference labels.
+        """
+        # Extract predictions
+        if request_state.result is None:
+            predictions = []
+        else:
+            predictions = [completion.text.strip() for completion in request_state.result.completions]
+        if not predictions:
+            hlog("Warning: No predictions found in completions")
+            return []
+
+        # Get the first prediction
+        prediction = predictions[0]
+
+        # Get references
+        references = getattr(request_state.instance, "references", None)
+
+        if not references or len(references) == 0:
+            hlog(f"Warning: Missing references for instance {request_state.instance}")
+            return []
+
+        # Extract codes from reference and prediction
+        ref_codes = []
+        for ref in references:
+            if ref.output.text:
+                ref_codes.extend(self.extract_icd_codes(ref.output.text))
+        ref_codes = list(set(ref_codes))  # Remove duplicates
+
+        pred_codes = self.extract_icd_codes(prediction)
+        pred_codes = list(set(pred_codes))  # Remove duplicates
+
+        # Convert to binary format for metrics
+        all_codes = sorted(list(set(ref_codes + pred_codes)))
+        mlb = MultiLabelBinarizer(classes=all_codes)
+
+        y_true_bin = mlb.fit_transform([ref_codes])
+        y_pred_bin = mlb.transform([pred_codes])
+
+        # Calculate metrics
+        precision = precision_score(y_true_bin, y_pred_bin, average="micro", zero_division=0)
+        recall = recall_score(y_true_bin, y_pred_bin, average="micro", zero_division=0)
+        f1 = f1_score(y_true_bin, y_pred_bin, average="micro", zero_division=0)
+
+        return [
+            Stat(MetricName("mimiciv_billing_code_precision")).add(precision),
+            Stat(MetricName("mimiciv_billing_code_recall")).add(recall),
+            Stat(MetricName("mimiciv_billing_code_f1")).add(f1),
+        ]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="mimiciv_billing_code_precision",
+                display_name="Precision for MIMIC Billing Codes",
+                short_display_name="MIMICBillingPre",
+                description="Measures the proportion of correctly predicted ICD codes among all ICD codes predicted by "
+                "the model.",
+                lower_is_better=False,
+                group=None,
+            ),
+            MetricMetadata(
+                name="mimiciv_billing_code_recall",
+                display_name="Recall for MIMIC Billing Codes",
+                short_display_name="MIMICBillingRec",
+                description="Measures the proportion of correctly predicted ICD codes among all ICD codes present in "
+                "the gold standard.",
+                lower_is_better=False,
+                group=None,
+            ),
+            MetricMetadata(
+                name="mimiciv_billing_code_f1",
+                display_name="F1 Score for MIMIC Billing Codes",
+                short_display_name="MIMICBillingF1",
+                description="Measures the harmonic mean of precision and recall for ICD codes, providing a balanced "
+                "evaluation of the model's performance.",
+                lower_is_better=False,
+                group=None,
+            ),
+        ]

helm/benchmark/metrics/nltk_helper.py

@@ -0,0 +1,32 @@
+import nltk
+from importlib.metadata import version
+
+
+def install_nltk_resources():
+    """Install resources for nltk tokenizers, which is required for bleu and rouge scores."""
+    # Install "punkt_tab" for nltk>=3.9.1 or "punkt" for nltk<=3.8.1
+    #
+    # Note that nltk 3.9.0 is disallowed due to https://github.com/nltk/nltk/issues/3308
+    #
+    # "punkt" is not longer supported for newer versions of nltk due to a security issue
+    # and has been replaced by "punkt_tab". For more information, see:
+    #
+    # - https://github.com/stanford-crfm/helm/issues/2926
+    # - https://github.com/nltk/nltk/issues/3293
+    # - https://github.com/nltk/nltk/issues/3266
+    # - https://nvd.nist.gov/vuln/detail/CVE-2024-39705
+    #
+    # TODO: Remove support for nltk<=3.8.1 and only install "punkt_tab"
+    nltk_major_version, nltk_minor_version = [int(v) for v in version("nltk").split(".")[0:2]]
+    if nltk_major_version < 3:
+        raise Exception("ntlk version <3 is not supported")
+    if nltk_major_version == 3 and nltk_minor_version <= 8:
+        try:
+            nltk.data.find("tokenizers/punkt")
+        except LookupError:
+            nltk.download("punkt")
+    else:
+        try:
+            nltk.data.find("tokenizers/punkt_tab")
+        except LookupError:
+            nltk.download("punkt_tab")

helm/benchmark/metrics/omni_math_metrics.py

@@ -0,0 +1,44 @@
+from typing import Any, Dict, List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class OmniMATHMetric(Metric):
+    """Score metrics for Omni-MATH."""
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        annotations: Dict[str, Any] = request_state.annotations["omni_math"]
+        scores: List[int] = []
+        for annotation_key, annotation_value in annotations.items():
+            if annotation_key.endswith("_equivalence_judgement") and annotation_value is not None:
+                scores.append(int(annotation_value))
+        if not scores:
+            raise ValueError("Could not compute Omni-MATH accuracy because all annotators failed.")
+        score = sum(scores) / len(scores)
+        return [
+            Stat(MetricName("omni_math_accuracy")).add(score),
+        ]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="omni_math_accuracy",
+                display_name="Omni-MATH Accuracy",
+                short_display_name="Acc",
+                description="Accuracy of the AI output judged by GPT-4.",
+                lower_is_better=False,
+                group="accuracy",
+            ),
+        ]

helm/benchmark/metrics/openai_mrcr_metrics.py

@@ -0,0 +1,52 @@
+from difflib import SequenceMatcher
+from typing import List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+from helm.benchmark.scenarios.scenario import CORRECT_TAG
+
+
+class OpenAIMRCRMetric(Metric):
+    """Accuracy metric for OpenAI MRCR.
+
+    The measured metric is the SequenceMatcher ratio as implemented in https://docs.python.org/3/library/difflib.html.
+    The model must prepend an alphanumeric hash to the beginning of its answer. If this hash is not included,
+    the match ratio is set to 0. If it is correctly included, the stripped sampled answer is compared to the
+    stripped ground truth answer.
+
+    Adapted from: https://huggingface.co/datasets/openai/mrcr/blob/204b0d4e8d9ca5c0a90bf942fdb2a5969094adc0/README.md
+    """
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.result
+        assert len(request_state.result.completions) == 1
+
+        response_text = request_state.result.completions[0].text
+
+        assert len(request_state.instance.references) == 1
+        assert len(request_state.instance.references[0].tags) == 1
+        assert request_state.instance.references[0].tags[0] == CORRECT_TAG
+
+        gold_text = request_state.instance.references[0].output.text
+
+        assert request_state.instance.extra_data
+        assert "random_string_to_prepend" in request_state.instance.extra_data
+        random_string_to_prepend = request_state.instance.extra_data["random_string_to_prepend"]
+
+        score = 0.0
+        if response_text.startswith(random_string_to_prepend):
+            response_sequence = response_text.removeprefix(random_string_to_prepend)
+            gold_sequence = gold_text.removeprefix(random_string_to_prepend)
+            score = float(SequenceMatcher(None, response_sequence, gold_sequence).ratio())
+
+        return [Stat(MetricName("openai_mrcr_accuracy")).add(score)]

helm/benchmark/metrics/output_processing_metric.py

@@ -0,0 +1,60 @@
+import dataclasses
+from typing import Any, Dict, List, TypedDict
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.adaptation.scenario_state import ScenarioState
+from helm.benchmark.metrics.metric import (
+    create_metric,
+    Metric,
+    MetricInterface,
+    MetricResult,
+    MetricSpec,
+    PerInstanceStats,
+)
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+from helm.common.object_spec import get_class_by_name
+from helm.common.request import GeneratedOutput
+
+
+class _MetricSpecDict(TypedDict):
+    class_name: str
+    args: Dict[str, Any]
+
+
+def _dict_to_metric_spec(metric_spec_dict: _MetricSpecDict) -> MetricSpec:
+    return MetricSpec(metric_spec_dict["class_name"], metric_spec_dict["args"])
+
+
+class OutputProcessingMetric(MetricInterface):
+    def __init__(self, processor: str, metric_specs: List[_MetricSpecDict]):
+        self.processor = get_class_by_name(processor)  # actually a function, not a class
+        self.metrics: List[Metric] = [create_metric(_dict_to_metric_spec(metric_spec)) for metric_spec in metric_specs]
+
+    def _process_request_state(self, request_state: RequestState) -> RequestState:
+        if not request_state.result:
+            return request_state
+        processed_completions: List[GeneratedOutput] = []
+        for completion in request_state.result.completions:
+            processed_completions.append(dataclasses.replace(completion, text=self.processor(completion.text)))
+        return dataclasses.replace(
+            request_state, result=dataclasses.replace(request_state.result, completions=processed_completions)
+        )
+
+    def evaluate(
+        self, scenario_state: ScenarioState, metric_service: MetricService, eval_cache_path: str, parallelism: int
+    ) -> MetricResult:
+        aggregated_stats: List[Stat] = []
+        per_instance_stats: List[PerInstanceStats] = []
+
+        processed_scenario_state = dataclasses.replace(
+            scenario_state,
+            request_states=[
+                self._process_request_state(request_state) for request_state in scenario_state.request_states
+            ],
+        )
+        for metric in self.metrics:
+            metric_result = metric.evaluate(processed_scenario_state, metric_service, eval_cache_path, parallelism)
+            aggregated_stats.extend(metric_result.aggregated_stats)
+            per_instance_stats.extend(metric_result.per_instance_stats)
+        return MetricResult(aggregated_stats=aggregated_stats, per_instance_stats=per_instance_stats)

helm/benchmark/metrics/output_processors.py

@@ -0,0 +1,15 @@
+import re
+
+
+def remove_deepseek_r1_thinking(input: str) -> str:
+    if "<think>" not in input:
+        return input
+
+    if "</think>\n\n" in input:
+        # The think block is usually followed by two newlines, so we should remove that
+        return re.sub("<think>.*</think>\n\n", "", input, flags=re.DOTALL)
+    elif "</think>" in input:
+        return re.sub("<think>.*</think>", "", input, flags=re.DOTALL)
+    else:
+        # Unclosed think block
+        return ""

helm/benchmark/metrics/paraphrase_generation_metrics.py

@@ -1,13 +1,13 @@
 from typing import List
 
 from helm.benchmark.adaptation.request_state import RequestState
-from .metric import Metric
-from .metric_name import MetricName
-from .statistic import Stat
+from helm.benchmark.metrics.evaluate_instances_metric import EvaluateInstancesMetric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.statistic import Stat
 from nltk.translate.bleu_score import corpus_bleu
 
 
-class CLEVAParaphraseGenerationMetric(Metric):
+class CLEVAParaphraseGenerationMetric(EvaluateInstancesMetric):
     """
     Compute the Chinese iBLEU score for Paraphrase Generation scenarios of CLEVA benchmark.
     This implementation allows variable number of references (i.e., golds).
@@ -21,8 +21,7 @@ class CLEVAParaphraseGenerationMetric(Metric):
     def __init__(self, alpha: float = 0.8):  # calculate iBLEU_0.8 by default
         self.alpha = alpha
 
-    def evaluate_instances(self, request_states: List[RequestState]) -> List[Stat]:
-
+    def evaluate_instances(self, request_states: List[RequestState], eval_cache_path: str) -> List[Stat]:
         inputs: List = []
         preds: List = []
         golds: List[List[str]] = []

helm/benchmark/metrics/prometheus_vision_critique_metrics.py

@@ -0,0 +1,185 @@
+from typing import Dict, List, Optional
+import re
+
+from tqdm import tqdm
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.adaptation.scenario_state import ScenarioState
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.metrics.metric import MetricInterface, MetricResult, PerInstanceStats, add_context
+from helm.benchmark.metrics.metric_name import MetricContext, MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat, merge_stat
+from helm.common.critique_request import CritiqueTaskTemplate, CritiqueQuestionTemplate, CritiqueRequest, QuestionType
+from helm.common.hierarchical_logger import hlog
+from helm.common.request import RequestResult, GeneratedOutput
+from helm.common.media_object import MultimediaObject, IMAGE_TYPE, MediaObject, TEXT_TYPE
+
+
+class PrometheusVisionCritiqueMetric(MetricInterface):
+    """
+    We compute the same metrics from the Prometheus-Vision: Vision-Language Model as a Judge for
+    Fine-Grained Evaluation paper:
+    https://arxiv.org/pdf/2401.06591.pdf
+
+    In this paper, the output of a Vision-Language Model named Prometheus-Vision is used to evaluate
+    the quality of the output of other Vision-Language Models to be evaluated.
+    """
+
+    # We can add more evaluation aspects here
+    METRIC_NAME: str = "prometheus_vision"
+    METRIC_PROMPT: str = """A chat between a curious human and an artificial intelligence assistant. \
+        The assistant gives helpful, detailed, and polite answers to the human's questions.\nUSER:<image>\
+###Task Description:
+An instruction (might include an Input inside it), a response to evaluate, a reference answer that gets a score of 5, \
+    image and a score rubric representing an evaluation criterion is given.
+1. Write a detailed feedback that assesses the quality of the response strictly based on the given score rubric, not \
+    evaluating in general.
+2. After writing a feedback, write a score that is an integer between 1 and 5. You should refer to the score rubric.
+3. The output format should look as follows: Feedback: (write a feedback for criteria) [RESULT] (an integer number \
+    between 1 and 5)
+4. Please do not generate any other opening, closing, and explanations.
+
+###The instruction to evaluate:
+{{orig_instruction}}
+
+###Response to evaluate:
+{{orig_response}}
+
+###Reference Answer (Score 5):
+{{orig_reference_answer}}
+
+###Score Rubrics:
+[{{orig_criteria}}]
+Score 1: {{orig_score1_description}}
+Score 2: {{orig_score2_description}}
+Score 3: {{orig_score3_description}}
+Score 4: {{orig_score4_description}}
+Score 5: {{orig_score5_description}}
+
+###Feedback:
+ASSISTANT:
+"""
+
+    def __init__(self, num_respondents: int, max_tokens: int):
+        self._num_respondents = num_respondents
+        self._max_tokens = max_tokens
+
+    def __repr__(self) -> str:
+        return "PrometheusVisionCritiqueMetric()"
+
+    def _extract_score_from_prometheus_vision_output(self, evaluator_response: str):
+        evaluator_response = evaluator_response.split("ASSISTANT:")[1]
+        re_match = re.search(r"\s*([1-5])", evaluator_response)
+        if re_match is None:
+            hlog(f"Error parsing answer: {evaluator_response}. Skipping question (and so the respondent entirely)")
+            return None
+        return int(re_match.group(1))
+
+    def evaluate(
+        self,
+        scenario_state: ScenarioState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+        parallelism: int,
+    ) -> MetricResult:
+        request_states: List[RequestState] = scenario_state.request_states
+
+        all_stats: Dict[MetricName, Stat] = {}
+        per_instance_stats: List[PerInstanceStats] = []
+        for request_state in tqdm(request_states):
+            context = MetricContext.from_instance(request_state.instance)
+            stats_without_context = self.evaluate_generation(
+                scenario_state.adapter_spec,
+                request_state,
+                metric_service,
+                eval_cache_path,
+            )
+            stats = [add_context(stat_without_context, context) for stat_without_context in stats_without_context]
+            for stat in stats:
+                merge_stat(all_stats, stat)
+            assert request_state.instance.id is not None
+            per_instance_stats.append(
+                PerInstanceStats(
+                    instance_id=request_state.instance.id,
+                    perturbation=request_state.instance.perturbation,
+                    train_trial_index=request_state.train_trial_index,
+                    stats=stats,
+                )
+            )
+        return MetricResult(aggregated_stats=list(all_stats.values()), per_instance_stats=per_instance_stats)
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        input_content = request_state.request
+        # Predicted outputs and their prometheus vision scores
+        assert request_state.result is not None
+        request_result: RequestResult = request_state.result
+        # Get input image and generated response for evaluation
+        assert input_content.multimodal_prompt is not None
+        completions: List[GeneratedOutput] = request_result.completions
+        generated_text: str = completions[0].text
+        input_media: MultimediaObject = input_content.multimodal_prompt
+        ref_text: str = request_state.instance.references[0].output.text
+        image_objects: List[MediaObject] = [
+            item for item in input_media.media_objects if item.is_type(IMAGE_TYPE) and item.location
+        ]
+        input_text: Optional[str] = [item for item in input_media.media_objects if item.is_type(TEXT_TYPE)][0].text
+
+        template = CritiqueTaskTemplate(
+            name="vhelm_prometheus_vision",
+            instructions=self.METRIC_PROMPT,
+            num_respondents=self._num_respondents,
+            max_tokens=self._max_tokens,
+            questions=[
+                CritiqueQuestionTemplate(
+                    name=self.METRIC_NAME,
+                    question_type=QuestionType.FREE_RESPONSE,
+                    text="",
+                    options=[],
+                    media_object=image_objects[0],  # we only take the first image as input
+                )
+            ],
+        )
+        request = CritiqueRequest(
+            template=template,
+            fields={
+                "orig_instruction": input_text if input_text is not None else "",
+                "orig_response": generated_text,
+                "orig_reference_answer": ref_text,
+                "orig_criteria": "similarity between the reponse and the reference.",
+                "orig_score1_description": "The model's responses do not follow the instructions provided.",
+                "orig_score2_description": "The resulting response follows the instructions, but the answer \
+                    is completely wrong relative to the reference answer.",
+                "orig_score3_description": "The resulting response follows the instructions, but the answer is \
+                    partially wrong relative to the reference answer.",
+                "orig_score4_description": "The resulting response follows the instructions, the overall answer \
+                    is relatively perfect with only a very few errors.",
+                "orig_score5_description": "The overall answer is completely correct compared to the reference \
+                    answer, and conforms to the instructions provided.",
+            },
+        )
+        # send to critique request
+        result = metric_service.make_critique_request(request)
+        if not result or not result.responses:
+            # Skip computing metrics if there aren't any responses yet
+            hlog("Waiting for responses to be generated.")
+            return []
+
+        stats: Dict[str, Stat] = {}
+        for question in template.questions:
+            stats[question.name] = Stat(MetricName(question.name))
+
+        for response in result.responses:
+            for answer_name, answer in response.answers.items():
+                assert isinstance(answer, str)
+                answer_value: float
+                answer_value = self._extract_score_from_prometheus_vision_output(answer)
+                stats[answer_name].add(answer_value)
+
+        return list(stats.values())

helm/benchmark/metrics/ranking_metrics.py

@@ -4,14 +4,14 @@ from typing import Callable, Dict, List, Tuple, Optional
 from helm.benchmark.adaptation.adapters.adapter_factory import ADAPT_RANKING_BINARY
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.metrics.reference_metric import ReferenceMetric
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.benchmark.scenarios.scenario import unpack_tag, CORRECT_TAG, Reference
 from helm.common.request import RequestResult
 from helm.common.general import assert_present, binarize_dict
-from .metric import Metric
-from .metric_name import MetricName
-from .metric_service import MetricService
-from .statistic import Stat
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
 
 try:
     import pytrec_eval
@@ -58,7 +58,7 @@ class RankingObject:
     model_relevance: Optional[int] = None
 
 
-class RankingMetric(Metric):
+class RankingMetric(ReferenceMetric):
     """Ranking metric."""
 
     """ Methods supported by this metric.