crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/metrics/language_modeling_metrics.py

@@ -0,0 +1,111 @@
+from collections import defaultdict
+from typing import List, Dict, Set
+
+from helm.benchmark.adaptation.scenario_state import ScenarioState
+from helm.benchmark.metrics.basic_metrics import (
+    compute_language_modeling_metrics,
+    compute_perplexity_metrics,
+    compute_request_state_metrics,
+)
+from helm.benchmark.metrics.efficiency_metrics import EfficiencyMetric
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.metrics.metric import MetricInterface, MetricMetadata, MetricResult, PerInstanceStats, add_context
+from helm.benchmark.metrics.metric_name import MetricContext, MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat, merge_stat
+
+
+class LanguageModelingMetric(MetricInterface):
+    """
+    Defines the basic metrics available when using the ADAPT_LANGUAGE_MODELING adapter.
+    This is parallel to BasicMetric and produces many of the same Stats.
+    """
+
+    def __init__(self, names: List[str]):
+        self.names: List[str] = names
+        self.efficiency_metric = EfficiencyMetric()
+
+    def __repr__(self):
+        return "LanguageModelingMetric"
+
+    def evaluate(
+        self, scenario_state: ScenarioState, metric_service: MetricService, eval_cache_path: str, parallelism: int
+    ) -> MetricResult:
+        global_stats: Dict[MetricName, Stat] = {}
+        # The first and only trial
+        trial_stats: Dict[MetricName, Stat] = {}
+        # Per-instance stats
+        all_per_instance_stats: List[PerInstanceStats] = []
+        instance_ids_per_context: Dict[MetricContext, Set[str]] = defaultdict(set)
+
+        for request_state in scenario_state.request_states:
+            # Evaluate request_state
+            request_stats = self.evaluate_generation(
+                scenario_state.adapter_spec, request_state, metric_service, eval_cache_path
+            )
+
+            # Add instance-related context (e.g., split, perturbation) to the metrics
+            for i, stat in enumerate(request_stats):
+                context = MetricContext.from_instance(request_state.instance)
+                request_stats[i] = add_context(stat, context)
+                assert request_state.instance.id is not None
+                instance_ids_per_context[context].add(request_state.instance.id)
+
+            # Use trial index of 0 here since we run only one trial for LM
+            assert request_state.instance.id is not None
+            all_per_instance_stats.append(
+                PerInstanceStats(request_state.instance.id, request_state.instance.perturbation, 0, request_stats)
+            )
+
+            for stat in request_stats:
+                merge_stat(trial_stats, stat)
+
+        # group stats according to the context (e.g., split, perturbation) and call derive_stats on each grouping
+        grouped_trial_stats: Dict[MetricContext, Dict[MetricName, Stat]] = defaultdict(dict)
+        for metric_name, stat in trial_stats.items():
+            grouped_trial_stats[MetricContext.from_metric_name(metric_name)][metric_name] = stat  # group by context
+
+        for context, stats_dict in grouped_trial_stats.items():
+            for stat in self.derive_stats(stats_dict):
+                merge_stat(trial_stats, add_context(stat, context))
+            # keep track of how many instances are in each subset
+            num_instances_stat = Stat(MetricName("num_instances")).add(len(instance_ids_per_context[context]))
+            merge_stat(trial_stats, add_context(num_instances_stat, context))
+
+        for stat in trial_stats.values():
+            merge_stat(global_stats, stat.take_mean())
+        return MetricResult(list(global_stats.values()), all_per_instance_stats)
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        """Compute all metrics."""
+        stats: List[Stat] = []
+        stats.extend(compute_request_state_metrics(self.efficiency_metric, adapter_spec, request_state, metric_service))
+        stats.extend(compute_language_modeling_metrics(adapter_spec, request_state, metric_service))
+
+        return stats
+
+    def derive_stats(self, stats_dict: Dict[MetricName, Stat]) -> List[Stat]:
+        """Derive perplexity metrics if applicable. We don't worry about splits and perturbations here."""
+        derived_stats: List[Stat] = []
+        derived_stats.extend(compute_perplexity_metrics(stats_dict))
+        return derived_stats
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="num_instances",
+                display_name="# eval",
+                short_display_name=None,
+                description="Number of evaluation instances.",
+                lower_is_better=None,
+                group="general_information",
+            ),
+        ]

helm/benchmark/metrics/live_qa_metrics.py

@@ -0,0 +1,35 @@
+from typing import List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class LiveQAScoreMetric(Metric):
+    """Score metrics for LiveQA."""
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        score = request_state.annotations["live_qa"]["score"]
+        return [Stat(MetricName("live_qa_score")).add(score)]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="live_qa_score",
+                display_name="Judge Score",
+                short_display_name=None,
+                description="LLM-as-judge score",
+                lower_is_better=False,
+                group=None,
+            ),
+        ]

helm/benchmark/metrics/llm_jury_metrics.py

@@ -0,0 +1,58 @@
+from typing import Any, Dict, List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.model_as_judge import AnnotatorModelInfo
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class LLMJuryMetric(Metric):
+    """Score metrics for LLM Jury."""
+
+    def __init__(
+        self,
+        metric_name: str,
+        scenario_name: str,
+        annotator_models: Dict[str, AnnotatorModelInfo],
+        default_score: float = 0.0,
+    ):
+        self.metric_name = metric_name
+        self.scenario_name = scenario_name
+        self.annotator_models = annotator_models
+        self.default_score = default_score
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        annotations: Dict[str, Any] = request_state.annotations[self.scenario_name]
+        scores: List[int] = []
+        score = self.default_score
+        for annotation_key, annotation_dict in annotations.items():
+            if annotation_key in self.annotator_models.keys() and annotation_dict is not None:
+                for val in annotation_dict.values():
+                    scores.append(int(val["score"]))
+        if scores:
+            score = sum(scores) / len(scores)
+        return [
+            Stat(MetricName(self.metric_name)).add(score),
+        ]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name=self.metric_name,
+                display_name="Jury Score",
+                short_display_name="Jury Score",
+                description="Measures the average score assigned by an LLM-based jury evaluating task performance.",
+                lower_is_better=False,
+                group=None,
+            ),
+        ]

helm/benchmark/metrics/lmkt_metric_specs.py

@@ -0,0 +1,12 @@
+from typing import List
+
+from helm.benchmark.metrics.metric import MetricSpec
+
+
+def get_semantic_similarity_metric_specs(similarity_fn_name: str = "cosine") -> List[MetricSpec]:
+    return [
+        MetricSpec(
+            class_name="helm.benchmark.metrics.lmkt_metrics.SemanticSimilarityMetric",
+            args={"similarity_fn_name": similarity_fn_name},
+        ),
+    ]

helm/benchmark/metrics/lmkt_metrics.py

@@ -0,0 +1,47 @@
+from typing import List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+from helm.benchmark.scenarios.scenario import CORRECT_TAG
+from sentence_transformers import SentenceTransformer
+
+
+class SemanticSimilarityMetric(Metric):
+    """Score metrics for LMKT semantic similarity measurement."""
+
+    def __init__(self, similarity_fn_name: str = "cosine"):
+        """
+        Initialize the SemanticSimilarityMetric with a SentenceTransformer model.
+        :param similarity_fn_name: The name of the similarity function to use.
+        Available options are "dot", "cosine", "manhattan" and "euclidean".
+        """
+        super().__init__()
+
+        self.model = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2", similarity_fn_name=similarity_fn_name)
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+
+        assert request_state.result
+
+        completions = [c.text for c in request_state.result.completions]
+        completion_embeddings = self.model.encode(completions)
+
+        references = [r.output.text for r in request_state.instance.references if CORRECT_TAG in r.tags]
+        reference_embeddings = self.model.encode(references)
+
+        similarities = self.model.similarity(completion_embeddings, reference_embeddings)
+        avg_similarity = similarities.mean().item()
+
+        return [
+            Stat(MetricName("semantic_similarity")).add(avg_similarity),
+        ]

helm/benchmark/metrics/machine_translation_metrics.py

@@ -0,0 +1,89 @@
+from typing import List
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.evaluate_instances_metric import EvaluateInstancesMetric
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.statistic import Stat
+
+try:
+    from sacrebleu.metrics import BLEU
+    from langdetect import detect
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e)
+
+
+class MachineTranslationMetric(EvaluateInstancesMetric):
+    """
+    Compute the BLEU score for Machine Translation scenarios. The implementation is based on sacrebleu.
+    """
+
+    def evaluate_instances(self, request_states: List[RequestState], eval_cache_path: str) -> List[Stat]:
+        """
+        Compute the corpus-level metric based on all reqeust_states.
+        """
+
+        bleu = BLEU()
+
+        refs: List[List[str]] = [[]]
+        sys: List = []
+        for request_state in request_states:
+            # Assume there is one referece per instance. TODO: Support multiple references after adding more scenarios.
+            num_references: int = len(request_state.instance.references)
+            if num_references != 1:
+                raise ValueError(f"This instance has {num_references} references, but we currently only support one.")
+            # Usually there is only one completion for each instance.
+            assert request_state.result is not None
+            if len(request_state.result.completions) != 1:
+                raise ValueError("Each request result should have only exactly one completion.")
+            sys.append(request_state.result.completions[0].text)
+            refs[0].append(request_state.instance.references[0].output.text)
+        bleu_score = bleu.corpus_score(sys, refs).score
+        return [Stat(MetricName("bleu")).add(bleu_score)]
+
+
+class CLEVAMachineTranslationMetric(EvaluateInstancesMetric):
+    """
+    Compute the BLEU score for Machine Translation scenarios of CLEVA benchmark.
+    Based on sacrebleu, this implementation distinguishes target language and allows variable number of references.
+    If there are more than one hypothesis, only the first one is adopted in the calculation.
+    """
+
+    def evaluate_instances(self, request_states: List[RequestState], eval_cache_path: str) -> List[Stat]:
+        """
+        Compute the corpus-level metric based on all reqeust_states.
+        """
+
+        def detect_language(request_states: List[RequestState]) -> str:
+            """
+            Determine the target language by detecting the language of references.
+            Currently, it only distinguishes if the target language is Chinese.
+            """
+
+            corpus: str = "".join(
+                [request_state.instance.references[0].output.text for request_state in request_states[:10]]
+            )
+            if detect(corpus) in ["zh-cn", "zh-tw"]:
+                return "zh"
+            else:
+                return "13a"  # Default tokenizer for sacrebleu.BLEU
+
+        bleu = BLEU(tokenize=detect_language(request_states))
+
+        max_num_references: int = max([len(request_state.instance.references) for request_state in request_states])
+        refs: List[List[str]] = [
+            [
+                request_state.instance.references[i].output.text if i < len(request_state.instance.references) else ""
+                for request_state in request_states
+            ]
+            for i in range(max_num_references)
+        ]
+
+        sys: List = []
+        for request_state in request_states:
+            assert request_state.result is not None
+            sys.append(request_state.result.completions[0].text)
+
+        bleu_score = bleu.corpus_score(sys, refs).score
+
+        return [Stat(MetricName("cleva_machine_translation_bleu")).add(bleu_score)]

helm/benchmark/metrics/medcalc_bench_metrics.py

@@ -0,0 +1,137 @@
+import re
+
+from datetime import datetime
+from typing import List, Dict, Any
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+from helm.common.hierarchical_logger import hlog
+
+
+class MedCalcBenchMetric(Metric):
+    """
+    Metric for evaluating the MedCalc Bench dataset, assessing the model's ability to
+    be a clinical calculator.
+
+    Exact match based on category:
+    1. Normal exact match: for categories "risk", "severity" or "diagnosis".
+    2. Variant exact match: for other categories, if the number calculated by the model falls between the values
+        in the Lower limit and Upper limit columns, we mark it as accurate.
+    """
+
+    def parse_duration(self, duration_str) -> int:
+        """Parses a duration tuple (weeks, days) from a string format like ('14 weeks', '2 days')."""
+        match = re.match(r"\('(\d+) weeks', '(\d+) days'\)", duration_str)
+        if match:
+            weeks, days = map(int, match.groups())
+            return weeks * 7 + days  # Convert to total days
+        else:
+            raise ValueError(f"Invalid format: {duration_str}")
+
+    def is_within_range(self, lower_bound, upper_bound, prediction) -> int:
+        """
+        Checks if a predicted duration falls within the given range.
+
+        Args:
+            lower_bound (str): The lower bound in format "('X weeks', 'Y days')".
+            upper_bound (str): The upper bound in format "('X weeks', 'Y days')".
+            prediction (str): The predicted duration in the same format.
+
+        Returns:
+            int: 1 if within range (inclusive), 0 otherwise.
+        """
+        lower_days = self.parse_duration(lower_bound)
+        upper_days = self.parse_duration(upper_bound)
+        prediction_days = self.parse_duration(prediction)
+        return 1 if lower_days <= prediction_days <= upper_days else 0
+
+    def check_date(self, prediction: str, reference: str, extra_data: Dict[str, Any]) -> int:
+        """Checks if prediction date is withing limits"""
+        if re.match(r"\('(\d+) weeks', '(\d+) days'\)", reference):
+            exact_match = self.is_within_range(extra_data["lower_limit"], extra_data["upper_limit"], prediction)
+        else:
+            prediction_date = self._str_to_date(prediction)
+            upper_limit_date = self._str_to_date(extra_data["upper_limit"])
+            lower_limit_date = self._str_to_date(extra_data["lower_limit"])
+            exact_match = 1 if lower_limit_date <= prediction_date <= upper_limit_date else 0
+        return exact_match
+
+    def _str_to_date(self, date_str: str) -> datetime:
+        """Convert string to datetime object."""
+        return datetime.strptime(date_str, "%m/%d/%Y")
+
+    def check_in_range(self, prediction: str, reference: str, extra_data: Dict[str, Any], category: str) -> int:
+        """Check if the prediction falls within the range specified by the reference."""
+        try:
+            if category == "date":
+                exact_match = self.check_date(prediction, reference, extra_data)
+            elif category in ["dosage conversion", "physical"]:
+                lower_limit = float(extra_data["lower_limit"])
+                upper_limit = float(extra_data["upper_limit"])
+                float_prediction = float(prediction)
+                exact_match = 1 if lower_limit <= float_prediction <= upper_limit else 0
+            else:
+                raise ValueError(f"Category {category} not supported")
+        except ValueError:
+            return 0
+
+        return exact_match
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        """
+        Evaluate a single generation against reference labels.
+        """
+        # Extract predictions
+        assert request_state.result, "request_state.result is unexpectedly None"
+        predictions = [completion.text.strip() for completion in request_state.result.completions]
+
+        if not predictions:
+            hlog("Warning: No predictions found in completions")
+            return []
+
+        # Get the first prediction
+        prediction = predictions[0]
+
+        # Get references
+        references = getattr(request_state.instance, "references", None)
+
+        if not references or len(references) == 0:
+            hlog(f"Warning: Missing references for instance {request_state.instance}")
+            return []
+
+        reference = references[0].output.text
+
+        # Extract category, upper limit and lower limit
+        assert request_state.instance.extra_data, "Extra data dict was expected but got None"
+        category = request_state.instance.extra_data["category"]
+
+        if category in ["risk", "severity", "diagnosis"]:
+            exact_match = 1 if prediction == reference else 0
+        else:
+            exact_match = self.check_in_range(prediction, reference, request_state.instance.extra_data, category)
+
+        return [
+            Stat(MetricName("medcalc_bench_accuracy")).add(exact_match),
+        ]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="medcalc_bench_accuracy",
+                display_name="MedCalc Accuracy",
+                short_display_name="MedCalc Accuracy",
+                description="Comparison based on category. Exact match for categories risk, severity and diagnosis. "
+                "Check if within range for the other categories.",
+                lower_is_better=False,
+                group=None,
+            ),
+        ]

helm/benchmark/metrics/medec_metrics.py

@@ -0,0 +1,124 @@
+from typing import List
+import re
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+from helm.common.hierarchical_logger import hlog
+from helm.benchmark.scenarios.scenario import CORRECT_TAG
+
+
+class MedecMetric(Metric):
+    """
+    Metric for evaluating the MEDEC dataset, assessing medical error detection and correction.
+
+    - **Error Flag Accuracy**: Whether the model correctly identifies if a medical note contains an error.
+    - **Error Sentence Detection Accuracy**: Whether the model correctly identifies the erroneous
+        sentence when an error is present.
+    """
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        """
+        Evaluate a single LLM generation against the ground truth labels.
+        """
+
+        # Extract predictions from the model output
+        if request_state.result is not None:
+            predictions = [completion.text.strip() for completion in request_state.result.completions]
+        else:
+            predictions = []
+
+        if not predictions:
+            raise ValueError("No predictions found in the completions.")
+
+        prediction = predictions[0]  # Use the first completion
+
+        # Extract references (ground truth)
+        references = getattr(request_state.instance, "references", None)
+
+        if not references or len(references) == 0:
+            hlog(f"Warning: Missing references for instance {request_state.instance}")
+            return []
+
+        ground_truth_reference = next((ref for ref in references if CORRECT_TAG in ref.tags), None)
+
+        if not ground_truth_reference:
+            hlog(f"Warning: No ground truth reference with CORRECT_TAG for instance {request_state.instance}")
+            return []
+
+        # Extract the ground truth error flag and sentence ID
+        # ground_truth_text_pre = ground_truth_reference.output.text.strip()
+        ground_truth_text = ground_truth_reference.output.text.replace(".", "").strip()
+
+        # Determine ground truth values
+        if ground_truth_text == "CORRECT":
+            ground_truth_flag = 0  # No medical error
+            ground_truth_sentence_id = -1  # No sentence should be flagged
+        else:
+            ground_truth_flag = 1  # Medical error exists
+            try:
+                ground_truth_sentence_id = int(ground_truth_text)
+            except ValueError:
+                ground_truth_sentence_id = -1
+
+        # Determine model predictions
+        prediction = prediction.strip()
+
+        if prediction == "CORRECT":
+            predicted_flag = 0
+            predicted_sentence_id = -1
+        else:
+            match = re.match(r"^(\d+)", prediction)  # Ensure first token is a number
+            if match:
+                predicted_sentence_id = int(match.group(1))
+                predicted_flag = 1  # A valid sentence ID implies the model thinks there's an error
+            else:
+                predicted_flag = 1  # The model attempted to correct something, but in an invalid format
+                predicted_sentence_id = -2  # Mark invalid outputs
+
+        # Calculate Error Flag Accuracy
+        flag_accuracy = 1 if predicted_flag == ground_truth_flag else 0
+
+        # Calculate Error Sentence Accuracy
+        if ground_truth_flag == 1 and predicted_flag == 1:
+            sentence_accuracy = 1 if predicted_sentence_id == ground_truth_sentence_id else 0
+        else:
+            sentence_accuracy = (
+                1 if ground_truth_flag == 0 and predicted_flag == 0 else 0
+            )  # Both must agree it's "CORRECT"
+
+        return [
+            Stat(MetricName("medec_error_flag_accuracy")).add(flag_accuracy),
+            Stat(MetricName("medec_error_sentence_accuracy")).add(sentence_accuracy),
+        ]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="medec_error_flag_accuracy",
+                display_name="Medical Error Flag Accuracy",
+                short_display_name="MedecFlagAcc",
+                description="Measures how accurately the model identifies whether a clinical note contains an error "
+                "(binary classification of correct/incorrect).",
+                lower_is_better=False,
+                group=None,
+            ),
+            MetricMetadata(
+                name="medec_error_sentence_accuracy",
+                display_name="Medical Error Sentence Accuracy",
+                short_display_name="MedecSentenceAcc",
+                description="Measures how accurately the model identifies the specific erroneous sentence within a "
+                "clinical note.",
+                lower_is_better=False,
+                group=None,
+            ),
+        ]