crfm-helm 0.5.4__py3-none-any.whl → 0.5.6__py3-none-any.whl

helm/benchmark/metrics/kpi_edgar_metrics.py

@@ -0,0 +1,121 @@
+from dataclasses import dataclass
+from typing import Iterable, List, Dict, Set, Tuple
+import re
+
+import numpy as np
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.evaluate_instances_metric import EvaluateInstancesMetric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.statistic import Stat
+from helm.benchmark.scenarios.kpi_edgar_scenario import KPIEDGARScenario
+
+
+@dataclass
+class _Entity:
+    phrase: str
+    tag: str
+
+
+@dataclass
+class _Counts:
+    num_overlap: int
+    num_pred: int
+    num_gold: int
+
+
+@dataclass
+class _Rates:
+    tp: float
+    fn: float
+    fp: float
+
+
+def _get_all_tags() -> Iterable[str]:
+    return KPIEDGARScenario.TAG_DICT.keys()
+
+
+def _parse_entities(text: str) -> List[_Entity]:
+    all_matches = re.findall(r"(?:^|(?<=\],))([^\[\]]+)\[([0-9a-z]+)\](?:$|(?=,))", text.strip())
+    return [_Entity(phrase=match[0].strip(), tag=match[1].strip()) for match in all_matches]
+
+
+def _compute_tag_to_words(entities: List[_Entity]) -> Dict[str, Set[str]]:
+    tag_to_words: Dict[str, Set[str]] = {tag: set() for tag in _get_all_tags()}
+    for entity in entities:
+        for word in entity.phrase.split():
+            tag_to_words[entity.tag].add(word)
+    return tag_to_words
+
+
+def _compute_tag_to_counts(pred_entities: List[_Entity], gold_entities: List[_Entity]) -> Dict[str, _Counts]:
+    tag_to_counts: Dict[str, _Counts] = {}
+    pred_tag_to_words = _compute_tag_to_words(pred_entities)
+    gold_tag_to_words = _compute_tag_to_words(gold_entities)
+    for tag in _get_all_tags():
+        tag_to_counts[tag] = _Counts(
+            num_overlap=len(pred_tag_to_words[tag] & gold_tag_to_words[tag]),
+            num_pred=len(pred_tag_to_words[tag]),
+            num_gold=len(gold_tag_to_words[tag]),
+        )
+    return tag_to_counts
+
+
+def _counts_to_rates(counts: _Counts, adjust: bool) -> _Rates:
+    if adjust:
+        return _Rates(
+            tp=counts.num_overlap / counts.num_gold if counts.num_gold > 0 else 0.0,
+            fn=1 - (counts.num_overlap / counts.num_gold) if counts.num_gold > 0 else 0.0,
+            fp=(counts.num_pred - counts.num_overlap) / counts.num_pred if counts.num_pred > 0 else 0.0,
+        )
+    else:
+        return _Rates(
+            tp=counts.num_overlap,
+            fn=counts.num_gold - counts.num_overlap,
+            fp=counts.num_pred - counts.num_overlap,
+        )
+
+
+def _compute_f1_score(rates: _Rates) -> float:
+    return (2 * rates.tp) / (2 * rates.tp + rates.fp + rates.fn) if rates.tp + rates.fp + rates.fn > 0 else 0.0
+
+
+def _compute_stats(pred_gold_pairs: List[Tuple[str, str]]) -> List[Stat]:
+    tag_to_counts: Dict[str, _Counts] = {tag: _Counts(0, 0, 0) for tag in _get_all_tags()}
+    for pred_text, gold_text in pred_gold_pairs:
+        pred_entities = _parse_entities(pred_text)
+        gold_entities = _parse_entities(gold_text)
+        instance_tag_to_counts = _compute_tag_to_counts(pred_entities=pred_entities, gold_entities=gold_entities)
+        for tag, instance_counts in instance_tag_to_counts.items():
+            tag_to_counts[tag].num_overlap += instance_counts.num_overlap
+            tag_to_counts[tag].num_pred += instance_counts.num_pred
+            tag_to_counts[tag].num_gold += instance_counts.num_gold
+    tag_word_f1_scores: List[float] = [
+        _compute_f1_score(_counts_to_rates(counts, adjust=False)) for counts in tag_to_counts.values()
+    ]
+    tag_adjusted_f1_scores: List[float] = [
+        _compute_f1_score(_counts_to_rates(counts, adjust=True)) for counts in tag_to_counts.values()
+    ]
+    return [
+        Stat(MetricName("word_macro_f1_score")).add(np.mean(tag_word_f1_scores)),
+        Stat(MetricName("adjusted_macro_f1_score")).add(np.mean(tag_adjusted_f1_scores)),
+    ]
+
+
+def _request_states_to_pred_gold_pairs(request_states: List[RequestState]) -> List[Tuple[str, str]]:
+    pred_gold_pairs: List[Tuple[str, str]] = []
+    for request_state in request_states:
+        assert request_state.result
+        assert len(request_state.result.completions) == 1
+        assert len(request_state.instance.references) == 1
+        pred_gold_pairs.append(
+            (request_state.instance.references[0].output.text, request_state.result.completions[0].text)
+        )
+    return pred_gold_pairs
+
+
+class KPIEdgarMetric(EvaluateInstancesMetric):
+    """Word-level entity type classification F1 score, macro-averaged across entity types."""
+
+    def evaluate_instances(self, request_states: List[RequestState], eval_cache_path: str) -> List[Stat]:
+        return _compute_stats(_request_states_to_pred_gold_pairs(request_states))

helm/benchmark/metrics/language_modeling_metrics.py

@@ -11,10 +11,10 @@ from helm.benchmark.metrics.efficiency_metrics import EfficiencyMetric
 
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
-from .metric import MetricInterface, MetricResult, PerInstanceStats, add_context
-from .metric_name import MetricContext, MetricName
-from .metric_service import MetricService
-from .statistic import Stat, merge_stat
+from helm.benchmark.metrics.metric import MetricInterface, MetricResult, PerInstanceStats, add_context
+from helm.benchmark.metrics.metric_name import MetricContext, MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat, merge_stat
 
 
 class LanguageModelingMetric(MetricInterface):

helm/benchmark/metrics/llm_jury_metrics.py

@@ -0,0 +1,46 @@
+from typing import Any, Dict, List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.model_as_judge import AnnotatorModelInfo
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class LLMJuryMetric(Metric):
+    """Score metrics for LLM Jury."""
+
+    def __init__(
+        self,
+        metric_name: str,
+        scenario_name: str,
+        annotator_models: Dict[str, AnnotatorModelInfo],
+        default_score: float = 0.0,
+    ):
+        self.metric_name = metric_name
+        self.scenario_name = scenario_name
+        self.annotator_models = annotator_models
+        self.default_score = default_score
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        annotations: Dict[str, Any] = request_state.annotations[self.scenario_name]
+        scores: List[int] = []
+        score = self.default_score
+        for annotation_key, annotation_dict in annotations.items():
+            if annotation_key in self.annotator_models.keys() and annotation_dict is not None:
+                for val in annotation_dict.values():
+                    scores.append(int(val["score"]))
+        if scores:
+            score = sum(scores) / len(scores)
+        return [
+            Stat(MetricName(self.metric_name)).add(score),
+        ]

helm/benchmark/metrics/machine_translation_metrics.py

@@ -3,8 +3,8 @@ from typing import List
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.metrics.evaluate_instances_metric import EvaluateInstancesMetric
 from helm.common.optional_dependencies import handle_module_not_found_error
-from .metric_name import MetricName
-from .statistic import Stat
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.statistic import Stat
 
 try:
     from sacrebleu.metrics import BLEU

helm/benchmark/metrics/med_dialog_metrics.py

@@ -0,0 +1,14 @@
+from helm.benchmark.annotation.med_dialog_annotator import ANNOTATOR_MODELS
+from helm.benchmark.metrics.llm_jury_metrics import LLMJuryMetric
+
+
+class MedDialogMetric(LLMJuryMetric):
+    """Score metrics for MedDialog."""
+
+    def __init__(self):
+        super().__init__(
+            metric_name="med_dialog_accuracy",
+            scenario_name="med_dialog",
+            annotator_models=ANNOTATOR_MODELS,
+            default_score=1.0,
+        )

helm/benchmark/metrics/medalign_metrics.py

@@ -0,0 +1,14 @@
+from helm.benchmark.annotation.medalign_annotator import ANNOTATOR_MODELS
+from helm.benchmark.metrics.llm_jury_metrics import LLMJuryMetric
+
+
+class MedalignMetric(LLMJuryMetric):
+    """Score metrics for Medalign."""
+
+    def __init__(self):
+        super().__init__(
+            metric_name="medalign_accuracy",
+            scenario_name="medalign",
+            annotator_models=ANNOTATOR_MODELS,
+            default_score=1.0,
+        )

helm/benchmark/metrics/medcalc_bench_metrics.py

@@ -0,0 +1,124 @@
+import re
+
+from datetime import datetime
+from typing import List, Dict, Any
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+from helm.common.hierarchical_logger import hlog
+
+
+class MedCalcBenchMetric(Metric):
+    """
+    Metric for evaluating the MedCalc Bench dataset, assessing the model's ability to
+    be a clinical calculator.
+
+    Exact match based on category:
+    1. Normal exact match: for categories "risk", "severity" or "diagnosis".
+    2. Variant exact match: for other categories, if the number calculated by the model falls between the values
+        in the Lower limit and Upper limit columns, we mark it as accurate.
+    """
+
+    def parse_duration(self, duration_str) -> int:
+        """Parses a duration tuple (weeks, days) from a string format like ('14 weeks', '2 days')."""
+        match = re.match(r"\('(\d+) weeks', '(\d+) days'\)", duration_str)
+        if match:
+            weeks, days = map(int, match.groups())
+            return weeks * 7 + days  # Convert to total days
+        else:
+            raise ValueError(f"Invalid format: {duration_str}")
+
+    def is_within_range(self, lower_bound, upper_bound, prediction) -> int:
+        """
+        Checks if a predicted duration falls within the given range.
+
+        Args:
+            lower_bound (str): The lower bound in format "('X weeks', 'Y days')".
+            upper_bound (str): The upper bound in format "('X weeks', 'Y days')".
+            prediction (str): The predicted duration in the same format.
+
+        Returns:
+            int: 1 if within range (inclusive), 0 otherwise.
+        """
+        lower_days = self.parse_duration(lower_bound)
+        upper_days = self.parse_duration(upper_bound)
+        prediction_days = self.parse_duration(prediction)
+        return 1 if lower_days <= prediction_days <= upper_days else 0
+
+    def check_date(self, prediction: str, reference: str, extra_data: Dict[str, Any]) -> int:
+        """Checks if prediction date is withing limits"""
+        if re.match(r"\('(\d+) weeks', '(\d+) days'\)", reference):
+            exact_match = self.is_within_range(extra_data["lower_limit"], extra_data["upper_limit"], prediction)
+        else:
+            prediction_date = self._str_to_date(prediction)
+            upper_limit_date = self._str_to_date(extra_data["upper_limit"])
+            lower_limit_date = self._str_to_date(extra_data["lower_limit"])
+            exact_match = 1 if lower_limit_date <= prediction_date <= upper_limit_date else 0
+        return exact_match
+
+    def _str_to_date(self, date_str: str) -> datetime:
+        """Convert string to datetime object."""
+        return datetime.strptime(date_str, "%m/%d/%Y")
+
+    def check_in_range(self, prediction: str, reference: str, extra_data: Dict[str, Any], category: str) -> int:
+        """Check if the prediction falls within the range specified by the reference."""
+        try:
+            if category == "date":
+                exact_match = self.check_date(prediction, reference, extra_data)
+            elif category in ["dosage conversion", "physical"]:
+                lower_limit = float(extra_data["lower_limit"])
+                upper_limit = float(extra_data["upper_limit"])
+                float_prediction = float(prediction)
+                exact_match = 1 if lower_limit <= float_prediction <= upper_limit else 0
+            else:
+                raise ValueError(f"Category {category} not supported")
+        except ValueError:
+            return 0
+
+        return exact_match
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        """
+        Evaluate a single generation against reference labels.
+        """
+        # Extract predictions
+        assert request_state.result, "request_state.result is unexpectedly None"
+        predictions = [completion.text.strip() for completion in request_state.result.completions]
+
+        if not predictions:
+            hlog("Warning: No predictions found in completions")
+            return []
+
+        # Get the first prediction
+        prediction = predictions[0]
+
+        # Get references
+        references = getattr(request_state.instance, "references", None)
+
+        if not references or len(references) == 0:
+            hlog(f"Warning: Missing references for instance {request_state.instance}")
+            return []
+
+        reference = references[0].output.text
+
+        # Extract category, upper limit and lower limit
+        assert request_state.instance.extra_data, "Extra data dict was expected but got None"
+        category = request_state.instance.extra_data["category"]
+
+        if category in ["risk", "severity", "diagnosis"]:
+            exact_match = 1 if prediction == reference else 0
+        else:
+            exact_match = self.check_in_range(prediction, reference, request_state.instance.extra_data, category)
+
+        return [
+            Stat(MetricName("medcalc_bench_accuracy")).add(exact_match),
+        ]

helm/benchmark/metrics/medec_metrics.py

@@ -0,0 +1,101 @@
+from typing import List
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+from helm.common.hierarchical_logger import hlog
+import re
+from helm.benchmark.scenarios.scenario import CORRECT_TAG
+
+
+class MedecMetric(Metric):
+    """
+    Metric for evaluating the MEDEC dataset, assessing medical error detection and correction.
+
+    - **Error Flag Accuracy**: Whether the model correctly identifies if a medical note contains an error.
+    - **Error Sentence Detection Accuracy**: Whether the model correctly identifies the erroneous
+        sentence when an error is present.
+    """
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        """
+        Evaluate a single LLM generation against the ground truth labels.
+        """
+
+        # Extract predictions from the model output
+        if request_state.result is not None:
+            predictions = [completion.text.strip() for completion in request_state.result.completions]
+        else:
+            predictions = []
+
+        if not predictions:
+            raise ValueError("No predictions found in the completions.")
+
+        prediction = predictions[0]  # Use the first completion
+
+        # Extract references (ground truth)
+        references = getattr(request_state.instance, "references", None)
+
+        if not references or len(references) == 0:
+            hlog(f"Warning: Missing references for instance {request_state.instance}")
+            return []
+
+        ground_truth_reference = next((ref for ref in references if CORRECT_TAG in ref.tags), None)
+
+        if not ground_truth_reference:
+            hlog(f"Warning: No ground truth reference with CORRECT_TAG for instance {request_state.instance}")
+            return []
+
+        # Extract the ground truth error flag and sentence ID
+        # ground_truth_text_pre = ground_truth_reference.output.text.strip()
+        ground_truth_text = ground_truth_reference.output.text.replace(".", "").strip()
+
+        # Determine ground truth values
+        if ground_truth_text == "CORRECT":
+            ground_truth_flag = 0  # No medical error
+            ground_truth_sentence_id = -1  # No sentence should be flagged
+        else:
+            ground_truth_flag = 1  # Medical error exists
+            try:
+                ground_truth_sentence_id = int(ground_truth_text)
+            except ValueError:
+                ground_truth_sentence_id = -1
+
+        # Determine model predictions
+        prediction = prediction.strip()
+
+        if prediction == "CORRECT":
+            predicted_flag = 0
+            predicted_sentence_id = -1
+        else:
+            match = re.match(r"^(\d+)", prediction)  # Ensure first token is a number
+            if match:
+                predicted_sentence_id = int(match.group(1))
+                predicted_flag = 1  # A valid sentence ID implies the model thinks there's an error
+            else:
+                predicted_flag = 1  # The model attempted to correct something, but in an invalid format
+                predicted_sentence_id = -2  # Mark invalid outputs
+
+        # Calculate Error Flag Accuracy
+        flag_accuracy = 1 if predicted_flag == ground_truth_flag else 0
+
+        # Calculate Error Sentence Accuracy
+        if ground_truth_flag == 1 and predicted_flag == 1:
+            sentence_accuracy = 1 if predicted_sentence_id == ground_truth_sentence_id else 0
+        else:
+            sentence_accuracy = (
+                1 if ground_truth_flag == 0 and predicted_flag == 0 else 0
+            )  # Both must agree it's "CORRECT"
+
+        return [
+            Stat(MetricName("medec_error_flag_accuracy")).add(flag_accuracy),
+            Stat(MetricName("medec_error_sentence_accuracy")).add(sentence_accuracy),
+        ]

helm/benchmark/metrics/medi_qa_metrics.py

@@ -0,0 +1,14 @@
+from helm.benchmark.annotation.medi_qa_annotator import ANNOTATOR_MODELS
+from helm.benchmark.metrics.llm_jury_metrics import LLMJuryMetric
+
+
+class MediQAMetric(LLMJuryMetric):
+    """Score metrics for MediQA."""
+
+    def __init__(self):
+        super().__init__(
+            metric_name="medi_qa_accuracy",
+            scenario_name="medi_qa",
+            annotator_models=ANNOTATOR_MODELS,
+            default_score=1.0,
+        )

helm/benchmark/metrics/medication_qa_metrics.py

@@ -1,23 +1,14 @@
-from typing import List
+from helm.benchmark.annotation.medication_qa_annotator import ANNOTATOR_MODELS
+from helm.benchmark.metrics.llm_jury_metrics import LLMJuryMetric
 
-from helm.benchmark.adaptation.adapter_spec import AdapterSpec
-from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.metrics.metric import Metric
-from helm.benchmark.metrics.metric_name import MetricName
-from helm.benchmark.metrics.metric_service import MetricService
-from helm.benchmark.metrics.statistic import Stat
 
-
-class MedicationQAScoreMetric(Metric):
+class MedicationQAMetric(LLMJuryMetric):
     """Score metrics for MedicationQA."""
 
-    def evaluate_generation(
-        self,
-        adapter_spec: AdapterSpec,
-        request_state: RequestState,
-        metric_service: MetricService,
-        eval_cache_path: str,
-    ) -> List[Stat]:
-        assert request_state.annotations
-        score = request_state.annotations["medication_qa"]["score"]
-        return [Stat(MetricName("medication_qa_score")).add(score)]
+    def __init__(self):
+        super().__init__(
+            metric_name="medication_qa_accuracy",
+            scenario_name="medication_qa",
+            annotator_models=ANNOTATOR_MODELS,
+            default_score=1.0,
+        )