crfm-helm 0.5.3__py3-none-any.whl → 0.5.5__py3-none-any.whl

helm/benchmark/metrics/med_dialog_metrics.py

@@ -0,0 +1,34 @@
+from typing import Any, Dict, List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.med_dialog_annotator import ANNOTATOR_MODELS
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class MedDialogMetric(Metric):
+    """Score metrics for MedDialog."""
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        annotations: Dict[str, Any] = request_state.annotations["med_dialog"]
+        scores: List[int] = []
+        score = 0.0
+        for annotation_key, annotation_dict in annotations.items():
+            if annotation_key in ANNOTATOR_MODELS.keys() and annotation_dict is not None:
+                for val in annotation_dict.values():
+                    scores.append(int(val["score"]))
+        if scores:
+            score = sum(scores) / len(scores)
+        return [
+            Stat(MetricName("med_dialog_accuracy")).add(score),
+        ]

helm/benchmark/metrics/medalign_metrics.py

@@ -0,0 +1,34 @@
+from typing import Any, Dict, List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.medalign_annotator import ANNOTATOR_MODELS
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class MedalignMetric(Metric):
+    """Score metrics for Medalign."""
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        annotations: Dict[str, Any] = request_state.annotations["medalign"]
+        scores: List[int] = []
+        score = 0.0
+        for annotation_key, annotation_dict in annotations.items():
+            if annotation_key in ANNOTATOR_MODELS.keys() and annotation_dict is not None:
+                for val in annotation_dict.values():
+                    scores.append(int(val["score"]))
+        if scores:
+            score = sum(scores) / len(scores)
+        return [
+            Stat(MetricName("medalign_accuracy")).add(score),
+        ]

helm/benchmark/metrics/medcalc_bench_metrics.py

@@ -0,0 +1,124 @@
+import re
+
+from datetime import datetime
+from typing import List, Dict, Any
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+from helm.common.hierarchical_logger import hlog
+
+
+class MedCalcBenchMetric(Metric):
+    """
+    Metric for evaluating the MedCalc Bench dataset, assessing the model's ability to
+    be a clinical calculator.
+
+    Exact match based on category:
+    1. Normal exact match: for categories "risk", "severity" or "diagnosis".
+    2. Variant exact match: for other categories, if the number calculated by the model falls between the values
+        in the Lower limit and Upper limit columns, we mark it as accurate.
+    """
+
+    def parse_duration(self, duration_str) -> int:
+        """Parses a duration tuple (weeks, days) from a string format like ('14 weeks', '2 days')."""
+        match = re.match(r"\('(\d+) weeks', '(\d+) days'\)", duration_str)
+        if match:
+            weeks, days = map(int, match.groups())
+            return weeks * 7 + days  # Convert to total days
+        else:
+            raise ValueError(f"Invalid format: {duration_str}")
+
+    def is_within_range(self, lower_bound, upper_bound, prediction) -> int:
+        """
+        Checks if a predicted duration falls within the given range.
+
+        Args:
+            lower_bound (str): The lower bound in format "('X weeks', 'Y days')".
+            upper_bound (str): The upper bound in format "('X weeks', 'Y days')".
+            prediction (str): The predicted duration in the same format.
+
+        Returns:
+            int: 1 if within range (inclusive), 0 otherwise.
+        """
+        lower_days = self.parse_duration(lower_bound)
+        upper_days = self.parse_duration(upper_bound)
+        prediction_days = self.parse_duration(prediction)
+        return 1 if lower_days <= prediction_days <= upper_days else 0
+
+    def check_date(self, prediction: str, reference: str, extra_data: Dict[str, Any]) -> int:
+        """Checks if prediction date is withing limits"""
+        if re.match(r"\('(\d+) weeks', '(\d+) days'\)", reference):
+            exact_match = self.is_within_range(extra_data["lower_limit"], extra_data["upper_limit"], prediction)
+        else:
+            prediction_date = self._str_to_date(prediction)
+            upper_limit_date = self._str_to_date(extra_data["upper_limit"])
+            lower_limit_date = self._str_to_date(extra_data["lower_limit"])
+            exact_match = 1 if lower_limit_date <= prediction_date <= upper_limit_date else 0
+        return exact_match
+
+    def _str_to_date(self, date_str: str) -> datetime:
+        """Convert string to datetime object."""
+        return datetime.strptime(date_str, "%m/%d/%Y")
+
+    def check_in_range(self, prediction: str, reference: str, extra_data: Dict[str, Any], category: str) -> int:
+        """Check if the prediction falls within the range specified by the reference."""
+        try:
+            if category == "date":
+                exact_match = self.check_date(prediction, reference, extra_data)
+            elif category in ["dosage conversion", "physical"]:
+                lower_limit = float(extra_data["lower_limit"])
+                upper_limit = float(extra_data["upper_limit"])
+                float_prediction = float(prediction)
+                exact_match = 1 if lower_limit <= float_prediction <= upper_limit else 0
+            else:
+                raise ValueError(f"Category {category} not supported")
+        except ValueError:
+            return 0
+
+        return exact_match
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        """
+        Evaluate a single generation against reference labels.
+        """
+        # Extract predictions
+        assert request_state.result, "request_state.result is unexpectedly None"
+        predictions = [completion.text.strip() for completion in request_state.result.completions]
+
+        if not predictions:
+            hlog("Warning: No predictions found in completions")
+            return []
+
+        # Get the first prediction
+        prediction = predictions[0]
+
+        # Get references
+        references = getattr(request_state.instance, "references", None)
+
+        if not references or len(references) == 0:
+            hlog(f"Warning: Missing references for instance {request_state.instance}")
+            return []
+
+        reference = references[0].output.text
+
+        # Extract category, upper limit and lower limit
+        assert request_state.instance.extra_data, "Extra data dict was expected but got None"
+        category = request_state.instance.extra_data["category"]
+
+        if category in ["risk", "severity", "diagnosis"]:
+            exact_match = 1 if prediction == reference else 0
+        else:
+            exact_match = self.check_in_range(prediction, reference, request_state.instance.extra_data, category)
+
+        return [
+            Stat(MetricName("medcalc_bench_accuracy")).add(exact_match),
+        ]

helm/benchmark/metrics/medec_metrics.py

@@ -0,0 +1,101 @@
+from typing import List
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+from helm.common.hierarchical_logger import hlog
+import re
+from helm.benchmark.scenarios.scenario import CORRECT_TAG
+
+
+class MedecMetric(Metric):
+    """
+    Metric for evaluating the MEDEC dataset, assessing medical error detection and correction.
+
+    - **Error Flag Accuracy**: Whether the model correctly identifies if a medical note contains an error.
+    - **Error Sentence Detection Accuracy**: Whether the model correctly identifies the erroneous
+        sentence when an error is present.
+    """
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        """
+        Evaluate a single LLM generation against the ground truth labels.
+        """
+
+        # Extract predictions from the model output
+        if request_state.result is not None:
+            predictions = [completion.text.strip() for completion in request_state.result.completions]
+        else:
+            predictions = []
+
+        if not predictions:
+            raise ValueError("No predictions found in the completions.")
+
+        prediction = predictions[0]  # Use the first completion
+
+        # Extract references (ground truth)
+        references = getattr(request_state.instance, "references", None)
+
+        if not references or len(references) == 0:
+            hlog(f"Warning: Missing references for instance {request_state.instance}")
+            return []
+
+        ground_truth_reference = next((ref for ref in references if CORRECT_TAG in ref.tags), None)
+
+        if not ground_truth_reference:
+            hlog(f"Warning: No ground truth reference with CORRECT_TAG for instance {request_state.instance}")
+            return []
+
+        # Extract the ground truth error flag and sentence ID
+        # ground_truth_text_pre = ground_truth_reference.output.text.strip()
+        ground_truth_text = ground_truth_reference.output.text.replace(".", "").strip()
+
+        # Determine ground truth values
+        if ground_truth_text == "CORRECT":
+            ground_truth_flag = 0  # No medical error
+            ground_truth_sentence_id = -1  # No sentence should be flagged
+        else:
+            ground_truth_flag = 1  # Medical error exists
+            try:
+                ground_truth_sentence_id = int(ground_truth_text)
+            except ValueError:
+                ground_truth_sentence_id = -1
+
+        # Determine model predictions
+        prediction = prediction.strip()
+
+        if prediction == "CORRECT":
+            predicted_flag = 0
+            predicted_sentence_id = -1
+        else:
+            match = re.match(r"^(\d+)", prediction)  # Ensure first token is a number
+            if match:
+                predicted_sentence_id = int(match.group(1))
+                predicted_flag = 1  # A valid sentence ID implies the model thinks there's an error
+            else:
+                predicted_flag = 1  # The model attempted to correct something, but in an invalid format
+                predicted_sentence_id = -2  # Mark invalid outputs
+
+        # Calculate Error Flag Accuracy
+        flag_accuracy = 1 if predicted_flag == ground_truth_flag else 0
+
+        # Calculate Error Sentence Accuracy
+        if ground_truth_flag == 1 and predicted_flag == 1:
+            sentence_accuracy = 1 if predicted_sentence_id == ground_truth_sentence_id else 0
+        else:
+            sentence_accuracy = (
+                1 if ground_truth_flag == 0 and predicted_flag == 0 else 0
+            )  # Both must agree it's "CORRECT"
+
+        return [
+            Stat(MetricName("medec_error_flag_accuracy")).add(flag_accuracy),
+            Stat(MetricName("medec_error_sentence_accuracy")).add(sentence_accuracy),
+        ]

helm/benchmark/metrics/medi_qa_metrics.py

@@ -0,0 +1,34 @@
+from typing import Any, Dict, List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.medi_qa_annotator import ANNOTATOR_MODELS
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class MediQAMetric(Metric):
+    """Score metrics for MediQA."""
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        annotations: Dict[str, Any] = request_state.annotations["medi_qa"]
+        scores: List[int] = []
+        score = 0.0
+        for annotation_key, annotation_dict in annotations.items():
+            if annotation_key in ANNOTATOR_MODELS.keys() and annotation_dict is not None:
+                for val in annotation_dict.values():
+                    scores.append(int(val["score"]))
+        if scores:
+            score = sum(scores) / len(scores)
+        return [
+            Stat(MetricName("medi_qa_accuracy")).add(score),
+        ]

helm/benchmark/metrics/medication_qa_metrics.py

@@ -1,14 +1,15 @@
-from typing import List
+from typing import Any, Dict, List
 
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.medi_qa_annotator import ANNOTATOR_MODELS
 from helm.benchmark.metrics.metric import Metric
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
 
 
-class MedicationQAScoreMetric(Metric):
+class MedicationQAMetric(Metric):
     """Score metrics for MedicationQA."""
 
     def evaluate_generation(
@@ -19,5 +20,15 @@ class MedicationQAScoreMetric(Metric):
         eval_cache_path: str,
     ) -> List[Stat]:
         assert request_state.annotations
-        score = request_state.annotations["medication_qa"]["score"]
-        return [Stat(MetricName("medication_qa_score")).add(score)]
+        annotations: Dict[str, Any] = request_state.annotations["medication_qa"]
+        scores: List[int] = []
+        score = 0.0
+        for annotation_key, annotation_dict in annotations.items():
+            if annotation_key in ANNOTATOR_MODELS.keys() and annotation_dict is not None:
+                for val in annotation_dict.values():
+                    scores.append(int(val["score"]))
+        if scores:
+            score = sum(scores) / len(scores)
+        return [
+            Stat(MetricName("medication_qa_accuracy")).add(score),
+        ]

helm/benchmark/metrics/mental_health_metrics.py

@@ -0,0 +1,34 @@
+from typing import Any, Dict, List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.mental_health_annotator import ANNOTATOR_MODELS
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class MentalHealthMetric(Metric):
+    """Score metrics for MentalHealth."""
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        annotations: Dict[str, Any] = request_state.annotations["mental_health"]
+        scores: List[int] = []
+        score = 0.0
+        for annotation_key, annotation_dict in annotations.items():
+            if annotation_key in ANNOTATOR_MODELS.keys() and annotation_dict is not None:
+                for val in annotation_dict.values():
+                    scores.append(int(val["score"]))
+        if scores:
+            score = sum(scores) / len(scores)
+        return [
+            Stat(MetricName("mental_health_accuracy")).add(score),
+        ]

helm/benchmark/metrics/metric.py

@@ -15,9 +15,9 @@ from helm.benchmark.adaptation.scenario_state import ScenarioState
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.scenarios.scenario import Instance
-from .metric_name import MetricName, MetricContext
-from .metric_service import MetricService
-from .statistic import Stat, merge_stat
+from helm.benchmark.metrics.metric_name import MetricName, MetricContext
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat, merge_stat
 
 
 @dataclass(frozen=True)

helm/benchmark/metrics/mimic_rrs_metrics.py

@@ -0,0 +1,34 @@
+from typing import Any, Dict, List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.mimic_rrs_annotator import ANNOTATOR_MODELS
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class MIMICRRSMetric(Metric):
+    """Score metrics for MIMICRRS."""
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        annotations: Dict[str, Any] = request_state.annotations["mimic_rrs"]
+        scores: List[int] = []
+        score = 0.0
+        for annotation_key, annotation_dict in annotations.items():
+            if annotation_key in ANNOTATOR_MODELS.keys() and annotation_dict is not None:
+                for val in annotation_dict.values():
+                    scores.append(int(val["score"]))
+        if scores:
+            score = sum(scores) / len(scores)
+        return [
+            Stat(MetricName("mimic_rrs_accuracy")).add(score),
+        ]

helm/benchmark/metrics/mimiciv_billing_code_metrics.py

@@ -0,0 +1,96 @@
+from typing import List
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+from helm.common.hierarchical_logger import hlog
+import re
+from sklearn.metrics import precision_score, recall_score, f1_score
+from sklearn.preprocessing import MultiLabelBinarizer
+
+
+class MIMICIVBillingCodeMetric(Metric):
+    """
+    Metric for evaluating the MIMIC Billing Code dataset, assessing the model's ability to match the
+    reference ICD codes. Handles cases where raw prediction output contains additional text.
+
+    Calculates:
+    1. Precision: proportion of correctly predicted ICD codes among all predicted codes
+    2. Recall: proportion of correctly predicted ICD codes among all reference codes
+    3. F1 score: harmonic mean of precision and recall
+
+    ICD codes format: letter followed by 1-3 digits, optional period, optional additional digits
+    Examples: "J18.9", "J45.909", "J47.1", "J96.01"
+    """
+
+    def extract_icd_codes(self, text: str) -> List[str]:
+        """Extract ICD codes from text, handling markdown and standardizing format."""
+        if not text:
+            return []
+
+        # Remove markdown bold formatting
+        cleaned_text = re.sub(r"\*\*", "", text)
+        # Match ICD code pattern with optional period and trailing digits
+        pattern = r"\b[A-Z]\d{1,3}(?:\.\d{1,4})?\.?\b"
+        codes = re.findall(pattern, cleaned_text)
+        # Standardize by removing trailing periods
+        return [code.rstrip(".").strip() for code in codes]
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        """
+        Evaluate a single generation against reference labels.
+        """
+        # Extract predictions
+        if request_state.result is None:
+            predictions = []
+        else:
+            predictions = [completion.text.strip() for completion in request_state.result.completions]
+        if not predictions:
+            hlog("Warning: No predictions found in completions")
+            return []
+
+        # Get the first prediction
+        prediction = predictions[0]
+
+        # Get references
+        references = getattr(request_state.instance, "references", None)
+
+        if not references or len(references) == 0:
+            hlog(f"Warning: Missing references for instance {request_state.instance}")
+            return []
+
+        # Extract codes from reference and prediction
+        ref_codes = []
+        for ref in references:
+            if ref.output.text:
+                ref_codes.extend(self.extract_icd_codes(ref.output.text))
+        ref_codes = list(set(ref_codes))  # Remove duplicates
+
+        pred_codes = self.extract_icd_codes(prediction)
+        pred_codes = list(set(pred_codes))  # Remove duplicates
+
+        # Convert to binary format for metrics
+        all_codes = sorted(list(set(ref_codes + pred_codes)))
+        mlb = MultiLabelBinarizer(classes=all_codes)
+
+        y_true_bin = mlb.fit_transform([ref_codes])
+        y_pred_bin = mlb.transform([pred_codes])
+
+        # Calculate metrics
+        precision = precision_score(y_true_bin, y_pred_bin, average="micro", zero_division=0)
+        recall = recall_score(y_true_bin, y_pred_bin, average="micro", zero_division=0)
+        f1 = f1_score(y_true_bin, y_pred_bin, average="micro", zero_division=0)
+
+        return [
+            Stat(MetricName("mimiciv_billing_code_precision")).add(precision),
+            Stat(MetricName("mimiciv_billing_code_recall")).add(recall),
+            Stat(MetricName("mimiciv_billing_code_f1")).add(f1),
+        ]

helm/benchmark/metrics/mtsamples_procedures_metrics.py

@@ -0,0 +1,34 @@
+from typing import Any, Dict, List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.mtsamples_procedures_annotator import ANNOTATOR_MODELS
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class MTSamplesProceduresMetric(Metric):
+    """Score metrics for MTSamplesProcedures."""
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        annotations: Dict[str, Any] = request_state.annotations["mtsamples_procedures"]
+        scores: List[int] = []
+        score = 0.0
+        for annotation_key, annotation_dict in annotations.items():
+            if annotation_key in ANNOTATOR_MODELS.keys() and annotation_dict is not None:
+                for val in annotation_dict.values():
+                    scores.append(int(val["score"]))
+        if scores:
+            score = sum(scores) / len(scores)
+        return [
+            Stat(MetricName("mtsamples_procedures_accuracy")).add(score),
+        ]

helm/benchmark/metrics/mtsamples_replicate_metrics.py

@@ -0,0 +1,34 @@
+from typing import Any, Dict, List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.mtsamples_replicate_annotator import ANNOTATOR_MODELS
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class MTSamplesReplicateMetric(Metric):
+    """Score metrics for MTSamplesReplicate."""
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        annotations: Dict[str, Any] = request_state.annotations["mtsamples_replicate"]
+        scores: List[int] = []
+        score = 0.0
+        for annotation_key, annotation_dict in annotations.items():
+            if annotation_key in ANNOTATOR_MODELS.keys() and annotation_dict is not None:
+                for val in annotation_dict.values():
+                    scores.append(int(val["score"]))
+        if scores:
+            score = sum(scores) / len(scores)
+        return [
+            Stat(MetricName("mtsamples_replicate_accuracy")).add(score),
+        ]

helm/benchmark/metrics/nltk_helper.py

@@ -0,0 +1,32 @@
+import nltk
+from importlib.metadata import version
+
+
+def install_nltk_resources():
+    """Install resources for nltk tokenizers, which is required for bleu and rouge scores."""
+    # Install "punkt_tab" for nltk>=3.9.1 or "punkt" for nltk<=3.8.1
+    #
+    # Note that nltk 3.9.0 is disallowed due to https://github.com/nltk/nltk/issues/3308
+    #
+    # "punkt" is not longer supported for newer versions of nltk due to a security issue
+    # and has been replaced by "punkt_tab". For more information, see:
+    #
+    # - https://github.com/stanford-crfm/helm/issues/2926
+    # - https://github.com/nltk/nltk/issues/3293
+    # - https://github.com/nltk/nltk/issues/3266
+    # - https://nvd.nist.gov/vuln/detail/CVE-2024-39705
+    #
+    # TODO: Remove support for nltk<=3.8.1 and only install "punkt_tab"
+    nltk_major_version, nltk_minor_version = [int(v) for v in version("nltk").split(".")[0:2]]
+    if nltk_major_version < 3:
+        raise Exception("ntlk version <3 is not supported")
+    if nltk_major_version == 3 and nltk_minor_version <= 8:
+        try:
+            nltk.data.find("tokenizers/punkt")
+        except LookupError:
+            nltk.download("punkt")
+    else:
+        try:
+            nltk.data.find("tokenizers/punkt_tab")
+        except LookupError:
+            nltk.download("punkt_tab")

helm/benchmark/metrics/numeracy_metrics.py

@@ -12,10 +12,10 @@ from helm.benchmark.scenarios.numeracy_scenario import (  # noqa
     distance_plane,
     distance_paraboloid,
 )
-from .metric import Metric
-from .metric_name import MetricName
-from .metric_service import MetricService
-from .statistic import Stat
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
 
 
 class DistanceMetric(Metric):