crfm-helm 0.5.6__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/metrics/numeracy_metrics.py

@@ -1,72 +0,0 @@
-from typing import List
-
-from helm.common.request import RequestResult
-from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.adaptation.adapter_spec import AdapterSpec
-from helm.benchmark.scenarios.numeracy_scenario import (  # noqa
-    NumeracyScenario,
-    Polynomial,
-    RELTYPE_INFO,
-    distance_linear,
-    distance_parabola,
-    distance_plane,
-    distance_paraboloid,
-)
-from helm.benchmark.metrics.metric import Metric
-from helm.benchmark.metrics.metric_name import MetricName
-from helm.benchmark.metrics.metric_service import MetricService
-from helm.benchmark.metrics.statistic import Stat
-
-
-class DistanceMetric(Metric):
-    """Returns the minimum geometric distance between the point represented by the completion
-    and the curve or surface specified by `rel_str`.
-
-    Expects `references.outputs` to be a list containing the following:
-
-     - val_GT (str): the last coordinate of the point lying on the given curve / surface
-         with first coordinates as given in the input
-     - rel_str (str): the relation
-     - relation_type (str): one of {'linear', 'parabola', 'plane', 'paraboloid'}
-
-    Returns:
-        The minimum geometric distance from the point to the curve / surface float.
-    """
-
-    def evaluate_generation(
-        self,
-        adapter_spec: AdapterSpec,
-        request_state: RequestState,
-        metric_service: MetricService,
-        eval_cache_path: str,
-    ) -> List[Stat]:
-        """For given request, compute the following two metrics:
-        1. geometric distance metric in range [0, ∞), calling the appropriate distance method, if possible, and
-        2. percent valid metric in range [0., 1.] of completions that are a valid number, ignoring commas.
-        """
-        references = request_state.instance.references
-        _, rel_str, relation_type = map(lambda _: _.output.text, references)
-        input_text: str = request_state.instance.input.text
-        datapoint_input = input_text.split("\n")[-1]
-        val = list(map(int, datapoint_input.split(NumeracyScenario.delimiter)))
-
-        distance_func = globals()[f"distance_{relation_type}"]
-        result = 0.0
-        num_valid = 0
-        assert request_state.result is not None
-        request_result: RequestResult = request_state.result
-        for completion_sequence in request_result.completions:
-            completion = completion_sequence.text.strip()
-            try:
-                pred = int(completion.replace(",", ""))  # ignore commas in numbers
-            except Exception:
-                continue
-            point = val + [pred]
-            result += distance_func(point, rel_str)
-            num_valid += 1
-        percent_valid = 1.0 * num_valid / len(request_result.completions)
-
-        return [
-            Stat(MetricName("distance")).add(result),
-            Stat(MetricName("percent_valid")).add(percent_valid),
-        ]

helm/benchmark/metrics/starr_patient_instructions_metrics.py

@@ -1,14 +0,0 @@
-from helm.benchmark.annotation.starr_patient_instructions_annotator import ANNOTATOR_MODELS
-from helm.benchmark.metrics.llm_jury_metrics import LLMJuryMetric
-
-
-class StarrPatientInstructionsMetric(LLMJuryMetric):
-    """Score metrics for StarrPatientInstructions."""
-
-    def __init__(self):
-        super().__init__(
-            metric_name="starr_patient_instructions_accuracy",
-            scenario_name="starr_patient_instructions",
-            annotator_models=ANNOTATOR_MODELS,
-            default_score=1.0,
-        )

helm/benchmark/metrics/test_numeracy_metrics.py

@@ -1,95 +0,0 @@
-from dataclasses import dataclass
-from typing import List, Callable
-
-from helm.benchmark.scenarios.numeracy_scenario import (
-    distance_linear,
-    distance_parabola,
-    distance_plane,
-    distance_paraboloid,
-)
-
-
-TOL = 1e-5  # note: different from TOL in numeracy_scenario.distance_<...> used for checking if real or complex
-
-
-@dataclass(frozen=True)
-class TestCase:
-    rel_str: str
-    point: List[int]
-    dist: float
-
-
-def check_test_cases(test_cases: List[TestCase], dist_func: Callable[[List[int], str], float]):
-    for test_case in test_cases:
-        dist = dist_func(test_case.point, test_case.rel_str)
-        dist_gt = test_case.dist
-        assert abs(dist - dist_gt) < TOL, f"{test_case.rel_str} {test_case.point}"
-        # print(f"{test_case.rel_str} {test_case.point} Dist: {dist}\tDist GT: {dist_gt}")
-
-
-def test_distance_linear():
-    test_cases = [
-        TestCase(
-            "y = 4x + 4", [59, 201], 9.458889376416986
-        ),  # https://www.wolframalpha.com/input?i=minimize+sqrt%28%28x+-+59%29%5E2+%2B+%284x+%2B+4+-+201%29%5E2%29
-        TestCase("y = x + 3 ", [30, 78], 31.819805153394636),
-        TestCase("y = 5x + 4", [-47, 2], 45.69505948719688),
-        TestCase("y = 4x + 3", [-65, -255], 0.48507125007266594),
-        TestCase("y = 4x + 3", [97, 391], 0.0),
-    ]
-    check_test_cases(test_cases, distance_linear)
-
-
-def test_distance_parabola():
-    test_cases = [
-        TestCase("y = 2x^2 + x + 1", [159, 50000], 1.137499072212397),
-        TestCase("y = 2x^2 + 2x + 4", [130, 28390], 11.364547837422966),
-        TestCase("y = 2x^2 + x + 4", [53, 10000], 17.4468675121177),
-        TestCase(
-            "y = 2x^2 + 2x + 2", [35, 1], 34.36171077312826
-        ),  # https://www.wolframalpha.com/input?i=minimize+%28x+-+35%29%5E2+%2B+%282x%5E2+%2B+2x+%2B+2+-+1%29%5E2
-        TestCase("y = x^2 + x + 2", [197, 39008], 0.0),
-    ]
-    check_test_cases(test_cases, distance_parabola)
-
-
-def test_distance_plane():
-    test_cases = [
-        TestCase(
-            "z = 4x + 4y + 1", [-4, 9, 1], 3.481553119113957
-        ),  # https://www.wolframalpha.com/input?i=minimize+sqrt%28%28x+%2B+4%29%5E2+%2B+%28y+-+9%29%5E2+%2B+%284x
-        # +%2B+4y+%2B+1+-+1%29%5E2%29
-        TestCase(
-            "z = 3x + 5y + 4", [-10, 4, 3], 1.52127765851133
-        ),  # https://www.wolframalpha.com/input?i=minimize+sqrt%28%28x+%2B+10%29%5E2+%2B+%28y+-+4%29%5E2+%2B+%283
-        # x+%2B+5y+%2B+4+-+3%29%5E2%29
-        TestCase("z = 4x + 3y + 4", [-5, 4, -7], 0.5883484054145521),
-        TestCase("z = 3x + 5y + 2", [-7, 10, 0], 5.239956379316803),
-        TestCase("z = 5x + 2y + 3", [-2, -1, -9], 0.0),
-    ]
-    check_test_cases(test_cases, distance_plane)
-
-
-def test_distance_paraboloid():
-    test_cases = [
-        TestCase("z = x^2 + y^2 + 2", [0, 0, 2], 0.0),
-        TestCase(
-            "z = 2x^2 + y^2 + 2", [0, 11, 151], 1.2055445093982982
-        ),  # https://www.wolframalpha.com/input?i=minimize+x%5E2+%2B+%28y+-+11%29%5E2+%2B+%28%282x%5E2+%2B+y%5E2+%2B+2%29+-+151%29%5E2  # noqa
-        TestCase(
-            "z = 2x^2 + 2y^2 + 2", [0, 0, 6], 1.3919410907075054
-        ),  # https://www.wolframalpha.com/input?i=minimize+x%5E2+%2By%5E2+%2B+%28%282x%5E2+%2B+2y%5E2+%2B+2%29+-+6%29%5E2  # noqa
-        TestCase(
-            "z = x^2 + y^2 + 2", [0, 0, 20], 4.2130748865881795
-        ),  # https://www.wolframalpha.com/input?i=x%5E2+%2B+y%5E2+%2B+%28%28x%5E2+%2B+y%5E2+%2B+2%29+-+20%29%5E2
-        TestCase("z = 2x^2 + xy + y^2 + 4", [6, 19, 519], 0.5290904095503263),
-        TestCase("z = 2x^2 + xy + 2y^2 + 3", [0, 14, 380], 0.26248531385619783),
-        TestCase("z = x^2 + 2y^2 + 1", [5, 14, 4], 13.354544558906934),
-        TestCase("z = x^2 + xy + 2y^2 + 4", [3, 20, 1001], 1.4206031238856873),
-        TestCase("z = x^2 + xy + 2y^2 + 4", [0, 0, 55], 51.0),
-        TestCase("z = x^2 + xy + 2y^2 + 4", [0, 9, 55], 3.8558889386410757),
-        TestCase("z = 2x^2 + 2y^2 + 1", [8, 9, 289], 0.04158555512549898),
-        TestCase("z = 2x^2 + 2y^2 + 1", [8, 9, 291], 0.0),
-        TestCase("z = x^2 + 2xy + 5y^2 + 4", [0, 9, 55], 5.7150737847649244),
-    ]
-    check_test_cases(test_cases, distance_paraboloid)

helm/benchmark/scenarios/audio_language/ultra_suite_asr_classification.py

@@ -1,103 +0,0 @@
-from typing import List, Tuple
-import os
-import json
-
-from tqdm import tqdm
-
-from helm.benchmark.scenarios.scenario import (
-    Scenario,
-    Instance,
-    Reference,
-    TEST_SPLIT,
-    CORRECT_TAG,
-    Input,
-    Output,
-)
-from helm.common.media_object import MediaObject, MultimediaObject
-from helm.common.general import ensure_file_downloaded
-
-
-def find_audio_json_pairs(directory: str) -> List[Tuple[str, str]]:
-    """
-    Find all pairs of MP3 and JSON files in the given directory and its subdirectories.
-    Each pair consists of an MP3 file and its corresponding JSON file with the same base name.
-
-    Args:
-        directory: Path to the directory containing the files
-
-    Returns:
-        List of tuples where each tuple contains (mp3_path, json_path)
-    """
-    pairs = []
-
-    # Walk through all directories and subdirectories
-    for root, _, files in os.walk(directory):
-        # Get all MP3 files in current directory
-        mp3_files = [f for f in files if f.endswith(".mp3")]
-
-        for mp3_file in mp3_files:
-            base_name = os.path.splitext(mp3_file)[0]
-            json_file = f"{base_name}.json"
-
-            # Check if corresponding JSON file exists in the same directory
-            if json_file in files:
-                mp3_path = os.path.join(root, mp3_file)
-                json_path = os.path.join(root, json_file)
-                pairs.append((mp3_path, json_path))
-
-    return pairs
-
-
-class UltraSuiteASRClassificationScenario(Scenario):
-    """
-    A scenario for evaluating whether a child speaker has a speech disorder or not.
-    The audio files contain speech from children, potentially with an adult present.
-    The task is to classify whether the child speaker is typically developing or has a speech disorder.
-    """
-
-    name = "speech_disorder"
-    description = "A scenario for evaluating speech disorders in children"
-    tags = ["audio", "classification", "speech_disorder", "asr"]
-    HF_MAPPING_URL = "https://https://huggingface.co/datasets/SAA-Lab/SLPHelmUltraSuite"
-
-    # Classification options
-    options: List[str] = ["Healthy", "Unhealthy"]
-
-    def get_instances(self, output_path: str) -> List[Instance]:
-        """
-        Create instances from the audio files and their corresponding JSON annotations.
-        The data directory should contain:
-        - Audio files (e.g., .mp3)
-        - A JSON file with annotations containing 'answer' field
-        """
-        print(f"Downloading dataset from {UltraSuiteASRClassificationScenario.HF_MAPPING_URL} to {output_path}")
-        ensure_file_downloaded(source_url=UltraSuiteASRClassificationScenario.HF_MAPPING_URL, target_path=output_path)
-
-        instances: List[Instance] = []
-        split: str = TEST_SPLIT
-
-        # Find all pairs of audio and JSON files
-        pairs = find_audio_json_pairs(output_path)
-
-        for audio_path, json_path in tqdm(pairs):
-
-            # Load the annotation
-            with open(json_path, "r") as f:
-                annotation = json.load(f)
-
-            # Get the correct answer and convert to label
-            answer = annotation["disorder_class"]
-            # Create references for each option
-            references: List[Reference] = []
-            reference = Reference(Output(text=answer), tags=[CORRECT_TAG])
-            references.append(reference)
-
-            # Create the input with audio and instruction
-            content = [
-                MediaObject(content_type="audio/mpeg", location=audio_path),
-            ]
-
-            input = Input(multimedia_content=MultimediaObject(content))
-            instances.append(Instance(input=input, references=references, split=split))
-
-        return instances