crfm-helm 0.5.6__py3-none-any.whl → 0.5.8__py3-none-any.whl

helm/benchmark/scenarios/test_healtha_br_scenario.py

@@ -0,0 +1,57 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.healthqa_br_scenario import HEALTHQA_BR_Scenario
+from helm.benchmark.scenarios.scenario import TEST_SPLIT, CORRECT_TAG, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_healthqa_br_instance():
+    scenario = HEALTHQA_BR_Scenario()
+    with TemporaryDirectory() as tmpdir:
+        instances = scenario.get_instances(tmpdir)
+
+    instance = instances[35]
+
+    assert instance.split == TEST_SPLIT
+
+    assert instance.input.text.startswith("Homem de 22 anos de idade procura a Unidade Básica")
+
+    assert instance.references == [
+        Reference(
+            output=Output(
+                text="administração de relaxante muscular, colocando o paciente em posição de Trendelenburg, com "
+                "tentativa de redução do volume."
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text="encaminhamento do paciente ao Serviço de Urgência do Hospital com o pedido de avaliação "
+                "imediata do cirurgião."
+            ),
+            tags=[CORRECT_TAG],
+        ),
+        Reference(
+            output=Output(
+                text="tentativa de redução manual do aumento de volume da região inguinescrotal para a cavidade "
+                "abdominal."
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text="transiluminação do escroto para tentar diferenciar hérnia inguinal de hidrocele comunicante."
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(text="prescrição de antiemético e solicitação de ecografia da região inguinescrotal."),
+            tags=[],
+        ),
+    ]
+
+    correct_refs = [ref for ref in instance.references if CORRECT_TAG in ref.tags]
+    assert len(correct_refs) == 1
+
+    assert instance.references[1].is_correct

helm/benchmark/scenarios/the_pile_scenario.py

@@ -5,9 +5,10 @@ import sys
 import requests
 from typing import Dict, List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 from helm.common.hierarchical_logger import hlog, htrack, htrack_block
-from helm.benchmark.scenarios.scenario import Scenario, Instance, TEST_SPLIT, Input
+from helm.benchmark.scenarios.scenario import Scenario, Instance, TEST_SPLIT, Input, ScenarioMetadata
 
 
 class ThePileScenario(Scenario):
@@ -146,3 +147,14 @@ class ThePileScenario(Scenario):
             instances = [instances[i] for i in indices]
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="the_pile",
+            display_name="The Pile",
+            description="The Pile corpus for measuring lanugage model performance across various "
+            "domains [(Gao et al., 2020)](https://arxiv.org/pdf/2101.00027.pdf).",
+            taxonomy=TaxonomyInfo(task="language modeling", what="?", when="?", who="?", language="English, code"),
+            main_metric="bits_per_byte",
+            main_split="test",
+        )

helm/benchmark/scenarios/truthful_qa_scenario.py

@@ -2,6 +2,7 @@ import csv
 import os
 from typing import List, Dict, Any
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -12,6 +13,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -154,3 +156,15 @@ class TruthfulQAScenario(Scenario):
         valid_instances: List[Instance] = get_split_instances(VALID_SPLIT, data[split_k:])
 
         return train_instances + valid_instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="truthful_qa",
+            display_name="TruthfulQA",
+            description="The TruthfulQA benchmarking for measuring model truthfulness and commonsense "
+            "knowledge in question answering [(Lin et al., "
+            "2022)](https://aclanthology.org/2022.acl-long.229/).",
+            taxonomy=TaxonomyInfo(task="question answering", what="?", when="?", who="?", language="English"),
+            main_metric="exact_match",
+            main_split="valid",
+        )

helm/benchmark/scenarios/twitter_aae_scenario.py

@@ -2,9 +2,10 @@ import csv
 import os
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 from helm.common.hierarchical_logger import hlog
-from helm.benchmark.scenarios.scenario import Scenario, Instance, TEST_SPLIT, Input
+from helm.benchmark.scenarios.scenario import Scenario, Instance, TEST_SPLIT, Input, ScenarioMetadata
 
 CODALAB_URI_TEMPLATE: str = (
     "https://worksheets.codalab.org/rest/bundles/0x31485f8c37ad481fb9f4e9bf7ccff6e5/contents/blob/"
@@ -56,3 +57,21 @@ class TwitterAAEScenario(Scenario):
                 instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="twitter_aae",
+            display_name="TwitterAAE",
+            description="The TwitterAAE corpus of [Blodgett et al. "
+            "(2016)](https://aclanthology.org/D16-1120/) for measuring language model "
+            "performance in tweets as a function of speaker dialect.",
+            taxonomy=TaxonomyInfo(
+                task="language modeling",
+                what="?",
+                when="?",
+                who="?",
+                language="English (AAE-aligned and White-aligned)",
+            ),
+            main_metric="bits_per_byte",
+            main_split="test",
+        )

helm/benchmark/scenarios/vicuna_scenario.py

@@ -2,8 +2,9 @@ import json
 import os
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
-from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT, ScenarioMetadata
 
 
 class VicunaScenario(Scenario):
@@ -47,3 +48,22 @@ class VicunaScenario(Scenario):
                 )
                 instances.append(instance)
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="vicuna",
+            display_name="Vicuna",
+            short_display_name="Vicuna",
+            description="The set of prompts used by the "
+            "[Vicuna](https://lmsys.org/blog/2023-03-30-vicuna/) team to evaluate "
+            "instruction-following models.",
+            taxonomy=TaxonomyInfo(
+                task="open-ended instruction following",
+                what="Instructions for LLMs",
+                when="Before 2023",
+                who="Unknown",
+                language="English",
+            ),
+            main_metric="Helpfulness",
+            main_split="test",
+        )

helm/benchmark/scenarios/wikifact_scenario.py

@@ -2,6 +2,7 @@ import os
 from typing import List, Dict
 import json
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_directory_exists, ensure_file_downloaded, flatten_list
 from helm.common.hierarchical_logger import hlog
 from helm.benchmark.scenarios.scenario import (
@@ -14,6 +15,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 PID_TO_NAME = {
@@ -183,3 +185,21 @@ class WIKIFactScenario(Scenario):
                 instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="wikifact",
+            display_name="WikiFact",
+            description="Scenario introduced in this work, inspired by [Petroni et al. "
+            "(2019)](https://aclanthology.org/D19-1250/), to more extensively test factual "
+            "knowledge.",
+            taxonomy=TaxonomyInfo(
+                task="knowledge base completion",
+                what="entity-relation-entity triples in natural language form",
+                when="?",
+                who="automatically generated from templates",
+                language="structured English",
+            ),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/wildbench_scenario.py

@@ -2,11 +2,13 @@ import datasets
 import os
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
     TEST_SPLIT,
     Input,
+    ScenarioMetadata,
 )
 from helm.common.general import ensure_directory_exists
 
@@ -81,3 +83,19 @@ class WildBenchScenario(Scenario):
             instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name=self.name,
+            display_name="WildBench",
+            description=self.description,
+            main_metric="wildbench_score_rescaled",
+            main_split="test",
+            taxonomy=TaxonomyInfo(
+                task="instruction following",
+                what="GPT-judged instruction following with instructions collected from real-user conversations",
+                who="real-world users",
+                when="2024",
+                language="English",
+            ),
+        )

helm/benchmark/scenarios/wmt_14_scenario.py

@@ -1,5 +1,6 @@
 from typing import List, Any
 from datasets import load_dataset
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.hierarchical_logger import htrack_block
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -11,6 +12,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -106,3 +108,13 @@ class WMT14Scenario(Scenario):
                     )
                 )
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="wmt_14",
+            display_name="WMT 2014",
+            description="WMT 2014 is a collection of machine translation datasets.",
+            taxonomy=TaxonomyInfo(task="machine translation", what="n/a", when="n/a", who="n/a", language="English"),
+            main_metric="bleu_4",
+            main_split="test",
+        )

helm/benchmark/slurm_jobs.py

@@ -13,7 +13,6 @@ except ModuleNotFoundError as e:
 
 
 class SlurmJobState:
-    # TODO: Convert to StrEnum after upgrading to Python 3.11
     # Non-exhaustive list of Slurm job states.
     # See: https://slurm.schedmd.com/squeue.html#SECTION_JOB-STATE-CODES
 
@@ -81,7 +80,7 @@ def get_slurm_job_state(job_id: int) -> str:
     except subprocess.CalledProcessError as e:
         # Default CalledProcessError message doesn't have output, so re-raise here to include the output.
         raise Exception(f"{str(e)} output: {e.output}")
-    search_result = re.search("JobState=(\w+)", scontrol_output.decode())
+    search_result = re.search(r"JobState=(\w+)", scontrol_output.decode())
     if not search_result:
         raise Exception(f"Could not extract JobState from scontrol: {scontrol_output.decode()}")
     return search_result.group(1)

helm/benchmark/slurm_runner.py

@@ -26,7 +26,7 @@ from helm.benchmark.slurm_jobs import (
     FAILURE_SLURM_JOB_STATES,
 )
 from helm.common.general import ensure_directory_exists
-from helm.common.hierarchical_logger import hlog, htrack_block
+from helm.common.hierarchical_logger import hlog, htrack_block, setup_default_logging
 
 from helm.benchmark.runner_config_registry import RUNNER_CONFIG
 
@@ -343,7 +343,14 @@ def main():
         help="Path to the RunSpec JSON file",
         required=True,
     )
+    parser.add_argument(
+        "--log-config",
+        type=str,
+        default=None,
+        help="PATH to a YAML file to customize logging",
+    )
     args = parser.parse_args()
+    setup_default_logging(args.log_config)
 
     # Deserialize SlurmRunner and RunSpec from the given files, then run the RunSpec with the SlurmRunner.
     with open(args.slurm_runner_spec_path, "r") as f:

helm/benchmark/static/schema_arabic.yaml

@@ -0,0 +1,271 @@
+---
+# Schema for Arabic scenarios
+############################################################
+metrics:
+  # Infrastructure metrics:
+  - name: num_perplexity_tokens
+    display_name: '# tokens'
+    description: Average number of tokens in the predicted output (for language modeling, the input too).
+  - name: num_bytes
+    display_name: '# bytes'
+    description: Average number of bytes in the predicted output (for language modeling, the input too).
+
+  - name: num_references
+    display_name: '# ref'
+    description: Number of references.
+  - name: num_train_trials
+    display_name: '# trials'
+    description: Number of trials, where in each trial we choose an independent, random set of training instances.
+  - name: estimated_num_tokens_cost
+    display_name: 'cost'
+    description: An estimate of the number of tokens (including prompt and output completions) needed to perform the request.
+  - name: num_prompt_tokens
+    display_name: '# prompt tokens'
+    description: Number of tokens in the prompt.
+  - name: num_prompt_characters
+    display_name: '# prompt chars'
+    description: Number of characters in the prompt.
+  - name: num_completion_tokens
+    display_name: '# completion tokens'
+    description: Actual number of completion tokens (over all completions).
+  - name: num_output_tokens
+    display_name: '# output tokens'
+    description: Actual number of output tokens.
+  - name: max_num_output_tokens
+    display_name: 'Max output tokens'
+    description: Maximum number of output tokens (overestimate since we might stop earlier due to stop sequences).
+  - name: num_requests
+    display_name: '# requests'
+    description: Number of distinct API requests.
+  - name: num_instances
+    display_name: '# eval'
+    description: Number of evaluation instances.
+  - name: num_train_instances
+    display_name: '# train'
+    description: Number of training instances (e.g., in-context examples).
+  - name: prompt_truncated
+    display_name: truncated
+    description: Fraction of instances where the prompt itself was truncated (implies that there were no in-context examples).
+  - name: finish_reason_length
+    display_name: finish b/c length
+    description: Fraction of instances where the the output was terminated because of the max tokens limit.
+  - name: finish_reason_stop
+    display_name: finish b/c stop
+    description: Fraction of instances where the the output was terminated because of the stop sequences.
+  - name: finish_reason_endoftext
+    display_name: finish b/c endoftext
+    description: Fraction of instances where the the output was terminated because the end of text token was generated.
+  - name: finish_reason_unknown
+    display_name: finish b/c unknown
+    description: Fraction of instances where the the output was terminated for unknown reasons.
+  - name: num_completions
+    display_name: '# completions'
+    description: Number of completions.
+  - name: predicted_index
+    display_name: Predicted index
+    description: Integer index of the reference (0, 1, ...) that was predicted by the model (for multiple-choice).
+  - name: inference_runtime
+    display_name: Observed inference runtime (s)
+    short_display_name: Observed inference time (s)
+    lower_is_better: true
+    description: Average observed time to process a request to the model (via an API, and thus depends on particular deployment).
+
+  # Accuracy metrics:
+  - name: exact_match
+    display_name: Exact match
+    short_display_name: EM
+    description: Fraction of instances that the predicted output matches a correct reference exactly.
+    lower_is_better: false
+  - name: quasi_exact_match
+    display_name: Quasi-exact match
+    short_display_name: EM
+    description: Fraction of instances that the predicted output matches a correct reference up to light processing.
+    lower_is_better: false
+  - name: prefix_exact_match
+    display_name: Prefix exact match
+    short_display_name: PEM
+    description: Fraction of instances that the predicted output matches the prefix of a correct reference exactly.
+    lower_is_better: false
+  - name: quasi_prefix_exact_match
+    # TODO: should call this prefix_quasi_exact_match
+    display_name: Prefix quasi-exact match
+    short_display_name: PEM
+    description: Fraction of instances that the predicted output matches the prefix of a correct reference up to light processing.
+    lower_is_better: false
+  - name: alrage_score
+    # TODO: should call this prefix_quasi_exact_match
+    display_name: ALRAGE Score
+    short_display_name: Score
+    description: Score of the output judged by GPT-4o.
+    lower_is_better: false
+
+############################################################
+perturbations: []
+
+############################################################
+metric_groups:
+  - name: accuracy
+    display_name: Accuracy
+    aggregation_strategies: 
+      - mean
+    metrics:
+      - name: ${main_name}
+        split: ${main_split}
+
+  - name: efficiency
+    display_name: Efficiency
+    aggregation_strategies: 
+      - mean
+    metrics:
+    - name: inference_runtime
+      split: ${main_split}
+
+  - name: general_information
+    display_name: General information
+    hide_win_rates: true
+    metrics:
+    - name: num_instances
+      split: ${main_split}
+    - name: num_train_instances
+      split: ${main_split}
+    - name: prompt_truncated
+      split: ${main_split}
+    - name: num_prompt_tokens
+      split: ${main_split}
+    - name: num_output_tokens
+      split: ${main_split}
+
+############################################################
+run_groups:
+  - name: arabic_scenarios
+    display_name: Arabic Scenarios
+    description: Arabic Scenarios
+    category: Scenarios
+    subgroups:
+      - alghafa
+      - arabic_mmlu
+      - arabic_exams
+      - madinah_qa
+      - aratrust
+      - alrage
+      - mbzuai_human_translated_arabic_mmlu
+
+  - name: mbzuai_human_translated_arabic_mmlu
+    display_name: MBZUAI Human-Translated Arabic MMLU
+    short_display_name: Translated MMLU
+    description: A translation from MBZUAI by human translators of the Massive Multitask Language Understanding benchmark.
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: multiple-choice question answering
+      what: math, science, history, etc.
+      who: various online sources
+      when: before 2021
+      language: Arabic
+
+  - name: arabic_mmlu
+    display_name: ArabicMMLU
+    description: ArabicMMLU
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: "question answering"
+      what: "academic questions across various disciplines"
+      who: "academic exams writers and takers"
+      when: "before 2024"
+      language: Arabic
+
+  - name: alghafa
+    display_name: AlGhafa
+    description: AlGhafa
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: "multiple choice question answering"
+      what: Various
+      who: Various
+      when: "before 2023"
+      language: Arabic
+
+  - name: arabic_exams
+    display_name: Arabic EXAMS
+    description: Arabic EXAMS
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: "multiple choice question answering"
+      what: High school examinations
+      who: High school examinations writers and test-takers
+      when: before 2020
+      language: Arabic
+
+  - name: aratrust
+    display_name: AraTrust
+    description: AraTrust
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: "question answering"
+      what: "academic questions across various disciplines"
+      who: "academic exams writers and takers"
+      when: "before 2024"
+      language: Arabic
+
+  - name: alrage
+    display_name: ALRAGE
+    description: ALRAGE
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: alrage_score
+      main_split: test
+    taxonomy:
+      task: "openbook (RAG) open-ended question answering"
+      what: "?"
+      who: "?"
+      when: "?"
+      language: Arabic
+
+  - name: madinah_qa
+    display_name: MadinahQA
+    description: Arabic language competency benchmark
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: "question answering"
+      what: "academic questions about Arabic language"
+      who: "academic exams writers and takers"
+      when: "before 2024"
+      language: Arabic

helm/benchmark/static/schema_classic.yaml

@@ -1683,23 +1683,6 @@ run_groups:
       when: n/a
       language: synthetic
 
-  - name: numeracy
-    display_name: Numerical reasoning
-    description: Scenario introduced in this work to test numerical reasoning via symbolic regression.
-    metric_groups:
-      - accuracy
-      - efficiency
-      - general_information
-    environment:
-      main_name: absolute_value_difference
-      main_split: test
-    taxonomy:
-      task: next-word prediction
-      what: Dyck formal language
-      who: n/a
-      when: n/a
-      language: synthetic
-
   - name: synthetic_reasoning
     display_name: Synthetic reasoning (abstract symbols)
     description: Synthetic reasoning tasks defined using abstract symbols based on LIME [(Wu et al., 2021)](https://proceedings.mlr.press/v139/wu21c.html).

helm/benchmark/static/schema_long_context.yaml

@@ -194,7 +194,8 @@ run_groups:
       - ruler_hotpotqa
       - ruler_squad
       - infinite_bench_en_sum
-      - infinite_bench_en_qa
+      # - infinite_bench_en_qa
+      - infinite_bench_en_mc
       - openai_mrcr
 
   - name: ruler_hotpotqa
@@ -232,18 +233,35 @@ run_groups:
       when: Before 2018
       language: English
 
-  - name: infinite_bench_en_qa
-    display_name: ∞Bench En.QA
-    description: ∞Bench En.QA is a question answering task that requires locating and processing information within a novel, performing reasoning through aggregation or filtering to derive answers. ([Zhang et al., 2024](https://arxiv.org/abs/2402.13718))
+  # - name: infinite_bench_en_qa
+  #   display_name: ∞Bench En.QA
+  #   description: ∞Bench En.QA is a open-ended question answering task that requires locating and processing information within a novel, performing reasoning through aggregation or filtering to derive answers. ([Zhang et al., 2024](https://arxiv.org/abs/2402.13718))
+  #   metric_groups:
+  #     - accuracy
+  #     - general_information
+  #     - annotation_metrics
+  #   environment:
+  #     main_name: f1_score
+  #     main_split: test
+  #   taxonomy:
+  #     task: question answering
+  #     what: Novels
+  #     who: Novel authors
+  #     when: Before 2024
+  #     language: English
+
+  - name: infinite_bench_en_mc
+    display_name: ∞Bench En.MC
+    description: ∞Bench En.MC is a multiple-choice question answering task that requires locating and processing information within a novel, performing reasoning through aggregation or filtering to derive answers. ([Zhang et al., 2024](https://arxiv.org/abs/2402.13718))
     metric_groups:
       - accuracy
       - general_information
       - annotation_metrics
     environment:
-      main_name: f1_score
+      main_name: exact_match
       main_split: test
     taxonomy:
-      task: question answering
+      task: multiple-choice question answering
       what: Novels
       who: Novel authors
       when: Before 2024