crfm-helm 0.5.6__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/scenarios/test_bluex_scenario.py

@@ -0,0 +1,59 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.bluex_scenario import BLUEXScenario
+from helm.benchmark.scenarios.scenario import TEST_SPLIT, CORRECT_TAG, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_bluex_scenario():
+    scenario = BLUEXScenario()
+    with TemporaryDirectory() as tmpdir:
+        instances = scenario.get_instances(tmpdir)
+
+    assert len(instances) > 100
+
+    assert instances[100].split == TEST_SPLIT
+
+    assert instances[0].input.text.startswith("Rubião fitava a enseada, - eram oito horas da manhã Quem o visse")
+
+    assert len(instances[0].input.text) == 1011
+
+    assert instances[0].references == [
+        Reference(
+            output=Output(
+                text='a contemplação das paisagens naturais, como se lê em "ele admirava aquele pedaço de água quieta".'
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text='a presença de um narrador-personagem, como se lê em "em verdade vos digo que pensava em '
+                'outra coisa".'
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text='a sobriedade do protagonista ao avaliar o seu percurso, como se lê em "Cotejava o passado com '
+                "o presente."
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text='o sentido místico e fatalista que rege os destinos, como se lê em "Deus escreve direito por '
+                'linhas tortas".'
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text='a reversibilidade entre o cômico e o trágico, como se lê em "de modo que o que parecia uma '
+                'desgraça...".'
+            ),
+            tags=[CORRECT_TAG],
+        ),
+    ]
+
+    assert instances[0].references[4].is_correct

helm/benchmark/scenarios/test_exams_multilingual_scenario.py

@@ -0,0 +1,29 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.exams_multilingual_scenario import EXAMSMultilingualScenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, TRAIN_SPLIT, Input
+
+
+@pytest.mark.scenarios
+def test_exam_multilingual_scenario_get_instances():
+    scenario = EXAMSMultilingualScenario(language="Bulgarian", subject="Physics")
+    with TemporaryDirectory() as tmpdir:
+        actual_instances = scenario.get_instances(tmpdir)
+    assert len(actual_instances) == 393
+    assert actual_instances[0].id == "4c05bbb8-7729-11ea-9116-54bef70b159e"
+    assert actual_instances[0].input == Input(text="Наелектризирането по индукция се обяснява с: ")
+    assert len(actual_instances[0].references) == 4
+    assert actual_instances[0].references[0].output.text == "преразпределение на положителните йони в тялото"
+    assert actual_instances[0].references[0].tags == []
+    assert (
+        actual_instances[0].references[1].output.text == "предаване на електрони от неутрално на наелектризирано тяло"
+    )
+    assert actual_instances[0].references[1].tags == []
+    assert (
+        actual_instances[0].references[2].output.text == "предаване на електрони от наелектризирано на неутрално тяло"
+    )
+    assert actual_instances[0].references[2].tags == []
+    assert actual_instances[0].references[3].output.text == "преразпределение на свободните електрони в тялото"
+    assert actual_instances[0].references[3].tags == [CORRECT_TAG]
+    assert actual_instances[0].split == TRAIN_SPLIT

helm/benchmark/scenarios/test_healtha_br_scenario.py

@@ -0,0 +1,57 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.healthqa_br_scenario import HEALTHQA_BR_Scenario
+from helm.benchmark.scenarios.scenario import TEST_SPLIT, CORRECT_TAG, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_healthqa_br_instance():
+    scenario = HEALTHQA_BR_Scenario()
+    with TemporaryDirectory() as tmpdir:
+        instances = scenario.get_instances(tmpdir)
+
+    instance = instances[35]
+
+    assert instance.split == TEST_SPLIT
+
+    assert instance.input.text.startswith("Homem de 22 anos de idade procura a Unidade Básica")
+
+    assert instance.references == [
+        Reference(
+            output=Output(
+                text="administração de relaxante muscular, colocando o paciente em posição de Trendelenburg, com "
+                "tentativa de redução do volume."
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text="encaminhamento do paciente ao Serviço de Urgência do Hospital com o pedido de avaliação "
+                "imediata do cirurgião."
+            ),
+            tags=[CORRECT_TAG],
+        ),
+        Reference(
+            output=Output(
+                text="tentativa de redução manual do aumento de volume da região inguinescrotal para a cavidade "
+                "abdominal."
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text="transiluminação do escroto para tentar diferenciar hérnia inguinal de hidrocele comunicante."
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(text="prescrição de antiemético e solicitação de ecografia da região inguinescrotal."),
+            tags=[],
+        ),
+    ]
+
+    correct_refs = [ref for ref in instance.references if CORRECT_TAG in ref.tags]
+    assert len(correct_refs) == 1
+
+    assert instance.references[1].is_correct

helm/benchmark/scenarios/thai_exam_scenario.py

@@ -2,6 +2,7 @@ import os
 from typing import Dict, List
 import json
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 from helm.common.hierarchical_logger import hlog
 from helm.benchmark.scenarios.scenario import (
@@ -13,6 +14,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -142,3 +144,96 @@ class ThaiExamScenario(Scenario):
             instances.extend(self.process_jsonl(jsonl_path, splits[split]))
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        if self.exam == "onet":
+            return ScenarioMetadata(
+                name="thai_exam_onet",
+                display_name="ONET",
+                description="The Ordinary National Educational Test (ONET) is an examination for students "
+                "in Thailand. We select the grade-12 ONET exam, which comprises 5 subjects and "
+                "each question has 5 choices. These subjects are Thai, English, Mathematics, "
+                "Social Studies, and Science. Amounting to a total of 170 questions and "
+                "options.\n",
+                taxonomy=TaxonomyInfo(
+                    task="question answering",
+                    what="high school / medical school academic knowledge",
+                    when="?",
+                    who="n/a",
+                    language="Thai and English",
+                ),
+                main_metric="exact_match",
+                main_split="test",
+            )
+        elif self.exam == "ic":
+            return ScenarioMetadata(
+                name="thai_exam_ic",
+                display_name="IC",
+                description="The Investment Consultant (IC) examination, a licensing test for investment "
+                "professionals in Thailand. Developed by the Stock Exchange of Thailand (SET), "
+                "features 4 choices per question. We extracted questions for levels 1, 2, and 3 "
+                "resulting in a total of 95 questions and options.\n",
+                taxonomy=TaxonomyInfo(
+                    task="question answering",
+                    what="licensing for investment professionals",
+                    when="?",
+                    who="n/a",
+                    language="Thai",
+                ),
+                main_metric="exact_match",
+                main_split="test",
+            )
+        elif self.exam == "tgat":
+            return ScenarioMetadata(
+                name="thai_exam_tgat",
+                display_name="TGAT",
+                description="The Thai General Aptitude Test (TGAT), a national high school examination in "
+                "Thailand. Focuses on critical and logical thinking skills. We collected a "
+                "total of 90 questions and answers. The TGAT consists of four choices per "
+                "question.\n",
+                taxonomy=TaxonomyInfo(
+                    task="question answering",
+                    what="high school level questions on reasoning",
+                    when="?",
+                    who="n/a",
+                    language="English",
+                ),
+                main_metric="exact_match",
+                main_split="test",
+            )
+        elif self.exam == "tpat1":
+            return ScenarioMetadata(
+                name="thai_exam_tpat1",
+                display_name="TPAT-1",
+                description="TBD",
+                taxonomy=TaxonomyInfo(
+                    task="question answering",
+                    what="high school / medical school academic knowledge",
+                    when="?",
+                    who="n/a",
+                    language="Thai",
+                ),
+                main_metric="exact_match",
+                main_split="test",
+            )
+        elif self.exam == "a_level":
+            return ScenarioMetadata(
+                name="thai_exam_a_level",
+                display_name="A-Level",
+                description="An academic knowledge assessment examination (Applied Knowledge Level) that "
+                "covers general foundational subjects taught in schools. The content assessed "
+                "in this examination aligns with the curriculum guidelines and emphasizes the "
+                "practical application of knowledge in daily life. We collected a total of 175 "
+                "questions and answers.\n",
+                taxonomy=TaxonomyInfo(
+                    task="question answering",
+                    what="high school academic knowledge",
+                    when="?",
+                    who="n/a",
+                    language="Thai and English",
+                ),
+                main_metric="exact_match",
+                main_split="test",
+            )
+        else:
+            raise ValueError(f"Unknown exam: {self.exam}")

helm/benchmark/scenarios/the_pile_scenario.py

@@ -5,9 +5,10 @@ import sys
 import requests
 from typing import Dict, List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 from helm.common.hierarchical_logger import hlog, htrack, htrack_block
-from helm.benchmark.scenarios.scenario import Scenario, Instance, TEST_SPLIT, Input
+from helm.benchmark.scenarios.scenario import Scenario, Instance, TEST_SPLIT, Input, ScenarioMetadata
 
 
 class ThePileScenario(Scenario):
@@ -146,3 +147,14 @@ class ThePileScenario(Scenario):
             instances = [instances[i] for i in indices]
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="the_pile",
+            display_name="The Pile",
+            description="The Pile corpus for measuring lanugage model performance across various "
+            "domains [(Gao et al., 2020)](https://arxiv.org/pdf/2101.00027.pdf).",
+            taxonomy=TaxonomyInfo(task="language modeling", what="?", when="?", who="?", language="English, code"),
+            main_metric="bits_per_byte",
+            main_split="test",
+        )

helm/benchmark/scenarios/truthful_qa_scenario.py

@@ -2,6 +2,7 @@ import csv
 import os
 from typing import List, Dict, Any
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -12,6 +13,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -154,3 +156,15 @@ class TruthfulQAScenario(Scenario):
         valid_instances: List[Instance] = get_split_instances(VALID_SPLIT, data[split_k:])
 
         return train_instances + valid_instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="truthful_qa",
+            display_name="TruthfulQA",
+            description="The TruthfulQA benchmarking for measuring model truthfulness and commonsense "
+            "knowledge in question answering [(Lin et al., "
+            "2022)](https://aclanthology.org/2022.acl-long.229/).",
+            taxonomy=TaxonomyInfo(task="question answering", what="?", when="?", who="?", language="English"),
+            main_metric="exact_match",
+            main_split="valid",
+        )

helm/benchmark/scenarios/twitter_aae_scenario.py

@@ -2,9 +2,10 @@ import csv
 import os
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 from helm.common.hierarchical_logger import hlog
-from helm.benchmark.scenarios.scenario import Scenario, Instance, TEST_SPLIT, Input
+from helm.benchmark.scenarios.scenario import Scenario, Instance, TEST_SPLIT, Input, ScenarioMetadata
 
 CODALAB_URI_TEMPLATE: str = (
     "https://worksheets.codalab.org/rest/bundles/0x31485f8c37ad481fb9f4e9bf7ccff6e5/contents/blob/"
@@ -56,3 +57,21 @@ class TwitterAAEScenario(Scenario):
                 instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="twitter_aae",
+            display_name="TwitterAAE",
+            description="The TwitterAAE corpus of [Blodgett et al. "
+            "(2016)](https://aclanthology.org/D16-1120/) for measuring language model "
+            "performance in tweets as a function of speaker dialect.",
+            taxonomy=TaxonomyInfo(
+                task="language modeling",
+                what="?",
+                when="?",
+                who="?",
+                language="English (AAE-aligned and White-aligned)",
+            ),
+            main_metric="bits_per_byte",
+            main_split="test",
+        )

helm/benchmark/scenarios/vicuna_scenario.py

@@ -2,8 +2,9 @@ import json
 import os
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
-from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT, ScenarioMetadata
 
 
 class VicunaScenario(Scenario):
@@ -47,3 +48,22 @@ class VicunaScenario(Scenario):
                 )
                 instances.append(instance)
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="vicuna",
+            display_name="Vicuna",
+            short_display_name="Vicuna",
+            description="The set of prompts used by the "
+            "[Vicuna](https://lmsys.org/blog/2023-03-30-vicuna/) team to evaluate "
+            "instruction-following models.",
+            taxonomy=TaxonomyInfo(
+                task="open-ended instruction following",
+                what="Instructions for LLMs",
+                when="Before 2023",
+                who="Unknown",
+                language="English",
+            ),
+            main_metric="Helpfulness",
+            main_split="test",
+        )

helm/benchmark/scenarios/wikifact_scenario.py

@@ -2,6 +2,7 @@ import os
 from typing import List, Dict
 import json
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_directory_exists, ensure_file_downloaded, flatten_list
 from helm.common.hierarchical_logger import hlog
 from helm.benchmark.scenarios.scenario import (
@@ -14,6 +15,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 PID_TO_NAME = {
@@ -183,3 +185,21 @@ class WIKIFactScenario(Scenario):
                 instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="wikifact",
+            display_name="WikiFact",
+            description="Scenario introduced in this work, inspired by [Petroni et al. "
+            "(2019)](https://aclanthology.org/D19-1250/), to more extensively test factual "
+            "knowledge.",
+            taxonomy=TaxonomyInfo(
+                task="knowledge base completion",
+                what="entity-relation-entity triples in natural language form",
+                when="?",
+                who="automatically generated from templates",
+                language="structured English",
+            ),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/wildbench_scenario.py

@@ -2,11 +2,13 @@ import datasets
 import os
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
     TEST_SPLIT,
     Input,
+    ScenarioMetadata,
 )
 from helm.common.general import ensure_directory_exists
 
@@ -81,3 +83,19 @@ class WildBenchScenario(Scenario):
             instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name=self.name,
+            display_name="WildBench",
+            description=self.description,
+            main_metric="wildbench_score_rescaled",
+            main_split="test",
+            taxonomy=TaxonomyInfo(
+                task="instruction following",
+                what="GPT-judged instruction following with instructions collected from real-user conversations",
+                who="real-world users",
+                when="2024",
+                language="English",
+            ),
+        )

helm/benchmark/scenarios/wmt_14_scenario.py

@@ -1,5 +1,6 @@
 from typing import List, Any
 from datasets import load_dataset
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.hierarchical_logger import htrack_block
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -11,6 +12,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -106,3 +108,20 @@ class WMT14Scenario(Scenario):
                     )
                 )
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="wmt_14",
+            display_name="WMT 2014",
+            description="WMT 2014 is a collection of machine translation datasets "
+            "[(website)](https://www.statmt.org/wmt14/index.html).",
+            taxonomy=TaxonomyInfo(
+                task="machine translation",
+                what="multilingual sentences",
+                when="before 2014",
+                who="Europarl, news, Common Crawl, etc.",
+                language="English, French, Czech, etc.",
+            ),
+            main_metric="bleu_4",
+            main_split="test",
+        )

helm/benchmark/slurm_jobs.py

@@ -13,7 +13,6 @@ except ModuleNotFoundError as e:
 
 
 class SlurmJobState:
-    # TODO: Convert to StrEnum after upgrading to Python 3.11
     # Non-exhaustive list of Slurm job states.
     # See: https://slurm.schedmd.com/squeue.html#SECTION_JOB-STATE-CODES
 
@@ -81,7 +80,7 @@ def get_slurm_job_state(job_id: int) -> str:
     except subprocess.CalledProcessError as e:
         # Default CalledProcessError message doesn't have output, so re-raise here to include the output.
         raise Exception(f"{str(e)} output: {e.output}")
-    search_result = re.search("JobState=(\w+)", scontrol_output.decode())
+    search_result = re.search(r"JobState=(\w+)", scontrol_output.decode())
     if not search_result:
         raise Exception(f"Could not extract JobState from scontrol: {scontrol_output.decode()}")
     return search_result.group(1)

helm/benchmark/slurm_runner.py

@@ -26,7 +26,7 @@ from helm.benchmark.slurm_jobs import (
     FAILURE_SLURM_JOB_STATES,
 )
 from helm.common.general import ensure_directory_exists
-from helm.common.hierarchical_logger import hlog, htrack_block
+from helm.common.hierarchical_logger import hlog, htrack_block, setup_default_logging
 
 from helm.benchmark.runner_config_registry import RUNNER_CONFIG
 
@@ -343,7 +343,14 @@ def main():
         help="Path to the RunSpec JSON file",
         required=True,
     )
+    parser.add_argument(
+        "--log-config",
+        type=str,
+        default=None,
+        help="PATH to a YAML file to customize logging",
+    )
     args = parser.parse_args()
+    setup_default_logging(args.log_config)
 
     # Deserialize SlurmRunner and RunSpec from the given files, then run the RunSpec with the SlurmRunner.
     with open(args.slurm_runner_spec_path, "r") as f: