crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/scenarios/image_generation/relational_understanding_scenario.py

@@ -0,0 +1,52 @@
+from typing import List, Set
+import csv
+import os
+
+from helm.common.general import ensure_file_downloaded
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT
+
+
+class RelationalUnderstandingScenario(Scenario):
+    """
+    From Testing Relational Understanding in Text-Guided Image Generation, based on existing cognitive,
+    linguistic, and developmental literature, the authors created a set of 15 relations (8 physical,
+    7 agentic) and a set of 12 entities (6 objects, 6 agents). The physical relations were: in, on,
+    under, covering, near, occluded by, hanging over, and tied to. The agentic relations were: pushing,
+    pulling, touching, hitting, kicking, helping, and hindering. The objects were: box, cylinder,
+    blanket, bowl, teacup, and knife. The agents were: man, woman, child, robot, monkey, and iguana.
+
+    The authors created 5 different prompts for each relation, by randomly sampling two entities five
+    times, resulting in 75 distinct basic relation prompts (e.g., a monkey touching an iguana). Withs
+    these prompts, the authors showed that DALL-E 2 suffers from a significant lack of commonsense
+    reasoning in the form of relational understanding.
+
+    Paper: https://arxiv.org/abs/2208.00005
+    Website: https://osf.io/sm68h
+    """
+
+    name = "relational_understanding"
+    description = (
+        "Consists of 75 basic relation prompts that tests commonsense reasoning "
+        "([paper](https://arxiv.org/abs/2208.00005))."
+    )
+    tags = ["text-to-image", "reasoning"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path: str = os.path.join(output_path, "choice_data.csv")
+        ensure_file_downloaded(source_url="https://osf.io/download/tb3a4", target_path=data_path)
+
+        instances: List[Instance] = []
+        seen_prompts: Set[str] = set()
+        with open(data_path) as csv_file:
+            csv_reader = csv.reader(csv_file, delimiter=",")
+            for i, row in enumerate(csv_reader):
+                if i == 0:
+                    # Skip the header
+                    continue
+
+                prompt: str = row[1]
+                if prompt not in seen_prompts:
+                    instances.append(Instance(Input(text=prompt), references=[], split=TEST_SPLIT))
+                    seen_prompts.add(prompt)
+
+        return instances

helm/benchmark/scenarios/image_generation/time_most_significant_historical_figures_scenario.py

@@ -0,0 +1,124 @@
+from typing import List
+
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT
+
+
+class TIMEMostSignificantHistoricalFigures(Scenario):
+    """
+    People from TIME's "The 100 Most Significant Figures in History" list.
+
+    https://ideas.time.com/2013/12/10/whos-biggest-the-100-most-significant-figures-in-history/
+    """
+
+    HISTORICAL_FIGURES: List[str] = [
+        "Jesus",
+        "Napoleon Bonaparte",
+        "Muhammad",
+        "William Shakespeare",
+        "Abraham Lincoln",
+        "George Washington",
+        "Adolf Hitler",
+        "Aristotle",
+        "Alexander the Great",
+        "Thomas Jefferson",
+        "Henry VIII of England",
+        "Charles Darwin",
+        "Elizabeth I of England",
+        "Karl Marx",
+        "Julius Caesar",
+        "Queen Victoria",
+        "Martin Luther",
+        "Joseph Stalin",
+        "Albert Einstein",
+        "Christopher Columbus",
+        "Isaac Newton",
+        "Charlemagne",
+        "Theodore Roosevelt",
+        "Wolfgang Amadeus Mozart",
+        "Plato",
+        "Louis XIV of France",
+        "Ludwig van Beethoven",
+        "Ulysses S.Grant",
+        "Leonardo da Vinci",
+        "Augustus",
+        "Carl Linnaeus",
+        "Ronald Reagan",
+        "Charles Dickens",
+        "Paul the Apostle",
+        "Benjamin Franklin",
+        # "George W.Bush",
+        "Winston Churchill",
+        "Genghis Khan",
+        "Charles I of England",
+        "Thomas Edison",
+        "James I of England",
+        "Friedrich Nietzsche",
+        "Franklin D.Roosevelt",
+        "Sigmund Freud",
+        "Alexander Hamilton",
+        "Mohandas Karamchand Gandhi",
+        "Woodrow Wilson",
+        "Johann Sebastian Bach",
+        "Galileo Galilei",
+        "Oliver Cromwell",
+        "James Madison",
+        "Gautama Buddha",
+        "Mark Twain",
+        "Edgar Allan Poe",
+        "Joseph Smith, Jr.",
+        "Adam Smith",
+        "David, King of Israel",
+        "George III of the United Kingdom",
+        "Immanuel Kant",
+        "James Cook",
+        "John Adams",
+        "Richard Wagner",
+        "Pyotr Ilyich Tchaikovsky",
+        "Voltaire",
+        "Saint Peter",
+        "Andrew Jackson",
+        "Constantine the Great",
+        "Socrates",
+        "Elvis Presley",
+        "William the Conqueror",
+        "John F.Kennedy",
+        "Augustine of Hippo",
+        "Vincent van Gogh",
+        "Nicolaus Copernicus",
+        "Vladimir Lenin",
+        "Robert E.Lee",
+        "Oscar Wilde",
+        "Charles II of England",
+        "Cicero",
+        "Jean-Jacques Rousseau",
+        "Francis Bacon",
+        "Richard Nixon",
+        "Louis XVI of France",
+        "Charles V, Holy Roman Emperor",
+        "King Arthur",
+        "Michelangelo",
+        "Philip II of Spain",
+        "Johann Wolfgang von Goethe",
+        "Ali, founder of Sufism",
+        "Thomas Aquinas",
+        "Pope John Paul II",
+        "René Descartes",
+        "Nikola Tesla",
+        "Harry S.Truman",
+        "Joan of Arc",
+        "Dante Alighieri",
+        "Otto von Bismarck",
+        "Grover Cleveland",
+        "John Calvin",
+        "John Locke",
+    ]
+
+    name = "time_most_significant_historical_figures"
+    description = 'People from TIME\'s "The 100 Most Significant Figures in History" list.'
+    tags = ["text-to-image", "knowledge"]
+
+    def get_instances(self, _) -> List[Instance]:
+        return [
+            Instance(Input(text=historical_figure), references=[], split=TEST_SPLIT)
+            for historical_figure in self.HISTORICAL_FIGURES
+        ]

helm/benchmark/scenarios/image_generation/winoground_scenario.py

@@ -0,0 +1,62 @@
+from typing import List
+import os
+
+from datasets import load_dataset
+
+from helm.common.general import get_file_name
+from helm.common.images_utils import copy_image
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, Output, Reference, CORRECT_TAG, TEST_SPLIT
+
+
+class WinogroundScenario(Scenario):
+    """
+    Winoground is a novel task and dataset for evaluating the ability of vision and language models
+    to conduct visio-linguistic compositional reasoning. Given two images and two captions, the
+    goal is to match them correctly—but crucially, both captions contain a completely identical set
+    of words/morphemes, only in a different order. The dataset was carefully hand-curated by
+    expert annotators and is labeled with a rich set of fine-grained tags to assist in analyzing
+    model performance.
+
+    Users must agree to share their contact information before downloading the dataset from
+    Hugging Face. Either agree to the terms and set HUGGING_FACE_ACCESS_TOKEN to an access token
+    of a valid Hugging Face account or have the dataset pre-downloaded at the Hugging Face cache
+    (default path: ~/.cache/huggingface/datasets).
+
+    Paper: https://arxiv.org/abs/2204.03162
+    Website: https://huggingface.co/datasets/facebook/winoground
+    """
+
+    name = "winoground"
+    description = (
+        "Winoground is a novel task and dataset for evaluating the ability of vision and language models "
+        "to conduct visio-linguistic compositional reasoning "
+        "([paper](https://arxiv.org/abs/2204.03162))."
+    )
+    tags = ["text-to-image", "image-to-text", "visual_reasoning"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        auth_token: str = os.environ.get("HUGGING_FACE_ACCESS_TOKEN", "")
+
+        instances: List[Instance] = []
+        for row in load_dataset("facebook/winoground", split="test", use_auth_token=auth_token):
+            # Use the first example of the pair for now (index 0)
+            caption: str = row["caption_0"]
+            image_path: str = row["image_0"].filename
+
+            # Create a copy of the image in the benchmark output folder for metrics computation
+            image_copy_path: str = os.path.join(output_path, get_file_name(image_path))
+            if not os.path.exists(image_copy_path):
+                copy_image(image_path, image_copy_path)
+            content: MultimediaObject = MultimediaObject(
+                [MediaObject(content_type="image/png", location=image_copy_path)]
+            )
+
+            instances.append(
+                Instance(
+                    input=Input(text=caption),
+                    references=[Reference(Output(multimedia_content=content), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+        return instances

helm/benchmark/scenarios/imdb_ptbr_scenario.py

@@ -0,0 +1,60 @@
+from typing import Any, List, Dict
+from pathlib import Path
+from datasets import load_dataset
+from helm.common.hierarchical_logger import hlog
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class IMDB_PTBRScenario(Scenario):
+    """
+    The IMDB dataset is a widely-used benchmark dataset for natural language processing (NLP)
+    particularly for text classification and sentiment analysis.
+    This is a translated version that is meant to evaluate PT-BR models.
+    It consists of movie reviews from the Internet Movie Database (IMDB) and
+    includes both positive and negative sentiments labeled for supervised learning.
+    """
+
+    name = "simple_classification"
+    description = "Classify movie reviews between positive or negative."
+    tags = ["classification"]
+
+    def process_dataset(self, dataset: Any, split: str) -> List[Instance]:
+        instances: List[Instance] = []
+        label_names = {0: "negativo", 1: "positivo"}
+        for example in dataset[split]:
+            input = Input(text=example["text"])
+            # NOTE: For classification scenarios, the reference outputs should be the same
+            # for all instances, and should include both correct and incorrect classes.
+            # HELM only supports single-label classification. Exactly one reference
+            # should have the CORRECT_TAG tag.
+            references = [
+                Reference(Output(text=label_names[example["label"]]), tags=[CORRECT_TAG]),
+            ]
+            instance = Instance(input=input, references=references, split=split)
+            instances.append(instance)
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        cache_dir = str(Path(output_path) / "data")
+        dataset = load_dataset("maritaca-ai/imdb_pt", cache_dir=cache_dir)
+        splits: Dict[str, str] = {
+            "train": TRAIN_SPLIT,
+            "test": TEST_SPLIT,
+        }
+        for split in splits:
+            if split not in splits.keys():
+                hlog(f"{split} split doesn't exist, skipping")
+                continue
+            instances.extend(self.process_dataset(dataset, splits[split]))
+
+        return instances

helm/benchmark/scenarios/imdb_scenario.py

@@ -1,9 +1,20 @@
 import os
 from typing import List, Dict, Optional
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
-from .scenario import Scenario, Instance, Reference, CORRECT_TAG, TRAIN_SPLIT, VALID_SPLIT, Input, Output
-from .imdb_scenario_pinned_file_order import listdir_with_pinned_file_order
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    CORRECT_TAG,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
+from helm.benchmark.scenarios.imdb_scenario_pinned_file_order import listdir_with_pinned_file_order
 
 
 class IMDBScenario(Scenario):
@@ -107,7 +118,6 @@ class IMDBScenario(Scenario):
         contrast_map = {}
 
         for orig_line, contrast_line in zip(orig_and_contrast_inputs[0], orig_and_contrast_inputs[1]):
-
             orig_label_name, orig_context = orig_line.strip().split("\t")
             orig_label = label_name_to_id[orig_label_name]
 
@@ -135,3 +145,16 @@ class IMDBScenario(Scenario):
         for split in [TRAIN_SPLIT, VALID_SPLIT]:
             instances.extend(self.get_split_instances(target_path, split, contrast_map))
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="imdb",
+            display_name="IMDB",
+            description="The IMDB benchmark for sentiment analysis in movie review [(Maas et al., "
+            "2011)](https://aclanthology.org/P11-1015/).",
+            taxonomy=TaxonomyInfo(
+                task="sentiment analysis", what="movie reviews", when="?", who="?", language="English"
+            ),
+            main_metric="quasi_exact_match",
+            main_split="valid",
+        )

helm/benchmark/scenarios/infinite_bench_en_mc_scenario.py

@@ -0,0 +1,111 @@
+import os
+import re
+from typing import List
+
+from datasets import load_dataset, Features, Value, Sequence, Dataset
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Input,
+    Reference,
+    Output,
+    CORRECT_TAG,
+    TEST_SPLIT,
+    ScenarioMetadata,
+)
+from helm.common.general import ensure_directory_exists
+
+
+class InfiniteBenchEnMCScenario(Scenario):
+    """InfiniteBench En.MC
+
+    InfiniteBench is a benchmark tailored for evaluating the capabilities of language models to process,
+    understand, and reason over long contexts (100k+ tokens). InfiniteBench En.MC is a subset of
+    InfiniteBench that requires models to perform multiple-choice question answering on questions that necessitate
+    long-range dependency and reasoning, beyond simple short passage retrieval.
+    """
+
+    name = "infinite_bench_en_mc"
+    description = "∞Bench En.MC is a multiple-choice question answering task that necessitates long-range dependency and reasoning. ([Zhang et al., 2024](https://arxiv.org/abs/2402.13718))"  # noqa: E501
+    tags = ["question_answering"]
+
+    def __init__(self, max_num_words: int):
+        self.max_num_words = max_num_words
+        super().__init__()
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Get InfiniteBench from HuggingFace
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+
+        # Define the features schema
+        ft = Features(
+            {
+                "id": Value("int64"),
+                "context": Value("string"),
+                "input": Value("string"),
+                "answer": Sequence(Value("string")),
+                "options": Sequence(Value("string")),
+            }
+        )
+
+        # Load the dataset with the specified features
+        dataset = load_dataset(
+            "xinrongzhang2022/InfiniteBench",
+            split="longbook_choice_eng",
+            features=ft,
+            revision="90f0394333616266d9fe85824ceaf505093cbaa5",
+        )
+
+        assert isinstance(dataset, Dataset)
+
+        def count_words(text: str) -> int:
+            return len(re.split(r"\s+", text.strip()))
+
+        dataset = dataset.filter(
+            lambda example: count_words(example["context"])
+            + count_words(example["input"])
+            + sum(count_words(option) for option in example["options"])
+            <= self.max_num_words
+        )
+
+        # Read all instances
+        instances: List[Instance] = []
+        for row in dataset:
+            assert len(row["answer"]) == 1
+            id = row["id"]
+            input = Input(text=row["context"] + "\n\n" + row["input"])
+            references = [
+                Reference(Output(text=option), tags=[CORRECT_TAG] if option == row["answer"][0] else [])
+                for option in row["options"]
+            ]
+            instance = Instance(
+                id=id,
+                input=input,
+                references=references,
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="infinite_bench_en_mc",
+            display_name="∞Bench En.MC",
+            description="∞Bench En.MC is a multiple-choice question answering task that requires "
+            "locating and processing information within a novel, performing reasoning "
+            "through aggregation or filtering to derive answers. ([Zhang et al., "
+            "2024](https://arxiv.org/abs/2402.13718))",
+            taxonomy=TaxonomyInfo(
+                task="multiple-choice question answering",
+                what="Novels",
+                when="Before 2024",
+                who="Novel authors",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/infinite_bench_en_qa_scenario.py

@@ -0,0 +1,85 @@
+import os
+import re
+from typing import List
+
+from datasets import load_dataset, Features, Value, Sequence, Dataset
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Input,
+    Reference,
+    Output,
+    CORRECT_TAG,
+    TEST_SPLIT,
+)
+from helm.common.general import ensure_directory_exists
+
+
+class InfiniteBenchEnQAScenario(Scenario):
+    """InfiniteBench En.QA
+
+    InfiniteBench is a benchmark tailored for evaluating the capabilities of language models to process,
+    understand, and reason over long contexts (100k+ tokens). InfiniteBench En.QA is a subset of
+    InfiniteBench that requires models to perform open-form question answering on questions that necessitate
+    long-range dependency and reasoning, beyond simple short passage retrieval.
+    """
+
+    name = "infinite_bench_en_qa"
+    description = "∞Bench En.QA is an open-ended question answering task that necessitates long-range dependency and reasoning.  ([Zhang et al., 2024](https://arxiv.org/abs/2402.13718))"  # noqa: E501
+    tags = ["question_answering"]
+
+    def __init__(self, max_num_words: int):
+        self.max_num_words = max_num_words
+        super().__init__()
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Get InfiniteBench from HuggingFace
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+
+        # Define the features schema
+        ft = Features(
+            {
+                "id": Value("int64"),
+                "context": Value("string"),
+                "input": Value("string"),
+                "answer": Sequence(Value("string")),
+                "options": Sequence(Value("string")),
+            }
+        )
+
+        # Load the dataset with the specified features
+        dataset = load_dataset(
+            "xinrongzhang2022/InfiniteBench",
+            split="longbook_qa_eng",
+            features=ft,
+            revision="90f0394333616266d9fe85824ceaf505093cbaa5",
+        )
+
+        assert isinstance(dataset, Dataset)
+
+        def count_words(text: str) -> int:
+            return len(re.split(r"\s+", text.strip()))
+
+        dataset = dataset.filter(
+            lambda example: count_words(example["context"])
+            + count_words(example["input"])
+            + sum(count_words(option) for option in example["options"])
+            <= self.max_num_words
+        )
+
+        # Read all instances
+        instances: List[Instance] = []
+        for row in dataset:
+            id = row["id"]
+            input = Input(text=row["context"] + "\n\n" + row["input"])
+            instance = Instance(
+                id=id,
+                input=input,
+                references=[Reference(Output(text=row["answer"][0]), tags=[CORRECT_TAG])],
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/infinite_bench_en_sum_scenario.py

@@ -0,0 +1,98 @@
+import os
+import re
+from typing import List
+from datasets import load_dataset, Features, Value, Sequence, Dataset
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Input,
+    Reference,
+    Output,
+    CORRECT_TAG,
+    TEST_SPLIT,
+    ScenarioMetadata,
+)
+from helm.common.general import ensure_directory_exists
+
+
+class InfiniteBenchEnSumScenario(Scenario):
+    """InfiniteBench En.Sum
+
+    InfiniteBench is a benchmark tailored for evaluating the capabilities of language models to process,
+    understand, and reason over super long contexts (100k+ tokens). InfiniteBench En.Sum is a subset of
+    InfiniteBench that requires models to generate a concise summary of the novel.
+    """
+
+    name = "infinite_bench_en_sum"
+    description = "∞Bench En.Sum is a summarization task that requires generating a concise summary of a novel. ([Zhang et al., 2024](https://arxiv.org/abs/2402.13718))"  # noqa: E501
+    tags = ["summarization"]
+
+    def __init__(self, max_num_words: int):
+        self.max_num_words = max_num_words
+        super().__init__()
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Get InfiniteBench from HuggingFace
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+
+        # Define the features schema
+        ft = Features(
+            {
+                "id": Value("int64"),
+                "context": Value("string"),
+                "input": Value("string"),
+                "answer": Sequence(Value("string")),
+                "options": Sequence(Value("string")),
+            }
+        )
+
+        # Load the dataset with the specified features
+        dataset = load_dataset(
+            "xinrongzhang2022/InfiniteBench",
+            split="longbook_sum_eng",
+            features=ft,
+            revision="90f0394333616266d9fe85824ceaf505093cbaa5",
+        )
+
+        assert isinstance(dataset, Dataset)
+
+        def count_words(text: str) -> int:
+            return len(re.split(r"\s+", text.strip()))
+
+        dataset = dataset.filter(
+            lambda example: count_words(example["context"]) + count_words(example["input"]) <= self.max_num_words
+        )
+
+        # Read all instances
+        instances: List[Instance] = []
+        for row in dataset:
+            id = row["id"]
+            input = Input(text=row["context"] + "\n\n" + row["input"])
+            instance = Instance(
+                id=id,
+                input=input,
+                references=[Reference(Output(text=row["answer"][0]), tags=[CORRECT_TAG])],
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="infinite_bench_en_sum",
+            display_name="∞Bench En.Sum",
+            description="∞Bench En.Sum is a summarization task that requires generating a concise "
+            "summary of a novel. ([Zhang et al., 2024](https://arxiv.org/abs/2402.13718))",
+            taxonomy=TaxonomyInfo(
+                task="multi-hop question answering",
+                what="Novels",
+                when="Before 2024",
+                who="Novel authors",
+                language="English",
+            ),
+            main_metric="rouge_l",
+            main_split="test",
+        )

helm/benchmark/scenarios/interactive_qa_mmlu_scenario.py

@@ -2,8 +2,8 @@ import os
 from typing import List
 
 from helm.common.general import ensure_file_downloaded
-from .scenario import Instance, TRAIN_SPLIT, TEST_SPLIT
-from .mmlu_scenario import MMLUScenario
+from helm.benchmark.scenarios.scenario import Instance, TRAIN_SPLIT, TEST_SPLIT
+from helm.benchmark.scenarios.mmlu_scenario import MMLUScenario
 
 
 class InteractiveQAMMLUScenario(MMLUScenario):