crfm-helm 0.4.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

helm/benchmark/scenarios/image_generation/parti_prompts_scenario.py

@@ -0,0 +1,94 @@
+import csv
+import os
+from typing import List
+
+from helm.common.general import ensure_file_downloaded
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT
+
+
+class PartiPromptsScenario(Scenario):
+    """
+    PartiPrompts (P2) is a set of 1600 diverse English prompts that allow us to more comprehensively
+    evaluate and test the limits of text-to-image synthesis models.
+
+    Each prompt in the P2 benchmark is associated with two labels:
+    1. Category: indicating a broad group that a prompt belongs to
+    2. Challenge: highlighting an aspect which makes a prompt difficult
+
+    Categories:
+    - Abstract: Descriptions that represent abstract concepts, including single words and simple numbers.
+    - World Knowledge: Descriptions focused on objects and places that exist in the real world.
+    - People: Descriptions where the primary participants are human beings (but not specific individuals,
+              living or dead).
+    - Animals: Descriptions in which the primary participants are animals.
+    - Illustrations: Descriptions of images that involve specific types of graphical representations,
+                     including geometrical objects, diagrams, and symbols.
+    - Artifacts: Descriptions that represent abstract concepts, including single words and simple numbers.
+    - Food & Beverage: Descriptions of things animals, especially human beings, eat or drink.
+    - Vehicles: Descriptions where the focus is on man-made devices for transportation.
+    - Arts: Descriptions of existing paintings or intended to produce novel images in the format of a painting.
+    - Indoor Scenes: Descriptions about objects and participants that occur indoors.
+    - Outdoor Scenes: Descriptions about objects and participants that occur outdoors.
+    - Produce & Plants: Descriptions focused on plants or their products (fruits, vegetables, seeds, etc).
+
+    Challenges:
+    - Simple Detail: Descriptions that include only simple or high-level details.
+    - Fine-grained Detail: Descriptions that include very detailed specifications of attributes or
+                           actions of entities or objects in a scene.
+    - Complex: Descriptions that include many fine-grained, interacting details or relationships between multiple
+               participants.
+    - Quantity: Descriptions that specify particular counts of occurrences of subjects in a scene.
+    - Style & Format: Descriptions that specifically focus on the visual manner in which a subject or scene
+                      must be depicted.
+    - Properties & Positioning: Descriptions that target precise assignment of properties to entities or
+                                objects (often in the context of multiple entities or objects), and/or the
+                                relative spatial arrangement of entities and objects with respect to one
+                                another or landmarks in the scene.
+    - Linguistic Structures: Long and/or abstract words or complex syntactic structures or semantic
+                             ambiguities.
+    - Writing & Symbols: Descriptions that require words or symbols to be accurately represented
+                         in the context of the visual scene.
+    - Imagination: Descriptions that include participants or interactions that are not, or are generally unlikely
+                   to be, found in the modern day world.
+    - Basic: Descriptions about a single subject or concept with little to no detail or embellishment.
+    - Perspective: Descriptions that specify particular viewpoints or positioning of the subjects in a scene.
+
+    Paper: https://arxiv.org/abs/2206.10789
+    Website: https://parti.research.google/
+    """
+
+    DATASET_DOWNLOAD_URL: str = "https://raw.githubusercontent.com/google-research/parti/main/PartiPrompts.tsv"
+    ALL_CATEGORY: str = "all"
+
+    name = "parti_prompts"
+    description = (
+        "PartiPrompts (P2) is a set of 1600 diverse English prompts that allow to more comprehensively "
+        "evaluate and test the limits of text-to-image synthesis models ([paper](https://arxiv.org/abs/2206.10789))."
+    )
+    tags = ["text-to-image"]
+
+    def __init__(self, category: str):
+        super().__init__()
+        self.category: str = category
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        prompts_path: str = os.path.join(output_path, "prompts.tsv")
+        ensure_file_downloaded(source_url=self.DATASET_DOWNLOAD_URL, target_path=prompts_path)
+
+        instances: List[Instance] = []
+        with open(prompts_path) as f:
+            tsv_reader = csv.reader(f, delimiter="\t")
+            for i, row in enumerate(tsv_reader):
+                if i == 0:
+                    # Skip the header
+                    continue
+
+                prompt: str = row[0]
+                category: str = row[1]
+
+                # P2 does not have reference images
+                instance = Instance(Input(text=prompt), references=[], split=TEST_SPLIT)
+                if category.startswith(self.category) or self.category == self.ALL_CATEGORY:
+                    instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/image_generation/radiology_scenario.py

@@ -0,0 +1,42 @@
+from typing import List
+
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT
+
+
+class RadiologyScenario(Scenario):
+    """
+    From "What Does DALL-E 2 Know About Radiology?", DALL-E 2 fails to generate realistic
+    X-ray, CT, MRI and ultrasound images. For example, generated images of X-rays have
+    smaller bones missing, extra joints added, etc.
+
+    Prompts are in the following format:
+
+    “An x-ray of {head, chest, shoulder, abdomen, pelvis, hand, knee, ankle}” => 8 prompts
+    “An MRI of the {heart, liver, kidney}” => 3 prompts
+    “A CT of the {heart, liver, kidney}” => 3 prompts
+    “Ultrasound of the {heart, liver, kidney}” => 3 prompts
+
+    Paper: https://arxiv.org/abs/2209.13696
+    """
+
+    X_RAY_ANATOMICAL_AREA: List[str] = ["head", "chest", "shoulder", "abdomen", "pelvis", "hand", "knee", "ankle"]
+    ORGANS: List[str] = ["heart", "liver", "kidney"]
+
+    name = "radiology"
+    description = "Prompts to generate radiological images ([paper](https://arxiv.org/abs/2209.13696))."
+    tags = ["text-to-image", "knowledge"]
+
+    def get_instances(self, _) -> List[Instance]:
+        prompts: List[str] = []
+
+        for anatomical_area in self.X_RAY_ANATOMICAL_AREA:
+            prompts.append(f"An x-ray of {anatomical_area}")
+
+        for organ in self.ORGANS:
+            prompts.append(f"An MRI of the {organ}")
+            prompts.append(f"A CT of the {organ}")
+            prompts.append(f"Ultrasound of the {organ}")
+
+        assert len(prompts) == 17, "Invalid number of prompts"
+        # There are no reference images
+        return [Instance(Input(text=prompt), references=[], split=TEST_SPLIT) for prompt in prompts]

helm/benchmark/scenarios/image_generation/relational_understanding_scenario.py

@@ -0,0 +1,52 @@
+from typing import List, Set
+import csv
+import os
+
+from helm.common.general import ensure_file_downloaded
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT
+
+
+class RelationalUnderstandingScenario(Scenario):
+    """
+    From Testing Relational Understanding in Text-Guided Image Generation, based on existing cognitive,
+    linguistic, and developmental literature, the authors created a set of 15 relations (8 physical,
+    7 agentic) and a set of 12 entities (6 objects, 6 agents). The physical relations were: in, on,
+    under, covering, near, occluded by, hanging over, and tied to. The agentic relations were: pushing,
+    pulling, touching, hitting, kicking, helping, and hindering. The objects were: box, cylinder,
+    blanket, bowl, teacup, and knife. The agents were: man, woman, child, robot, monkey, and iguana.
+
+    The authors created 5 different prompts for each relation, by randomly sampling two entities five
+    times, resulting in 75 distinct basic relation prompts (e.g., a monkey touching an iguana). Withs
+    these prompts, the authors showed that DALL-E 2 suffers from a significant lack of commonsense
+    reasoning in the form of relational understanding.
+
+    Paper: https://arxiv.org/abs/2208.00005
+    Website: https://osf.io/sm68h
+    """
+
+    name = "relational_understanding"
+    description = (
+        "Consists of 75 basic relation prompts that tests commonsense reasoning "
+        "([paper](https://arxiv.org/abs/2208.00005))."
+    )
+    tags = ["text-to-image", "reasoning"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path: str = os.path.join(output_path, "choice_data.csv")
+        ensure_file_downloaded(source_url="https://osf.io/download/tb3a4", target_path=data_path)
+
+        instances: List[Instance] = []
+        seen_prompts: Set[str] = set()
+        with open(data_path) as csv_file:
+            csv_reader = csv.reader(csv_file, delimiter=",")
+            for i, row in enumerate(csv_reader):
+                if i == 0:
+                    # Skip the header
+                    continue
+
+                prompt: str = row[1]
+                if prompt not in seen_prompts:
+                    instances.append(Instance(Input(text=prompt), references=[], split=TEST_SPLIT))
+                    seen_prompts.add(prompt)
+
+        return instances

helm/benchmark/scenarios/image_generation/time_most_significant_historical_figures_scenario.py

@@ -0,0 +1,124 @@
+from typing import List
+
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT
+
+
+class TIMEMostSignificantHistoricalFigures(Scenario):
+    """
+    People from TIME's "The 100 Most Significant Figures in History" list.
+
+    https://ideas.time.com/2013/12/10/whos-biggest-the-100-most-significant-figures-in-history/
+    """
+
+    HISTORICAL_FIGURES: List[str] = [
+        "Jesus",
+        "Napoleon Bonaparte",
+        "Muhammad",
+        "William Shakespeare",
+        "Abraham Lincoln",
+        "George Washington",
+        "Adolf Hitler",
+        "Aristotle",
+        "Alexander the Great",
+        "Thomas Jefferson",
+        "Henry VIII of England",
+        "Charles Darwin",
+        "Elizabeth I of England",
+        "Karl Marx",
+        "Julius Caesar",
+        "Queen Victoria",
+        "Martin Luther",
+        "Joseph Stalin",
+        "Albert Einstein",
+        "Christopher Columbus",
+        "Isaac Newton",
+        "Charlemagne",
+        "Theodore Roosevelt",
+        "Wolfgang Amadeus Mozart",
+        "Plato",
+        "Louis XIV of France",
+        "Ludwig van Beethoven",
+        "Ulysses S.Grant",
+        "Leonardo da Vinci",
+        "Augustus",
+        "Carl Linnaeus",
+        "Ronald Reagan",
+        "Charles Dickens",
+        "Paul the Apostle",
+        "Benjamin Franklin",
+        # "George W.Bush",
+        "Winston Churchill",
+        "Genghis Khan",
+        "Charles I of England",
+        "Thomas Edison",
+        "James I of England",
+        "Friedrich Nietzsche",
+        "Franklin D.Roosevelt",
+        "Sigmund Freud",
+        "Alexander Hamilton",
+        "Mohandas Karamchand Gandhi",
+        "Woodrow Wilson",
+        "Johann Sebastian Bach",
+        "Galileo Galilei",
+        "Oliver Cromwell",
+        "James Madison",
+        "Gautama Buddha",
+        "Mark Twain",
+        "Edgar Allan Poe",
+        "Joseph Smith, Jr.",
+        "Adam Smith",
+        "David, King of Israel",
+        "George III of the United Kingdom",
+        "Immanuel Kant",
+        "James Cook",
+        "John Adams",
+        "Richard Wagner",
+        "Pyotr Ilyich Tchaikovsky",
+        "Voltaire",
+        "Saint Peter",
+        "Andrew Jackson",
+        "Constantine the Great",
+        "Socrates",
+        "Elvis Presley",
+        "William the Conqueror",
+        "John F.Kennedy",
+        "Augustine of Hippo",
+        "Vincent van Gogh",
+        "Nicolaus Copernicus",
+        "Vladimir Lenin",
+        "Robert E.Lee",
+        "Oscar Wilde",
+        "Charles II of England",
+        "Cicero",
+        "Jean-Jacques Rousseau",
+        "Francis Bacon",
+        "Richard Nixon",
+        "Louis XVI of France",
+        "Charles V, Holy Roman Emperor",
+        "King Arthur",
+        "Michelangelo",
+        "Philip II of Spain",
+        "Johann Wolfgang von Goethe",
+        "Ali, founder of Sufism",
+        "Thomas Aquinas",
+        "Pope John Paul II",
+        "René Descartes",
+        "Nikola Tesla",
+        "Harry S.Truman",
+        "Joan of Arc",
+        "Dante Alighieri",
+        "Otto von Bismarck",
+        "Grover Cleveland",
+        "John Calvin",
+        "John Locke",
+    ]
+
+    name = "time_most_significant_historical_figures"
+    description = 'People from TIME\'s "The 100 Most Significant Figures in History" list.'
+    tags = ["text-to-image", "knowledge"]
+
+    def get_instances(self, _) -> List[Instance]:
+        return [
+            Instance(Input(text=historical_figure), references=[], split=TEST_SPLIT)
+            for historical_figure in self.HISTORICAL_FIGURES
+        ]

helm/benchmark/scenarios/image_generation/winoground_scenario.py

@@ -0,0 +1,62 @@
+from typing import List
+import os
+
+from datasets import load_dataset
+
+from helm.common.general import get_file_name
+from helm.common.images_utils import copy_image
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, Output, Reference, CORRECT_TAG, TEST_SPLIT
+
+
+class WinogroundScenario(Scenario):
+    """
+    Winoground is a novel task and dataset for evaluating the ability of vision and language models
+    to conduct visio-linguistic compositional reasoning. Given two images and two captions, the
+    goal is to match them correctly—but crucially, both captions contain a completely identical set
+    of words/morphemes, only in a different order. The dataset was carefully hand-curated by
+    expert annotators and is labeled with a rich set of fine-grained tags to assist in analyzing
+    model performance.
+
+    Users must agree to share their contact information before downloading the dataset from
+    Hugging Face. Either agree to the terms and set HUGGING_FACE_ACCESS_TOKEN to an access token
+    of a valid Hugging Face account or have the dataset pre-downloaded at the Hugging Face cache
+    (default path: ~/.cache/huggingface/datasets).
+
+    Paper: https://arxiv.org/abs/2204.03162
+    Website: https://huggingface.co/datasets/facebook/winoground
+    """
+
+    name = "winoground"
+    description = (
+        "Winoground is a novel task and dataset for evaluating the ability of vision and language models "
+        "to conduct visio-linguistic compositional reasoning "
+        "([paper](https://arxiv.org/abs/2204.03162))."
+    )
+    tags = ["text-to-image", "image-to-text", "visual_reasoning"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        auth_token: str = os.environ.get("HUGGING_FACE_ACCESS_TOKEN", "")
+
+        instances: List[Instance] = []
+        for row in load_dataset("facebook/winoground", split="test", use_auth_token=auth_token):
+            # Use the first example of the pair for now (index 0)
+            caption: str = row["caption_0"]
+            image_path: str = row["image_0"].filename
+
+            # Create a copy of the image in the benchmark output folder for metrics computation
+            image_copy_path: str = os.path.join(output_path, get_file_name(image_path))
+            if not os.path.exists(image_copy_path):
+                copy_image(image_path, image_copy_path)
+            content: MultimediaObject = MultimediaObject(
+                [MediaObject(content_type="image/png", location=image_copy_path)]
+            )
+
+            instances.append(
+                Instance(
+                    input=Input(text=caption),
+                    references=[Reference(Output(multimedia_content=content), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+        return instances

helm/benchmark/scenarios/imdb_scenario.py

@@ -107,7 +107,6 @@ class IMDBScenario(Scenario):
         contrast_map = {}
 
         for orig_line, contrast_line in zip(orig_and_contrast_inputs[0], orig_and_contrast_inputs[1]):
-
             orig_label_name, orig_context = orig_line.strip().split("\t")
             orig_label = label_name_to_id[orig_label_name]
 

helm/benchmark/scenarios/legalbench_scenario.py

@@ -96,8 +96,12 @@ class LegalBenchScenario(Scenario):
 
         # Download data from Huggingface. LegalBench provides splits for samples to
         # be used for prompt construction and for testing.
-        train_dataset = datasets.load_dataset("nguha/legalbench", self.subset, cache_dir=cache_dir, split="train")
-        test_dataset = datasets.load_dataset("nguha/legalbench", self.subset, cache_dir=cache_dir, split="test")
+        train_dataset = datasets.load_dataset(
+            "nguha/legalbench", self.subset, trust_remote_code=True, cache_dir=cache_dir, split="train"
+        )
+        test_dataset = datasets.load_dataset(
+            "nguha/legalbench", self.subset, trust_remote_code=True, cache_dir=cache_dir, split="test"
+        )
         assert isinstance(train_dataset, datasets.Dataset)
         assert isinstance(test_dataset, datasets.Dataset)
 

helm/benchmark/scenarios/live_qa_scenario.py

@@ -0,0 +1,94 @@
+import os
+from typing import List
+from xml.etree.ElementTree import Element
+import xml.etree.ElementTree as ET
+
+from helm.common.general import ensure_file_downloaded
+from .scenario import CORRECT_TAG, TEST_SPLIT, Input, Instance, Output, Reference, Scenario
+
+
+class LiveQAScenario(Scenario):
+    """
+    TREC-2017 LiveQA: Medical Question Answering Task
+
+    The LiveQA'17 medical task focuses on consumer health question answering.
+    Please refer to the original paper for more information about the constructed datasets and the LiveQA Track:
+    https://trec.nist.gov/pubs/trec26/papers/Overview-QA.pdf
+
+    Paper citation:
+
+        @inproceedings{LiveMedQA2017,
+          author    = {Asma {Ben Abacha} and Eugene Agichtein and Yuval Pinter and Dina Demner{-}Fushman},
+          title     = {Overview of the Medical Question Answering Task at TREC 2017 LiveQA},
+          booktitle = {TREC 2017},
+          year      = {2017}
+        }
+    """
+
+    SOURCE_REPO_URL = "https://raw.githubusercontent.com/abachaa/LiveQA_MedicalTask_TREC2017/master/TestDataset/"
+    FILENAME = "TREC-2017-LiveQA-Medical-Test-Questions-w-summaries.xml"
+
+    name = "live_qa"
+    description = "TREC-2017 LiveQA: Medical Question Answering Task"
+    tags = ["knowledge", "generation", "question_answering", "biomedical"]
+
+    def download_liveqa(self, path: str):
+        """Download the XML file containing the questions & reference answers"""
+        ensure_file_downloaded(
+            source_url=os.path.join(self.SOURCE_REPO_URL, self.FILENAME),
+            target_path=os.path.join(path, self.FILENAME),
+            unpack=False,
+        )
+
+    @staticmethod
+    def remove_whitespace(s: str) -> str:
+        """Just remove all whitespace from a string"""
+        return " ".join(s.strip().split())
+
+    @staticmethod
+    def _extract_question_id(element: Element):
+        return element.attrib["qid"]
+
+    @classmethod
+    def _extract_question(cls, element: Element) -> str:
+        """Given an XML Element representing a question, extract just the question as text"""
+        return cls.remove_whitespace(element.find("NLM-Summary").text)  # type: ignore
+
+    @classmethod
+    def _extract_answers(cls, element: Element) -> List[str]:
+        """Given an XML Element representing a question, extract the reference answers"""
+        answers = []
+        for answer in element.iter("ANSWER"):
+            answers.append(cls.remove_whitespace(answer.text))  # type: ignore
+
+        return answers
+
+    def process_xml(self, base_path: str) -> List[Instance]:
+        """Parse the XMLs into question-answer(s) pairs"""
+        xml_path = os.path.join(base_path, self.FILENAME)
+        tree = ET.parse(xml_path)
+        root = tree.getroot()
+
+        instances = []
+        for question_root in root:
+            # get the actual question and question ID
+            id = self._extract_question_id(question_root)
+            question = Input(self._extract_question(question_root))
+
+            # parse out the reference answers
+            answers = self._extract_answers(question_root)
+            references = [Reference(Output(answer), tags=[CORRECT_TAG]) for answer in answers]
+
+            # stitch it all together
+            instances.append(Instance(question, references, split=TEST_SPLIT, id=id))
+
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """entrypoint to creating this scenario's instances"""
+        # get the dataset
+        self.download_liveqa(output_path)
+
+        # get the instances by parsing the XML
+        instances = self.process_xml(output_path)
+        return instances