crfm-helm 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

helm/benchmark/scenarios/image_generation/paint_skills_scenario.py

@@ -0,0 +1,72 @@
+import json
+import os
+from typing import Dict, List, Set
+
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_file_downloaded
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, Output, Reference, CORRECT_TAG, VALID_SPLIT
+
+
+class PaintSkillsScenario(Scenario):
+    """
+    PaintSkills is a compositional diagnostic dataset an evaluation toolkit that measures three
+    fundamental visual reasoning capabilities:
+
+    - object recognition => object
+    - object counting => count
+    - spatial relation understanding => spatial
+
+    Paper: https://arxiv.org/abs/2202.04053
+    Website: https://github.com/j-min/DallEval/tree/main/paintskills
+    """
+
+    METADATA_DOWNLOAD_URL: str = "https://drive.google.com/uc?export=download&id=12jsHDzEcBr-Et3FhLq-HckI5cmLB_rxC"
+    SKILL_TO_DOWNLOAD_URL: Dict[str, str] = {
+        "object": "https://drive.google.com/uc?export=download&id=1lpvSpBNfEg5EJt16prumXiuEO99byjzw&confirm=t",
+        "count": "https://drive.google.com/uc?export=download&id=1koA-5xiZbAUDh65jpYaylG3IOA-mZTH2&confirm=t",
+        "spatial": "https://drive.google.com/uc?export=download&id=1g-L0dVQjBTWp1uRwJLYXIj2xYIlQ2knu&confirm=t",
+    }
+
+    name = "paint_skills"
+    description = (
+        "A compositional diagnostic dataset an evaluation toolkit that measures visual reasoning skills "
+        "([paper](https://arxiv.org/abs/2202.04053))."
+    )
+    tags = ["text-to-image", "image-to-text"]
+
+    def __init__(self, skill: str):
+        super().__init__()
+        assert skill in self.SKILL_TO_DOWNLOAD_URL, f"Invalid skill: {skill}"
+        self.skill: str = skill
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        skills_data_path: str = os.path.join(output_path, self.skill)
+        ensure_file_downloaded(
+            source_url=self.SKILL_TO_DOWNLOAD_URL[self.skill],
+            target_path=skills_data_path,
+            unpack=True,
+            unpack_type="unzip",
+        )
+
+        images_path: str = os.path.join(skills_data_path, "images")
+        with open(os.path.join(skills_data_path, "scenes", f"{self.skill}_val.json"), "r") as f:
+            examples: Dict = json.load(f)
+
+        instances: List[Instance] = []
+        seen_captions: Set[str] = set()
+        for example in examples["data"]:
+            caption: str = example["text"]
+            if caption in seen_captions:
+                continue
+
+            seen_captions.add(caption)
+            image_path: str = os.path.join(images_path, f"image_{example['id']}.png")
+            content: MultimediaObject = MultimediaObject([MediaObject(content_type="image/png", location=image_path)])
+            instance = Instance(
+                Input(text=caption),
+                references=[Reference(Output(multimedia_content=content), tags=[CORRECT_TAG])],
+                split=VALID_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/image_generation/parti_prompts_scenario.py

@@ -0,0 +1,94 @@
+import csv
+import os
+from typing import List
+
+from helm.common.general import ensure_file_downloaded
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT
+
+
+class PartiPromptsScenario(Scenario):
+    """
+    PartiPrompts (P2) is a set of 1600 diverse English prompts that allow us to more comprehensively
+    evaluate and test the limits of text-to-image synthesis models.
+
+    Each prompt in the P2 benchmark is associated with two labels:
+    1. Category: indicating a broad group that a prompt belongs to
+    2. Challenge: highlighting an aspect which makes a prompt difficult
+
+    Categories:
+    - Abstract: Descriptions that represent abstract concepts, including single words and simple numbers.
+    - World Knowledge: Descriptions focused on objects and places that exist in the real world.
+    - People: Descriptions where the primary participants are human beings (but not specific individuals,
+              living or dead).
+    - Animals: Descriptions in which the primary participants are animals.
+    - Illustrations: Descriptions of images that involve specific types of graphical representations,
+                     including geometrical objects, diagrams, and symbols.
+    - Artifacts: Descriptions that represent abstract concepts, including single words and simple numbers.
+    - Food & Beverage: Descriptions of things animals, especially human beings, eat or drink.
+    - Vehicles: Descriptions where the focus is on man-made devices for transportation.
+    - Arts: Descriptions of existing paintings or intended to produce novel images in the format of a painting.
+    - Indoor Scenes: Descriptions about objects and participants that occur indoors.
+    - Outdoor Scenes: Descriptions about objects and participants that occur outdoors.
+    - Produce & Plants: Descriptions focused on plants or their products (fruits, vegetables, seeds, etc).
+
+    Challenges:
+    - Simple Detail: Descriptions that include only simple or high-level details.
+    - Fine-grained Detail: Descriptions that include very detailed specifications of attributes or
+                           actions of entities or objects in a scene.
+    - Complex: Descriptions that include many fine-grained, interacting details or relationships between multiple
+               participants.
+    - Quantity: Descriptions that specify particular counts of occurrences of subjects in a scene.
+    - Style & Format: Descriptions that specifically focus on the visual manner in which a subject or scene
+                      must be depicted.
+    - Properties & Positioning: Descriptions that target precise assignment of properties to entities or
+                                objects (often in the context of multiple entities or objects), and/or the
+                                relative spatial arrangement of entities and objects with respect to one
+                                another or landmarks in the scene.
+    - Linguistic Structures: Long and/or abstract words or complex syntactic structures or semantic
+                             ambiguities.
+    - Writing & Symbols: Descriptions that require words or symbols to be accurately represented
+                         in the context of the visual scene.
+    - Imagination: Descriptions that include participants or interactions that are not, or are generally unlikely
+                   to be, found in the modern day world.
+    - Basic: Descriptions about a single subject or concept with little to no detail or embellishment.
+    - Perspective: Descriptions that specify particular viewpoints or positioning of the subjects in a scene.
+
+    Paper: https://arxiv.org/abs/2206.10789
+    Website: https://parti.research.google/
+    """
+
+    DATASET_DOWNLOAD_URL: str = "https://raw.githubusercontent.com/google-research/parti/main/PartiPrompts.tsv"
+    ALL_CATEGORY: str = "all"
+
+    name = "parti_prompts"
+    description = (
+        "PartiPrompts (P2) is a set of 1600 diverse English prompts that allow to more comprehensively "
+        "evaluate and test the limits of text-to-image synthesis models ([paper](https://arxiv.org/abs/2206.10789))."
+    )
+    tags = ["text-to-image"]
+
+    def __init__(self, category: str):
+        super().__init__()
+        self.category: str = category
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        prompts_path: str = os.path.join(output_path, "prompts.tsv")
+        ensure_file_downloaded(source_url=self.DATASET_DOWNLOAD_URL, target_path=prompts_path)
+
+        instances: List[Instance] = []
+        with open(prompts_path) as f:
+            tsv_reader = csv.reader(f, delimiter="\t")
+            for i, row in enumerate(tsv_reader):
+                if i == 0:
+                    # Skip the header
+                    continue
+
+                prompt: str = row[0]
+                category: str = row[1]
+
+                # P2 does not have reference images
+                instance = Instance(Input(text=prompt), references=[], split=TEST_SPLIT)
+                if category.startswith(self.category) or self.category == self.ALL_CATEGORY:
+                    instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/image_generation/radiology_scenario.py

@@ -0,0 +1,42 @@
+from typing import List
+
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT
+
+
+class RadiologyScenario(Scenario):
+    """
+    From "What Does DALL-E 2 Know About Radiology?", DALL-E 2 fails to generate realistic
+    X-ray, CT, MRI and ultrasound images. For example, generated images of X-rays have
+    smaller bones missing, extra joints added, etc.
+
+    Prompts are in the following format:
+
+    “An x-ray of {head, chest, shoulder, abdomen, pelvis, hand, knee, ankle}” => 8 prompts
+    “An MRI of the {heart, liver, kidney}” => 3 prompts
+    “A CT of the {heart, liver, kidney}” => 3 prompts
+    “Ultrasound of the {heart, liver, kidney}” => 3 prompts
+
+    Paper: https://arxiv.org/abs/2209.13696
+    """
+
+    X_RAY_ANATOMICAL_AREA: List[str] = ["head", "chest", "shoulder", "abdomen", "pelvis", "hand", "knee", "ankle"]
+    ORGANS: List[str] = ["heart", "liver", "kidney"]
+
+    name = "radiology"
+    description = "Prompts to generate radiological images ([paper](https://arxiv.org/abs/2209.13696))."
+    tags = ["text-to-image", "knowledge"]
+
+    def get_instances(self, _) -> List[Instance]:
+        prompts: List[str] = []
+
+        for anatomical_area in self.X_RAY_ANATOMICAL_AREA:
+            prompts.append(f"An x-ray of {anatomical_area}")
+
+        for organ in self.ORGANS:
+            prompts.append(f"An MRI of the {organ}")
+            prompts.append(f"A CT of the {organ}")
+            prompts.append(f"Ultrasound of the {organ}")
+
+        assert len(prompts) == 17, "Invalid number of prompts"
+        # There are no reference images
+        return [Instance(Input(text=prompt), references=[], split=TEST_SPLIT) for prompt in prompts]

helm/benchmark/scenarios/image_generation/relational_understanding_scenario.py

@@ -0,0 +1,52 @@
+from typing import List, Set
+import csv
+import os
+
+from helm.common.general import ensure_file_downloaded
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT
+
+
+class RelationalUnderstandingScenario(Scenario):
+    """
+    From Testing Relational Understanding in Text-Guided Image Generation, based on existing cognitive,
+    linguistic, and developmental literature, the authors created a set of 15 relations (8 physical,
+    7 agentic) and a set of 12 entities (6 objects, 6 agents). The physical relations were: in, on,
+    under, covering, near, occluded by, hanging over, and tied to. The agentic relations were: pushing,
+    pulling, touching, hitting, kicking, helping, and hindering. The objects were: box, cylinder,
+    blanket, bowl, teacup, and knife. The agents were: man, woman, child, robot, monkey, and iguana.
+
+    The authors created 5 different prompts for each relation, by randomly sampling two entities five
+    times, resulting in 75 distinct basic relation prompts (e.g., a monkey touching an iguana). Withs
+    these prompts, the authors showed that DALL-E 2 suffers from a significant lack of commonsense
+    reasoning in the form of relational understanding.
+
+    Paper: https://arxiv.org/abs/2208.00005
+    Website: https://osf.io/sm68h
+    """
+
+    name = "relational_understanding"
+    description = (
+        "Consists of 75 basic relation prompts that tests commonsense reasoning "
+        "([paper](https://arxiv.org/abs/2208.00005))."
+    )
+    tags = ["text-to-image", "reasoning"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path: str = os.path.join(output_path, "choice_data.csv")
+        ensure_file_downloaded(source_url="https://osf.io/download/tb3a4", target_path=data_path)
+
+        instances: List[Instance] = []
+        seen_prompts: Set[str] = set()
+        with open(data_path) as csv_file:
+            csv_reader = csv.reader(csv_file, delimiter=",")
+            for i, row in enumerate(csv_reader):
+                if i == 0:
+                    # Skip the header
+                    continue
+
+                prompt: str = row[1]
+                if prompt not in seen_prompts:
+                    instances.append(Instance(Input(text=prompt), references=[], split=TEST_SPLIT))
+                    seen_prompts.add(prompt)
+
+        return instances

helm/benchmark/scenarios/image_generation/time_most_significant_historical_figures_scenario.py

@@ -0,0 +1,124 @@
+from typing import List
+
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT
+
+
+class TIMEMostSignificantHistoricalFigures(Scenario):
+    """
+    People from TIME's "The 100 Most Significant Figures in History" list.
+
+    https://ideas.time.com/2013/12/10/whos-biggest-the-100-most-significant-figures-in-history/
+    """
+
+    HISTORICAL_FIGURES: List[str] = [
+        "Jesus",
+        "Napoleon Bonaparte",
+        "Muhammad",
+        "William Shakespeare",
+        "Abraham Lincoln",
+        "George Washington",
+        "Adolf Hitler",
+        "Aristotle",
+        "Alexander the Great",
+        "Thomas Jefferson",
+        "Henry VIII of England",
+        "Charles Darwin",
+        "Elizabeth I of England",
+        "Karl Marx",
+        "Julius Caesar",
+        "Queen Victoria",
+        "Martin Luther",
+        "Joseph Stalin",
+        "Albert Einstein",
+        "Christopher Columbus",
+        "Isaac Newton",
+        "Charlemagne",
+        "Theodore Roosevelt",
+        "Wolfgang Amadeus Mozart",
+        "Plato",
+        "Louis XIV of France",
+        "Ludwig van Beethoven",
+        "Ulysses S.Grant",
+        "Leonardo da Vinci",
+        "Augustus",
+        "Carl Linnaeus",
+        "Ronald Reagan",
+        "Charles Dickens",
+        "Paul the Apostle",
+        "Benjamin Franklin",
+        # "George W.Bush",
+        "Winston Churchill",
+        "Genghis Khan",
+        "Charles I of England",
+        "Thomas Edison",
+        "James I of England",
+        "Friedrich Nietzsche",
+        "Franklin D.Roosevelt",
+        "Sigmund Freud",
+        "Alexander Hamilton",
+        "Mohandas Karamchand Gandhi",
+        "Woodrow Wilson",
+        "Johann Sebastian Bach",
+        "Galileo Galilei",
+        "Oliver Cromwell",
+        "James Madison",
+        "Gautama Buddha",
+        "Mark Twain",
+        "Edgar Allan Poe",
+        "Joseph Smith, Jr.",
+        "Adam Smith",
+        "David, King of Israel",
+        "George III of the United Kingdom",
+        "Immanuel Kant",
+        "James Cook",
+        "John Adams",
+        "Richard Wagner",
+        "Pyotr Ilyich Tchaikovsky",
+        "Voltaire",
+        "Saint Peter",
+        "Andrew Jackson",
+        "Constantine the Great",
+        "Socrates",
+        "Elvis Presley",
+        "William the Conqueror",
+        "John F.Kennedy",
+        "Augustine of Hippo",
+        "Vincent van Gogh",
+        "Nicolaus Copernicus",
+        "Vladimir Lenin",
+        "Robert E.Lee",
+        "Oscar Wilde",
+        "Charles II of England",
+        "Cicero",
+        "Jean-Jacques Rousseau",
+        "Francis Bacon",
+        "Richard Nixon",
+        "Louis XVI of France",
+        "Charles V, Holy Roman Emperor",
+        "King Arthur",
+        "Michelangelo",
+        "Philip II of Spain",
+        "Johann Wolfgang von Goethe",
+        "Ali, founder of Sufism",
+        "Thomas Aquinas",
+        "Pope John Paul II",
+        "René Descartes",
+        "Nikola Tesla",
+        "Harry S.Truman",
+        "Joan of Arc",
+        "Dante Alighieri",
+        "Otto von Bismarck",
+        "Grover Cleveland",
+        "John Calvin",
+        "John Locke",
+    ]
+
+    name = "time_most_significant_historical_figures"
+    description = 'People from TIME\'s "The 100 Most Significant Figures in History" list.'
+    tags = ["text-to-image", "knowledge"]
+
+    def get_instances(self, _) -> List[Instance]:
+        return [
+            Instance(Input(text=historical_figure), references=[], split=TEST_SPLIT)
+            for historical_figure in self.HISTORICAL_FIGURES
+        ]

helm/benchmark/scenarios/image_generation/winoground_scenario.py

@@ -0,0 +1,62 @@
+from typing import List
+import os
+
+from datasets import load_dataset
+
+from helm.common.general import get_file_name
+from helm.common.images_utils import copy_image
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, Output, Reference, CORRECT_TAG, TEST_SPLIT
+
+
+class WinogroundScenario(Scenario):
+    """
+    Winoground is a novel task and dataset for evaluating the ability of vision and language models
+    to conduct visio-linguistic compositional reasoning. Given two images and two captions, the
+    goal is to match them correctly—but crucially, both captions contain a completely identical set
+    of words/morphemes, only in a different order. The dataset was carefully hand-curated by
+    expert annotators and is labeled with a rich set of fine-grained tags to assist in analyzing
+    model performance.
+
+    Users must agree to share their contact information before downloading the dataset from
+    Hugging Face. Either agree to the terms and set HUGGING_FACE_ACCESS_TOKEN to an access token
+    of a valid Hugging Face account or have the dataset pre-downloaded at the Hugging Face cache
+    (default path: ~/.cache/huggingface/datasets).
+
+    Paper: https://arxiv.org/abs/2204.03162
+    Website: https://huggingface.co/datasets/facebook/winoground
+    """
+
+    name = "winoground"
+    description = (
+        "Winoground is a novel task and dataset for evaluating the ability of vision and language models "
+        "to conduct visio-linguistic compositional reasoning "
+        "([paper](https://arxiv.org/abs/2204.03162))."
+    )
+    tags = ["text-to-image", "image-to-text", "visual_reasoning"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        auth_token: str = os.environ.get("HUGGING_FACE_ACCESS_TOKEN", "")
+
+        instances: List[Instance] = []
+        for row in load_dataset("facebook/winoground", split="test", use_auth_token=auth_token):
+            # Use the first example of the pair for now (index 0)
+            caption: str = row["caption_0"]
+            image_path: str = row["image_0"].filename
+
+            # Create a copy of the image in the benchmark output folder for metrics computation
+            image_copy_path: str = os.path.join(output_path, get_file_name(image_path))
+            if not os.path.exists(image_copy_path):
+                copy_image(image_path, image_copy_path)
+            content: MultimediaObject = MultimediaObject(
+                [MediaObject(content_type="image/png", location=image_copy_path)]
+            )
+
+            instances.append(
+                Instance(
+                    input=Input(text=caption),
+                    references=[Reference(Output(multimedia_content=content), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+        return instances

helm/benchmark/scenarios/imdb_scenario.py

@@ -107,7 +107,6 @@ class IMDBScenario(Scenario):
         contrast_map = {}
 
         for orig_line, contrast_line in zip(orig_and_contrast_inputs[0], orig_and_contrast_inputs[1]):
-
             orig_label_name, orig_context = orig_line.strip().split("\t")
             orig_label = label_name_to_id[orig_label_name]
 

helm/benchmark/scenarios/legalbench_scenario.py

@@ -0,0 +1,123 @@
+import random
+import os
+import json
+import datasets
+from pathlib import Path
+from typing import List, Dict
+
+from helm.common.general import ensure_file_downloaded, ensure_directory_exists
+from .scenario import Scenario, Instance, Reference, CORRECT_TAG, TRAIN_SPLIT, TEST_SPLIT, Input, Output
+
+PROMPT_SETTINGS_URL = "https://raw.githubusercontent.com/HazyResearch/legalbench/main/helm_prompt_settings.jsonl"
+
+SUBSETS = [
+    "abercrombie",
+    "corporate_lobbying",
+    "international_citizenship_questions",
+    "function_of_decision_section",
+    "proa",
+]
+
+
+def get_legalbench_prompt_settings(subset: str, cache_dir: str):
+    """
+    Loads prompt construction settings for all subsets.
+    """
+    assert subset in SUBSETS, "Unknown subset: {}".format(subset)
+
+    prompt_construction_settings_path = os.path.join(cache_dir, "prompt_construction_settings.json")
+    ensure_directory_exists(cache_dir)
+    ensure_file_downloaded(
+        source_url=PROMPT_SETTINGS_URL,
+        target_path=prompt_construction_settings_path,
+    )
+    with open(prompt_construction_settings_path, "r") as f:
+        field_ordering, instructions, label_keys, output_nouns, _ = map(json.loads, f.read().strip().split("\n"))
+    return (
+        field_ordering[subset],
+        instructions[subset],
+        label_keys[subset],
+        output_nouns[subset],
+    )
+
+
+def get_legalbench_instructions(subset: str, cache_dir: str):
+    return get_legalbench_prompt_settings(subset, cache_dir)[1]
+
+
+def get_legalbench_output_nouns(subset: str, cache_dir: str):
+    return get_legalbench_prompt_settings(subset, cache_dir)[3]
+
+
+class LegalBenchScenario(Scenario):
+    """
+    LegalBench is benchmark containing different legal reasoning tasks. We use a subset of the tasks, selected
+    to represent different legal reasoning patterns.
+
+    LegalBench: A Collaboratively Built Benchmark for Measuring Legal Reasoning in Large Language Models
+    https://arxiv.org/abs/2308.11462
+
+    Official website for LegalBench:
+    http://hazyresearch.stanford.edu/legalbench/
+
+    Dataset summary:
+    https://huggingface.co/datasets/nguha/legalbench
+
+    Prompts are adapted from:
+    https://github.com/HazyResearch/legalbench/
+
+    Subsets:
+
+    - abercrombie
+    - corporate_lobbying
+    - international_citizenship_questions
+    - function_of_decision_section
+    - proa
+    """
+
+    name = "legalbench"
+    description = "LegalBench"
+    tags = ["text_classification", "robustness"]
+
+    def __init__(self, subset: str, random_seed=42):
+        super().__init__()
+        assert subset in SUBSETS, "Unknown subset: {}".format(subset)
+        self.subset = subset
+        self.random_seed = random_seed
+
+    def load_prompt_construction_settings(self, output_path: str):
+        # Load from prompt construction settings
+        cache_dir = str(Path(output_path) / "data")
+        return get_legalbench_prompt_settings(self.subset, cache_dir)
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        fields, _, label_key, _ = self.load_prompt_construction_settings(output_path)
+        cache_dir = str(Path(output_path) / "data")
+
+        # Download data from Huggingface. LegalBench provides splits for samples to
+        # be used for prompt construction and for testing.
+        train_dataset = datasets.load_dataset("nguha/legalbench", self.subset, cache_dir=cache_dir, split="train")
+        test_dataset = datasets.load_dataset("nguha/legalbench", self.subset, cache_dir=cache_dir, split="test")
+        assert isinstance(train_dataset, datasets.Dataset)
+        assert isinstance(test_dataset, datasets.Dataset)
+
+        dataset_splits: Dict[str, datasets.Dataset] = {
+            TRAIN_SPLIT: train_dataset,
+            TEST_SPLIT: test_dataset,
+        }
+
+        # Read all instances
+        random.seed(self.random_seed)
+        instances: List[Instance] = []
+        for split, subset in dataset_splits.items():
+            for x in subset:
+                assert fields is not None, "Field ordering not loaded"
+                prompt: str = "\n".join([f"{field[0]}: {x[field[1]]}" for field in fields])
+                instance = Instance(
+                    input=Input(text=prompt),
+                    references=[Reference(Output(text=x[label_key]), tags=[CORRECT_TAG])],
+                    split=split,
+                )
+                instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/live_qa_scenario.py

@@ -0,0 +1,94 @@
+import os
+from typing import List
+from xml.etree.ElementTree import Element
+import xml.etree.ElementTree as ET
+
+from helm.common.general import ensure_file_downloaded
+from .scenario import CORRECT_TAG, TEST_SPLIT, Input, Instance, Output, Reference, Scenario
+
+
+class LiveQAScenario(Scenario):
+    """
+    TREC-2017 LiveQA: Medical Question Answering Task
+
+    The LiveQA'17 medical task focuses on consumer health question answering.
+    Please refer to the original paper for more information about the constructed datasets and the LiveQA Track:
+    https://trec.nist.gov/pubs/trec26/papers/Overview-QA.pdf
+
+    Paper citation:
+
+        @inproceedings{LiveMedQA2017,
+          author    = {Asma {Ben Abacha} and Eugene Agichtein and Yuval Pinter and Dina Demner{-}Fushman},
+          title     = {Overview of the Medical Question Answering Task at TREC 2017 LiveQA},
+          booktitle = {TREC 2017},
+          year      = {2017}
+        }
+    """
+
+    SOURCE_REPO_URL = "https://raw.githubusercontent.com/abachaa/LiveQA_MedicalTask_TREC2017/master/TestDataset/"
+    FILENAME = "TREC-2017-LiveQA-Medical-Test-Questions-w-summaries.xml"
+
+    name = "live_qa"
+    description = "TREC-2017 LiveQA: Medical Question Answering Task"
+    tags = ["knowledge", "generation", "question_answering", "biomedical"]
+
+    def download_liveqa(self, path: str):
+        """Download the XML file containing the questions & reference answers"""
+        ensure_file_downloaded(
+            source_url=os.path.join(self.SOURCE_REPO_URL, self.FILENAME),
+            target_path=os.path.join(path, self.FILENAME),
+            unpack=False,
+        )
+
+    @staticmethod
+    def remove_whitespace(s: str) -> str:
+        """Just remove all whitespace from a string"""
+        return " ".join(s.strip().split())
+
+    @staticmethod
+    def _extract_question_id(element: Element):
+        return element.attrib["qid"]
+
+    @classmethod
+    def _extract_question(cls, element: Element) -> str:
+        """Given an XML Element representing a question, extract just the question as text"""
+        return cls.remove_whitespace(element.find("NLM-Summary").text)  # type: ignore
+
+    @classmethod
+    def _extract_answers(cls, element: Element) -> List[str]:
+        """Given an XML Element representing a question, extract the reference answers"""
+        answers = []
+        for answer in element.iter("ANSWER"):
+            answers.append(cls.remove_whitespace(answer.text))  # type: ignore
+
+        return answers
+
+    def process_xml(self, base_path: str) -> List[Instance]:
+        """Parse the XMLs into question-answer(s) pairs"""
+        xml_path = os.path.join(base_path, self.FILENAME)
+        tree = ET.parse(xml_path)
+        root = tree.getroot()
+
+        instances = []
+        for question_root in root:
+            # get the actual question and question ID
+            id = self._extract_question_id(question_root)
+            question = Input(self._extract_question(question_root))
+
+            # parse out the reference answers
+            answers = self._extract_answers(question_root)
+            references = [Reference(Output(answer), tags=[CORRECT_TAG]) for answer in answers]
+
+            # stitch it all together
+            instances.append(Instance(question, references, split=TEST_SPLIT, id=id))
+
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """entrypoint to creating this scenario's instances"""
+        # get the dataset
+        self.download_liveqa(output_path)
+
+        # get the instances by parsing the XML
+        instances = self.process_xml(output_path)
+        return instances