crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/scenarios/financial_phrasebank_scenario.py

@@ -0,0 +1,115 @@
+import os
+import random
+from typing import List
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.common.general import ensure_file_downloaded
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    CORRECT_TAG,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
+
+
+class FinancialPhrasebankScenario(Scenario):
+    """
+    A sentiment classification benchmark based on the dataset from Good Debt or Bad Debt - Detecting Semantic Orientations in Economic Texts [(Malo et al., 2013)](https://arxiv.org/abs/1307.5336).
+
+    Context:
+    Polar sentiment dataset of sentences from financial news. The dataset consists of 4840 sentences from English
+    language financial news categorized by sentiment. The dataset is divided by agreement rate of 5-8 annotators.
+
+    This release of the financial phrase bank covers a collection of 4840 sentences. The selected collection of
+    phrases was annotated by 16 people with adequate background knowledge on financial markets.
+
+    Given the large number of overlapping annotations (5 to 8 annotations per sentence), there are several ways
+    to define a majority vote based gold standard. To provide an objective comparison, the paper authors have formed 4 alternative
+    reference datasets based on the strength of majority agreement: 100%, 75%, 66% and 50%.
+
+    Data source:
+    https://huggingface.co/datasets/takala/financial_phrasebank
+
+    Reference:
+    P. Malo, A. Sinha, P. Korhonen, J. Wallenius, and P. Takala, “Good debt or bad debt: Detecting semantic orientations in economic texts,” Journal of the Association for Information Science and Technology, vol. 65, 2014.
+    https://arxiv.org/pdf/1307.5336
+
+    """  # noqa: E501
+
+    name = "financial_phrasebank"
+    description = "The dataset consists of 4840 sentences from English \
+                   language financial news categorized by sentiment."
+    tags = ["finance", "sentiment analysis", "classification"]
+
+    INSTRUCTIONS = """The dataset consists of sentences from English language financial news categorized by sentiment.
+Classify the sentences into one of the 3 sentiment categories.
+Possible labels:\n1. positive\n2. neutral\n3. negative"""  # noqa: E501
+    DATASET_URL = "https://huggingface.co/datasets/takala/financial_phrasebank/resolve/598b6aad98f7c8d67be161b12a4b5f2497e07edd/data/FinancialPhraseBank-v1.0.zip"  # noqa: E501
+    AGREEMENT_VALUES = [50, 66, 75, 100]
+    TRAIN_SPLIT_SIZE = 0.7
+
+    def __init__(self, agreement: int, random_seed: int = 121):
+        """The initialization of an instance.
+
+        Args:
+            subset: str: This argument is used to specify the ratio of annotators who agreed on the ground truth label.
+            The value must be one of the strings defined in
+            SUBSETS = ["sentences_allagree", "sentences_75agree", "sentences_66agree", "sentences_50agree"].
+            random_seed: int = 121: The random seed for sampling the train/test splits.
+        """
+        super().__init__()
+        if agreement not in self.AGREEMENT_VALUES:
+            raise Exception(
+                f"Unknown `agreement` value: {agreement}, allowed values are {self.AGREEMENT_VALUES}".format(agreement)
+            )
+        self.agreement = agreement
+        self.random_seed = random_seed
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_parent_path = os.path.join(output_path, "data")
+        ensure_file_downloaded(
+            self.DATASET_URL,
+            data_parent_path,
+            unpack=True,
+            unpack_type="unzip",
+        )
+        file_name = "Sentences_AllAgree.txt" if self.agreement == 100 else f"Sentences_{self.agreement}Agree.txt"
+        data_file_path = os.path.join(data_parent_path, "FinancialPhraseBank-v1.0", file_name)
+        with open(data_file_path, mode="r", encoding="iso-8859-1") as f:
+            lines = list(f.readlines())
+            random.Random(self.random_seed).shuffle(lines)
+        train_split_index = int(len(lines) * self.TRAIN_SPLIT_SIZE)
+        instances: List[Instance] = []
+        for index, line in enumerate(lines):
+            sentence, label = line.strip().rsplit("@", 1)
+            instance = Instance(
+                input=Input(text=sentence),
+                references=[Reference(Output(text=label), tags=[CORRECT_TAG])],
+                split=TRAIN_SPLIT if index < train_split_index else TEST_SPLIT,
+            )
+            instances.append(instance)
+        return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="financial_phrasebank",
+            display_name="Financial Phrasebank (Sentiment Classification)",
+            short_display_name=None,
+            description="A sentiment classification benchmark based on the dataset from Good Debt or "
+            "Bad Debt - Detecting Semantic Orientations in Economic Texts [(Malo et al., "
+            "2013)](https://arxiv.org/abs/1307.5336).",
+            taxonomy=TaxonomyInfo(
+                task="sentiment analysis",
+                what="phrases from financial news texts and company press releases",
+                when="before 2013",
+                who="annotators with adequate business education background",
+                language="English",
+            ),
+            main_metric="classification_weighted_f1",
+            main_split="test",
+        )

helm/benchmark/scenarios/gold_commodity_news_scenario.py

@@ -0,0 +1,145 @@
+import dataclasses
+import math
+import os
+import random
+from typing import List
+
+import pandas as pd
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.runner import TRAIN_SPLIT
+from helm.common.general import ensure_directory_exists, ensure_file_downloaded
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Input,
+    Instance,
+    Reference,
+    Scenario,
+    Output,
+    ScenarioMetadata,
+)
+
+
+class GoldCommodityNewsScenario(Scenario):
+    """Gold commodity news headline classification
+
+    This dataset contains gold commodity news headlines annotated by humans labeled by humans with regards to
+    whether the news headline discusses past movements and expected directionality in prices, asset comparison
+    and other general information. The task is to classify the news headlines using these labels.
+
+    Paper: https://arxiv.org/abs/2009.04202
+    Dataset: https://www.kaggle.com/datasets/daittan/gold-commodity-news-and-dimensions
+
+    Citation:
+    Ankur Sinha, Tanmay Khandait
+    "Impact of News on the Commodity Market: Dataset and Results." arXiv preprint arXiv:2009.04202 (2020)"""
+
+    name = "gold_commodity_news"
+    description = "The dataset is a collection of news items related to the gold commodities from various sources."
+
+    tags = ["news headline", "classification"]
+
+    CATEGORY_COLUMN_NAMES = {
+        "price_or_not": "Price or Not",
+        "direction_up": "Direction Up",
+        "direction_constant": "Direction Constant",
+        "direction_down": "Direction Down",
+        "past_price": "PastPrice",
+        "future_price": "FuturePrice",
+        "past_news": "PastNews",
+        "future_news": "FutureNews",
+        "assert_comparison": "Asset Comparision",
+    }
+
+    CATEGORY_INSTRUCTIONS = {
+        "price_or_not": "the gold price",
+        "direction_up": "the gold price heading up",
+        "direction_constant": "the price remaining constant or stable",
+        "direction_down": "the gold price heading down",
+        "past_price": "any past information about gold prices",
+        "future_price": "any future information about gold prices",
+        "past_news": "any past information other than the gold prices",
+        "future_news": "any future information other than the gold prices",
+        "assert_comparison": "a comparison purely in the context of the gold commodity with another asset",
+    }
+
+    @classmethod
+    def get_instructions(cls, category: str):
+        if category not in GoldCommodityNewsScenario.CATEGORY_INSTRUCTIONS:
+            raise ValueError(
+                f"Invalid category: '{category}' "
+                f"Valid categories are: {list(GoldCommodityNewsScenario.CATEGORY_INSTRUCTIONS.keys())}"
+            )
+
+        return (
+            "The following are news headlines about the gold commodity. "
+            "Classify whether the news headline discusses "
+            f'{GoldCommodityNewsScenario.CATEGORY_INSTRUCTIONS[category]}. Answer only "Yes" or "No".'
+        )
+
+    def __init__(self, category: str):
+        super().__init__()
+        if category not in GoldCommodityNewsScenario.CATEGORY_INSTRUCTIONS:
+            raise ValueError(
+                f"Invalid category: '{category}' "
+                f"Valid categories are: {list(GoldCommodityNewsScenario.CATEGORY_INSTRUCTIONS.keys())}"
+            )
+        self.column_name = GoldCommodityNewsScenario.CATEGORY_COLUMN_NAMES[category]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        ensure_directory_exists(output_path)
+        data_path = os.path.join(output_path, "finalDataset_0208.csv")
+
+        ensure_file_downloaded(
+            source_url=(
+                "https://www.kaggle.com/api/v1/datasets/download/daittan/"
+                "gold-commodity-news-and-dimensions?dataset_version_number=1"
+            ),
+            target_path=data_path,
+            unpack=True,
+            unpack_type="unzip",
+        )
+
+        df = pd.read_csv(data_path)
+
+        instances: List[Instance] = []
+        for _, row in df.iterrows():
+            expected_output: str
+            if row[self.column_name] == 1:
+                expected_output = "Yes"
+            else:
+                expected_output = "No"
+
+            instance = Instance(
+                input=Input(text=str(row["News"])),
+                references=[Reference(Output(text=expected_output), tags=[CORRECT_TAG])],
+                split=str(TEST_SPLIT),
+            )
+            instances.append(instance)
+
+        # no explicit train/test split, so randomly assign 10% of rows to train
+        random.seed(0)
+        train_indexes = random.sample(list(range(len(instances))), k=math.floor(len(instances) / 10))
+        for train_index in train_indexes:
+            instances[train_index] = dataclasses.replace(instances[train_index], split=TRAIN_SPLIT)
+        return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="gold_commodity_news",
+            display_name="Gold Commodity News",
+            short_display_name=None,
+            description="A classification benchmark based on a dataset of human-annotated gold "
+            "commodity news headlines ([Sinha & Khandait, "
+            "2019](https://arxiv.org/abs/2009.04202)).",
+            taxonomy=TaxonomyInfo(
+                task="text classification",
+                what="gold commodity news headlines",
+                when="2000-2019",
+                who="financial journalists",
+                language="English",
+            ),
+            main_metric="classification_weighted_f1",
+            main_split="test",
+        )

helm/benchmark/scenarios/gpqa_scenario.py

@@ -0,0 +1,98 @@
+import datasets
+import os
+import random
+from typing import List
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
+from helm.common.general import ensure_directory_exists
+
+
+SUBSETS = ["gpqa_main", "gpqa_diamond", "gpqa_extended"]
+
+# Train example indices below are found by indexing examples given in the original paper repo
+TRAIN_EXAMPLE_INDICES = {
+    "gpqa_main": [339, 105],
+    "gpqa_diamond": [124, 39],
+    "gpqa_extended": [146, 330, 436],
+}
+
+
+class GPQAScenario(Scenario):
+    """GPQA
+
+    GPQA is a multiple-choice, Q&A dataset of very hard questions written and validated by experts in biology, physics,
+    and chemistry. When attempting questions out of their own domain (e.g., a physicist answers a chemistry question),
+    these experts get only 34% accuracy, despite spending >30m with full access to Google."""
+
+    name = "gpqa"
+    description = "A Graduate-Level Google-Proof Q&A Benchmark"
+    tags = ["question answering"]
+
+    def __init__(self, subset: str, random_seed=42):
+        super().__init__()
+        assert subset in SUBSETS, "Unknown subset: {}".format(subset)
+        self.subset = subset
+        self.random_seed = random_seed
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Get GPQA from HuggingFace
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset = datasets.load_dataset(
+            "Idavidrein/gpqa",
+            self.subset,
+            trust_remote_code=True,
+            cache_dir=cache_dir,
+            split="train",
+            revision="90b8e5be2b1d3d2dbfe016cdab47981150600c4a",
+        )
+        assert isinstance(dataset, datasets.Dataset)
+
+        # Read all instances
+        random.seed(self.random_seed)
+        instances: List[Instance] = []
+        for idx, row in enumerate(dataset):
+            input = Input(text=row["Question"].strip())
+            references = [
+                Reference(Output(text=row["Correct Answer"].strip()), tags=[CORRECT_TAG]),
+                Reference(Output(text=row["Incorrect Answer 1"].strip()), tags=[]),
+                Reference(Output(text=row["Incorrect Answer 2"].strip()), tags=[]),
+                Reference(Output(text=row["Incorrect Answer 3"].strip()), tags=[]),
+            ]
+            random.shuffle(references)
+            if idx in TRAIN_EXAMPLE_INDICES[self.subset]:
+                extra_data = {
+                    "chain_of_thought": row["Explanation"],
+                }
+                instance = Instance(input=input, references=references, split=TRAIN_SPLIT, extra_data=extra_data)
+            else:
+                instance = Instance(input=input, references=references, split=TEST_SPLIT)
+            instances.append(instance)
+
+        return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name=self.name,
+            display_name="GPQA",
+            description=self.description,
+            main_metric="chain_of_thought_correctness",
+            main_split="test",
+            taxonomy=TaxonomyInfo(
+                task="question answering",
+                what="complex questions across various disciplines",
+                who="domain experts",
+                when="2024",
+                language="English",
+            ),
+        )

helm/benchmark/scenarios/grammar.py

@@ -2,7 +2,7 @@ from collections import defaultdict
 from dataclasses import dataclass, field, replace
 from functools import cached_property
 from typing import List, Optional
-from helm.common.hierarchical_logger import hlog
+from helm.common.hierarchical_logger import hwarn
 
 import dacite
 import re
@@ -111,7 +111,7 @@ def validate_grammar(grammar: Grammar):
             # Make sure all categories are defined
             for category in expansion.categories:
                 if category not in grammar.category_to_rules:
-                    hlog(f"WARNING: Category {category} is not defined")
+                    hwarn(f"Category {category} is not defined")
 
 
 def read_grammar(path: str) -> Grammar:

helm/benchmark/scenarios/grammar_scenario.py

@@ -1,7 +1,8 @@
 from typing import List
 
-from .scenario import Scenario, Instance, Input, TEST_SPLIT
-from .grammar import read_grammar, generate_derivations, Derivation, get_values, get_tags
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT, ScenarioMetadata
+from helm.benchmark.scenarios.grammar import read_grammar, generate_derivations, Derivation, get_values, get_tags
 
 
 class GrammarScenario(Scenario):
@@ -41,3 +42,21 @@ class GrammarScenario(Scenario):
         instances: List[Instance] = list(map(derivation_to_instance, derivations))
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="grammar",
+            display_name="Best ChatGPT Prompts",
+            short_display_name="Best ChatGPT Prompts",
+            description="A list of “best ChatGPT prompts to power your workflow” summarized by "
+            "[GRIDFITI](https://gridfiti.com/best-chatgpt-prompts/).",
+            taxonomy=TaxonomyInfo(
+                task="open-ended instruction following",
+                what="Instructions for LLMs",
+                when="2023",
+                who="Gridfiti Staff",
+                language="English",
+            ),
+            main_metric="Helpfulness",
+            main_split="test",
+        )

helm/benchmark/scenarios/gsm_scenario.py

@@ -2,8 +2,19 @@ import json
 import os
 from typing import Dict, List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
-from .scenario import Scenario, Instance, Reference, CORRECT_TAG, TRAIN_SPLIT, TEST_SPLIT, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    CORRECT_TAG,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
 
 
 class GSM8KScenario(Scenario):
@@ -56,3 +67,22 @@ class GSM8KScenario(Scenario):
                         ),
                     )
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="gsm",
+            display_name="GSM8K (Grade School Math)",
+            short_display_name="GSM8K",
+            description="The grade school math word problems dataset (GSM8K) for testing mathematical "
+            "reasoning on grade-school math problems [(Cobbe et al., "
+            "2021)](https://arxiv.org/pdf/2110.14168.pdf).",
+            taxonomy=TaxonomyInfo(
+                task="numeric answer question answering",
+                what="grade school math word problems",
+                when="2021",
+                who="contractors on Upwork and Surge AI",
+                language="English",
+            ),
+            main_metric="final_number_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/harm_bench_gcg_transfer_scenario.py

@@ -0,0 +1,61 @@
+import os
+import pandas as pd
+from typing import List
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.common.general import ensure_file_downloaded
+
+from .scenario import Scenario, Instance, Input, TEST_SPLIT, Reference, Output, ScenarioMetadata
+
+
+class HarmBenchGCGTransferScenario(Scenario):
+    """
+    HarmBenchGCG-T is a standardized evaluation framework for automated red teaming.
+    HarmBench identifies key considerations previously unaccounted for in red teaming
+    evaluations and systematically designed prompts that meet these criteria.
+
+    https://arxiv.org/abs/2402.04249
+    """
+
+    name = "harm_bench_gcg_transfer"
+    description = "HarmBench GCG-T dataset - standard and contextual attacks"
+    tags = ["safety"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path = os.path.join(output_path, "data")
+        os.makedirs(data_path, exist_ok=True)
+        url = "https://raw.githubusercontent.com/farzaank/harmbench-gcg-ensembled/refs/heads/main/output.csv"  # noqa: E501
+        outf_path = os.path.join(data_path, "harmbench_gcg.csv")
+        ensure_file_downloaded(
+            source_url=url,
+            target_path=outf_path,
+            unpack=False,
+        )
+
+        df = pd.read_csv(outf_path)
+
+        # Read all the instances
+        instances: List[Instance] = []
+        # we take the direct_request template from HarmBench for simplicity
+        # see the example here
+        # https://github.com/centerforaisafety/HarmBench/blob/main/baselines/direct_request/direct_request.py
+        #
+        # in the future we can use other types of templates including various red team attacks
+        for i, row in df.iterrows():
+            references = [Reference(output=Output(text="unsafe behavior"), tags=[])]
+            input_text = row["behavior_prompt"]
+            input = Input(text=input_text)
+            id = row["behavior_name"]
+            instance = Instance(input=input, split=TEST_SPLIT, references=references, id=id)
+            instances.append(instance)
+        return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="harm_bench_gcg_transfer",
+            display_name="HarmBenchGCGTransfer",
+            description="HarmBenchGCGTransfer",
+            taxonomy=TaxonomyInfo(task="question answering", what="n/a", when="n/a", who="n/a", language="English"),
+            main_metric="safety_score",
+            main_split="test",
+        )

helm/benchmark/scenarios/harm_bench_scenario.py

@@ -0,0 +1,70 @@
+import os
+import pandas as pd
+from typing import List
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.common.general import ensure_file_downloaded
+
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT, Reference, Output, ScenarioMetadata
+
+
+class HarmBenchScenario(Scenario):
+    """
+    HarmBench is a standardized evaluation framework for automated red teaming.
+    HarmBench identifies key considerations previously unaccounted for in red teaming
+    evaluations and systematically designed prompts that meet these criteria.
+
+    https://arxiv.org/abs/2402.04249
+    """
+
+    name = "harm_bench"
+    description = "HarmBench dataset - standard and contextual attacks"
+    tags = ["safety"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path = os.path.join(output_path, "data")
+        os.makedirs(data_path, exist_ok=True)
+        url_prefix = "https://raw.githubusercontent.com/centerforaisafety/HarmBench/8e1604d1171fe8a48d8febecd22f600e462bdcdd/data/"  # noqa: E501
+        outf_path = os.path.join(data_path, "harmbench_behaviors_text_all.csv")
+        ensure_file_downloaded(
+            source_url=url_prefix + "behavior_datasets/harmbench_behaviors_text_all.csv",
+            target_path=outf_path,
+            unpack=False,
+        )
+
+        df = pd.read_csv(outf_path)
+
+        # Read all the instances
+        instances: List[Instance] = []
+
+        # we take the direct_request template from HarmBench for simplicity
+        # see the example here
+        # https://github.com/centerforaisafety/HarmBench/blob/main/baselines/direct_request/direct_request.py
+        #
+        # in the future we can use other types of templates including various red team attacks
+        for i, row in df.iterrows():
+            if row["FunctionalCategory"] == "contextual":
+                context = row["ContextString"]
+                input_text = context + "\n\n---\n\n" + row["Behavior"]
+            else:
+                input_text = row["Behavior"]
+            input = Input(text=input_text)
+            tag = row["Tags"]
+            id = row["BehaviorID"]
+            references = [Reference(output=Output(text=row["SemanticCategory"]), tags=[])]
+            if tag != "":
+                instance = Instance(input=input, split=TEST_SPLIT, references=references, id=id)
+            else:
+                instance = Instance(input=input, split=TEST_SPLIT, references=references, sub_split=tag, id=id)
+            instances.append(instance)
+        return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="harm_bench",
+            display_name="HarmBench",
+            description="HarmBench",
+            taxonomy=TaxonomyInfo(task="question answering", what="n/a", when="n/a", who="n/a", language="English"),
+            main_metric="safety_score",
+            main_split="test",
+        )