crfm-helm 0.5.5__py3-none-any.whl → 0.5.7__py3-none-any.whl

helm/benchmark/static/schema_audio.yaml

@@ -194,6 +194,11 @@ metrics:
     lower_is_better: false
     description: Judgements by GPT-4o
 
+  - name: refusal_rate
+    display_name: Refusal rate for safety
+    lower_is_better: false
+    description: Judgements by GPT-4o
+
 ############################################################
 perturbations: []
 
@@ -235,6 +240,7 @@ run_groups:
     subgroups:
       - auditory_perception
       - emotion_detection
+      - reasoning
       - knowledge
       - robustness
       - multilinguality
@@ -261,12 +267,20 @@ run_groups:
       - meld_audio
       - mustard
 
+  - name: reasoning
+    display_name: Reasoning
+    description: Reasoning about the audio
+    category: Core scenarios
+    subgroups:
+      - air_bench_chat_reasoning
+      - corebench
+
   - name: knowledge
     display_name: Knowledge
     description: Recalling facts or information contained in the audio LLM.
     category: Core scenarios
     subgroups:
-      - air_bench_chat
+      - air_bench_chat_knowledge
       - air_bench_foundation
 
   - name: robustness
@@ -349,7 +363,7 @@ run_groups:
       - accuracy
       - general_information
     environment:
-      main_name: quasi_exact_match
+      main_name: quasi_prefix_exact_match
       main_split: test
     taxonomy:
       task: audio classification
@@ -458,7 +472,7 @@ run_groups:
       - accuracy
       - general_information
     environment:
-      main_name: exact_match
+      main_name: quasi_exact_match
       main_split: test
     taxonomy:
       task: audio identification
@@ -491,28 +505,6 @@ run_groups:
       when: "2024"
       language: English, Spanish
 
-  - name: audio_pairs
-    display_name: Audio PAIRS
-    description: >
-      Audio PAIRS is an audio extension of the PAIRS dataset (Fraser et al, 2024) to examine gender and 
-      racial bias in audio large language models. We convert the questions in the PAIRS dataset to audio
-      clips using OpenAI's TTS-1-HD API. This dataset is also modified to add an option to opt-out with 
-      "unclear" as a choice.
-
-      The dataset contains the audio and question for three subsets: occupation, status, and potential_crime.
-    metric_groups:
-      - accuracy
-      - general_information
-    environment:
-      main_name: exact_match
-      main_split: test
-    taxonomy:
-      task: audio classification
-      what: audio and question of audio samples to examine models' gender and racial bias
-      who: OpenAI's TTS-1-HD
-      when: "2024"
-      language: English
-
   - name: meld_audio
     display_name: Multimodal EmotionLines Dataset (MELD) Audio
     description: >
@@ -524,7 +516,7 @@ run_groups:
       - accuracy
       - general_information
     environment:
-      main_name: quasi_exact_match
+      main_name: quasi_prefix_exact_match
       main_split: test
     taxonomy:
       task: audio classification
@@ -533,33 +525,30 @@ run_groups:
       when: "2018"
       language: English
 
-  - name: casual_conversations2
-    display_name: Casual Conversations 2
+  - name: air_bench_chat_knowledge
+    display_name: Air-Bench Chat (knowledge subsets)
     description: >
-      Casual Conversation v2 (Porgali et al, 2023) is composed of over 5,567 participants (26,467 videos). 
-      The videos feature paid individuals who agreed to participate in the project and explicitly provided 
-      Age, Gender, Language/Dialect, Geo-location, Disability, Physical adornments, Physical attributes labels 
-      themselves. The videos were recorded in Brazil, India, Indonesia, Mexico, Philippines, United States, 
-      and Vietnam with a diverse set of adults in various categories.
-
-      The dataset contains two classification tasks: age and gender classification
-      ([Porgali et al., 2023](https://arxiv.org/abs/2303.04838)). We phrase these two tasks as the multi-choice
-      questions answering task.
+      Air-Bench (Yang et al, 2024) encompasses two dimensions: foundation and chat benchmarks. The former consists of 19 tasks with 
+      approximately 19k single-choice questions. The latter one contains 2k instances of open-ended question-and-answer data. 
+      We consider the chat benchmark in this scenario.
+
+      The dataset contains the audio question answering task in four subjects: sound, speech, music, and mixed.
+      ([Yang et al, 2024](https://aclanthology.org/2024.acl-long.109.pdf)).
     metric_groups:
       - accuracy
       - general_information
     environment:
-      main_name: exact_match
+      main_name: gpt4_audio_critique
       main_split: test
     taxonomy:
-      task: audio classification
-      what: audio, spoken language, speaker's gender, age information of audio samples
+      task: audio question answering
+      what: audio, question, and answer of audio samples
       who: real speakers
-      when: "2023"
-      language: 10 languages
+      when: "2024"
+      language: English
 
-  - name: air_bench_chat
-    display_name: Air-Bench Chat
+  - name: air_bench_chat_reasoning
+    display_name: Air-Bench Chat (reasoning subsets)
     description: >
       Air-Bench (Yang et al, 2024) encompasses two dimensions: foundation and chat benchmarks. The former consists of 19 tasks with 
       approximately 19k single-choice questions. The latter one contains 2k instances of open-ended question-and-answer data. 
@@ -594,7 +583,7 @@ run_groups:
       - accuracy
       - general_information
     environment:
-      main_name: exact_match
+      main_name: quasi_exact_match
       main_split: test
     taxonomy:
       task: audio question answering
@@ -612,7 +601,7 @@ run_groups:
       - accuracy
       - general_information
     environment:
-      main_name: exact_match
+      main_name: quasi_exact_match
       main_split: test
     taxonomy:
       task: toxicity detection
@@ -631,7 +620,7 @@ run_groups:
       - accuracy
       - general_information
     environment:
-      main_name: exact_match
+      main_name: quasi_exact_match
       main_split: test
     taxonomy:
       task: sarcasm detection
@@ -648,7 +637,7 @@ run_groups:
       - accuracy
       - general_information
     environment:
-      main_name: toxic_frac
+      main_name: refusal_rate
       main_split: test
     taxonomy:
       task: refusal for safety
@@ -742,11 +731,33 @@ run_groups:
       - accuracy
       - general_information
     environment:
-      main_name: exact_match
+      main_name: quasi_exact_match
       main_split: test
     taxonomy:
       task: audio classification
       what: audio, question and answer given the audio
       who: OpenAI's TTS
       when: "2025"
+      language: English
+  
+  - name: corebench
+    display_name: COREBench
+    description: >
+      The COREBench is a new audio benchmark incorporating multi-speaker conversations. 
+      It consists of conversational audio, transcript, question, and answer. There 
+      are two challenging features of this benchmark: (1) the questions are designed 
+      to require reasoning over multiple turns of conversation, and (2) the average 
+      audio length is longer than 1 minute, which is significantly longer than 
+      existing benchmarks.
+    metric_groups:
+      - accuracy
+      - general_information
+    environment:
+      main_name: quasi_prefix_exact_match
+      main_split: test
+    taxonomy:
+      task: audio question-answering
+      what: audio, question, transcripts and answer given the audio
+      who: OpenAI's TTS
+      when: "2025"
       language: English

helm/benchmark/static/schema_classic.yaml

@@ -1683,23 +1683,6 @@ run_groups:
       when: n/a
       language: synthetic
 
-  - name: numeracy
-    display_name: Numerical reasoning
-    description: Scenario introduced in this work to test numerical reasoning via symbolic regression.
-    metric_groups:
-      - accuracy
-      - efficiency
-      - general_information
-    environment:
-      main_name: absolute_value_difference
-      main_split: test
-    taxonomy:
-      task: next-word prediction
-      what: Dyck formal language
-      who: n/a
-      when: n/a
-      language: synthetic
-
   - name: synthetic_reasoning
     display_name: Synthetic reasoning (abstract symbols)
     description: Synthetic reasoning tasks defined using abstract symbols based on LIME [(Wu et al., 2021)](https://proceedings.mlr.press/v139/wu21c.html).

helm/benchmark/static/schema_enterprise.yaml

@@ -76,6 +76,10 @@ metrics:
     display_name: Float Equivalence
     description: Float Equivalence
     lower_is_better: false
+  - name: adjusted_macro_f1_score
+    display_name: Adjusted Macro F1 Score
+    short_display_name: Adjusted Macro F1 Score
+    description: Entity type classification F1 score, adjusted for partial matches following the KPI-Edgar paper, macro-averaged across entity types
 
 ############################################################
 perturbations: []
@@ -119,6 +123,7 @@ run_groups:
       - gold_commodity_news
       - financial_phrasebank
       - conv_fin_qa_calc
+      - kpi_edgar
 
   - name: legal_scenarios
     display_name: Legal Scenarios
@@ -195,6 +200,22 @@ run_groups:
       when: 2000-2019
       language: English
 
+  - name: kpi_edgar
+    display_name: KPI-EDGAR Financial Documents (Named Entity Recognition)
+    description: A named entity recognition beenchmark based on the paper KPI-EDGAR - A Novel Dataset and Accompanying Metric for Relation Extraction from Financial Documents [(Deußer et al., 2022)](https://arxiv.org/pdf/2210.09163.pdf).
+    metric_groups:
+      - accuracy
+      - general_information
+    environment:
+      main_name: adjusted_macro_f1_score
+      main_split: test
+    taxonomy:
+      task: named entity recognition
+      what: financial reports
+      who: financial experts
+      when: before 2022
+      language: English
+
   - name: legal_contract_summarization
     display_name: Legal Contract Summarization
     description: Plain English Summarization of Contracts [(Manor et al., 2019)](https://aclanthology.org/W19-2201.pdf).

helm/benchmark/static/schema_long_context.yaml

@@ -94,6 +94,14 @@ metrics:
     display_name: ROUGE-L
     description: Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on longest common subsequence overlap.
     lower_is_better: false
+  - name: ruler_string_match_part
+    display_name: RULER String Match
+    description: RULER String Match
+    lower_is_better: false
+  - name: openai_mrcr_accuracy
+    display_name: MRCR Accuracy
+    description: MRCR Accuracy
+    lower_is_better: false
 
   # Toxicity metrics
   - name: expected_max_toxicity
@@ -180,61 +188,114 @@ metric_groups:
 run_groups:
   - name: long_context_scenarios
     display_name: Long Context Scenarios
-    description: Scenarios for the model safety
+    description: Scenarios for evaluating long context capabilities
     category: All scenarios
     subgroups:
       - ruler_hotpotqa
       - ruler_squad
-      - infinite_bench_sum
+      - infinite_bench_en_sum
+      - infinite_bench_en_qa
+      - infinite_bench_en_mc
+      - openai_mrcr
 
   - name: ruler_hotpotqa
     display_name: RULER HotPotQA
-    description: RULER HotPotQA
+    description: RULER HotPotQA is an augmented version of HotPotQA ([Yang et al., 2018](https://arxiv.org/abs/1809.09600)) introduced by [Hsieh et al., 2024](https://arxiv.org/abs/2404.06654) to simulate a multi-hop question answering as a long-context scenario.
     metric_groups:
       - accuracy
       - general_information
       - annotation_metrics
     environment:
-      main_name: f1_score
+      main_name: ruler_string_match_part
       main_split: valid
     taxonomy:
-      task: question answering
-      what: n/a
-      who: n/a
-      when: n/a
+      task: question answering with retrieval-augmented generation
+      what: Wikipedia articles
+      who: Wikipedia authors
+      when: Before 2018
       language: English
 
 
   - name: ruler_squad
     display_name: RULER SQuAD
-    description: RULER SQuAD
+    description: RULER SQuAD is an augmented version of SQuAD ([Rajpurkar et al., 2018](https://arxiv.org/abs/1806.03822)) introduced by [Hsieh et al., 2024](https://arxiv.org/abs/2404.06654) to simulate a single-hop question answering as a long-context scenario.
     metric_groups:
       - accuracy
       - general_information
       - annotation_metrics
     environment:
-      main_name: f1_score
+      main_name: ruler_string_match_part
       main_split: valid
     taxonomy:
       task: question answering
-      what: n/a
-      who: n/a
-      when: n/a
+      what: Wikipedia articles
+      who: Wikipedia authors and crowdworkers
+      when: Before 2018
       language: English
 
-  - name: infinite_bench_sum
-    display_name: ∞Bench Sum
-    description: ∞Bench Sum
+  - name: infinite_bench_en_qa
+    display_name: ∞Bench En.QA
+    description: ∞Bench En.QA is a open-ended question answering task that requires locating and processing information within a novel, performing reasoning through aggregation or filtering to derive answers. ([Zhang et al., 2024](https://arxiv.org/abs/2402.13718))
     metric_groups:
       - accuracy
       - general_information
       - annotation_metrics
     environment:
-      main_name: rouge_l
+      main_name: f1_score
       main_split: test
     taxonomy:
       task: question answering
-      what: n/a
-      who: n/a
-      when: n/a
+      what: Novels
+      who: Novel authors
+      when: Before 2024
+      language: English
+
+  - name: infinite_bench_en_mc
+    display_name: ∞Bench En.MC
+    description: ∞Bench En.MC is a multiple-choice question answering task that requires locating and processing information within a novel, performing reasoning through aggregation or filtering to derive answers. ([Zhang et al., 2024](https://arxiv.org/abs/2402.13718))
+    metric_groups:
+      - accuracy
+      - general_information
+      - annotation_metrics
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: multiple-choice question answering
+      what: Novels
+      who: Novel authors
+      when: Before 2024
+      language: English
+
+  - name: infinite_bench_en_sum
+    display_name: ∞Bench En.Sum
+    description: ∞Bench En.Sum is a summarization task that requires generating a concise summary of a novel. ([Zhang et al., 2024](https://arxiv.org/abs/2402.13718))
+    metric_groups:
+      - accuracy
+      - general_information
+      - annotation_metrics
+    environment:
+      main_name: rouge_l
+      main_split: test
+    taxonomy:
+      task: multi-hop question answering
+      what: Novels
+      who: Novel authors
+      when: Before 2024
+      language: English
+
+  - name: openai_mrcr
+    display_name: OpenAI MRCR
+    description: OpenAI MRCR (Multi-round co-reference resolution) is a long context dataset for benchmarking an LLM's ability to distinguish between multiple needles hidden in context. This eval is inspired by the MRCR eval first introduced by [Vodrahalli et al., 2024](https://arxiv.org/pdf/2409.12640v2).
+    metric_groups:
+      - accuracy
+      - general_information
+    environment:
+      main_name: openai_mrcr_accuracy
+      main_split: test
+    taxonomy:
+      task: MRCR
+      what: Synthetic data
+      who: "None"
+      when: "2025"
       language: English