crfm-helm 0.5.3__py3-none-any.whl → 0.5.5__py3-none-any.whl

helm/benchmark/static/schema_safety.yaml

@@ -76,7 +76,7 @@ metrics:
     lower_is_better: false
   - name: safety_score
     display_name: LM Evaluated Safety score
-    description: LM Evaluated Safety score (mean of Claude Sonnet3.5 and GPT4o ratings)
+    description: LM Evaluated Safety score (mean of Llama3.1 and GPT4o ratings) representing whether the output constitutes a successful refusal of unsafe actions (0 is worst, 1 is best).
     lower_is_better: false
   - name: bbq_accuracy
     display_name: BBQ accuracy
@@ -98,6 +98,13 @@ metrics:
     lower_is_better: true
     description: Fraction of model outputs that are toxic (based on the PerspectiveAPI toxicity classifier).
 
+  # Annotation metrics
+  - name: safety_llama_annotator_success
+    display_name: Llama Annotator Success Rate
+    description: Fraction of annotator requests to Llama that succeeded.
+  - name: safety_gpt_annotator_success
+    display_name: GPT Annotator Success Rate
+    description: Fraction of annotator requests to GPT that succeeded.
 
 ############################################################
 perturbations: []
@@ -106,6 +113,8 @@ perturbations: []
 metric_groups:
   - name: accuracy
     display_name: Accuracy
+    aggregation_strategies: 
+      - mean
     metrics:
       - name: ${main_name}
         split: ${main_split}
@@ -131,6 +140,15 @@ metric_groups:
     - name: num_output_tokens
       split: ${main_split}
 
+  - name: annotation_metrics
+    display_name: Annotation
+    hide_win_rates: true
+    metrics:
+    - name: safety_llama_annotator_success
+      split: ${main_split}
+    - name: safety_gpt_annotator_success
+      split: ${main_split}
+
   - name: toxicity
     display_name: Toxicity
     metrics:
@@ -153,6 +171,7 @@ run_groups:
     description: Scenarios for the model safety
     category: All scenarios
     subgroups:
+      # - harm_bench_gcg_transfer
       - harm_bench
       - simple_safety_tests
       - bbq
@@ -164,8 +183,25 @@ run_groups:
     description: HarmBench
     metric_groups:
       - accuracy
-      - efficiency
       - general_information
+      - annotation_metrics
+    environment:
+      main_name: safety_score
+      main_split: test
+    taxonomy:
+      task: question answering
+      what: n/a
+      who: n/a
+      when: n/a
+      language: English
+
+  - name: harm_bench_gcg_transfer
+    display_name: HarmBenchGCGTransfer
+    description: HarmBenchGCGTransfer
+    metric_groups:
+      - accuracy
+      - general_information
+      - annotation_metrics
     environment:
       main_name: safety_score
       main_split: test
@@ -181,8 +217,8 @@ run_groups:
     description: SimpleSafetyTests
     metric_groups:
       - accuracy
-      - efficiency
       - general_information
+      - annotation_metrics
     environment:
       main_name: safety_score
       main_split: test
@@ -198,8 +234,8 @@ run_groups:
     description: XSTest
     metric_groups:
       - accuracy
-      - efficiency
       - general_information
+      - annotation_metrics
     environment:
       main_name: safety_score
       main_split: test
@@ -215,9 +251,9 @@ run_groups:
     description: The Bias Benchmark for Question Answering (BBQ) for measuring social bias in question answering in ambiguous and unambigous context [(Parrish et al., 2022)](https://aclanthology.org/2022.findings-acl.165/).
     metric_groups:
       - accuracy
-      - efficiency
       - general_information
       - bbq_metrics
+      - annotation_metrics
     environment:
       main_name: bbq_accuracy
       main_split: test
@@ -234,8 +270,8 @@ run_groups:
     description: Anthropic Red Team
     metric_groups:
       - accuracy
-      - efficiency
       - general_information
+      - annotation_metrics
     environment:
       main_name: safety_score
       main_split: test

helm/benchmark/static/{schema_bhasa.yaml → schema_seahelm.yaml}

@@ -69,15 +69,25 @@ metrics:
   - name: chr_f_plus_plus
     display_name: ChrF++
     description: Character n-gram F-score with word n-gram order (ChrF++) [(Popovic, 2015)](https://aclanthology.org/W15-3049/). Code can be found [here](https://github.com/mjpost/sacrebleu).
+    lower_is_better: false
   - name: squad_exact_match_score
     display_name: SQuAD exact match
     description: SQuAD exact match score [(Rajpurkar, 2016)](https://aclanthology.org/D16-1264). Code can be found [here](https://github.com/allenai/bi-att-flow/blob/master/squad/evaluate-v1.1.py)
+    lower_is_better: false
   - name: squad_f1_score
     display_name: SQuAD macro-averaged F1 score
     description: SQuAD macro-averaged F1 score [(Rajpurkar, 2016)](https://aclanthology.org/D16-1264). Code can be found [here](https://github.com/allenai/bi-att-flow/blob/master/squad/evaluate-v1.1.py)
+    lower_is_better: false
   - name: classification_macro_f1
     display_name: Macro F1 score
     description: Macro F1 score
+    lower_is_better: false
+  - name: inference_runtime
+    display_name: Observed inference runtime (s)
+    short_display_name: Observed inference time (s)
+    lower_is_better: true
+    description: Average observed time to process a request to the model (via an API, and thus depends on particular deployment).
+
 
 ############################################################
 perturbations: []
@@ -86,12 +96,16 @@ perturbations: []
 metric_groups:
   - name: accuracy
     display_name: Accuracy
+    aggregation_strategies: 
+      - mean
     metrics:
       - name: ${main_name}
         split: ${main_split}
 
   - name: efficiency
     display_name: Efficiency
+    aggregation_strategies: 
+      - mean
     metrics:
     - name: inference_runtime
       split: ${main_split}
@@ -113,10 +127,10 @@ metric_groups:
 ############################################################
 
 run_groups:
-  - name: bhasa_nlu
-    display_name: BHASA natural language understanding (NLU)
-    description: BHASA natural language understanding (NLU) scenarios
-    category: BHASA scenarios
+  - name: seahelm_nlu
+    display_name: SEA-HELM natural language understanding (NLU)
+    description: SEA-HELM natural language understanding (NLU) scenarios
+    category: SEA-HELM scenarios
     subgroups:
       - tydiqa
       - xquad_vi
@@ -130,10 +144,10 @@ run_groups:
       - vihsd
       - thaitoxicitytweets
 
-  - name: bhasa_nlg
-    display_name: BHASA natural language generation (NLG)
-    description: BHASA natural language generation (NLG) scenarios
-    category: BHASA scenarios
+  - name: seahelm_nlg
+    display_name: SEA-HELM natural language generation (NLG)
+    description: SEA-HELM natural language generation (NLG) scenarios
+    category: SEA-HELM scenarios
     subgroups:
     - flores_id_en
     - flores_vi_en
@@ -144,10 +158,10 @@ run_groups:
     - flores_en_th
     - flores_en_ta
 
-  - name: bhasa_nlr
-    display_name: BHASA natural language reasoning (NLR)
-    description: BHASA natural language reasoning (NLR) scenarios
-    category: BHASA scenarios
+  - name: seahelm_nlr
+    display_name: SEA-HELM natural language reasoning (NLR)
+    description: SEA-HELM natural language reasoning (NLR) scenarios
+    category: SEA-HELM scenarios
     subgroups:
     - indonli
     - xnli_vi
@@ -158,14 +172,14 @@ run_groups:
     - xcopa_th
     - xcopa_ta
 
-  - name: bhasa_lindsea
-    display_name: BHASA Linguistic Diagnostics for Southeast Asian Languages (LINDSEA)
-    description: BHASA Linguistic Diagnostics for Southeast Asian Languages (LINDSEA) scenarios
-    category: BHASA scenarios
+  - name: seahelm_lindsea
+    display_name: SEA-HELM Linguistic Diagnostics for Southeast Asian Languages (LINDSEA)
+    description: SEA-HELM Linguistic Diagnostics for Southeast Asian Languages (LINDSEA) scenarios
+    category: SEA-HELM scenarios
     subgroups:
     - lindsea_syntax_minimal_pairs_id
-    - lindsea_pragmatics_pragmatic_reasoning_single_id
-    - lindsea_pragmatics_pragmatic_reasoning_pair_id
+    - lindsea_pragmatics_presuppositions_id
+    - lindsea_pragmatics_scalar_implicatures_id
 
   - name: tydiqa
     display_name: TyDiQA
@@ -672,10 +686,10 @@ run_groups:
       when: "?"
       language: Indonesian
 
-  - name: lindsea_pragmatics_pragmatic_reasoning_single_id
-    display_name: LINDSEA Pragmatics Pragmatic Reasoning (single sentence)
+  - name: lindsea_pragmatics_presuppositions_id
+    display_name: LINDSEA Pragmatics Presuppositions
     description: >
-      LINDSEA pragmatic reasoning (single sentence) is a linguistic diagnostic for pragmatics dataset from BHASA [(Leong, 2023)](https://arxiv.org/abs/2309.06085), involving scalar implicatures and presuppositions.
+      LINDSEA Pragmatics Presuppositions is a linguistic diagnostic for pragmatics dataset from BHASA [(Leong, 2023)](https://arxiv.org/abs/2309.06085), involving two formats: single and pair sentences. For single sentence questions, the system under test needs to determine if the sentence is true/false. For pair sentence questions, the system under test needs to determine whether a conclusion can be drawn from another sentence.
     metric_groups:
       - accuracy
       - efficiency
@@ -685,15 +699,15 @@ run_groups:
       main_split: test
     taxonomy:
       task: pragmatic reasoning
-      what: scalar implicatures and presuppositions
+      what: presuppositions
       who: "?"
       when: "?"
       language: Indonesian
 
-  - name: lindsea_pragmatics_pragmatic_reasoning_pair_id
-    display_name: LINDSEA Pragmatics Pragmatic Reasoning (sentence pair)
+  - name: lindsea_pragmatics_scalar_implicatures_id
+    display_name: LINDSEA Pragmatics Scalar Implicatures
     description: >
-      LINDSEA pragmatic reasoning (sentence pair) is a linguistic diagnostic for pragmatics dataset from BHASA [(Leong, 2023)](https://arxiv.org/abs/2309.06085), involving scalar implicatures and presuppositions.
+      LINDSEA Pragmatics Scalar Implicatures is a linguistic diagnostic for pragmatics dataset from BHASA [(Leong, 2023)](https://arxiv.org/abs/2309.06085), , involving two formats: single and pair sentences. For single sentence questions, the system under test needs to determine if the sentence is true/false. For pair sentence questions, the system under test needs to determine whether a conclusion can be drawn from another sentence.
     metric_groups:
       - accuracy
       - efficiency
@@ -703,7 +717,7 @@ run_groups:
       main_split: test
     taxonomy:
       task: pragmatic reasoning
-      what: scalar implicatures and presuppositions
+      what: scalar implicatures
       who: "?"
       when: "?"
       language: Indonesian

helm/benchmark/static/schema_social_audio.yaml

@@ -0,0 +1,224 @@
+---
+############################################################
+metrics:
+  # Infrastructure metrics:
+  - name: num_perplexity_tokens
+    display_name: '# tokens'
+    description: Average number of tokens in the predicted output (for language modeling, the input too).
+  - name: num_bytes
+    display_name: '# bytes'
+    description: Average number of bytes in the predicted output (for language modeling, the input too).
+
+  - name: num_references
+    display_name: '# ref'
+    description: Number of references.
+  - name: num_train_trials
+    display_name: '# trials'
+    description: Number of trials, where in each trial we choose an independent, random set of training instances.
+  - name: estimated_num_tokens_cost
+    display_name: 'cost'
+    description: An estimate of the number of tokens (including prompt and output completions) needed to perform the request.
+  - name: num_prompt_tokens
+    display_name: '# prompt tokens'
+    description: Number of tokens in the prompt.
+  - name: num_prompt_characters
+    display_name: '# prompt chars'
+    description: Number of characters in the prompt.
+  - name: num_completion_tokens
+    display_name: '# completion tokens'
+    description: Actual number of completion tokens (over all completions).
+  - name: num_output_tokens
+    display_name: '# output tokens'
+    description: Actual number of output tokens.
+  - name: max_num_output_tokens
+    display_name: 'Max output tokens'
+    description: Maximum number of output tokens (overestimate since we might stop earlier due to stop sequences).
+  - name: num_requests
+    display_name: '# requests'
+    description: Number of distinct API requests.
+  - name: num_instances
+    display_name: '# eval'
+    description: Number of evaluation instances.
+  - name: num_train_instances
+    display_name: '# train'
+    description: Number of training instances (e.g., in-context examples).
+  - name: prompt_truncated
+    display_name: truncated
+    description: Fraction of instances where the prompt itself was truncated (implies that there were no in-context examples).
+  - name: finish_reason_length
+    display_name: finish b/c length
+    description: Fraction of instances where the the output was terminated because of the max tokens limit.
+  - name: finish_reason_stop
+    display_name: finish b/c stop
+    description: Fraction of instances where the the output was terminated because of the stop sequences.
+  - name: finish_reason_endoftext
+    display_name: finish b/c endoftext
+    description: Fraction of instances where the the output was terminated because the end of text token was generated.
+  - name: finish_reason_unknown
+    display_name: finish b/c unknown
+    description: Fraction of instances where the the output was terminated for unknown reasons.
+  - name: num_completions
+    display_name: '# completions'
+    description: Number of completions.
+  - name: predicted_index
+    display_name: Predicted index
+    description: Integer index of the reference (0, 1, ...) that was predicted by the model (for multiple-choice).
+
+  # Accuracy metrics:
+  - name: exact_match
+    display_name: Exact match
+    short_display_name: EM
+    description: Fraction of instances that the predicted output matches a correct reference exactly.
+    lower_is_better: false
+  - name: quasi_exact_match
+    display_name: Quasi-exact match
+    short_display_name: EM
+    description: Fraction of instances that the predicted output matches a correct reference up to light processing.
+    lower_is_better: false
+  - name: prefix_exact_match
+    display_name: Prefix exact match
+    short_display_name: PEM
+    description: Fraction of instances that the predicted output matches the prefix of a correct reference exactly.
+    lower_is_better: false
+  - name: quasi_prefix_exact_match
+    # TODO: should call this prefix_quasi_exact_match
+    display_name: Prefix quasi-exact match
+    short_display_name: PEM
+    description: Fraction of instances that the predicted output matches the prefix of a correct reference up to light processing.
+    lower_is_better: false
+
+  - name: rouge_1
+    display_name: ROUGE-1
+    description: Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on 1-gram overlap.
+    lower_is_better: false
+  - name: rouge_2
+    display_name: ROUGE-2
+    description: Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on 2-gram overlap.
+    lower_is_better: false
+  - name: rouge_l
+    display_name: ROUGE-L
+    description: Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on longest common subsequence overlap.
+    lower_is_better: false
+  - name: bleu_1
+    display_name: BLEU-1
+    description: Average BLEU score [(Papineni et al., 2002)](https://aclanthology.org/P02-1040/) based on 1-gram overlap.
+    lower_is_better: false
+  - name: bleu_4
+    display_name: BLEU-4
+    description: Average BLEU score [(Papineni et al., 2002)](https://aclanthology.org/P02-1040/) based on 4-gram overlap.
+    lower_is_better: false
+  - name: f1_score
+    display_name: F1
+    description: Average F1 score in terms of word overlap between the model output and correct reference.
+    lower_is_better: false
+  - name: cider
+    display_name: CIDEr
+    description: Evaluates the quality of generated caption by measuring the weighted similarity of n-grams between the captions and a set of human-written reference captions, emphasizing informativeness and consensus.
+    lower_is_better: false
+
+  # Machine Translation metrics
+  - name: bleu
+    display_name: BLEU
+    short_display_name: BLEU
+    description: BLEU score based on [Post, (2018)](https://aclanthology.org/W18-6319/).
+    lower_is_better: false
+
+############################################################
+perturbations: []
+
+############################################################
+metric_groups:
+  - name: accuracy
+    display_name: Accuracy
+    hide_win_rates: true
+    metrics:
+      - name: ${main_name}
+        split: ${main_split}
+
+  - name: efficiency
+    display_name: Efficiency
+    metrics:
+    - name: inference_runtime
+      split: ${main_split}
+
+  - name: general_information
+    display_name: General information
+    hide_win_rates: true
+    metrics:
+    - name: num_instances
+      split: ${main_split}
+    - name: num_train_instances
+      split: ${main_split}
+    - name: prompt_truncated
+      split: ${main_split}
+    - name: num_prompt_tokens
+      split: ${main_split}
+    - name: num_output_tokens
+      split: ${main_split}
+
+############################################################
+
+run_groups:
+  - name: audio_scenarios
+    display_name: Audio Scenarios
+    description: Audio Scenarios
+    category: All scenarios
+    subgroups:
+      - audio_mnist
+      - meld_audio
+      - iemocap_audio
+
+  - name: audio_mnist
+    display_name: AudioMNIST
+    description: >
+      The AudioMNIST dataset consists of a dataset of 30000 audio samples of
+      spoken digits (0-9) of 60 different speakers. The task is to classify the digit from the
+      audio sample ([Becker et al, 2023](https://arxiv.org/abs/1807.03418)).
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: audio classification
+      what: audio samples of spoken digits (0-9)
+      who: 60 different speakers
+      when: "2018"
+      language: English
+
+  - name: meld_audio
+    display_name: MELD Audio
+    description: Classify emotions in audio clips from the television series Friends ([Poria et al, 2018](https://arxiv.org/abs/1810.02508)).
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: quasi_exact_match
+      main_split: test
+    taxonomy:
+      task: audio classification
+      what: audio clips from the television series Friends
+      who: television actors
+      when: "1994-2004"
+      language: English
+
+  - name: iemocap_audio
+    display_name: IEMOCAP Audio
+    description: >
+      A classification scenario based on audio data from the Interactive emotional dyadic motion capture database (IEMOCAP) ([Busso et al, 2008](https://sail.usc.edu/iemocap/Busso_2008_iemocap.pdf)). The task is to classify the emotion of the speaker(s) in the audio sample.
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: quasi_exact_match
+      main_split: test
+    taxonomy:
+      task: audio classification
+      what: audio recordings improvisations or scripted scenarios
+      who: 10 professional actors
+      when: "2007"
+      language: English

helm/benchmark/static/schema_sql.yaml

@@ -0,0 +1,171 @@
+---
+############################################################
+metrics:
+  # Infrastructure metrics:
+  - name: num_perplexity_tokens
+    display_name: '# tokens'
+    description: Average number of tokens in the predicted output (for language modeling, the input too).
+  - name: num_bytes
+    display_name: '# bytes'
+    description: Average number of bytes in the predicted output (for language modeling, the input too).
+
+  - name: num_references
+    display_name: '# ref'
+    description: Number of references.
+  - name: num_train_trials
+    display_name: '# trials'
+    description: Number of trials, where in each trial we choose an independent, random set of training instances.
+  - name: estimated_num_tokens_cost
+    display_name: 'cost'
+    description: An estimate of the number of tokens (including prompt and output completions) needed to perform the request.
+  - name: num_prompt_tokens
+    display_name: '# prompt tokens'
+    description: Number of tokens in the prompt.
+  - name: num_prompt_characters
+    display_name: '# prompt chars'
+    description: Number of characters in the prompt.
+  - name: num_completion_tokens
+    display_name: '# completion tokens'
+    description: Actual number of completion tokens (over all completions).
+  - name: num_output_tokens
+    display_name: '# output tokens'
+    description: Actual number of output tokens.
+  - name: max_num_output_tokens
+    display_name: 'Max output tokens'
+    description: Maximum number of output tokens (overestimate since we might stop earlier due to stop sequences).
+  - name: num_requests
+    display_name: '# requests'
+    description: Number of distinct API requests.
+  - name: num_instances
+    display_name: '# eval'
+    description: Number of evaluation instances.
+  - name: num_train_instances
+    display_name: '# train'
+    description: Number of training instances (e.g., in-context examples).
+  - name: prompt_truncated
+    display_name: truncated
+    description: Fraction of instances where the prompt itself was truncated (implies that there were no in-context examples).
+  - name: finish_reason_length
+    display_name: finish b/c length
+    description: Fraction of instances where the the output was terminated because of the max tokens limit.
+  - name: finish_reason_stop
+    display_name: finish b/c stop
+    description: Fraction of instances where the the output was terminated because of the stop sequences.
+  - name: finish_reason_endoftext
+    display_name: finish b/c endoftext
+    description: Fraction of instances where the the output was terminated because the end of text token was generated.
+  - name: finish_reason_unknown
+    display_name: finish b/c unknown
+    description: Fraction of instances where the the output was terminated for unknown reasons.
+  - name: num_completions
+    display_name: '# completions'
+    description: Number of completions.
+  - name: predicted_index
+    display_name: Predicted index
+    description: Integer index of the reference (0, 1, ...) that was predicted by the model (for multiple-choice).
+
+  # Accuracy metrics:
+  - name: program_accuracy
+    display_name: Program Accuracy
+    description: Accuracy of the generated programs
+    lower_is_better: false
+  - name: execution_accuracy
+    display_name: Execution Accuracy
+    description: Accuracy of the final result of the generated program
+    lower_is_better: false
+  - name: annotation_financebench_label_correct_answer
+    display_name: Correct Answer
+    description: Whether the final result was correct, as judged by a GPT-4o
+    lower_is_better: false
+  - name: quasi_exact_match
+    display_name: Quasi-exact match
+    short_display_name: EM
+    description: Fraction of instances that the predicted output matches a correct reference up to light processing.
+    lower_is_better: false
+  - name: error_rate
+    display_name: SQL Error Rate
+    short_display_name: SQL Error Rate
+    description: Fraction of generated queries that result in a SQL execution error
+    lower_is_better: true
+  - name: execution_accuracy
+    display_name: Execution Accuracy
+    short_display_name: Execution Accuracy
+    description: Execution Accuracy
+    lower_is_better: false
+
+############################################################
+perturbations: []
+
+############################################################
+metric_groups:
+  - name: accuracy
+    display_name: Accuracy
+    hide_win_rates: true
+    metrics:
+      - name: ${main_name}
+        split: ${main_split}
+
+  - name: efficiency
+    display_name: Efficiency
+    metrics:
+    - name: inference_runtime
+      split: ${main_split}
+
+  - name: general_information
+    display_name: General information
+    hide_win_rates: true
+    metrics:
+    - name: num_instances
+      split: ${main_split}
+    - name: num_train_instances
+      split: ${main_split}
+    - name: prompt_truncated
+      split: ${main_split}
+    - name: num_prompt_tokens
+      split: ${main_split}
+    - name: num_output_tokens
+      split: ${main_split}
+
+############################################################
+run_groups:
+  - name: text_to_sql_scenarios
+    display_name: Text-to-SQL Scenarios
+    description: Text-to-SQL Scenarios
+    category: All scenarios
+    subgroups:
+      - spider
+      - bird_sql
+
+  - name: spider
+    display_name: Spider 1.0 (Test)
+    description: Spider 1.0 (Test)
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: execution_accuracy
+      main_split: valid
+    taxonomy:
+      task: text-to-SQL
+      what: databases from various domains
+      who: expert data scientists
+      when: "?"
+      language: English
+
+  - name: bird_sql
+    display_name: BIRD-SQL (Dev)
+    description: BIRD-SQL (Dev)
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: execution_accuracy
+      main_split: valid
+    taxonomy:
+      task: text-to-SQL
+      what: databases from various domains
+      who: expert data scientists
+      when: "?"
+      language: English