crfm-helm 0.5.5__py3-none-any.whl → 0.5.7__py3-none-any.whl

helm/benchmark/static/schema_slphelm.yaml

@@ -0,0 +1,162 @@
+---
+############################################################
+metrics:
+  # Accuracy metrics:
+  - name: exact_match
+    display_name: Exact match
+    short_display_name: EM
+    description: Fraction of instances that the predicted output matches a correct reference exactly.
+    lower_is_better: false
+  - name: quasi_exact_match
+    display_name: Quasi-exact match
+    short_display_name: EM
+    description: Fraction of instances that the predicted output matches a correct reference up to light processing.
+    lower_is_better: false
+
+  # Classification metrics:
+  - name: classification_macro_f1
+    display_name: Macro-F1
+    description: Population-level macro-averaged F1 score.
+    lower_is_better: false
+  - name: classification_micro_f1
+    display_name: Micro-F1
+    description: Population-level micro-averaged F1 score.
+    lower_is_better: false
+
+  # Speech-specific metrics:
+  - name: wer
+    display_name: Word Error Rate
+    short_display_name: WER
+    description: Word Error Rate for automatic speech recognition evaluation.
+    lower_is_better: true
+  - name: mer
+    display_name: Match Error Rate
+    short_display_name: MER
+    description: Match Error Rate for automatic speech recognition evaluation.
+    lower_is_better: true
+  - name: wip
+    display_name: Word Information Preserved
+    short_display_name: WIP
+    description: Word Information Preserved for automatic speech recognition evaluation.
+    lower_is_better: false
+
+############################################################
+metric_groups:
+  - name: accuracy
+    display_name: Accuracy
+    metrics:
+      - name: ${main_name}
+        split: ${main_split}
+
+  - name: classification_metrics
+    display_name: Classification metrics
+    metrics:
+      - name: classification_macro_f1
+        split: ${main_split}
+      - name: classification_micro_f1
+        split: ${main_split}
+      - name: exact_match
+        split: ${main_split}
+
+  - name: speech_metrics
+    display_name: Speech metrics
+    metrics:
+      - name: wer
+        split: ${main_split}
+      - name: mer
+        split: ${main_split}
+      - name: wip
+        split: ${main_split}
+
+#######################################################
+run_groups:
+  - name: slphelm
+    display_name: SLPHelm Scenarios
+    description: Scenarios for speech language processing evaluation
+    category: All scenarios
+    subgroups:
+      - slphelm_disorder_diagnosis
+      - slphelm_asr_disorder_diagnosis
+      - slphelm_asr_transcription
+      - slphelm_disorder_type_classification
+      - slphelm_disorder_symptom_classification
+
+  - name: slphelm_disorder_diagnosis
+    display_name: Disorder Diagnosis
+    description: Speech-based disorder diagnosis evaluation using audio input.
+    category: Disorder Diagnosis
+    metric_groups:
+      - classification_metrics
+    environment:
+      main_name: classification_macro_f1
+      main_split: test
+    taxonomy:
+      task: disorder diagnosis
+      what: "Speech disorder classification from audio"
+      who: "Children with speech disorders"
+      when: "Clinical assessment"
+      language: English
+
+  - name: slphelm_asr_disorder_diagnosis
+    display_name: ASR-Based Disorder Diagnosis
+    description: Disorder diagnosis based on automatic speech recognition transcription accuracy.
+    category: ASR-Based Disorder Diagnosis
+    metric_groups:
+      - classification_metrics
+    environment:
+      main_name: classification_macro_f1
+      main_split: test
+    taxonomy:
+      task: asr-based disorder diagnosis
+      what: "Disorder classification from transcription accuracy"
+      who: "Children with speech disorders"
+      when: "Clinical assessment"
+      language: English
+
+  - name: slphelm_asr_transcription
+    display_name: ASR-Transcription
+    description: Automatic speech recognition transcription quality evaluation.
+    category: ASR-Transcription
+    metric_groups:
+      - speech_metrics
+    environment:
+      main_name: wer
+      main_split: test
+    taxonomy:
+      task: automatic speech recognition
+      what: "Speech transcription accuracy"
+      who: "Children with speech disorders"
+      when: "Clinical assessment"
+      language: English
+
+  - name: slphelm_disorder_type_classification
+    display_name: Disorder Type Classification
+    description: Classification of specific types of speech disorders.
+    category: Disorder Type Classification
+    metric_groups:
+      - classification_metrics
+    environment:
+      main_name: classification_macro_f1
+      main_split: test
+    taxonomy:
+      task: disorder type classification
+      what: "Specific speech disorder type identification"
+      who: "Children with speech disorders"
+      when: "Clinical assessment"
+      language: English
+
+  - name: slphelm_disorder_symptom_classification
+    display_name: Disorder Symptom Classification
+    description: Classification of specific symptoms within speech disorders.
+    category: Disorder Symptom Classification
+    metric_groups:
+      - classification_metrics
+    environment:
+      main_name: classification_macro_f1
+      main_split: test
+    taxonomy:
+      task: disorder symptom classification
+      what: "Speech disorder symptom identification"
+      who: "Children with speech disorders"
+      when: "Clinical assessment"
+      language: English

helm/benchmark/static/schema_vhelm.yaml

@@ -307,6 +307,8 @@ run_groups:
     description: Does the model understand objects, counts and spatial relations? Can the model reason about both the text and image input?
     category: Core scenarios
     subgroups:
+      - mmmu
+      - exams_v
       - gqa
       - math_vista
       - seed_bench
@@ -320,7 +322,6 @@ run_groups:
     category: Core scenarios
     subgroups:
       - a_okvqa_base
-      - mmmu
       - mme
       - vibe_eval
       - mm_star_knowledge
@@ -369,7 +370,6 @@ run_groups:
       - a_okvqa_hindi
       - a_okvqa_spanish
       - a_okvqa_swahili
-      - exams_v
       - bingo_multilinguality
   - name: a_okvqa_base
     display_name: A-OKVQA
@@ -378,7 +378,7 @@ run_groups:
       - accuracy
       - general_information
     environment:
-      main_name: exact_match
+      main_name: quasi_prefix_exact_match
       main_split: valid
     taxonomy:
       task: multiple-choice question answering
@@ -394,7 +394,7 @@ run_groups:
       - fairness
       - general_information
     environment:
-      main_name: exact_match
+      main_name: quasi_prefix_exact_match
       main_split: valid
     taxonomy:
       task: multiple-choice question answering
@@ -410,7 +410,7 @@ run_groups:
       - translate
       - general_information
     environment:
-      main_name: exact_match
+      main_name: quasi_prefix_exact_match
       main_split: valid
     taxonomy:
       task: multiple-choice question answering
@@ -426,7 +426,7 @@ run_groups:
       - translate
       - general_information
     environment:
-      main_name: exact_match
+      main_name: quasi_prefix_exact_match
       main_split: valid
     taxonomy:
       task: multiple-choice question answering
@@ -442,7 +442,7 @@ run_groups:
       - translate
       - general_information
     environment:
-      main_name: exact_match
+      main_name: quasi_prefix_exact_match
       main_split: valid
     taxonomy:
       task: multiple-choice question answering
@@ -458,7 +458,7 @@ run_groups:
       - translate
       - general_information
     environment:
-      main_name: exact_match
+      main_name: quasi_prefix_exact_match
       main_split: valid
     taxonomy:
       task: multiple-choice question answering
@@ -474,7 +474,7 @@ run_groups:
       - accuracy
       - general_information
     environment:
-      main_name: exact_match
+      main_name: quasi_prefix_exact_match
       main_split: valid
     taxonomy:
       task: multiple-choice question answering
@@ -490,7 +490,7 @@ run_groups:
       - accuracy
       - general_information
     environment:
-      main_name: exact_match
+      main_name: quasi_prefix_exact_match
       main_split: valid
     taxonomy:
       task: multiple-choice question answering
@@ -506,7 +506,7 @@ run_groups:
       - accuracy
       - general_information
     environment:
-      main_name: exact_match
+      main_name: quasi_prefix_exact_match
       main_split: valid
     taxonomy:
       task: multiple-choice question answering
@@ -522,7 +522,7 @@ run_groups:
       - accuracy
       - general_information
     environment:
-      main_name: exact_match
+      main_name: quasi_prefix_exact_match
       main_split: valid
     taxonomy:
       task: multiple-choice question answering
@@ -538,7 +538,7 @@ run_groups:
       - accuracy
       - general_information
     environment:
-      main_name: exact_match
+      main_name: quasi_prefix_exact_match
       main_split: valid
     taxonomy:
       task: multiple-choice question answering
@@ -554,7 +554,7 @@ run_groups:
       - accuracy
       - general_information
     environment:
-      main_name: exact_match
+      main_name: quasi_prefix_exact_match
       main_split: valid
     taxonomy:
       task: multiple-choice question answering
@@ -602,7 +602,7 @@ run_groups:
       - accuracy
       - general_information
     environment:
-      main_name: quasi_exact_match
+      main_name: quasi_prefix_exact_match
       main_split: valid
     taxonomy:
       task: short-answer question answering
@@ -618,7 +618,7 @@ run_groups:
       - accuracy
       - general_information
     environment:
-      main_name: exact_match
+      main_name: quasi_prefix_exact_match
       main_split: test
     taxonomy:
       task: toxicity identification
@@ -651,7 +651,7 @@ run_groups:
       - accuracy
       - general_information
     environment:
-      main_name: quasi_exact_match
+      main_name: quasi_prefix_exact_match
       main_split: valid
     taxonomy:
       task: short-answer question answering
@@ -667,7 +667,7 @@ run_groups:
       - accuracy
       - general_information
     environment:
-      main_name: quasi_exact_match
+      main_name: quasi_prefix_exact_match
       main_split: valid
     taxonomy:
       task: short-answer question answering
@@ -683,7 +683,7 @@ run_groups:
       - fairness
       - general_information
     environment:
-      main_name: quasi_exact_match
+      main_name: quasi_prefix_exact_match
       main_split: valid
     taxonomy:
       task: short-answer question answering
@@ -715,7 +715,7 @@ run_groups:
       - accuracy
       - general_information
     environment:
-      main_name: exact_match
+      main_name: quasi_prefix_exact_match
       main_split: valid
     taxonomy:
       task: multiple-choice question answering
@@ -795,7 +795,7 @@ run_groups:
       - accuracy
       - general_information
     environment:
-      main_name: exact_match
+      main_name: quasi_prefix_exact_match
       main_split: test
     taxonomy:
       task: short-answer question answering
@@ -811,7 +811,7 @@ run_groups:
       - accuracy
       - general_information
     environment:
-      main_name: exact_match
+      main_name: quasi_prefix_exact_match
       main_split: test
     taxonomy:
       task: multiple-choice question answering
@@ -827,7 +827,7 @@ run_groups:
       - accuracy
       - general_information
     environment:
-      main_name: exact_match
+      main_name: quasi_prefix_exact_match
       main_split: test
     taxonomy:
       task: multiple-choice question answering
@@ -875,7 +875,7 @@ run_groups:
       - accuracy
       - general_information
     environment:
-      main_name: exact_match
+      main_name: quasi_prefix_exact_match
       main_split: test
     taxonomy:
       task: multiple-choice question answering
@@ -891,7 +891,7 @@ run_groups:
       - accuracy
       - general_information
     environment:
-      main_name: exact_match
+      main_name: quasi_prefix_exact_match
       main_split: valid
     taxonomy:
       task: multiple-choice question answering
@@ -923,7 +923,7 @@ run_groups:
       - accuracy
       - general_information
     environment:
-      main_name: exact_match
+      main_name: quasi_prefix_exact_match
       main_split: test
     taxonomy:
       task: multiple-choice question answering

helm/benchmark/static/schema_video.yaml

@@ -0,0 +1,219 @@
+---
+############################################################
+perturbations: []
+############################################################
+metrics:
+  - name: num_references
+    display_name: '# ref'
+    description: Number of references.
+  - name: num_train_trials
+    display_name: '# trials'
+    description: Number of trials, where in each trial we choose an independent, random set of training instances.
+  - name: estimated_num_tokens_cost
+    display_name: 'cost'
+    description: An estimate of the number of tokens (including prompt and output completions) needed to perform the request.
+  - name: num_prompt_tokens
+    display_name: '# prompt tokens'
+    description: Number of tokens in the prompt.
+  - name: num_prompt_characters
+    display_name: '# prompt chars'
+    description: Number of characters in the prompt.
+  - name: num_completion_tokens
+    display_name: '# completion tokens'
+    description: Actual number of completion tokens (over all completions).
+  - name: num_output_tokens
+    display_name: '# output tokens'
+    description: Actual number of output tokens.
+  - name: max_num_output_tokens
+    display_name: 'Max output tokens'
+    description: Maximum number of output tokens (overestimate since we might stop earlier due to stop sequences).
+  - name: num_requests
+    display_name: '# requests'
+    description: Number of distinct API requests.
+  - name: num_instances
+    display_name: '# eval'
+    description: Number of evaluation instances.
+  - name: num_train_instances
+    display_name: '# train'
+    description: Number of training instances (e.g., in-context examples).
+  - name: prompt_truncated
+    display_name: truncated
+    description: Fraction of instances where the prompt itself was truncated (implies that there were no in-context examples).
+  - name: finish_reason_length
+    display_name: finish b/c length
+    description: Fraction of instances where the the output was terminated because of the max tokens limit.
+  - name: finish_reason_stop
+    display_name: finish b/c stop
+    description: Fraction of instances where the the output was terminated because of the stop sequences.
+  - name: finish_reason_endoftext
+    display_name: finish b/c endoftext
+    description: Fraction of instances where the the output was terminated because the end of text token was generated.
+  - name: finish_reason_unknown
+    display_name: finish b/c unknown
+    description: Fraction of instances where the the output was terminated for unknown reasons.
+  - name: num_completions
+    display_name: '# completions'
+    description: Number of completions.
+  - name: predicted_index
+    display_name: Predicted index
+    description: Integer index of the reference (0, 1, ...) that was predicted by the model (for multiple-choice).
+
+  # Vision Language metrics [image]:
+  - name: earth_mover_similarity
+    display_name: Earth Mover Similarity
+    short_display_name: EMD-Sim
+    description: 1 - Earth Mover Distance [(Rubner and Tomasi, 2000)](https://www.cs.cmu.edu/~efros/courses/LBMV07/Papers/rubner-jcviu-00.pdf) between an image generated by the model and the target image.
+    lower_is_better: false
+  - name: pixel_similarity
+    display_name: Pixel Similarity
+    short_display_name: PS
+    description: Pixel Similarity between an image generated by the model and the target image.
+    lower_is_better: false
+  - name: sift_similarity
+    display_name: SIFT Similarity
+    short_display_name: SIFT
+    description: SIFT Similarity (Scale-Invariant Feature Transform) [(Lowe, 1999)](https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=790410) between an image generated by the model and the target image.
+    lower_is_better: false
+  - name: compilation_success
+    display_name: Compilation success
+    description: Fraction of instances where the generated code compiles successfully.
+    lower_is_better: false
+  - name: lpips_similarity
+    display_name: LPIPS similarity
+    short_display_name: LPIPS
+    description: LPIPS similarity (Learned Perceptual Image Patch Similarity) [(Zhang et al., 2018)](https://arxiv.org/abs/1801.03924) between an image generated by the model and the target image.
+    lower_is_better: false
+  - name: fid_similarity
+    display_name: FID similarity
+    short_display_name: FID
+    description: FID similarity (Fréchet Inception Distance) [(Heusel et al., 2017)](https://arxiv.org/abs/1706.08500) between an image generated by the model and the target image.
+    lower_is_better: false
+  - name: ssim_similarity
+    display_name: SSIM
+    short_display_name: SSIM
+    description: SSIM similarity (Structural Similarity Index) [(Wang et al., 2004)](https://www.cns.nyu.edu/pub/eero/wang03-reprint.pdf) between an image generated by the model and the target image.
+    lower_is_better: false
+
+  # Accuracy metrics:
+  - name: exact_match
+    display_name: Exact match
+    short_display_name: EM
+    description: Fraction of instances that the predicted output matches a correct reference exactly.
+    lower_is_better: false
+  - name: quasi_exact_match
+    display_name: Quasi-exact match
+    short_display_name: EM
+    description: Fraction of instances that the predicted output matches a correct reference up to light processing.
+    lower_is_better: false
+  - name: quasi_leave_articles_exact_match
+    display_name: Quasi-exact match
+    short_display_name: EM
+    description: Fraction of instances that the predicted output matches a correct reference up to light processing.
+    lower_is_better: false
+  - name: prefix_exact_match
+    display_name: Prefix exact match
+    short_display_name: PEM
+    description: Fraction of instances that the predicted output matches the prefix of a correct reference exactly.
+    lower_is_better: false
+  - name: quasi_prefix_exact_match
+    # TODO: should call this prefix_quasi_exact_match
+    display_name: Prefix quasi-exact match
+    short_display_name: PEM
+    description: Fraction of instances that the predicted output matches the prefix of a correct reference up to light processing.
+    lower_is_better: false
+
+  - name: rouge_1
+    display_name: ROUGE-1
+    description: Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on 1-gram overlap.
+    lower_is_better: false
+  - name: rouge_2
+    display_name: ROUGE-2
+    description: Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on 2-gram overlap.
+    lower_is_better: false
+  - name: rouge_l
+    display_name: ROUGE-L
+    description: Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on longest common subsequence overlap.
+    lower_is_better: false
+  - name: bleu_1
+    display_name: BLEU-1
+    description: Average BLEU score [(Papineni et al., 2002)](https://aclanthology.org/P02-1040/) based on 1-gram overlap.
+    lower_is_better: false
+  - name: bleu_4
+    display_name: BLEU-4
+    description: Average BLEU score [(Papineni et al., 2002)](https://aclanthology.org/P02-1040/) based on 4-gram overlap.
+    lower_is_better: false
+  - name: f1_score
+    display_name: F1
+    description: Average F1 score in terms of word overlap between the model output and correct reference.
+    lower_is_better: false
+  - name: cider
+    display_name: CIDEr
+    description: Evaluates the quality of generated caption by measuring the weighted similarity of n-grams between the captions and a set of human-written reference captions, emphasizing informativeness and consensus.
+    lower_is_better: false
+  - name: prometheus_vision
+    display_name: Prometheus Vision rating
+    description: Scores are from 1 to 5, where 5 is the best.
+    lower_is_better: false
+
+  # Toxicity metrics
+  - name: expected_max_toxicity
+    display_name: Average max toxicity
+    short_display_name: Avg. max toxicity
+    lower_is_better: true
+    description: Average max toxicity score across all completions (based on the PerspectiveAPI toxicity classifier; [Gehman et al. (2020)](https://aclanthology.org/2020.findings-emnlp.301/)).
+  - name: max_toxicity_probability
+    display_name: max toxicity
+    lower_is_better: true
+    description: Fraction of instances where most toxic completion is above 0.5 toxicity (based on the PerspectiveAPI toxicity classifier; [Gehman et al. (2020)](https://aclanthology.org/2020.findings-emnlp.301/)).
+  - name: toxic_frac
+    display_name: Toxic fraction
+    lower_is_better: true
+    description: Fraction of model outputs that are toxic (based on the PerspectiveAPI toxicity classifier).
+
+############################################################
+metric_groups:
+  - name: accuracy
+    display_name: Accuracy
+    metrics:
+      - name: ${main_name}
+        split: ${main_split}
+
+  - name: general_information
+    display_name: General information
+    metrics:
+    - name: num_instances
+      split: ${main_split}
+    - name: num_train_instances
+      split: ${main_split}
+    - name: prompt_truncated
+      split: ${main_split}
+    - name: num_prompt_tokens
+      split: ${main_split}
+    - name: num_output_tokens
+      split: ${main_split}
+
+
+############################################################
+run_groups:
+  - name: core_scenarios
+    display_name: All
+    description: All scenarios across capabilities
+    category: All scenarios
+    subgroups:
+      - msr_vtt
+
+  - name: msr_vtt
+    display_name: MST-VTT
+    description: A large-scale video benchmark for video understanding, especially the emerging task of translating video to text.
+    metric_groups:
+      - accuracy
+      - general_information
+    environment:
+      main_name: f1_score
+      main_split: test
+    taxonomy:
+      task: captioning
+      what: Real-world videos
+      who: Human experts
+      when: "2016"
+      language: English