crfm-helm 0.5.1__py3-none-any.whl → 0.5.2__py3-none-any.whl

helm/benchmark/static/schema_classic.yaml

@@ -1,64 +1,8 @@
 ---
 ############################################################
-adapter:
-  - name: method
-    description: The high-level strategy for converting instances into a prompt for the language model.
-    values:
-      - name: generation
-        description: Given the input, the model generates the output free-form.
-      - name: multiple_choice_joint
-        description: Given the input, the model selects from multiple-choice options (A., B., C., D., E.).
-      - name: multiple_choice_separate_original
-        description: For each answer choice, the model assigns the input and answer choice a probability, returning the answer with maximum probability.
-      - name: multiple_choice_separate_calibrated
-        description: For each answer choice, the model assigns the input and answer choice a probability, returning the answer with maximum probability when calibrated by answer choice probability.
-      - name: language_modeling
-        description: Given the input, the model assigns the sequence a probability.
-  - name: instructions
-    description: The description of the task that is included at the very beginning of the prompt.
-  - name: global_prefix
-    description: The string that is prepended to the prompt.
-  - name: instance_prefix
-    description: The string that is included before each instance (e.g., '\n\n').
-  - name: input_prefix
-    description: The string that is included before each input (e.g., 'Question:').
-  - name: input_suffix
-    description: The string that is included after each input (e.g., '\n').
-  - name: reference_prefix
-    description: The string that is included before each reference (for multiple-choice questions).
-  - name: reference_suffix
-    description: The string that is included after each reference (for multiple-choice questions).
-  - name: output_prefix
-    description: The string that is included before the correct answer/predicted output (e.g., 'Answer:').
-  - name: output_suffix
-    description: The string that is included after the correct answer/predicted output (e.g., '\n').
-  - name: substitutions
-    description: A list of regular expression substitutions (e.g., replacing '\n' with ';\n') to perform at the very end on the prompt.
-  - name: max_train_instances
-    description: Maximum number of training instances to include in the prompt (currently by randomly sampling).
-  - name: max_eval_instances
-    description: Maximum number of instances to evaluate on (over all splits - test, valid, etc.).
-  - name: num_outputs
-    description: Maximum number of possible outputs to generate by sampling multiple outputs.
-  - name: num_train_trials
-    description: Number of trials, where in each trial we choose an independent, random set of training instances. Used to compute variance.
-  - name: sample_train
-    description: If true, randomly sample N training examples; if false, select N consecutive training examples
-  - name: model
-    description: Name of the language model (<creator_organization>/<model name>) to send requests to.
-  - name: model_deployment
-    description: Name of the language model deployment (<host_organization>/<model name>) to send requests to.
-  - name: temperature
-    description: Temperature parameter used in generation.
-  - name: max_tokens
-    description: Maximum number of tokens to generate.
-  - name: stop_sequences
-    description: List of sequences, where we stop generation if we encounter any of them.
-  - name: random
-    description: Random seed (string), which guarantees reproducibility.
-  - name: multi_label
-    description: If true, for instances with multiple correct reference, the gold answer should be considered to be all of the correct references rather than any of the correct references.
-
+# For backwards compatibility with older versions of HELM.
+# TODO: Remove this after 2024-09-01.
+adapter: []
 ############################################################
 metrics:
   # Infrastructure metrics:

helm/benchmark/static/schema_finance.yaml

@@ -0,0 +1,143 @@
+---
+############################################################
+metrics:
+  # Infrastructure metrics:
+  - name: num_perplexity_tokens
+    display_name: '# tokens'
+    description: Average number of tokens in the predicted output (for language modeling, the input too).
+  - name: num_bytes
+    display_name: '# bytes'
+    description: Average number of bytes in the predicted output (for language modeling, the input too).
+
+  - name: num_references
+    display_name: '# ref'
+    description: Number of references.
+  - name: num_train_trials
+    display_name: '# trials'
+    description: Number of trials, where in each trial we choose an independent, random set of training instances.
+  - name: estimated_num_tokens_cost
+    display_name: 'cost'
+    description: An estimate of the number of tokens (including prompt and output completions) needed to perform the request.
+  - name: num_prompt_tokens
+    display_name: '# prompt tokens'
+    description: Number of tokens in the prompt.
+  - name: num_prompt_characters
+    display_name: '# prompt chars'
+    description: Number of characters in the prompt.
+  - name: num_completion_tokens
+    display_name: '# completion tokens'
+    description: Actual number of completion tokens (over all completions).
+  - name: num_output_tokens
+    display_name: '# output tokens'
+    description: Actual number of output tokens.
+  - name: max_num_output_tokens
+    display_name: 'Max output tokens'
+    description: Maximum number of output tokens (overestimate since we might stop earlier due to stop sequences).
+  - name: num_requests
+    display_name: '# requests'
+    description: Number of distinct API requests.
+  - name: num_instances
+    display_name: '# eval'
+    description: Number of evaluation instances.
+  - name: num_train_instances
+    display_name: '# train'
+    description: Number of training instances (e.g., in-context examples).
+  - name: prompt_truncated
+    display_name: truncated
+    description: Fraction of instances where the prompt itself was truncated (implies that there were no in-context examples).
+  - name: finish_reason_length
+    display_name: finish b/c length
+    description: Fraction of instances where the the output was terminated because of the max tokens limit.
+  - name: finish_reason_stop
+    display_name: finish b/c stop
+    description: Fraction of instances where the the output was terminated because of the stop sequences.
+  - name: finish_reason_endoftext
+    display_name: finish b/c endoftext
+    description: Fraction of instances where the the output was terminated because the end of text token was generated.
+  - name: finish_reason_unknown
+    display_name: finish b/c unknown
+    description: Fraction of instances where the the output was terminated for unknown reasons.
+  - name: num_completions
+    display_name: '# completions'
+    description: Number of completions.
+  - name: predicted_index
+    display_name: Predicted index
+    description: Integer index of the reference (0, 1, ...) that was predicted by the model (for multiple-choice).
+
+  # Accuracy metrics:
+  - name: program_accuracy
+    display_name: Program Accuracy
+    description: Accuracy of the generated programs
+    lower_is_better: false
+  - name: execution_accuracy
+    display_name: Execution Accuracy
+    description: Accuracy of the final result of the generated program
+    lower_is_better: false
+
+############################################################
+perturbations: []
+
+############################################################
+metric_groups:
+  - name: accuracy
+    display_name: Accuracy
+    metrics:
+      - name: ${main_name}
+        split: ${main_split}
+
+  - name: efficiency
+    display_name: Efficiency
+    metrics:
+    - name: inference_runtime
+      split: ${main_split}
+
+  - name: general_information
+    display_name: General information
+    hide_win_rates: true
+    metrics:
+    - name: num_instances
+      split: ${main_split}
+    - name: num_train_instances
+      split: ${main_split}
+    - name: prompt_truncated
+      split: ${main_split}
+    - name: num_prompt_tokens
+      split: ${main_split}
+    - name: num_output_tokens
+      split: ${main_split}
+
+############################################################
+run_groups:
+  - name: financial_scenarios
+    display_name: Financial Scenarios
+    description: Scenarios for the financial domain
+    category: All scenarios
+    subgroups:
+      - fin_qa
+
+  - name: fin_qa
+    display_name: FinQA
+    description: The FinQA benchmark for numeric reasoning over financial data, with question answering pairs written by financial experts over financial reports [(Chen et al., 2021)](https://arxiv.org/abs/2109.00122/).
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: program_accuracy
+      main_split: test
+    taxonomy:
+      task: question answering with numeric reasoning
+      what: financial reports
+      who: financial experts
+      when: 1999 to 2019
+      language: English
+
+  - name: financial_scenarios_ablations
+    display_name: Financial Scenarios Ablations
+    description: Scenarios for the financial domain with ablations
+    category: All scenarios
+    subgroups:
+      - fin_qa
+    adapter_keys_shown:
+      - model
+      - max_train_instances

helm/benchmark/static/schema_image2structure.yaml

@@ -1,78 +1,11 @@
 ---
 ############################################################
-adapter:
-  - name: method
-    description: The high-level strategy for converting instances into a prompt for the language model.
-    values:
-      - name: generation
-        description: Given the input, the model generates the output free-form.
-      - name: generation_multimodal
-        description: Given the multimodal input, the model generates the output free-form.
-      - name: multiple_choice_joint
-        description: Given the input, the model selects from multiple-choice options (A., B., C., D., E.).
-      - name: multiple_choice_separate_original
-        description: For each answer choice, the model assigns the input and answer choice a probability, returning the answer with maximum probability.
-      - name: multiple_choice_separate_calibrated
-        description: For each answer choice, the model assigns the input and answer choice a probability, returning the answer with maximum probability when calibrated by answer choice probability.
-      - name: language_modeling
-        description: Given the input, the model assigns the sequence a probability.
-  - name: instructions
-    description: The description of the task that is included at the very beginning of the prompt.
-  - name: global_prefix
-    description: The string that is prepended to the prompt.
-  - name: global_suffix
-    description: The string that is appended to the prompt.
-  - name: instance_prefix
-    description: The string that is included before each instance (e.g., '\n\n').
-  - name: input_prefix
-    description: The string that is included before each input (e.g., 'Question:').
-  - name: input_suffix
-    description: The string that is included after each input (e.g., '\n').
-  - name: reference_prefix
-    description: The string that is included before each reference (for multiple-choice questions).
-  - name: reference_suffix
-    description: The string that is included after each reference (for multiple-choice questions).
-  - name: output_prefix
-    description: The string that is included before the correct answer/predicted output (e.g., 'Answer:').
-  - name: output_suffix
-    description: The string that is included after the correct answer/predicted output (e.g., '\n').
-  - name: substitutions
-    description: A list of regular expression substitutions (e.g., replacing '\n' with ';\n') to perform at the very end on the prompt.
-  - name: max_train_instances
-    description: Maximum number of training instances to include in the prompt (currently by randomly sampling).
-  - name: max_eval_instances
-    description: Maximum number of instances to evaluate on (over all splits - test, valid, etc.).
-  - name: num_outputs
-    description: Maximum number of possible outputs to generate by sampling multiple outputs.
-  - name: num_train_trials
-    description: Number of trials, where in each trial we choose an independent, random set of training instances. Used to compute variance.
-  - name: sample_train
-    description: If true, randomly sample N training examples; if false, select N consecutive training examples
-  - name: model
-    description: Name of the language model (<creator_organization>/<model name>) to send requests to.
-  - name: model_deployment
-    description: Name of the language model deployment (<host_organization>/<model name>) to send requests to.
-  - name: temperature
-    description: Temperature parameter used in generation.
-  - name: max_tokens
-    description: Maximum number of tokens to generate.
-  - name: stop_sequences
-    description: List of sequences, where we stop generation if we encounter any of them.
-  - name: random
-    description: Random seed (string), which guarantees reproducibility.
-  - name: multi_label
-    description: If true, for instances with multiple correct reference, the gold answer should be considered to be all of the correct references rather than any of the correct references.
-
+# For backwards compatibility with older versions of HELM.
+# TODO: Remove this after 2024-09-01.
+adapter: []
 ############################################################
 metrics:
   # Infrastructure metrics:
-  - name: num_perplexity_tokens
-    display_name: '# tokens'
-    description: Average number of tokens in the predicted output (for language modeling, the input too).
-  - name: num_bytes
-    display_name: '# bytes'
-    description: Average number of bytes in the predicted output (for language modeling, the input too).
-
   - name: num_references
     display_name: '# ref'
     description: Number of references.
@@ -136,50 +69,25 @@ metrics:
     description: Average Levenshtein edit similarity (1 - distance normalized by length of longer sequence) between model generation and reference.
 
   # Vision Language metrics [image]:
-  - name: block_emd_similarity
-    display_name: Block Earth Mover Similarity
-    short_display_name: Block EMS
-    description: Block Earth Mover Similarity
-    lower_is_better: false
-  - name: block_emd_similarity_white
-    display_name: Block Earth Mover Similarity (white)
-    short_display_name: Block EMS (white)
-    description: Block Earth Mover Similarity (white)
-    lower_is_better: false
-  - name: block_emd_similarity_median_color
-    display_name: Block Earth Mover Similarity (median)
-    short_display_name: Block EMS (median)
-    description: Block Earth Mover Similarity (median)
+  - name: earth_mover_similarity
+    display_name: Earth Mover Similarity
+    short_display_name: EMS
+    description: Earth Mover Similarity (EMD adapted to speed up calculations)
     lower_is_better: false
   - name: pixel_similarity
     display_name: Pixel Similarity
     short_display_name: PS
     description: Pixel Similarity between an image generated by the model and the target image.
     lower_is_better: false
-  - name: sift_similarity
-    display_name: SIFT Similarity
-    short_display_name: SIFT
-    description: SIFT Similarity (Scale-Invariant Feature Transform) [(Lowe, 1999)](https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=790410) between an image generated by the model and the target image.
-    lower_is_better: false
   - name: compilation_success
     display_name: Compilation success
     description: Fraction of instances where the generated code compiles successfully.
     lower_is_better: false
-  - name: lpips_similarity
-    display_name: LPIPS similarity
-    short_display_name: LPIPS
-    description: LPIPS similarity (Learned Perceptual Image Patch Similarity) [(Zhang et al., 2018)](https://arxiv.org/abs/1801.03924) between an image generated by the model and the target image.
-    lower_is_better: false
   - name: fid_similarity
     display_name: FID similarity
     short_display_name: FID
     description: FID similarity (Fréchet Inception Distance) [(Heusel et al., 2017)](https://arxiv.org/abs/1706.08500) between an image generated by the model and the target image.
     lower_is_better: false
-  - name: ssim_similarity
-    display_name: SSIM
-    short_display_name: SSIM
-    description: SSIM similarity (Structural Similarity Index) [(Wang et al., 2004)](https://www.cns.nyu.edu/pub/eero/wang03-reprint.pdf) between an image generated by the model and the target image.
-    lower_is_better: false
 
   # Accuracy metrics:
   - name: exact_match
@@ -215,11 +123,37 @@ metric_groups:
   - name: accuracy
     display_name: Compilation Rate and Earth Mover Similarity
     metrics:
-      - name: ${main_name}
+      - name: earth_mover_similarity
         split: ${main_split}
       - name: compilation_success
         split: ${main_split}
 
+  - name: general_information
+    display_name: General information
+    metrics:
+      - name: num_instances
+        split: ${main_split}
+      - name: num_train_instances
+        split: ${main_split}
+      - name: prompt_truncated
+        split: ${main_split}
+      - name: num_prompt_tokens
+        split: ${main_split}
+      - name: num_output_tokens
+        split: ${main_split}
+
+  - name: accuracy_simple
+    display_name: Earth Mover Similarity
+    metrics:
+      - name: earth_mover_similarity
+        split: ${main_split}
+
+  - name: compilation
+    display_name: Compilation Rate
+    metrics:
+      - name: compilation_success
+        split: ${main_split}
+
   - name: generation_image
     display_name: Generation (image)
     metrics:
@@ -229,11 +163,7 @@ metric_groups:
         split: ${main_split}
       - name: fid_similarity
         split: ${main_split}
-      - name: block_emd_similarity
-        split: ${main_split}
-      - name: block_emd_similarity_white
-        split: ${main_split}
-      - name: block_emd_similarity_median_color
+      - name: earth_mover_similarity
         split: ${main_split}
 
   - name: generation_text
@@ -253,6 +183,14 @@ run_groups:
       - image2webpage
       - image2musicsheet
 
+  - name: image2structure_real
+    display_name: Image2Structure (Wild)
+    description: Scenarios for evaluating the ability of Vision-Language models to generate structured outputs from images. These scenarios contain images that do not have a ground truth.
+    category: All scenarios
+    subgroups:
+      - image2latex_real
+      - image2webpage_real
+
   - name: image2latex
     display_name: Image2LaTeX
     description: The Image2LaTeX benchmark for converting images of mathematical equations, tables. algorithms and tikz to LaTeX.
@@ -260,13 +198,88 @@ run_groups:
       - accuracy
       - generation_image
       - generation_text
+      - general_information
     environment:
-      main_name: block_emd_similarity
+      main_name: earth_mover_similarity
       main_split: valid
     taxonomy:
       task: image-to-text
       what: mathematical equations, tables, algorithms, tikz
-      who: n/a
+      who: dataset authors
+      when: "2024"
+      language: English
+
+  - name: image2latex_easy
+    display_name: I2LaTeX (Easy)
+    description: The 1/3 easiest examples of the Image2LaTeX benchmark according to a simple heuristic counting the number of characters in the ground truth LaTeX code.
+    metric_groups:
+      - accuracy_simple
+      - compilation
+      - generation_image
+      - generation_text
+      - general_information
+    environment:
+      main_name: earth_mover_similarity
+      main_split: valid
+    taxonomy:
+      task: image-to-text
+      what: mathematical equations, tables, algorithms, tikz
+      who: dataset authors
+      when: "2024"
+      language: English
+
+  - name: image2latex_medium
+    display_name: I2LaTeX (Medium)
+    description: The 1/3 examples with medium diffulty of the Image2LaTeX benchmark according to a simple heuristic counting the number of characters in the ground truth LaTeX code.
+    metric_groups:
+      - accuracy_simple
+      - compilation
+      - generation_image
+      - generation_text
+      - general_information
+    environment:
+      main_name: earth_mover_similarity
+      main_split: valid
+    taxonomy:
+      task: image-to-text
+      what: mathematical equations, tables, algorithms, tikz
+      who: dataset authors
+      when: "2024"
+      language: English
+
+  - name: image2latex_hard
+    display_name: I2LaTeX (Hard)
+    description: The 1/3 hardest examples of the Image2LaTeX benchmark according to a simple heuristic counting the number of characters in the ground truth LaTeX code.
+    metric_groups:
+      - accuracy_simple
+      - compilation
+      - generation_image
+      - generation_text
+      - general_information
+    environment:
+      main_name: earth_mover_similarity
+      main_split: valid
+    taxonomy:
+      task: image-to-text
+      what: mathematical equations, tables, algorithms, tikz
+      who: dataset authors
+      when: "2024"
+      language: English
+
+  - name: image2latex_real
+    display_name: Image2LaTeX (Wild)
+    description: Images of mathematical equations gathered from Wikipedia that do not have a LaTeX ground truth.
+    metric_groups:
+      - accuracy
+      - generation_image
+      - general_information
+    environment:
+      main_name: earth_mover_similarity
+      main_split: valid
+    taxonomy:
+      task: image-to-text
+      what: mathematical equations, tables, algorithms, tikz
+      who: dataset authors
       when: "2024"
       language: English
 
@@ -277,13 +290,88 @@ run_groups:
       - accuracy
       - generation_image
       - generation_text
+      - general_information
     environment:
-      main_name: block_emd_similarity
+      main_name: earth_mover_similarity
       main_split: valid
     taxonomy:
       task: image-to-text
       what: css, html, javascript
-      who: n/a
+      who: dataset authors
+      when: "2024"
+      language: English
+
+  - name: image2webpage_easy
+    display_name: I2webpage (Easy)
+    description: The 1/3 easiest examples of the Image2webpage benchmark according to a simple heuristic counting the number of characters in the ground truth HTML/CSS/Javascript code.
+    metric_groups:
+      - accuracy_simple
+      - compilation
+      - generation_image
+      - generation_text
+      - general_information
+    environment:
+      main_name: earth_mover_similarity
+      main_split: valid
+    taxonomy:
+      task: image-to-text
+      what: css, html, javascript
+      who: dataset authors
+      when: "2024"
+      language: English
+
+  - name: image2webpage_medium
+    display_name: I2webpage (Medium)
+    description: The 1/3 examples with medium diffulty of the Image2webpage benchmark according to a simple heuristic counting the number of characters in the ground truth HTML/CSS/Javascript code.
+    metric_groups:
+      - accuracy_simple
+      - compilation
+      - generation_image
+      - generation_text
+      - general_information
+    environment:
+      main_name: earth_mover_similarity
+      main_split: valid
+    taxonomy:
+      task: image-to-text
+      what: css, html, javascript
+      who: dataset authors
+      when: "2024"
+      language: English
+
+  - name: image2webpage_hard
+    display_name: I2webpage (Hard)
+    description: The 1/3 hardest examples of the Image2webpage benchmark according to a simple heuristic counting the number of characters in the ground truth HTML/CSS/Javascript code.
+    metric_groups:
+      - accuracy_simple
+      - compilation
+      - generation_image
+      - generation_text
+      - general_information
+    environment:
+      main_name: earth_mover_similarity
+      main_split: valid
+    taxonomy:
+      task: image-to-text
+      what: css, html, javascript
+      who: dataset authors
+      when: "2024"
+      language: English
+
+  - name: image2webpage_real
+    display_name: Image2webpage (Wild)
+    description: Images of webpages gathered from the internet by taking sceenshots and so on that do not have a HTML/CSS/Javascript ground truth.
+    metric_groups:
+      - accuracy
+      - generation_image
+      - general_information
+    environment:
+      main_name: earth_mover_similarity
+      main_split: valid
+    taxonomy:
+      task: image-to-text
+      what: css, html, javascript
+      who: dataset authors
       when: "2024"
       language: English
 
@@ -293,12 +381,67 @@ run_groups:
     metric_groups:
       - accuracy
       - generation_image
+      - general_information
+    environment:
+      main_name: earth_mover_similarity
+      main_split: valid
+    taxonomy:
+      task: image-to-text
+      what: music sheets
+      who: dataset authors
+      when: "2024"
+      language: English
+
+  - name: image2musicsheet_easy
+    display_name: I2musicsheet (Easy)
+    description: The 1/3 easiest examples of the Image2musicsheet benchmark according to a simple heuristic counting the number of black pixels in the target image.
+    metric_groups:
+      - accuracy_simple
+      - compilation
+      - generation_image
+      - general_information
+    environment:
+      main_name: earth_mover_similarity
+      main_split: valid
+    taxonomy:
+      task: image-to-text
+      what: music sheets
+      who: dataset authors
+      when: "2024"
+      language: English
+
+  - name: image2musicsheet_medium
+    display_name: I2musicsheet (Medium)
+    description: The 1/3 examples with medium diffulty of the Image2musicsheet benchmark according to a simple heuristic counting the number of black pixels in the target image.
+    metric_groups:
+      - accuracy_simple
+      - compilation
+      - generation_image
+      - general_information
+    environment:
+      main_name: earth_mover_similarity
+      main_split: valid
+    taxonomy:
+      task: image-to-text
+      what: music sheets
+      who: dataset authors
+      when: "2024"
+      language: English
+
+  - name: image2musicsheet_hard
+    display_name: I2musicsheet (Hard)
+    description: The 1/3 hardest examples of the Image2musicsheet benchmark according to a simple heuristic counting the number of black pixels in the target image.
+    metric_groups:
+      - accuracy_simple
+      - compilation
+      - generation_image
+      - general_information
     environment:
-      main_name: block_emd_similarity
+      main_name: earth_mover_similarity
       main_split: valid
     taxonomy:
       task: image-to-text
       what: music sheets
-      who: n/a
+      who: dataset authors
       when: "2024"
       language: English