crfm-helm 0.4.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

helm/benchmark/static/schema_unitxt.yaml

@@ -0,0 +1,428 @@
+---
+############################################################
+adapter:
+  - name: method
+    description: The high-level strategy for converting instances into a prompt for the language model.
+    values:
+      - name: generation
+        description: Given the input, the model generates the output free-form.
+      - name: multiple_choice_joint
+        description: Given the input, the model selects from multiple-choice options (A., B., C., D., E.).
+      - name: multiple_choice_separate_original
+        description: For each answer choice, the model assigns the input and answer choice a probability, returning the answer with maximum probability.
+      - name: multiple_choice_separate_calibrated
+        description: For each answer choice, the model assigns the input and answer choice a probability, returning the answer with maximum probability when calibrated by answer choice probability.
+      - name: language_modeling
+        description: Given the input, the model assigns the sequence a probability.
+  - name: instructions
+    description: The description of the task that is included at the very beginning of the prompt.
+  - name: global_prefix
+    description: The string that is prepended to the prompt.
+  - name: global_suffix
+    description: The string that is appended to the prompt.
+  - name: instance_prefix
+    description: The string that is included before each instance (e.g., '\n\n').
+  - name: input_prefix
+    description: The string that is included before each input (e.g., 'Question:').
+  - name: input_suffix
+    description: The string that is included after each input (e.g., '\n').
+  - name: reference_prefix
+    description: The string that is included before each reference (for multiple-choice questions).
+  - name: reference_suffix
+    description: The string that is included after each reference (for multiple-choice questions).
+  - name: output_prefix
+    description: The string that is included before the correct answer/predicted output (e.g., 'Answer:').
+  - name: output_suffix
+    description: The string that is included after the correct answer/predicted output (e.g., '\n').
+  - name: substitutions
+    description: A list of regular expression substitutions (e.g., replacing '\n' with ';\n') to perform at the very end on the prompt.
+  - name: max_train_instances
+    description: Maximum number of training instances to include in the prompt (currently by randomly sampling).
+  - name: max_eval_instances
+    description: Maximum number of instances to evaluate on (over all splits - test, valid, etc.).
+  - name: num_outputs
+    description: Maximum number of possible outputs to generate by sampling multiple outputs.
+  - name: num_train_trials
+    description: Number of trials, where in each trial we choose an independent, random set of training instances. Used to compute variance.
+  - name: sample_train
+    description: If true, randomly sample N training examples; if false, select N consecutive training examples
+  - name: model
+    description: Name of the language model (<creator_organization>/<model name>) to send requests to.
+  - name: model_deployment
+    description: Name of the language model deployment (<host_organization>/<model name>) to send requests to.
+  - name: temperature
+    description: Temperature parameter used in generation.
+  - name: max_tokens
+    description: Maximum number of tokens to generate.
+  - name: stop_sequences
+    description: List of sequences, where we stop generation if we encounter any of them.
+  - name: random
+    description: Random seed (string), which guarantees reproducibility.
+  - name: multi_label
+    description: If true, for instances with multiple correct reference, the gold answer should be considered to be all of the correct references rather than any of the correct references.
+
+############################################################
+metrics:
+  # Infrastructure metrics:
+  - name: num_perplexity_tokens
+    display_name: '# tokens'
+    description: Average number of tokens in the predicted output (for language modeling, the input too).
+  - name: num_bytes
+    display_name: '# bytes'
+    description: Average number of bytes in the predicted output (for language modeling, the input too).
+  - name: num_references
+    display_name: '# ref'
+    description: Number of references.
+  - name: num_train_trials
+    display_name: '# trials'
+    description: Number of trials, where in each trial we choose an independent, random set of training instances.
+  - name: estimated_num_tokens_cost
+    display_name: 'cost'
+    description: An estimate of the number of tokens (including prompt and output completions) needed to perform the request.
+  - name: num_prompt_tokens
+    display_name: '# prompt tokens'
+    description: Number of tokens in the prompt.
+  - name: num_prompt_characters
+    display_name: '# prompt chars'
+    description: Number of characters in the prompt.
+  - name: num_completion_tokens
+    display_name: '# completion tokens'
+    description: Actual number of completion tokens (over all completions).
+  - name: num_output_tokens
+    display_name: '# output tokens'
+    description: Actual number of output tokens.
+  - name: max_num_output_tokens
+    display_name: 'Max output tokens'
+    description: Maximum number of output tokens (overestimate since we might stop earlier due to stop sequences).
+  - name: num_requests
+    display_name: '# requests'
+    description: Number of distinct API requests.
+  - name: num_instances
+    display_name: '# eval'
+    description: Number of evaluation instances.
+  - name: num_train_instances
+    display_name: '# train'
+    description: Number of training instances (e.g., in-context examples).
+  - name: prompt_truncated
+    display_name: truncated
+    description: Fraction of instances where the prompt itself was truncated (implies that there were no in-context examples).
+  - name: finish_reason_length
+    display_name: finish b/c length
+    description: Fraction of instances where the the output was terminated because of the max tokens limit.
+  - name: finish_reason_stop
+    display_name: finish b/c stop
+    description: Fraction of instances where the the output was terminated because of the stop sequences.
+  - name: finish_reason_endoftext
+    display_name: finish b/c endoftext
+    description: Fraction of instances where the the output was terminated because the end of text token was generated.
+  - name: finish_reason_unknown
+    display_name: finish b/c unknown
+    description: Fraction of instances where the the output was terminated for unknown reasons.
+  - name: num_completions
+    display_name: '# completions'
+    description: Number of completions.
+  - name: predicted_index
+    display_name: Predicted index
+    description: Integer index of the reference (0, 1, ...) that was predicted by the model (for multiple-choice).
+
+  # Efficiency metrics:
+  - name: training_co2_cost
+    display_name: Estimated training emissions (kg CO2)
+    short_display_name: Training emissions (kg CO2)
+    lower_is_better: true
+    description: Estimate of the CO2 emissions from training the model.
+  - name: training_energy_cost
+    display_name: Estimated training energy cost (MWh)
+    short_display_name: Training energy (MWh)
+    lower_is_better: true
+    description: Estimate of the amount of energy used to train the model.
+  - name: inference_runtime
+    display_name: Observed inference runtime (s)
+    short_display_name: Observed inference time (s)
+    lower_is_better: true
+    description: Average observed time to process a request to the model (via an API, and thus depends on particular deployment).
+  - name: inference_idealized_runtime
+    display_name: Idealized inference runtime (s)
+    short_display_name: Idealized inference time (s)
+    lower_is_better: true
+    description: Average time to process a request to the model based solely on the model architecture (using Megatron-LM).
+  - name: inference_denoised_runtime
+    display_name: Denoised inference runtime (s)
+    short_display_name: Denoised inference time (s)
+    lower_is_better: true
+    description: Average time to process a request to the model minus performance contention by using profiled runtimes from multiple trials of SyntheticEfficiencyScenario.
+  - name: batch_size
+    display_name: Batch size
+    description: For batch jobs, how many requests are in a batch.
+
+  # Unitxt Metrics
+  - name: accuracy
+    display_name: accuracy
+    short_display_name: accuracy
+    description: accuracy
+  - name: accuracy_ci_high
+    display_name: accuracy_ci_high
+    short_display_name: accuracy_ci_high
+    description: accuracy_ci_high
+  - name: accuracy_ci_low
+    display_name: accuracy_ci_low
+    short_display_name: accuracy_ci_low
+    description: accuracy_ci_low
+  - name: f1_audio_volume_up
+    display_name: f1_audio_volume_up
+    short_display_name: f1_audio_volume_up
+    description: f1_audio_volume_up
+  - name: f1_calendar_remove
+    display_name: f1_calendar_remove
+    short_display_name: f1_calendar_remove
+    description: f1_calendar_remove
+  - name: f1_contradiction
+    display_name: f1_contradiction
+    short_display_name: f1_contradiction
+    description: f1_contradiction
+  - name: f1_cooking_recipe
+    display_name: f1_cooking_recipe
+    short_display_name: f1_cooking_recipe
+    description: f1_cooking_recipe
+  - name: f1_datetime_query
+    display_name: f1_datetime_query
+    short_display_name: f1_datetime_query
+    description: f1_datetime_query
+  - name: f1_entailment
+    display_name: f1_entailment
+    short_display_name: f1_entailment
+    description: f1_entailment
+  - name: f1_lists_remove
+    display_name: f1_lists_remove
+    short_display_name: f1_lists_remove
+    description: f1_lists_remove
+  - name: f1_macro
+    display_name: f1_macro
+    short_display_name: f1_macro
+    description: f1_macro
+  - name: f1_macro_ci_high
+    display_name: f1_macro_ci_high
+    short_display_name: f1_macro_ci_high
+    description: f1_macro_ci_high
+  - name: f1_macro_ci_low
+    display_name: f1_macro_ci_low
+    short_display_name: f1_macro_ci_low
+    description: f1_macro_ci_low
+  - name: f1_micro
+    display_name: f1_micro
+    short_display_name: f1_micro
+    description: f1_micro
+  - name: f1_micro_ci_high
+    display_name: f1_micro_ci_high
+    short_display_name: f1_micro_ci_high
+    description: f1_micro_ci_high
+  - name: f1_micro_ci_low
+    display_name: f1_micro_ci_low
+    short_display_name: f1_micro_ci_low
+    description: f1_micro_ci_low
+  - name: f1_music_query
+    display_name: f1_music_query
+    short_display_name: f1_music_query
+    description: f1_music_query
+  - name: f1_neutral
+    display_name: f1_neutral
+    short_display_name: f1_neutral
+    description: f1_neutral
+  - name: f1_news_query
+    display_name: f1_news_query
+    short_display_name: f1_news_query
+    description: f1_news_query
+  - name: f1_play_game
+    display_name: f1_play_game
+    short_display_name: f1_play_game
+    description: f1_play_game
+  - name: f1_play_music
+    display_name: f1_play_music
+    short_display_name: f1_play_music
+    description: f1_play_music
+  - name: fairness
+    display_name: fairness
+    short_display_name: fairness
+    description: fairness
+  - name: groups_mean_score
+    display_name: groups_mean_score
+    short_display_name: groups_mean_score
+    description: groups_mean_score
+  - name: max_prob
+    display_name: max_prob
+    short_display_name: max_prob
+    description: max_prob
+  - name: perplexity
+    display_name: perplexity
+    short_display_name: perplexity
+    description: perplexity
+  - name: robustness
+    display_name: robustness
+    short_display_name: robustness
+    description: robustness
+  - name: rouge1
+    display_name: rouge1
+    short_display_name: rouge1
+    description: rouge1
+  - name: rouge2
+    display_name: rouge2
+    short_display_name: rouge2
+    description: rouge2
+  - name: rougeL
+    display_name: rougeL
+    short_display_name: rougeL
+    description: rougeL
+  - name: rougeLsum
+    display_name: rougeLsum
+    short_display_name: rougeLsum
+    description: rougeLsum
+  - name: score_ci_high
+    display_name: score_ci_high
+    short_display_name: score_ci_high
+    description: score_ci_high
+  - name: score_ci_low
+    display_name: score_ci_low
+    short_display_name: score_ci_low
+    description: score_ci_low
+
+perturbations: []
+
+metric_groups:
+  - name: main_score
+    display_name: Main Score
+    metrics:
+      - name: ${main_name}
+        split: __all__
+
+  - name: efficiency
+    display_name: Efficiency
+    metrics:
+    - name: inference_runtime
+      split: ${main_split}
+
+  - name: general_information
+    display_name: General information
+    metrics:
+    - name: num_instances
+      split: ${main_split}
+    - name: num_train_instances
+      split: ${main_split}
+    - name: prompt_truncated
+      split: ${main_split}
+    - name: num_prompt_tokens
+      split: ${main_split}
+    - name: num_output_tokens
+      split: ${main_split}
+
+  - name: classification_metrics
+    display_name: Main Score
+    metrics:
+      # Not included because already in main_score
+      # - name: accuracy
+      #   split: __all__
+      - name: f1_macro
+        split: __all__
+      - name: f1_micro
+        split: __all__
+
+  - name: summarization_metrics
+    display_name: Main Score
+    metrics:
+      - name: rouge1
+        split: __all__
+      - name: rouge2
+        split: __all__
+      # Not included because already in main_score
+      # - name: rougeL
+      #   split: __all__
+      - name: rougeLsum
+        split: __all__
+
+run_groups:
+  - name: spanish_scenarios
+    display_name: Spanish Scenarios
+    description: Spanish Scenarios
+    category: Multi-lingual Scenarios
+    subgroups:
+      - unitxt_cards.amazon_mass.es_ES
+      - unitxt_cards.xnli.es
+      - unitxt_cards.xlsum.spanish
+      - unitxt_cards.mlsum.es
+
+  - name: unitxt_cards.amazon_mass.es_ES
+    display_name: Amazon MASS
+    short_display_name: Amazon MASS
+    description: Amazon MASS
+    metric_groups:
+      - main_score
+      - classification_metrics
+      - efficiency
+      - general_information
+    environment:
+      # TODO: f1_macro instead?
+      main_name: accuracy
+      main_split: test
+    taxonomy:
+      task: "?"
+      what: "?"
+      who: "?"
+      when: "?"
+      language: Spanish
+
+  - name: unitxt_cards.xnli.es
+    display_name: XNLI
+    short_display_name: XNLI
+    description: XNLI
+    metric_groups:
+      - main_score
+      - classification_metrics
+      - efficiency
+      - general_information
+    environment:
+      # TODO: f1_macro instead?
+      main_name: accuracy
+      main_split: test
+    taxonomy:
+      task: "?"
+      what: "?"
+      who: "?"
+      when: "?"
+      language: Spanish
+
+  - name: unitxt_cards.xlsum.spanish
+    display_name: XL-Sum
+    short_display_name: XL-Sum
+    description: XL-Sum
+    metric_groups:
+      - main_score
+      - summarization_metrics
+      - efficiency
+      - general_information
+    environment:
+      main_name: rougeL
+      main_split: test
+    taxonomy:
+      task: "?"
+      what: "?"
+      who: "?"
+      when: "?"
+      language: Spanish
+
+  - name: unitxt_cards.mlsum.es
+    display_name: MLSUM
+    short_display_name: MLSUM
+    description: MLSUM
+    metric_groups:
+      - main_score
+      - summarization_metrics
+      - efficiency
+      - general_information
+    environment:
+      main_name: rougeL
+      main_split: test
+    taxonomy:
+      task: "?"
+      what: "?"
+      who: "?"
+      when: "?"
+      language: Spanish

helm/benchmark/static/schema_vhelm_lite.yaml

@@ -0,0 +1,164 @@
+---
+############################################################
+adapter:
+  - name: method
+    description: The high-level strategy for converting instances into a prompt for the language model.
+    values:
+      - name: generation
+        description: Given the input, the model generates the output free-form.
+      - name: generation_multimodal
+        description: Given the multimodal input, the model generates the output free-form.
+      - name: multiple_choice_joint
+        description: Given the input, the model selects from multiple-choice options (A., B., C., D., E.).
+      - name: multiple_choice_joint_multimodal
+        description: Given the multimodal input, the model selects from multiple-choice options (A., B., C., D., E.).
+  - name: instructions
+    description: The description of the task that is included at the very beginning of the prompt.
+  - name: global_prefix
+    description: The string that is prepended to the prompt.
+  - name: global_suffix
+    description: The string that is appended to the prompt.
+  - name: instance_prefix
+    description: The string that is included before each instance (e.g., '\n\n').
+  - name: input_prefix
+    description: The string that is included before each input (e.g., 'Question:').
+  - name: input_suffix
+    description: The string that is included after each input (e.g., '\n').
+  - name: reference_prefix
+    description: The string that is included before each reference (for multiple-choice questions).
+  - name: reference_suffix
+    description: The string that is included after each reference (for multiple-choice questions).
+  - name: output_prefix
+    description: The string that is included before the correct answer/predicted output (e.g., 'Answer:').
+  - name: output_suffix
+    description: The string that is included after the correct answer/predicted output (e.g., '\n').
+  - name: substitutions
+    description: A list of regular expression substitutions (e.g., replacing '\n' with ';\n') to perform at the very end on the prompt.
+  - name: max_train_instances
+    description: Maximum number of training instances to include in the prompt (currently by randomly sampling).
+  - name: max_eval_instances
+    description: Maximum number of instances to evaluate on (over all splits - test, valid, etc.).
+  - name: num_outputs
+    description: Maximum number of possible outputs to generate by sampling multiple outputs.
+  - name: num_train_trials
+    description: Number of trials, where in each trial we choose an independent, random set of training instances. Used to compute variance.
+  - name: sample_train
+    description: If true, randomly sample N training examples; if false, select N consecutive training examples
+  - name: model
+    description: Name of the language model (<creator_organization>/<model name>) to send requests to.
+  - name: model_deployment
+    description: Name of the language model deployment (<host_organization>/<model name>) to send requests to.
+  - name: temperature
+    description: Temperature parameter used in generation.
+  - name: max_tokens
+    description: Maximum number of tokens to generate.
+  - name: stop_sequences
+    description: List of sequences, where we stop generation if we encounter any of them.
+  - name: random
+    description: Random seed (string), which guarantees reproducibility.
+  - name: multi_label
+    description: If true, for instances with multiple correct reference, the gold answer should be considered to be all of the correct references rather than any of the correct references.
+
+perturbations:
+  - name: robustness
+    display_name: Robustness
+    description: Computes worst case over different robustness perturbations (misspellings, formatting, contrast sets).
+
+############################################################
+metrics:
+  # Accuracy metrics:
+  - name: exact_match
+    display_name: Exact match
+    short_display_name: EM
+    description: Fraction of instances that the predicted output matches a correct reference exactly.
+    lower_is_better: false
+  - name: quasi_exact_match
+    display_name: Quasi-exact match
+    short_display_name: EM
+    description: Fraction of instances that the predicted output matches a correct reference up to light processing.
+    lower_is_better: false
+  - name: prefix_exact_match
+    display_name: Prefix exact match
+    short_display_name: PEM
+    description: Fraction of instances that the predicted output matches the prefix of a correct reference exactly.
+    lower_is_better: false
+  - name: quasi_prefix_exact_match
+    # TODO: should call this prefix_quasi_exact_match
+    display_name: Prefix quasi-exact match
+    short_display_name: PEM
+    description: Fraction of instances that the predicted output matches the prefix of a correct reference up to light processing.
+    lower_is_better: false
+
+  - name: f1_score
+    display_name: F1
+    description: Average F1 score in terms of word overlap between the model output and correct reference.
+    lower_is_better: false
+  - name: cider
+    display_name: CIDEr
+    description: Evaluates the quality of generated caption by measuring the weighted similarity of n-grams between the captions and a set of human-written reference captions, emphasizing informativeness and consensus.
+    lower_is_better: false
+
+
+############################################################
+metric_groups:
+  - name: accuracy
+    display_name: Accuracy
+    metrics:
+      - name: ${main_name}
+        split: ${main_split}
+
+############################################################
+run_groups:
+  - name: core_scenarios
+    display_name: Core scenarios
+    description: The scenarios where we evaluate all the models.
+    category: All scenarios
+    subgroups:
+      - viz_wiz
+      - vqa
+      - mmmu
+
+  - name: viz_wiz
+    display_name: VizWiz
+    description: A benchmark for visual question answering with images and questions created by visually impaired people [(Gurari et al., 2018)](https://arxiv.org/abs/1802.08218).
+    metric_groups:
+      - accuracy
+    environment:
+      main_name: quasi_exact_match
+      main_split: valid
+    taxonomy:
+      task: multimodal short answer question answering
+      what: Real-world images
+      who: Visually impaired people
+      when: "2018"
+      language: English
+
+  - name: vqa
+    display_name: VQAv2
+    description: Open-ended questions about real-world images [(Goyal et al., 2017)](https://arxiv.org/abs/1612.00837).
+    metric_groups:
+      - accuracy
+    environment:
+      main_name: quasi_exact_match
+      main_split: valid
+    taxonomy:
+      task: multimodal short answer question answering
+      what: Real-world images
+      who: Human experts
+      when: "2017"
+      language: English
+
+  - name: mmmu
+    display_name: MMMU
+    description: A benchmark designed to evaluate multimodal models on massive multi-discipline tasks demanding college-level subject knowledge and deliberate reasoning [(Yue et al., 2023)](https://arxiv.org/abs/2311.16502).
+    metric_groups:
+      - accuracy
+    environment:
+      main_name: exact_match
+      main_split: valid
+    taxonomy:
+      task: multimodal multiple-choice question answering
+      what: Art & Design, Business, Science, Health & Medicine, Humanities & Social Science, and Tech & Engineering
+      who: Human experts
+      when: "2023"
+      language: English