crfm-helm 0.5.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

helm/benchmark/static/schema_tables.yaml

@@ -0,0 +1,317 @@
+---
+
+############################################################
+metrics:
+  # Infrastructure metrics:
+  - name: num_perplexity_tokens
+    display_name: '# tokens'
+    description: Average number of tokens in the predicted output (for language modeling, the input too).
+  - name: num_bytes
+    display_name: '# bytes'
+    description: Average number of bytes in the predicted output (for language modeling, the input too).
+  - name: num_references
+    display_name: '# ref'
+    description: Number of references.
+  - name: num_train_trials
+    display_name: '# trials'
+    description: Number of trials, where in each trial we choose an independent, random set of training instances.
+  - name: estimated_num_tokens_cost
+    display_name: 'cost'
+    description: An estimate of the number of tokens (including prompt and output completions) needed to perform the request.
+  - name: num_prompt_tokens
+    display_name: '# prompt tokens'
+    description: Number of tokens in the prompt.
+  - name: num_prompt_characters
+    display_name: '# prompt chars'
+    description: Number of characters in the prompt.
+  - name: num_completion_tokens
+    display_name: '# completion tokens'
+    description: Actual number of completion tokens (over all completions).
+  - name: num_output_tokens
+    display_name: '# output tokens'
+    description: Actual number of output tokens.
+  - name: max_num_output_tokens
+    display_name: 'Max output tokens'
+    description: Maximum number of output tokens (overestimate since we might stop earlier due to stop sequences).
+  - name: num_requests
+    display_name: '# requests'
+    description: Number of distinct API requests.
+  - name: num_instances
+    display_name: '# eval'
+    description: Number of evaluation instances.
+  - name: num_train_instances
+    display_name: '# train'
+    description: Number of training instances (e.g., in-context examples).
+  - name: prompt_truncated
+    display_name: truncated
+    description: Fraction of instances where the prompt itself was truncated (implies that there were no in-context examples).
+  - name: finish_reason_length
+    display_name: finish b/c length
+    description: Fraction of instances where the the output was terminated because of the max tokens limit.
+  - name: finish_reason_stop
+    display_name: finish b/c stop
+    description: Fraction of instances where the the output was terminated because of the stop sequences.
+  - name: finish_reason_endoftext
+    display_name: finish b/c endoftext
+    description: Fraction of instances where the the output was terminated because the end of text token was generated.
+  - name: finish_reason_unknown
+    display_name: finish b/c unknown
+    description: Fraction of instances where the the output was terminated for unknown reasons.
+  - name: num_completions
+    display_name: '# completions'
+    description: Number of completions.
+  - name: predicted_index
+    display_name: Predicted index
+    description: Integer index of the reference (0, 1, ...) that was predicted by the model (for multiple-choice).
+
+  # Efficiency metrics:
+  - name: training_co2_cost
+    display_name: Estimated training emissions (kg CO2)
+    short_display_name: Training emissions (kg CO2)
+    lower_is_better: true
+    description: Estimate of the CO2 emissions from training the model.
+  - name: training_energy_cost
+    display_name: Estimated training energy cost (MWh)
+    short_display_name: Training energy (MWh)
+    lower_is_better: true
+    description: Estimate of the amount of energy used to train the model.
+  - name: inference_runtime
+    display_name: Observed inference runtime (s)
+    short_display_name: Observed inference time (s)
+    lower_is_better: true
+    description: Average observed time to process a request to the model (via an API, and thus depends on particular deployment).
+  - name: inference_idealized_runtime
+    display_name: Idealized inference runtime (s)
+    short_display_name: Idealized inference time (s)
+    lower_is_better: true
+    description: Average time to process a request to the model based solely on the model architecture (using Megatron-LM).
+  - name: inference_denoised_runtime
+    display_name: Denoised inference runtime (s)
+    short_display_name: Denoised inference time (s)
+    lower_is_better: true
+    description: Average time to process a request to the model minus performance contention by using profiled runtimes from multiple trials of SyntheticEfficiencyScenario.
+  - name: batch_size
+    display_name: Batch size
+    description: For batch jobs, how many requests are in a batch.
+
+  # Unitxt Metrics
+  - name: meteor
+    display_name: METEOR
+    short_display_name: METEOR
+    description: METEOR
+    lower_is_better: false
+  - name: f1
+    display_name: BERTScore F1
+    short_display_name: BERTScore F1
+    description: BERTScore F1
+    lower_is_better: false
+  - name: precision
+    display_name: Precision
+    short_display_name: Precision
+    description: Precision
+    lower_is_better: false
+  - name: recall
+    display_name: Recall
+    short_display_name: Recall
+    description: Recall
+    lower_is_better: false
+  - name: rouge1
+    display_name: ROUGE-1
+    short_display_name: ROUGE-1
+    description: ROUGE-1
+    lower_is_better: false
+  - name: rouge2
+    display_name: ROUGE-2
+    short_display_name: ROUGE-2
+    description: ROUGE-2
+    lower_is_better: false
+  - name: rougeL
+    display_name: ROUGE-L
+    short_display_name: ROUGE-L
+    description: ROUGE-L
+    lower_is_better: false
+  - name: rougeLsum
+    display_name: ROUGE-Lsum
+    short_display_name: ROUGE-Lsum
+    description: ROUGE-Lsum
+    lower_is_better: false
+  - name: bleu
+    display_name: BLEU
+    short_display_name: BLEU
+    description: BLEU
+    lower_is_better: false
+  - name: accuracy
+    display_name: Accuracy
+    short_display_name: Accuracy
+    description: Accuracy
+    lower_is_better: false
+  - name: f1_macro
+    display_name: Macro F1
+    short_display_name: Macro F1
+    description: Macro F1
+    lower_is_better: false
+  - name: f1_micro
+    display_name: Micro F1
+    short_display_name: Micro F1
+    description: Micro F1
+    lower_is_better: false
+  - name: unsorted_list_exact_match
+    display_name: Unsorted List Exact Match
+    short_display_name: Exact Match
+    description: Unsorted List Exact Match
+    lower_is_better: false
+
+  # FinQA Accuracy
+  - name: program_accuracy
+    display_name: Program Accuracy
+    short_display_name: Program Accuracy
+    description: Program Accuracy
+    lower_is_better: false
+  - name: execution_accuracy
+    display_name: Execution Accuracy
+    short_display_name: Execution Accuracy
+    description: Execution Accuracy
+    lower_is_better: false
+
+perturbations: []
+
+metric_groups:
+  - name: main_metrics
+    display_name: Main Metrics
+    metrics:
+    - name: ${main_name}
+      split: __all__
+
+  - name: generation_metrics
+    display_name: Other Generation Metrics
+    hide_win_rates: true
+    metrics:
+    - name: f1
+      split: __all__
+    - name: rouge1
+      split: __all__
+    - name: rouge2
+      split: __all__
+    - name: rougeL
+      split: __all__
+    - name: rougeLsum
+      split: __all__
+    - name: bleu
+      split: __all__
+
+  - name: classification_metrics
+    display_name: Classification Metrics
+    hide_win_rates: true
+    metrics:
+    - name: accuracy
+      split: __all__
+    - name: f1_macro
+      split: __all__
+    - name: f1_micro
+      split: __all__
+
+  - name: efficiency
+    display_name: Efficiency
+    metrics:
+    - name: inference_runtime
+      split: ${main_split}
+
+  - name: general_information
+    display_name: General information
+    hide_win_rates: true
+    metrics:
+    - name: num_instances
+      split: ${main_split}
+    - name: num_train_instances
+      split: ${main_split}
+    - name: prompt_truncated
+      split: ${main_split}
+    - name: num_prompt_tokens
+      split: ${main_split}
+    - name: num_output_tokens
+      split: ${main_split}
+
+run_groups:
+  - name: table_scenarios
+    display_name: Table  Scenarios
+    description: Table Scenarios
+    category: All Scenarios
+    subgroups:
+      - unitxt_cards.numeric_nlg
+      - unitxt_cards.tab_fact
+      - unitxt_cards.wikitq
+      - unitxt_cards.fin_qa
+
+  - name: unitxt_cards.numeric_nlg
+    display_name: NumericNLG
+    short_display_name: NumericNLG
+    description: "NumericNLG is a dataset for numerical table-to-text generation using pairs of a table and a paragraph of a table description with richer inference from scientific papers."
+    metric_groups:
+      - main_metrics
+      - generation_metrics
+      - efficiency
+      - general_information
+    environment:
+      main_name: meteor
+      main_split: test
+    taxonomy:
+      task: "?"
+      what: "?"
+      who: "?"
+      when: "?"
+      language: English
+
+  - name: unitxt_cards.tab_fact
+    display_name: TabFact
+    short_display_name: TabFact
+    description: "tab_fact is a large-scale dataset for the task of fact-checking on tables."
+    metric_groups:
+      - main_metrics
+      - classification_metrics
+      - efficiency
+      - general_information
+    environment:
+      main_name: accuracy
+      main_split: test
+    taxonomy:
+      task: "?"
+      what: "?"
+      who: "?"
+      when: "?"
+      language: English
+
+  - name: unitxt_cards.wikitq
+    display_name: WikiTableQuestions
+    short_display_name: WikiTableQuestions
+    description: "This WikiTableQuestions dataset is a large-scale dataset for the task of question answering on semi-structured tables."
+    metric_groups:
+      - main_metrics
+      - classification_metrics
+      - efficiency
+      - general_information
+    environment:
+      main_name: unsorted_list_exact_match
+      main_split: test
+    taxonomy:
+      task: "?"
+      what: "?"
+      who: "?"
+      when: "?"
+      language: English
+
+  - name: unitxt_cards.fin_qa
+    display_name: FinQA
+    description: The FinQA benchmark for numeric reasoning over financial data, with question answering pairs written by financial experts over financial reports [(Chen et al., 2021)](https://arxiv.org/abs/2109.00122/).
+    metric_groups:
+      - main_metrics
+      - efficiency
+      - general_information
+    environment:
+      main_name: program_accuracy
+      main_split: test
+    taxonomy:
+      task: question answering with numeric reasoning
+      what: financial reports
+      who: financial experts
+      when: 1999 to 2019
+      language: English

helm/benchmark/static/schema_thai.yaml

@@ -0,0 +1,244 @@
+---
+############################################################
+metrics:
+  # Infrastructure metrics:
+  - name: num_perplexity_tokens
+    display_name: '# tokens'
+    description: Average number of tokens in the predicted output (for language modeling, the input too).
+  - name: num_bytes
+    display_name: '# bytes'
+    description: Average number of bytes in the predicted output (for language modeling, the input too).
+
+  - name: num_references
+    display_name: '# ref'
+    description: Number of references.
+  - name: num_train_trials
+    display_name: '# trials'
+    description: Number of trials, where in each trial we choose an independent, random set of training instances.
+  - name: estimated_num_tokens_cost
+    display_name: 'cost'
+    description: An estimate of the number of tokens (including prompt and output completions) needed to perform the request.
+  - name: num_prompt_tokens
+    display_name: '# prompt tokens'
+    description: Number of tokens in the prompt.
+  - name: num_prompt_characters
+    display_name: '# prompt chars'
+    description: Number of characters in the prompt.
+  - name: num_completion_tokens
+    display_name: '# completion tokens'
+    description: Actual number of completion tokens (over all completions).
+  - name: num_output_tokens
+    display_name: '# output tokens'
+    description: Actual number of output tokens.
+  - name: max_num_output_tokens
+    display_name: 'Max output tokens'
+    description: Maximum number of output tokens (overestimate since we might stop earlier due to stop sequences).
+  - name: num_requests
+    display_name: '# requests'
+    description: Number of distinct API requests.
+  - name: num_instances
+    display_name: '# eval'
+    description: Number of evaluation instances.
+  - name: num_train_instances
+    display_name: '# train'
+    description: Number of training instances (e.g., in-context examples).
+  - name: prompt_truncated
+    display_name: truncated
+    description: Fraction of instances where the prompt itself was truncated (implies that there were no in-context examples).
+  - name: finish_reason_length
+    display_name: finish b/c length
+    description: Fraction of instances where the the output was terminated because of the max tokens limit.
+  - name: finish_reason_stop
+    display_name: finish b/c stop
+    description: Fraction of instances where the the output was terminated because of the stop sequences.
+  - name: finish_reason_endoftext
+    display_name: finish b/c endoftext
+    description: Fraction of instances where the the output was terminated because the end of text token was generated.
+  - name: finish_reason_unknown
+    display_name: finish b/c unknown
+    description: Fraction of instances where the the output was terminated for unknown reasons.
+  - name: num_completions
+    display_name: '# completions'
+    description: Number of completions.
+  - name: predicted_index
+    display_name: Predicted index
+    description: Integer index of the reference (0, 1, ...) that was predicted by the model (for multiple-choice).
+
+  # Accuracy metrics:
+  - name: exact_match
+    display_name: Exact match
+    short_display_name: EM
+    description: Fraction of instances that the predicted output matches a correct reference exactly.
+    lower_is_better: false
+
+############################################################
+perturbations: []
+
+############################################################
+metric_groups:
+  - name: accuracy
+    display_name: Accuracy
+    hide_win_rates: true
+    metrics:
+      - name: ${main_name}
+        split: ${main_split}
+
+  - name: efficiency
+    display_name: Efficiency
+    metrics:
+    - name: inference_runtime
+      split: ${main_split}
+
+  - name: general_information
+    display_name: General information
+    hide_win_rates: true
+    metrics:
+    - name: num_instances
+      split: ${main_split}
+    - name: num_train_instances
+      split: ${main_split}
+    - name: prompt_truncated
+      split: ${main_split}
+    - name: num_prompt_tokens
+      split: ${main_split}
+    - name: num_output_tokens
+      split: ${main_split}
+
+############################################################
+
+run_groups:
+  - name: thai_scenarios
+    display_name: Thai Scenarios
+    description: Thai-language scenarios
+    category: All scenarios
+    subgroups:
+      - thai_exam
+      - thai_exam_onet
+      - thai_exam_ic
+      - thai_exam_tgat
+      - thai_exam_tpat1
+      - thai_exam_a_level
+
+
+  - name: thai_exam
+    display_name: ThaiExam
+    description: >
+      Macro-averaged accuracy on all ThaiExam examinations.
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: question answering
+      what: n/a
+      who: n/a
+      when: "?"
+      language: Thai and English
+
+  - name: thai_exam_onet
+    display_name: ONET
+    description: >
+      The Ordinary National Educational Test (ONET) is an examination for students in Thailand.
+      We select the grade-12 ONET exam, which comprises 5 subjects and each question has 5 choices.
+      These subjects are Thai, English, Mathematics, Social Studies, and Science.
+      Amounting to a total of 170 questions and options.
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: question answering
+      what: high school / medical school academic knowledge
+      who: n/a
+      when: "?"
+      language: Thai and English
+
+  - name: thai_exam_ic
+    display_name: IC
+    description: >
+      The Investment Consultant (IC) examination, a licensing test for investment professionals in Thailand.
+      Developed by the Stock Exchange of Thailand (SET), features 4 choices per question.
+      We extracted questions for levels 1, 2, and 3 resulting in a total of 95 questions and options.
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: question answering
+      what: licensing for investment professionals
+      who: n/a
+      when: "?"
+      language: Thai
+
+  - name: thai_exam_tgat
+    display_name: TGAT
+    description: >
+      The Thai General Aptitude Test (TGAT), a national high school examination in Thailand.
+      Focuses on critical and logical thinking skills.
+      We collected a total of 90 questions and answers. The TGAT consists of four choices per question.
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: question answering
+      what: high school level questions on reasoning
+      who: n/a
+      when: "?"
+      language: English
+
+  - name: thai_exam_tpat1
+    display_name: TPAT-1
+    description: >
+      The Thai Professional Aptitude Test 1 (TPAT-1) is a national high school examination in Thailand.
+      The Exam assesses students’ professional skills requirement in medical schools.
+      This subset contains reasoning and medical ethics. We collected a total of 116 questions and answers.
+      The TPAT-1 consists of 5 choices per question.
+    description: TBD
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: question answering
+      what: high school / medical school academic knowledge
+      who: n/a
+      when: "?"
+      language: Thai
+
+  - name: thai_exam_a_level
+    display_name: A-Level
+    description: >
+      An academic knowledge assessment examination (Applied Knowledge Level)
+      that covers general foundational subjects taught in schools.
+      The content assessed in this examination aligns with the curriculum guidelines
+      and emphasizes the practical application of knowledge in daily life.
+      We collected a total of 175 questions and answers.
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: question answering
+      what: high school academic knowledge
+      who: n/a
+      when: "?"
+      language: Thai and English

helm/benchmark/static/schema_unitxt.yaml

@@ -1,66 +1,8 @@
 ---
 ############################################################
-adapter:
-  - name: method
-    description: The high-level strategy for converting instances into a prompt for the language model.
-    values:
-      - name: generation
-        description: Given the input, the model generates the output free-form.
-      - name: multiple_choice_joint
-        description: Given the input, the model selects from multiple-choice options (A., B., C., D., E.).
-      - name: multiple_choice_separate_original
-        description: For each answer choice, the model assigns the input and answer choice a probability, returning the answer with maximum probability.
-      - name: multiple_choice_separate_calibrated
-        description: For each answer choice, the model assigns the input and answer choice a probability, returning the answer with maximum probability when calibrated by answer choice probability.
-      - name: language_modeling
-        description: Given the input, the model assigns the sequence a probability.
-  - name: instructions
-    description: The description of the task that is included at the very beginning of the prompt.
-  - name: global_prefix
-    description: The string that is prepended to the prompt.
-  - name: global_suffix
-    description: The string that is appended to the prompt.
-  - name: instance_prefix
-    description: The string that is included before each instance (e.g., '\n\n').
-  - name: input_prefix
-    description: The string that is included before each input (e.g., 'Question:').
-  - name: input_suffix
-    description: The string that is included after each input (e.g., '\n').
-  - name: reference_prefix
-    description: The string that is included before each reference (for multiple-choice questions).
-  - name: reference_suffix
-    description: The string that is included after each reference (for multiple-choice questions).
-  - name: output_prefix
-    description: The string that is included before the correct answer/predicted output (e.g., 'Answer:').
-  - name: output_suffix
-    description: The string that is included after the correct answer/predicted output (e.g., '\n').
-  - name: substitutions
-    description: A list of regular expression substitutions (e.g., replacing '\n' with ';\n') to perform at the very end on the prompt.
-  - name: max_train_instances
-    description: Maximum number of training instances to include in the prompt (currently by randomly sampling).
-  - name: max_eval_instances
-    description: Maximum number of instances to evaluate on (over all splits - test, valid, etc.).
-  - name: num_outputs
-    description: Maximum number of possible outputs to generate by sampling multiple outputs.
-  - name: num_train_trials
-    description: Number of trials, where in each trial we choose an independent, random set of training instances. Used to compute variance.
-  - name: sample_train
-    description: If true, randomly sample N training examples; if false, select N consecutive training examples
-  - name: model
-    description: Name of the language model (<creator_organization>/<model name>) to send requests to.
-  - name: model_deployment
-    description: Name of the language model deployment (<host_organization>/<model name>) to send requests to.
-  - name: temperature
-    description: Temperature parameter used in generation.
-  - name: max_tokens
-    description: Maximum number of tokens to generate.
-  - name: stop_sequences
-    description: List of sequences, where we stop generation if we encounter any of them.
-  - name: random
-    description: Random seed (string), which guarantees reproducibility.
-  - name: multi_label
-    description: If true, for instances with multiple correct reference, the gold answer should be considered to be all of the correct references rather than any of the correct references.
-
+# For backwards compatibility with older versions of HELM.
+# TODO: Remove this after 2024-09-01.
+adapter: []
 ############################################################
 metrics:
   # Infrastructure metrics: