crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/static/schema_autobencher.yaml

@@ -0,0 +1,150 @@
+---
+############################################################
+metrics:
+  # Infrastructure metrics:
+  - name: num_perplexity_tokens
+    display_name: '# tokens'
+    description: Average number of tokens in the predicted output (for language modeling, the input too).
+  - name: num_bytes
+    display_name: '# bytes'
+    description: Average number of bytes in the predicted output (for language modeling, the input too).
+
+  - name: num_references
+    display_name: '# ref'
+    description: Number of references.
+  - name: num_train_trials
+    display_name: '# trials'
+    description: Number of trials, where in each trial we choose an independent, random set of training instances.
+  - name: num_prompt_tokens
+    display_name: '# prompt tokens'
+    description: Number of tokens in the prompt.
+  - name: num_completion_tokens
+    display_name: '# completion tokens'
+    description: Actual number of completion tokens (over all completions).
+  - name: num_output_tokens
+    display_name: '# output tokens'
+    description: Actual number of output tokens.
+  - name: num_instances
+    display_name: '# eval'
+    description: Number of evaluation instances.
+  - name: num_train_instances
+    display_name: '# train'
+    description: Number of training instances (e.g., in-context examples).
+  - name: prompt_truncated
+    display_name: truncated
+    description: Fraction of instances where the prompt itself was truncated (implies that there were no in-context examples).
+  - name: finish_reason_length
+    display_name: finish b/c length
+    description: Fraction of instances where the the output was terminated because of the max tokens limit.
+  - name: finish_reason_stop
+    display_name: finish b/c stop
+    description: Fraction of instances where the the output was terminated because of the stop sequences.
+  - name: finish_reason_endoftext
+    display_name: finish b/c endoftext
+    description: Fraction of instances where the the output was terminated because the end of text token was generated.
+  - name: finish_reason_unknown
+    display_name: finish b/c unknown
+    description: Fraction of instances where the the output was terminated for unknown reasons.
+  # Accuracy metrics:
+  - name: exact_match
+    display_name: Exact match
+    short_display_name: EM
+    description: Fraction of instances that the predicted output matches a correct reference exactly.
+    lower_is_better: false
+  - name: quasi_exact_match
+    display_name: Quasi-exact match
+    short_display_name: EM
+    description: Fraction of instances that the predicted output matches a correct reference up to light processing.
+    lower_is_better: false
+  - name: rouge_1
+    display_name: ROUGE-1
+    description: Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on 1-gram overlap.
+    lower_is_better: false
+  - name: rouge_2
+    display_name: ROUGE-2
+    description: Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on 2-gram overlap.
+    lower_is_better: false
+  - name: rouge_l
+    display_name: ROUGE-L
+    description: Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on longest common subsequence overlap.
+    lower_is_better: false
+  - name: annotation_autobencher_capabilities_score
+    display_name: Correct
+    description: Model-judged correctness for AutoBencher Capabilities
+    lower_is_better: false
+
+############################################################
+perturbations: []
+
+############################################################
+metric_groups:
+  - name: accuracy
+    display_name: Accuracy
+    metrics:
+      - name: ${main_name}
+        split: ${main_split}
+
+  - name: efficiency
+    display_name: Efficiency
+    metrics:
+    - name: inference_runtime
+      split: ${main_split}
+
+  - name: general_information
+    display_name: General information
+    hide_win_rates: true
+    metrics:
+    - name: num_instances
+      split: ${main_split}
+    - name: num_train_instances
+      split: ${main_split}
+    - name: prompt_truncated
+      split: ${main_split}
+    - name: num_prompt_tokens
+      split: ${main_split}
+    - name: num_output_tokens
+      split: ${main_split}
+
+############################################################
+run_groups:
+  - name: autobencher_scenarios
+    display_name: AutoBencher Scenarios
+    description: AutoBencher Scenarios
+    category: All scenarios
+    subgroups:
+      - autobencher_capabilities
+      - autobencher_safety
+
+  - name: autobencher_capabilities
+    display_name: AutoBencher Capabilities
+    description: AutoBencher Capabilities consists of question answering datasets for math, multilingual, and knowledge-intensive question answering created by AutoBencher. ([paper](https://arxiv.org/abs/2407.08351))
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: annotation_autobencher_capabilities_score
+      main_split: test
+    taxonomy:
+      task: question answering
+      what: questions about various 
+      who: synthetic model-generated questions
+      when: "2024"
+      language: English and various languages
+
+  - name: autobencher_safety
+    display_name: AutoBencher Safety
+    description: AutoBencher Safety consists of harmful question answering datasets created by AutoBencher. ([paper](https://arxiv.org/abs/2407.08351))
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: annotation_autobencher_capabilities_score
+      main_split: test
+    taxonomy:
+      task: question answering
+      what: harmful questions 
+      who: synthetic model-generated questions
+      when: "2024"
+      language: English and various languages

helm/benchmark/static/schema_call_center.yaml

@@ -0,0 +1,269 @@
+---
+############################################################
+metrics:
+  # Infrastructure metrics:
+  - name: num_perplexity_tokens
+    display_name: '# tokens'
+    description: Average number of tokens in the predicted output (for language modeling, the input too).
+  - name: num_bytes
+    display_name: '# bytes'
+    description: Average number of bytes in the predicted output (for language modeling, the input too).
+
+  - name: num_references
+    display_name: '# ref'
+    description: Number of references.
+  - name: num_train_trials
+    display_name: '# trials'
+    description: Number of trials, where in each trial we choose an independent, random set of training instances.
+  - name: estimated_num_tokens_cost
+    display_name: 'cost'
+    description: An estimate of the number of tokens (including prompt and output completions) needed to perform the request.
+  - name: num_prompt_tokens
+    display_name: '# prompt tokens'
+    description: Number of tokens in the prompt.
+  - name: num_prompt_characters
+    display_name: '# prompt chars'
+    description: Number of characters in the prompt.
+  - name: num_completion_tokens
+    display_name: '# completion tokens'
+    description: Actual number of completion tokens (over all completions).
+  - name: num_output_tokens
+    display_name: '# output tokens'
+    description: Actual number of output tokens.
+  - name: max_num_output_tokens
+    display_name: 'Max output tokens'
+    description: Maximum number of output tokens (overestimate since we might stop earlier due to stop sequences).
+  - name: num_requests
+    display_name: '# requests'
+    description: Number of distinct API requests.
+  - name: num_instances
+    display_name: '# eval'
+    description: Number of evaluation instances.
+  - name: num_train_instances
+    display_name: '# train'
+    description: Number of training instances (e.g., in-context examples).
+  - name: prompt_truncated
+    display_name: truncated
+    description: Fraction of instances where the prompt itself was truncated (implies that there were no in-context examples).
+  - name: finish_reason_length
+    display_name: finish b/c length
+    description: Fraction of instances where the the output was terminated because of the max tokens limit.
+  - name: finish_reason_stop
+    display_name: finish b/c stop
+    description: Fraction of instances where the the output was terminated because of the stop sequences.
+  - name: finish_reason_endoftext
+    display_name: finish b/c endoftext
+    description: Fraction of instances where the the output was terminated because the end of text token was generated.
+  - name: finish_reason_unknown
+    display_name: finish b/c unknown
+    description: Fraction of instances where the the output was terminated for unknown reasons.
+  - name: num_completions
+    display_name: '# completions'
+    description: Number of completions.
+  - name: predicted_index
+    display_name: Predicted index
+    description: Integer index of the reference (0, 1, ...) that was predicted by the model (for multiple-choice).
+
+  # Accuracy metrics:
+  - name: exact_match
+    display_name: Exact match
+    short_display_name: EM
+    description: Fraction of instances that the predicted output matches a correct reference exactly.
+    lower_is_better: false
+
+  # Summarization metrics:
+  - name: annotation_call_center_summarization_faithfulness
+    display_name: Faithfulness
+    short_display_name: Faithfulness
+    description: Whether all the information expressed by the summary can be inferred from the source transcript.
+    lower_is_better: false
+  - name: annotation_call_center_summarization_relevance
+    display_name: Relevance
+    short_display_name: Relevance
+    description: Whether the summary includes only important information from the source.
+    lower_is_better: false
+  - name: annotation_call_center_summarization_coherence
+    display_name: Coherence
+    short_display_name: Coherence
+    description: Whether the summary organizes the relevant information into a well-structured summary.
+    lower_is_better: false
+
+  - name: annotation_call_center_summarization_pairwise_comparison_score
+    display_name: Pairwise
+    short_display_name: Pairwise
+    description: Whether the model's summary was preferred by the evaluator model
+    lower_is_better: false
+    
+  - name: annotation_call_center_summarization_key_points_recall_score
+    display_name: Recall
+    short_display_name: Recall
+    description: How many key items were recalled
+    lower_is_better: false
+
+  - name: annotation_helpdesk_call_center_summarization_score
+    display_name: Score
+    short_display_name: Score
+    description: Score
+    lower_is_better: false
+
+
+  - name: call_summarization_score
+    display_name: Score
+    short_display_name: Score
+    description: Score
+    lower_is_better: false
+
+
+############################################################
+perturbations: []
+
+############################################################
+metric_groups:
+  - name: summarization_metrics
+    display_name: Summarization
+    hide_win_rates: true
+    metrics:
+      - name: call_summarization_score
+        split: ${main_split}
+      # - name: annotation_helpdesk_call_center_summarization_score
+      #   split: ${main_split}
+      # - name: annotation_call_center_summarization_faithfulness
+      #   split: ${main_split}
+      # - name: annotation_call_center_summarization_relevance
+      #   split: ${main_split}
+      # - name: annotation_call_center_summarization_coherence
+      #   split: ${main_split}
+
+  - name: pairwise_comparison_metrics
+    display_name: Pairwise Comparison
+    hide_win_rates: true
+    metrics:
+      - name: annotation_call_center_summarization_pairwise_comparison_score
+        split: ${main_split}
+
+  - name: key_points_recall_metrics
+    display: Key Points Recall
+    hide_win_rates: true
+    metrics:
+      - name: annotation_call_center_summarization_key_points_recall_score
+        split: ${main_split}
+
+  - name: efficiency
+    display_name: Efficiency
+    metrics:
+    - name: inference_runtime
+      split: ${main_split}
+
+  - name: general_information
+    display_name: General information
+    hide_win_rates: true
+    metrics:
+    - name: num_instances
+      split: ${main_split}
+    - name: num_train_instances
+      split: ${main_split}
+    - name: prompt_truncated
+      split: ${main_split}
+    - name: num_prompt_tokens
+      split: ${main_split}
+    - name: num_output_tokens
+      split: ${main_split}
+
+############################################################
+
+run_groups:
+  - name: call_center_scenarios
+    display_name: Call Center Scenarios
+    description: Scenarios representating realistic tasks from the call center.
+    category: All scenarios
+    subgroups:
+      - helpdesk_call_summarization
+      # - call_center_summarization
+      # - call_center_summarization_real_call_transcripts
+      # - call_center_summarization_pairwise_comparison
+      # - call_center_summarization_key_points_recall
+
+  # - name: call_center_summarization
+  #   display_name: Summarization
+  #   description: summarization
+  #   metric_groups:
+  #     # - accuracy
+  #     - summarization_metrics
+  #     - efficiency
+  #     - general_information
+  #   environment:
+  #     main_split: test
+  #   taxonomy:
+  #     task: summarization
+  #     what: n/a
+  #     who: n/a
+  #     when: "?"
+  #     language: English
+
+  - name: helpdesk_call_summarization
+    display_name: Helpdesk Call summarization
+    description: Helpdesk Call summarization
+    metric_groups:
+      # - accuracy
+      - summarization_metrics
+      - efficiency
+      - general_information
+    environment:
+      main_split: test
+    taxonomy:
+      task: summarization
+      what: n/a
+      who: n/a
+      when: "?"
+      language: English
+
+  # - name: call_center_summarization_real_call_transcripts
+  #   display_name: Summarization (Real)
+  #   description: Summarization with real call transcripts
+  #   metric_groups:
+  #     # - accuracy
+  #     - summarization_metrics
+  #     - efficiency
+  #     - general_information
+  #   environment:
+  #     main_split: test
+  #   taxonomy:
+  #     task: summarization
+  #     what: n/a
+  #     who: n/a
+  #     when: "?"
+  #     language: English
+
+  # - name: call_center_summarization_pairwise_comparison
+  #   display_name: Summarization (Pairwise)
+  #   description: summarization
+  #   metric_groups:
+  #     # - accuracy
+  #     - pairwise_comparison_metrics
+  #     - efficiency
+  #     - general_information
+  #   environment:
+  #     main_split: test
+  #   taxonomy:
+  #     task: summarization
+  #     what: n/a
+  #     who: n/a
+  #     when: "?"
+  #     language: English
+
+  # - name: call_center_summarization_key_points_recall
+  #   display_name: Summarization (Key Points Recall)
+  #   description: summarization
+  #   metric_groups:
+  #     # - accuracy
+  #     - key_points_recall_metrics
+  #     - efficiency
+  #     - general_information
+  #   environment:
+  #     main_split: test
+  #   taxonomy:
+  #     task: summarization
+  #     what: n/a
+  #     who: n/a
+  #     when: "?"
+  #     language: English

helm/benchmark/static/schema_capabilities.yaml

@@ -0,0 +1,254 @@
+---
+# EXPERIMENTAL: DO NOT USE IN PROD
+# Temporary schema for prototyping HELM Lite v2
+############################################################
+metrics:
+  # Infrastructure metrics:
+  - name: num_perplexity_tokens
+    display_name: '# tokens'
+    description: Average number of tokens in the predicted output (for language modeling, the input too).
+  - name: num_bytes
+    display_name: '# bytes'
+    description: Average number of bytes in the predicted output (for language modeling, the input too).
+
+  - name: num_references
+    display_name: '# ref'
+    description: Number of references.
+  - name: num_train_trials
+    display_name: '# trials'
+    description: Number of trials, where in each trial we choose an independent, random set of training instances.
+  - name: estimated_num_tokens_cost
+    display_name: 'cost'
+    description: An estimate of the number of tokens (including prompt and output completions) needed to perform the request.
+  - name: num_prompt_tokens
+    display_name: '# prompt tokens'
+    description: Number of tokens in the prompt.
+  - name: num_prompt_characters
+    display_name: '# prompt chars'
+    description: Number of characters in the prompt.
+  - name: num_completion_tokens
+    display_name: '# completion tokens'
+    description: Actual number of completion tokens (over all completions).
+  - name: num_output_tokens
+    display_name: '# output tokens'
+    description: Actual number of output tokens.
+  - name: max_num_output_tokens
+    display_name: 'Max output tokens'
+    description: Maximum number of output tokens (overestimate since we might stop earlier due to stop sequences).
+  - name: num_requests
+    display_name: '# requests'
+    description: Number of distinct API requests.
+  - name: num_instances
+    display_name: '# eval'
+    description: Number of evaluation instances.
+  - name: num_train_instances
+    display_name: '# train'
+    description: Number of training instances (e.g., in-context examples).
+  - name: prompt_truncated
+    display_name: truncated
+    description: Fraction of instances where the prompt itself was truncated (implies that there were no in-context examples).
+  - name: finish_reason_length
+    display_name: finish b/c length
+    description: Fraction of instances where the the output was terminated because of the max tokens limit.
+  - name: finish_reason_stop
+    display_name: finish b/c stop
+    description: Fraction of instances where the the output was terminated because of the stop sequences.
+  - name: finish_reason_endoftext
+    display_name: finish b/c endoftext
+    description: Fraction of instances where the the output was terminated because the end of text token was generated.
+  - name: finish_reason_unknown
+    display_name: finish b/c unknown
+    description: Fraction of instances where the the output was terminated for unknown reasons.
+  - name: num_completions
+    display_name: '# completions'
+    description: Number of completions.
+  - name: predicted_index
+    display_name: Predicted index
+    description: Integer index of the reference (0, 1, ...) that was predicted by the model (for multiple-choice).
+  - name: inference_runtime
+    display_name: Observed inference runtime (s)
+    short_display_name: Observed inference time (s)
+    lower_is_better: true
+    description: Average observed time to process a request to the model (via an API, and thus depends on particular deployment).
+
+  # Accuracy metrics:
+  - name: exact_match
+    display_name: Exact match
+    short_display_name: EM
+    description: Fraction of instances that the predicted output matches a correct reference exactly.
+    lower_is_better: false
+  - name: quasi_exact_match
+    display_name: Quasi-exact match
+    short_display_name: EM
+    description: Fraction of instances that the predicted output matches a correct reference up to light processing.
+    lower_is_better: false
+  - name: prefix_exact_match
+    display_name: Prefix exact match
+    short_display_name: PEM
+    description: Fraction of instances that the predicted output matches the prefix of a correct reference exactly.
+    lower_is_better: false
+  - name: quasi_prefix_exact_match
+    # TODO: should call this prefix_quasi_exact_match
+    display_name: Prefix quasi-exact match
+    short_display_name: PEM
+    description: Fraction of instances that the predicted output matches the prefix of a correct reference up to light processing.
+    lower_is_better: false
+  - name: ifeval_strict_accuracy
+    display_name: IFEval strict accuracy
+    short_display_name: IFEval Strict Acc
+    description: Fraction of instructions in the instance that are correctly followed.
+    lower_is_better: false
+  - name: chain_of_thought_correctness
+    display_name: COT correct
+    short_display_name: COT correct
+    description: Fraction of correct answers after chain of thought
+    lower_is_better: false
+  - name: wildbench_score
+    display_name: WildBench Score
+    short_display_name: WB Score
+    description: Score of the AI output judged by GPT-4o.
+    lower_is_better: false
+  - name: wildbench_score_rescaled
+    display_name: WildBench Score
+    short_display_name: WB Score
+    description: Score of the AI output judged by GPT-4o, rescaled to be between 0 and 1.
+    lower_is_better: false
+  - name: omni_math_accuracy
+    display_name: Omni-MATH Accuracy
+    short_display_name: Acc
+    description: Accuracy of the AI output judged by GPT-4.
+    lower_is_better: false
+
+############################################################
+perturbations: []
+
+############################################################
+metric_groups:
+  - name: accuracy
+    display_name: Accuracy
+    aggregation_strategies: 
+      - mean
+    metrics:
+      - name: ${main_name}
+        split: ${main_split}
+
+  - name: efficiency
+    display_name: Efficiency
+    aggregation_strategies: 
+      - mean
+    metrics:
+    - name: inference_runtime
+      split: ${main_split}
+
+  - name: general_information
+    display_name: General information
+    hide_win_rates: true
+    metrics:
+    - name: num_instances
+      split: ${main_split}
+    - name: num_train_instances
+      split: ${main_split}
+    - name: prompt_truncated
+      split: ${main_split}
+    - name: num_prompt_tokens
+      split: ${main_split}
+    - name: num_output_tokens
+      split: ${main_split}
+
+############################################################
+run_groups:
+  - name: core_scenarios
+    display_name: Core Scenarios
+    description: Core Scenarios
+    category: All scenarios
+    subgroups:
+      - mmlu_pro
+      - gpqa
+      - ifeval
+      - wildbench
+      - omni_math
+
+  - name: mmlu_pro
+    display_name: MMLU-Pro
+    description: MMLU-Pro
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: chain_of_thought_correctness
+      main_split: test
+    taxonomy:
+      task: "question answering"
+      what: "complex questions across various disciplines"
+      who: "2024"
+      when: "domain experts"
+      language: English
+
+  - name: gpqa
+    display_name: GPQA
+    description: GPQA
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: chain_of_thought_correctness
+      main_split: test
+    taxonomy:
+      task: "question answering"
+      what: "graduate-level questions in biology, physics, and chemistry"
+      who: "domain experts"
+      when: "2023"
+      language: English
+  
+  - name: ifeval
+    display_name: IFEval
+    description: IFEval
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: ifeval_strict_accuracy
+      main_split: test
+    taxonomy:
+      task: "instruction following"
+      what: "verifiable general domain instruction following"
+      who: "human annotators"
+      when: "2023"
+      language: English
+  
+  - name: wildbench
+    display_name: WildBench
+    description: WildBench
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: wildbench_score_rescaled
+      main_split: test
+    taxonomy:
+      task: "instruction following"
+      what: "GPT-judged instruction following with instructions collected from real-user conversations"
+      who: "real-world users"
+      when: "2024"
+      language: English
+
+  - name: omni_math
+    display_name: Omni-MATH
+    description: Omni-MATH
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: omni_math_accuracy
+      main_split: test
+    taxonomy:
+      task: "mathematics"
+      what: "universal Olympiad level mathematic benchmark"
+      who: "human annotators"
+      when: "2024"
+      language: English