crfm-helm 0.4.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

helm/benchmark/static/schema_image2structure.yaml

@@ -0,0 +1,304 @@
+---
+############################################################
+adapter:
+  - name: method
+    description: The high-level strategy for converting instances into a prompt for the language model.
+    values:
+      - name: generation
+        description: Given the input, the model generates the output free-form.
+      - name: generation_multimodal
+        description: Given the multimodal input, the model generates the output free-form.
+      - name: multiple_choice_joint
+        description: Given the input, the model selects from multiple-choice options (A., B., C., D., E.).
+      - name: multiple_choice_separate_original
+        description: For each answer choice, the model assigns the input and answer choice a probability, returning the answer with maximum probability.
+      - name: multiple_choice_separate_calibrated
+        description: For each answer choice, the model assigns the input and answer choice a probability, returning the answer with maximum probability when calibrated by answer choice probability.
+      - name: language_modeling
+        description: Given the input, the model assigns the sequence a probability.
+  - name: instructions
+    description: The description of the task that is included at the very beginning of the prompt.
+  - name: global_prefix
+    description: The string that is prepended to the prompt.
+  - name: global_suffix
+    description: The string that is appended to the prompt.
+  - name: instance_prefix
+    description: The string that is included before each instance (e.g., '\n\n').
+  - name: input_prefix
+    description: The string that is included before each input (e.g., 'Question:').
+  - name: input_suffix
+    description: The string that is included after each input (e.g., '\n').
+  - name: reference_prefix
+    description: The string that is included before each reference (for multiple-choice questions).
+  - name: reference_suffix
+    description: The string that is included after each reference (for multiple-choice questions).
+  - name: output_prefix
+    description: The string that is included before the correct answer/predicted output (e.g., 'Answer:').
+  - name: output_suffix
+    description: The string that is included after the correct answer/predicted output (e.g., '\n').
+  - name: substitutions
+    description: A list of regular expression substitutions (e.g., replacing '\n' with ';\n') to perform at the very end on the prompt.
+  - name: max_train_instances
+    description: Maximum number of training instances to include in the prompt (currently by randomly sampling).
+  - name: max_eval_instances
+    description: Maximum number of instances to evaluate on (over all splits - test, valid, etc.).
+  - name: num_outputs
+    description: Maximum number of possible outputs to generate by sampling multiple outputs.
+  - name: num_train_trials
+    description: Number of trials, where in each trial we choose an independent, random set of training instances. Used to compute variance.
+  - name: sample_train
+    description: If true, randomly sample N training examples; if false, select N consecutive training examples
+  - name: model
+    description: Name of the language model (<creator_organization>/<model name>) to send requests to.
+  - name: model_deployment
+    description: Name of the language model deployment (<host_organization>/<model name>) to send requests to.
+  - name: temperature
+    description: Temperature parameter used in generation.
+  - name: max_tokens
+    description: Maximum number of tokens to generate.
+  - name: stop_sequences
+    description: List of sequences, where we stop generation if we encounter any of them.
+  - name: random
+    description: Random seed (string), which guarantees reproducibility.
+  - name: multi_label
+    description: If true, for instances with multiple correct reference, the gold answer should be considered to be all of the correct references rather than any of the correct references.
+
+############################################################
+metrics:
+  # Infrastructure metrics:
+  - name: num_perplexity_tokens
+    display_name: '# tokens'
+    description: Average number of tokens in the predicted output (for language modeling, the input too).
+  - name: num_bytes
+    display_name: '# bytes'
+    description: Average number of bytes in the predicted output (for language modeling, the input too).
+
+  - name: num_references
+    display_name: '# ref'
+    description: Number of references.
+  - name: num_train_trials
+    display_name: '# trials'
+    description: Number of trials, where in each trial we choose an independent, random set of training instances.
+  - name: estimated_num_tokens_cost
+    display_name: 'cost'
+    description: An estimate of the number of tokens (including prompt and output completions) needed to perform the request.
+  - name: num_prompt_tokens
+    display_name: '# prompt tokens'
+    description: Number of tokens in the prompt.
+  - name: num_prompt_characters
+    display_name: '# prompt chars'
+    description: Number of characters in the prompt.
+  - name: num_completion_tokens
+    display_name: '# completion tokens'
+    description: Actual number of completion tokens (over all completions).
+  - name: num_output_tokens
+    display_name: '# output tokens'
+    description: Actual number of output tokens.
+  - name: max_num_output_tokens
+    display_name: 'Max output tokens'
+    description: Maximum number of output tokens (overestimate since we might stop earlier due to stop sequences).
+  - name: num_requests
+    display_name: '# requests'
+    description: Number of distinct API requests.
+  - name: num_instances
+    display_name: '# eval'
+    description: Number of evaluation instances.
+  - name: num_train_instances
+    display_name: '# train'
+    description: Number of training instances (e.g., in-context examples).
+  - name: prompt_truncated
+    display_name: truncated
+    description: Fraction of instances where the prompt itself was truncated (implies that there were no in-context examples).
+  - name: finish_reason_length
+    display_name: finish b/c length
+    description: Fraction of instances where the the output was terminated because of the max tokens limit.
+  - name: finish_reason_stop
+    display_name: finish b/c stop
+    description: Fraction of instances where the the output was terminated because of the stop sequences.
+  - name: finish_reason_endoftext
+    display_name: finish b/c endoftext
+    description: Fraction of instances where the the output was terminated because the end of text token was generated.
+  - name: finish_reason_unknown
+    display_name: finish b/c unknown
+    description: Fraction of instances where the the output was terminated for unknown reasons.
+  - name: num_completions
+    display_name: '# completions'
+    description: Number of completions.
+  - name: predicted_index
+    display_name: Predicted index
+    description: Integer index of the reference (0, 1, ...) that was predicted by the model (for multiple-choice).
+
+  # Vision Language metrics [text]:
+  - name: edit_similarity
+    display_name: Edit similarity (Levenshtein)
+    short_display_name: Edit sim.
+    lower_is_better: false
+    description: Average Levenshtein edit similarity (1 - distance normalized by length of longer sequence) between model generation and reference.
+
+  # Vision Language metrics [image]:
+  - name: block_emd_similarity
+    display_name: Block Earth Mover Similarity
+    short_display_name: Block EMS
+    description: Block Earth Mover Similarity
+    lower_is_better: false
+  - name: block_emd_similarity_white
+    display_name: Block Earth Mover Similarity (white)
+    short_display_name: Block EMS (white)
+    description: Block Earth Mover Similarity (white)
+    lower_is_better: false
+  - name: block_emd_similarity_median_color
+    display_name: Block Earth Mover Similarity (median)
+    short_display_name: Block EMS (median)
+    description: Block Earth Mover Similarity (median)
+    lower_is_better: false
+  - name: pixel_similarity
+    display_name: Pixel Similarity
+    short_display_name: PS
+    description: Pixel Similarity between an image generated by the model and the target image.
+    lower_is_better: false
+  - name: sift_similarity
+    display_name: SIFT Similarity
+    short_display_name: SIFT
+    description: SIFT Similarity (Scale-Invariant Feature Transform) [(Lowe, 1999)](https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=790410) between an image generated by the model and the target image.
+    lower_is_better: false
+  - name: compilation_success
+    display_name: Compilation success
+    description: Fraction of instances where the generated code compiles successfully.
+    lower_is_better: false
+  - name: lpips_similarity
+    display_name: LPIPS similarity
+    short_display_name: LPIPS
+    description: LPIPS similarity (Learned Perceptual Image Patch Similarity) [(Zhang et al., 2018)](https://arxiv.org/abs/1801.03924) between an image generated by the model and the target image.
+    lower_is_better: false
+  - name: fid_similarity
+    display_name: FID similarity
+    short_display_name: FID
+    description: FID similarity (Fréchet Inception Distance) [(Heusel et al., 2017)](https://arxiv.org/abs/1706.08500) between an image generated by the model and the target image.
+    lower_is_better: false
+  - name: ssim_similarity
+    display_name: SSIM
+    short_display_name: SSIM
+    description: SSIM similarity (Structural Similarity Index) [(Wang et al., 2004)](https://www.cns.nyu.edu/pub/eero/wang03-reprint.pdf) between an image generated by the model and the target image.
+    lower_is_better: false
+
+  # Accuracy metrics:
+  - name: exact_match
+    display_name: Exact match
+    short_display_name: EM
+    description: Fraction of instances that the predicted output matches a correct reference exactly.
+    lower_is_better: false
+  - name: quasi_exact_match
+    display_name: Quasi-exact match
+    short_display_name: EM
+    description: Fraction of instances that the predicted output matches a correct reference up to light processing.
+    lower_is_better: false
+  - name: prefix_exact_match
+    display_name: Prefix exact match
+    short_display_name: PEM
+    description: Fraction of instances that the predicted output matches the prefix of a correct reference exactly.
+    lower_is_better: false
+  - name: quasi_prefix_exact_match
+    # TODO: should call this prefix_quasi_exact_match
+    display_name: Prefix quasi-exact match
+    short_display_name: PEM
+    description: Fraction of instances that the predicted output matches the prefix of a correct reference up to light processing.
+    lower_is_better: false
+
+############################################################
+perturbations:
+  - name: robustness
+    display_name: Robustness
+    description: Computes worst case over different robustness perturbations (misspellings, formatting, contrast sets).
+
+############################################################
+metric_groups:
+  - name: accuracy
+    display_name: Compilation Rate and Earth Mover Similarity
+    metrics:
+      - name: ${main_name}
+        split: ${main_split}
+      - name: compilation_success
+        split: ${main_split}
+
+  - name: generation_image
+    display_name: Generation (image)
+    metrics:
+      - name: pixel_similarity
+        split: ${main_split}
+      - name: compilation_success
+        split: ${main_split}
+      - name: fid_similarity
+        split: ${main_split}
+      - name: block_emd_similarity
+        split: ${main_split}
+      - name: block_emd_similarity_white
+        split: ${main_split}
+      - name: block_emd_similarity_median_color
+        split: ${main_split}
+
+  - name: generation_text
+    display_name: Generation (text)
+    metrics:
+      - name: edit_similarity
+        split: ${main_split}
+
+############################################################
+run_groups:
+  - name: core_scenarios
+    display_name: Image2Structure
+    description: Scenarios for evaluating the ability of Vision-Language models to generate structured outputs from images.
+    category: All scenarios
+    subgroups:
+      - image2latex
+      - image2webpage
+      - image2musicsheet
+
+  - name: image2latex
+    display_name: Image2LaTeX
+    description: The Image2LaTeX benchmark for converting images of mathematical equations, tables. algorithms and tikz to LaTeX.
+    metric_groups:
+      - accuracy
+      - generation_image
+      - generation_text
+    environment:
+      main_name: block_emd_similarity
+      main_split: valid
+    taxonomy:
+      task: image-to-text
+      what: mathematical equations, tables, algorithms, tikz
+      who: n/a
+      when: "2024"
+      language: English
+
+  - name: image2webpage
+    display_name: Image2webpage
+    description: The Image2webpage benchmark for converting images of webpages to HTML/CSS/Javascript.
+    metric_groups:
+      - accuracy
+      - generation_image
+      - generation_text
+    environment:
+      main_name: block_emd_similarity
+      main_split: valid
+    taxonomy:
+      task: image-to-text
+      what: css, html, javascript
+      who: n/a
+      when: "2024"
+      language: English
+
+  - name: image2musicsheet
+    display_name: Image2musicsheet
+    description: The Image2musicsheet benchmark for converting images of music sheets to LilyPond.
+    metric_groups:
+      - accuracy
+      - generation_image
+    environment:
+      main_name: block_emd_similarity
+      main_split: valid
+    taxonomy:
+      task: image-to-text
+      what: music sheets
+      who: n/a
+      when: "2024"
+      language: English

helm/benchmark/static/schema_instruction_following.yaml

@@ -0,0 +1,210 @@
+---
+############################################################
+perturbations: []
+adapter:
+  - name: method
+    description: The high-level strategy for converting instances into a prompt for the language model.
+    values:
+      - name: generation
+        description: Given the input, the model generates the output free-form.
+  - name: instructions
+    description: The description of the task that is included at the very beginning of the prompt.
+  - name: global_prefix
+    description: The string that is prepended to the prompt.
+  - name: instance_prefix
+    description: The string that is included before each instance (e.g., '\n\n').
+  - name: input_prefix
+    description: The string that is included before each input (e.g., 'Question:').
+  - name: input_suffix
+    description: The string that is included after each input (e.g., '\n').
+  - name: reference_prefix
+    description: The string that is included before each reference (for multiple-choice questions).
+  - name: reference_suffix
+    description: The string that is included after each reference (for multiple-choice questions).
+  - name: output_prefix
+    description: The string that is included before the correct answer/predicted output (e.g., 'Answer:').
+  - name: output_suffix
+    description: The string that is included after the correct answer/predicted output (e.g., '\n').
+  - name: substitutions
+    description: A list of regular expression substitutions (e.g., replacing '\n' with ';\n') to perform at the very end on the prompt.
+  - name: max_train_instances
+    description: Maximum number of training instances to include in the prompt (currently by randomly sampling).
+  - name: max_eval_instances
+    description: Maximum number of instances to evaluate on (over all splits - test, valid, etc.).
+  - name: num_outputs
+    description: Maximum number of possible outputs to generate by sampling multiple outputs.
+  - name: num_train_trials
+    description: Number of trials, where in each trial we choose an independent, random set of training instances. Used to compute variance.
+  - name: sample_train
+    description: If true, randomly sample N training examples; if false, select N consecutive training examples
+  - name: model
+    description: Name of the language model (<creator_organization>/<model name>) to send requests to.
+  - name: model_deployment
+    description: Name of the language model deployment (<host_organization>/<model name>) to send requests to.
+  - name: temperature
+    description: Temperature parameter used in generation.
+  - name: max_tokens
+    description: Maximum number of tokens to generate.
+  - name: stop_sequences
+    description: List of sequences, where we stop generation if we encounter any of them.
+  - name: random
+    description: Random seed (string), which guarantees reproducibility.
+  - name: multi_label
+    description: If true, for instances with multiple correct reference, the gold answer should be considered to be all of the correct references rather than any of the correct references.
+
+############################################################
+metrics:
+  - name: Helpfulness
+    display_name: Helpfulness
+    short_display_name: Helpfulness
+    description: Whether the model appears to do what it is instructed to.
+    lower_is_better: false
+  - name: Understandability
+    display_name: Understandability
+    short_display_name: Understandability
+    description: Whether the answer is easy to comprehend.
+    lower_is_better: false
+  - name: Completeness
+    display_name: Completeness
+    short_display_name: Completeness
+    description: Whether the response provides a comprehensive answer to the question or request.
+    lower_is_better: false
+  - name: Conciseness
+    display_name: Conciseness
+    short_display_name: Conciseness
+    description: Whether the response is concise and to the point, without unnecessary repetition or elaboration.
+    lower_is_better: false
+  - name: Harmlessness
+    display_name: Harmlessness
+    short_display_name: Harmlessness
+    description: Whether the response avoids using toxic language or helping the user with harmful goals.
+    lower_is_better: false
+
+############################################################
+metric_groups:
+  - name: instruction_following_metrics
+    display_name: Instruction Following
+    metrics:
+      - name: Helpfulness
+        split: ${main_split}
+      - name: Understandability
+        split: ${main_split}
+      - name: Completeness
+        split: ${main_split}
+      - name: Conciseness
+        split: ${main_split}
+      - name: Harmlessness
+        split: ${main_split}
+
+############################################################
+run_groups:
+  - name: instruction_following
+    display_name: Instruction Following
+    description: Given an open-ended instruction in natural language, the goal is to produce a text response that is helpful, understandable, complete, concise and harmless.
+    subgroups:
+      - anthropic_hh_rlhf
+      - grammar
+      - koala
+      - open_assistant
+      - self_instruct
+      - vicuna
+
+  - name: anthropic_hh_rlhf
+    display_name: Anthropic RLHF dataset
+    short_display_name: Anthropic RLHF dataset
+    description: The dialogue datasets released by Anthropic to facilitate research in model helpfulness and harmlessness ([Bai et al., 2022](https://arxiv.org/pdf/2204.05862.pdf); [Ganguli et al., 2022](https://arxiv.org/pdf/2209.07858.pdf)). We only use the first utterance of each dialogue.
+    metric_groups:
+      - instruction_following_metrics
+    environment:
+      main_name: Helpfulness
+      main_split: test
+    taxonomy:
+      task: open-ended instruction following
+      what: "Human-LM dialogues and preference labels"
+      who: "Workers from MTurk and Upwork, language models from Anthropic"
+      when: "2022"
+      language: English
+
+  # Ideally, the name should be "best_chatgpt_prompts".
+  # But unfortunately the group name in the results is "grammar",
+  # so the schema has to match the same group name.
+  # TODO: Change the group name in the "grammar" run spec, and then change this group name.
+  - name: grammar
+    display_name: Best ChatGPT Prompts
+    short_display_name: Best ChatGPT Prompts
+    description: A list of “best ChatGPT prompts to power your workflow” summarized by [GRIDFITI](https://gridfiti.com/best-chatgpt-prompts/).
+    metric_groups:
+      - instruction_following_metrics
+    environment:
+      main_name: Helpfulness
+      main_split: test
+    taxonomy:
+      task: open-ended instruction following
+      what: "Instructions for LLMs"
+      who: "Gridfiti Staff"
+      when: "2023"
+      language: English
+
+  - name: koala
+    display_name: Koala test dataset
+    short_display_name: Koala test dataset
+    description: The test dataset from the [Koala paper](https://bair.berkeley.edu/blog/2023/04/03/koala/) for evaluating instruction-following models.
+    metric_groups:
+      - instruction_following_metrics
+    environment:
+      main_name: Helpfulness
+      main_split: test
+    taxonomy:
+      task: open-ended instruction following
+      what: "Instructions for LLMs"
+      who: "Web users"
+      when: "Before 2023"
+      language: English
+
+  - name: open_assistant
+    display_name: Open Assistant
+    short_display_name: Open Assistant
+    description: LAION’s OpenAssistant Conversations Dataset (OASST1) that consists of 66,497 conversation trees ([Köpf et al., 2023](https://openreview.net/forum?id=VSJotgbPHF)). We only use the initial prompt in each conversation.
+    metric_groups:
+      - instruction_following_metrics
+    environment:
+      main_name: Helpfulness
+      main_split: valid
+    taxonomy:
+      task: open-ended instruction following
+      what: "Human-written dialogues and response rankings"
+      who: "Open Assistant participants"
+      when: "2023"
+      language: "35 languages"
+
+  - name: self_instruct
+    display_name: Self Instruct
+    short_display_name: Self Instruct
+    description: The manually-curated instructions from the Self-Instruct paper ([Wang et al., 2023](https://aclanthology.org/2023.acl-long.754.pdf)).
+    metric_groups:
+      - instruction_following_metrics
+    environment:
+      main_name: Helpfulness
+      main_split: test
+    taxonomy:
+      task: open-ended instruction following
+      what: "Instructions for LLMs"
+      who: "Authors of the research paper"
+      when: "2022"
+      language: English
+
+  - name: vicuna
+    display_name: Vicuna
+    short_display_name: Vicuna
+    description: The set of prompts used by the [Vicuna](https://lmsys.org/blog/2023-03-30-vicuna/) team to evaluate instruction-following models.
+    metric_groups:
+      - instruction_following_metrics
+    environment:
+      main_name: Helpfulness
+      main_split: test
+    taxonomy:
+      task: open-ended instruction following
+      what: "Instructions for LLMs"
+      who: "Unknown"
+      when: "Before 2023"
+      language: English