crfm-helm 0.5.0__py3-none-any.whl → 0.5.2__py3-none-any.whl

helm/benchmark/static/schema_vhelm.yaml

@@ -0,0 +1,824 @@
+---
+############################################################
+# For backwards compatibility with older versions of HELM.
+# TODO: Remove this after 2024-09-01.
+adapter: []
+############################################################
+metrics:
+  # Infrastructure metrics:
+  - name: num_perplexity_tokens
+    display_name: '# tokens'
+    description: Average number of tokens in the predicted output (for language modeling, the input too).
+  - name: num_bytes
+    display_name: '# bytes'
+    description: Average number of bytes in the predicted output (for language modeling, the input too).
+
+  - name: num_references
+    display_name: '# ref'
+    description: Number of references.
+  - name: num_train_trials
+    display_name: '# trials'
+    description: Number of trials, where in each trial we choose an independent, random set of training instances.
+  - name: estimated_num_tokens_cost
+    display_name: 'cost'
+    description: An estimate of the number of tokens (including prompt and output completions) needed to perform the request.
+  - name: num_prompt_tokens
+    display_name: '# prompt tokens'
+    description: Number of tokens in the prompt.
+  - name: num_prompt_characters
+    display_name: '# prompt chars'
+    description: Number of characters in the prompt.
+  - name: num_completion_tokens
+    display_name: '# completion tokens'
+    description: Actual number of completion tokens (over all completions).
+  - name: num_output_tokens
+    display_name: '# output tokens'
+    description: Actual number of output tokens.
+  - name: max_num_output_tokens
+    display_name: 'Max output tokens'
+    description: Maximum number of output tokens (overestimate since we might stop earlier due to stop sequences).
+  - name: num_requests
+    display_name: '# requests'
+    description: Number of distinct API requests.
+  - name: num_instances
+    display_name: '# eval'
+    description: Number of evaluation instances.
+  - name: num_train_instances
+    display_name: '# train'
+    description: Number of training instances (e.g., in-context examples).
+  - name: prompt_truncated
+    display_name: truncated
+    description: Fraction of instances where the prompt itself was truncated (implies that there were no in-context examples).
+  - name: finish_reason_length
+    display_name: finish b/c length
+    description: Fraction of instances where the the output was terminated because of the max tokens limit.
+  - name: finish_reason_stop
+    display_name: finish b/c stop
+    description: Fraction of instances where the the output was terminated because of the stop sequences.
+  - name: finish_reason_endoftext
+    display_name: finish b/c endoftext
+    description: Fraction of instances where the the output was terminated because the end of text token was generated.
+  - name: finish_reason_unknown
+    display_name: finish b/c unknown
+    description: Fraction of instances where the the output was terminated for unknown reasons.
+  - name: num_completions
+    display_name: '# completions'
+    description: Number of completions.
+  - name: predicted_index
+    display_name: Predicted index
+    description: Integer index of the reference (0, 1, ...) that was predicted by the model (for multiple-choice).
+
+  # Vision Language metrics [text]:
+  - name: edit_similarity
+    display_name: Edit similarity (Levenshtein)
+    short_display_name: Edit sim.
+    lower_is_better: false
+    description: Average Levenshtein edit similarity (1 - distance normalized by length of longer sequence) between model generation and reference.
+
+  # Vision Language metrics [image]:
+  - name: earth_mover_similarity
+    display_name: Earth Mover Similarity
+    short_display_name: EMD-Sim
+    description: 1 - Earth Mover Distance [(Rubner and Tomasi, 2000)](https://www.cs.cmu.edu/~efros/courses/LBMV07/Papers/rubner-jcviu-00.pdf) between an image generated by the model and the target image.
+    lower_is_better: false
+  - name: pixel_similarity
+    display_name: Pixel Similarity
+    short_display_name: PS
+    description: Pixel Similarity between an image generated by the model and the target image.
+    lower_is_better: false
+  - name: sift_similarity
+    display_name: SIFT Similarity
+    short_display_name: SIFT
+    description: SIFT Similarity (Scale-Invariant Feature Transform) [(Lowe, 1999)](https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=790410) between an image generated by the model and the target image.
+    lower_is_better: false
+  - name: compilation_success
+    display_name: Compilation success
+    description: Fraction of instances where the generated code compiles successfully.
+    lower_is_better: false
+  - name: lpips_similarity
+    display_name: LPIPS similarity
+    short_display_name: LPIPS
+    description: LPIPS similarity (Learned Perceptual Image Patch Similarity) [(Zhang et al., 2018)](https://arxiv.org/abs/1801.03924) between an image generated by the model and the target image.
+    lower_is_better: false
+  - name: fid_similarity
+    display_name: FID similarity
+    short_display_name: FID
+    description: FID similarity (Fréchet Inception Distance) [(Heusel et al., 2017)](https://arxiv.org/abs/1706.08500) between an image generated by the model and the target image.
+    lower_is_better: false
+  - name: ssim_similarity
+    display_name: SSIM
+    short_display_name: SSIM
+    description: SSIM similarity (Structural Similarity Index) [(Wang et al., 2004)](https://www.cns.nyu.edu/pub/eero/wang03-reprint.pdf) between an image generated by the model and the target image.
+    lower_is_better: false
+
+  # Accuracy metrics:
+  - name: exact_match
+    display_name: Exact match
+    short_display_name: EM
+    description: Fraction of instances that the predicted output matches a correct reference exactly.
+    lower_is_better: false
+  - name: quasi_exact_match
+    display_name: Quasi-exact match
+    short_display_name: EM
+    description: Fraction of instances that the predicted output matches a correct reference up to light processing.
+    lower_is_better: false
+  - name: prefix_exact_match
+    display_name: Prefix exact match
+    short_display_name: PEM
+    description: Fraction of instances that the predicted output matches the prefix of a correct reference exactly.
+    lower_is_better: false
+  - name: quasi_prefix_exact_match
+    # TODO: should call this prefix_quasi_exact_match
+    display_name: Prefix quasi-exact match
+    short_display_name: PEM
+    description: Fraction of instances that the predicted output matches the prefix of a correct reference up to light processing.
+    lower_is_better: false
+
+  - name: rouge_1
+    display_name: ROUGE-1
+    description: Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on 1-gram overlap.
+    lower_is_better: false
+  - name: rouge_2
+    display_name: ROUGE-2
+    description: Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on 2-gram overlap.
+    lower_is_better: false
+  - name: rouge_l
+    display_name: ROUGE-L
+    description: Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on longest common subsequence overlap.
+    lower_is_better: false
+  - name: bleu_1
+    display_name: BLEU-1
+    description: Average BLEU score [(Papineni et al., 2002)](https://aclanthology.org/P02-1040/) based on 1-gram overlap.
+    lower_is_better: false
+  - name: bleu_4
+    display_name: BLEU-4
+    description: Average BLEU score [(Papineni et al., 2002)](https://aclanthology.org/P02-1040/) based on 4-gram overlap.
+    lower_is_better: false
+  - name: f1_score
+    display_name: F1
+    description: Average F1 score in terms of word overlap between the model output and correct reference.
+    lower_is_better: false
+  - name: cider
+    display_name: CIDEr
+    description: Evaluates the quality of generated caption by measuring the weighted similarity of n-grams between the captions and a set of human-written reference captions, emphasizing informativeness and consensus.
+    lower_is_better: false
+  - name: prometheus_vision
+    display_name: Prometheus Vision rating
+    description: Scores are from 1 to 5, where 5 is the best.
+    lower_is_better: false
+
+  # Toxicity metrics
+  - name: expected_max_toxicity
+    display_name: Average max toxicity
+    short_display_name: Avg. max toxicity
+    lower_is_better: true
+    description: Average max toxicity score across all completions (based on the PerspectiveAPI toxicity classifier; [Gehman et al. (2020)](https://aclanthology.org/2020.findings-emnlp.301/)).
+  - name: max_toxicity_probability
+    display_name: max toxicity
+    lower_is_better: true
+    description: Fraction of instances where most toxic completion is above 0.5 toxicity (based on the PerspectiveAPI toxicity classifier; [Gehman et al. (2020)](https://aclanthology.org/2020.findings-emnlp.301/)).
+  - name: toxic_frac
+    display_name: Toxic fraction
+    lower_is_better: true
+    description: Fraction of model outputs that are toxic (based on the PerspectiveAPI toxicity classifier).
+
+############################################################
+perturbations:
+  - name: translate
+    display_name: Translate
+    description: Translate prompts to different languages.
+  - name: robustness
+    display_name: Robustness
+    description: Computes worst case over different robustness perturbations (misspellings, formatting, contrast sets).
+  - name: fairness
+    display_name: Fairness
+    description: Computes worst case over different fairness perturbations (changing dialect, race of names, gender).
+  - name: typos
+    display_name: Typos
+    description: >
+      Randomly adds typos to each token in the input with probability 0.05 and computes the per-instance worst-case
+      performance between perturbed and unperturbed versions.
+  - name: synonym
+    display_name: Synonyms
+    description: >
+      Randomly substitutes words in the input with WordNet synonyms with probability 0.5 and computes the per-instance
+      worst-case performance between perturbed and unperturbed versions.
+  - name: dialect
+    display_name: SAE -> AAE
+    short_display_name: Dialect
+    description: >
+      Deterministically substitutes SAE words in input with AAE counterparts using validated dictionary of [Ziems et al. (2022)](https://aclanthology.org/2022.acl-long.258/) and computes the per-instance worst-case performance between perturbed and unperturbed versions.
+  - name: race
+    display_name: First names by race (White -> Black)
+    short_display_name: Race
+    description: >
+      Deterministically substitutes White first names with Black first names sampled from the lists of [Caliskan et al. (2017)](https://www.science.org/doi/10.1126/science.aal4230) and computes the per-instance worst-case performance between perturbed and unperturbed versions.
+  - name: gender
+    display_name: Pronouns by gender (Male -> Female)
+    short_display_name: Gender
+    description: >
+      Deterministically substitutes male pronouns with female pronouns and computes the per-instance worst-case
+      performance between perturbed and unperturbed versions.
+
+############################################################
+metric_groups:
+  - name: accuracy
+    display_name: Accuracy
+    metrics:
+      - name: ${main_name}
+        split: ${main_split}
+
+  - name: general_information
+    display_name: General information
+    metrics:
+    - name: num_instances
+      split: ${main_split}
+    - name: num_train_instances
+      split: ${main_split}
+    - name: prompt_truncated
+      split: ${main_split}
+    - name: num_prompt_tokens
+      split: ${main_split}
+    - name: num_output_tokens
+      split: ${main_split}
+
+  - name: toxicity
+    display_name: Toxicity
+    metrics:
+      - name: toxic_frac
+        split: ${main_split}
+
+  - name: fairness
+    display_name: Fairness
+    metrics:
+      - name: ${main_name}
+        split: ${main_split}
+        perturbation_name: fairness
+
+  - name: robustness
+    display_name: Robustness
+    metrics:
+      - name: ${main_name}
+        split: ${main_split}
+        perturbation_name: robustness
+
+  - name: translate
+    display_name: Translate
+    metrics:
+      - name: ${main_name}
+        split: ${main_split}
+        perturbation_name: translate
+
+
+############################################################
+run_groups:
+  - name: core_scenarios
+    display_name: All
+    description: All scenarios across capabilities
+    category: All scenarios
+    subgroups:
+      - visual_perception
+      - reasoning
+      - knowledge
+      - bias
+      - fairness
+      - toxicity
+      - robustness
+      - multilinguality
+  - name: visual_perception
+    display_name: Visual perception
+    description: Is the output semantically correct, given the text and image inputs?
+    category: Core scenarios
+    subgroups:
+      - vqa_base
+      - viz_wiz
+      - flickr30k
+  - name: reasoning
+    display_name: Reasoning
+    description: Does the model understand objects, counts and spatial relations? Can the model reason about both the text and image input?
+    category: Core scenarios
+    subgroups:
+      - gqa
+      - math_vista
+      - seed_bench
+  - name: real_world_reasoning
+    display_name: Real-world Reasoning
+    description: Reasoning in the real-world
+    category: Core scenarios
+    subgroups:
+      - gqa
+      - seed_bench
+      - mementos
+  - name: knowledge
+    display_name: Knowledge
+    description: Does the model have knowledge about the world and common sense?
+    category: Core scenarios
+    subgroups:
+      - a_okvqa_base
+      - mmmu
+      - mme
+      - vibe_eval
+  - name: bias
+    display_name: Bias
+    description: Are the generations biased in demographic representation? We focus on gender and skin tone bias.
+    category: Core scenarios
+    subgroups:
+      - pairs
+  - name: fairness
+    display_name: Fairness
+    description: Does the model exhibit performance disparities across different groups? We focus on gender, dialect and geographic bias.
+    category: Core scenarios
+    subgroups:
+      - vqa_dialect
+      - a_okvqa_dialect
+      - crossmodal_3600
+  - name: toxicity
+    display_name: Toxicity
+    description: Does the model generate toxic or inappropriate content? Can the model identify toxic or inappropriate content?
+    category: Core scenarios
+    subgroups:
+      - mm_safety_bench
+      - hateful_memes
+  - name: robustness
+    display_name: Robustness
+    description: Is the model robust to perturbations? We focus on both text and image perturbations.
+    category: Core scenarios
+    subgroups:
+      - vqa_robustness
+      - a_okvqa_robustness
+      - unicorn
+      - bingo
+      - pope
+  - name: multilinguality
+    display_name: Multilinguality
+    description: Do the model support non-English languages?
+    category: Core scenarios
+    subgroups:
+      - a_okvqa_chinese
+      - a_okvqa_hindi
+      - a_okvqa_spanish
+      - a_okvqa_swahili
+
+  - name: a_okvqa_base
+    display_name: A-OKVQA
+    description: A crowdsourced dataset composed of a diverse set of about 25K questions requiring a broad base of commonsense and world knowledge to answer ([paper](https://arxiv.org/abs/2206.01718)).
+    metric_groups:
+      - accuracy
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: valid
+    taxonomy:
+      task: multiple-choice question answering
+      what: Real-world images
+      who: Human experts
+      when: "2023"
+      language: English
+
+  - name: a_okvqa_dialect
+    display_name: A-OKVQA (AAE)
+    description: A crowdsourced dataset composed of a diverse set of about 25K questions requiring a broad base of commonsense and world knowledge to answer ([paper](https://arxiv.org/abs/2206.01718)).
+    metric_groups:
+      - fairness
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: valid
+    taxonomy:
+      task: multiple-choice question answering
+      what: Real-world images
+      who: Human experts
+      when: "2023"
+      language: English
+
+  - name: a_okvqa_robustness
+    display_name: A-OKVQA (robustness)
+    description: A crowdsourced dataset composed of a diverse set of about 25K questions requiring a broad base of commonsense and world knowledge to answer ([paper](https://arxiv.org/abs/2206.01718)).
+    metric_groups:
+      - robustness
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: valid
+    taxonomy:
+      task: multiple-choice question answering
+      what: Real-world images
+      who: Human experts
+      when: "2023"
+      language: English
+
+  - name: a_okvqa_chinese
+    display_name: A-OKVQA (chinese)
+    description: A crowdsourced dataset composed of a diverse set of about 25K questions requiring a broad base of commonsense and world knowledge to answer ([paper](https://arxiv.org/abs/2206.01718)).
+    metric_groups:
+      - translate
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: valid
+    taxonomy:
+      task: multiple-choice question answering
+      what: Real-world images
+      who: Human experts
+      when: "2023"
+      language: Chinese
+
+  - name: a_okvqa_hindi
+    display_name: A-OKVQA (hindi)
+    description: A crowdsourced dataset composed of a diverse set of about 25K questions requiring a broad base of commonsense and world knowledge to answer ([paper](https://arxiv.org/abs/2206.01718)).
+    metric_groups:
+      - translate
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: valid
+    taxonomy:
+      task: multiple-choice question answering
+      what: Real-world images
+      who: Human experts
+      when: "2023"
+      language: Hindi
+
+  - name: a_okvqa_spanish
+    display_name: A-OKVQA (spanish)
+    description: A crowdsourced dataset composed of a diverse set of about 25K questions requiring a broad base of commonsense and world knowledge to answer ([paper](https://arxiv.org/abs/2206.01718)).
+    metric_groups:
+      - translate
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: valid
+    taxonomy:
+      task: multiple-choice question answering
+      what: Real-world images
+      who: Human experts
+      when: "2023"
+      language: Spanish
+
+  - name: a_okvqa_swahili
+    display_name: A-OKVQA (swahili)
+    description: A crowdsourced dataset composed of a diverse set of about 25K questions requiring a broad base of commonsense and world knowledge to answer ([paper](https://arxiv.org/abs/2206.01718)).
+    metric_groups:
+      - translate
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: valid
+    taxonomy:
+      task: multiple-choice question answering
+      what: Real-world images
+      who: Human experts
+      when: "2023"
+      language: Swahili
+
+  - name: crossmodal_3600
+    display_name: Crossmodal 3600
+    description: Crossmodal-3600 dataset (XM3600 in short), a geographically-diverse set of 3600 images annotated with human-generated reference captions in 36 languages. ([paper](https://arxiv.org/abs/2205.12522))
+    metric_groups:
+      - accuracy
+      - general_information
+    environment:
+      main_name: prometheus_vision
+      main_split: test
+    taxonomy:
+      task: multilingual captioning
+      what: Real-world images
+      who: Human experts
+      when: "2022"
+      language: 36 languages
+
+  - name: flickr30k
+    display_name: Flickr30k
+    description: An image caption corpus consisting of 158,915 crowd-sourced captions describing 31,783 Flickr images. ([paper](https://shannon.cs.illinois.edu/DenotationGraph/TACLDenotationGraph.pdf))
+    metric_groups:
+      - accuracy
+      - general_information
+    environment:
+      main_name: prometheus_vision
+      main_split: test
+    taxonomy:
+      task: image captioning
+      what: Flickr images
+      who: Human experts
+      when: "2014"
+      language: English
+
+  - name: gqa
+    display_name: GQA
+    description: Questions about real-world visual reasoning and compositional QA
+    metric_groups:
+      - accuracy
+      - general_information
+    environment:
+      main_name: quasi_exact_match
+      main_split: valid
+    taxonomy:
+      task: short answer question answering
+      what: Real-world images
+      who: Human experts
+      when: "2019"
+      language: English
+
+  - name: hateful_memes
+    display_name: Hateful Memes
+    description: The Hateful Memes benchmark for multimodal hate speech detection [(Dwibedi et al., 2020)](https://arxiv.org/pdf/2005.04790.pdf).
+    metric_groups:
+      - accuracy
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: multimodal classification
+      what: images and text
+      who: annotators from Amazon Mechanical Turk
+      when: "2020"
+      language: English
+
+  - name: mm_safety_bench
+    display_name: MM-SafetyBench
+    description: Expose the vulnerability of open-source VLMs with toxic and biased content ([paper](https://arxiv.org/abs/2311.17600))
+    metric_groups:
+      - accuracy
+      - toxicity
+      - general_information
+    environment:
+      main_name: toxic_frac
+      main_split: test
+    taxonomy:
+      task: safety
+      what: safety images
+      who: Human experts
+      when: "2023"
+      language: English
+
+  - name: viz_wiz
+    display_name: VizWiz
+    description: A benchmark for visual question answering with images and questions created by visually impaired people [(Gurari et al., 2018)](https://arxiv.org/abs/1802.08218).
+    metric_groups:
+      - accuracy
+    environment:
+      main_name: quasi_exact_match
+      main_split: valid
+    taxonomy:
+      task: multimodal short answer question answering
+      what: Real-world images
+      who: Visually impaired people
+      when: "2018"
+      language: English
+
+  - name: vqa_base
+    display_name: VQAv2
+    description: Open-ended questions about real-world images [(Goyal et al., 2017)](https://arxiv.org/abs/1612.00837).
+    metric_groups:
+      - accuracy
+      - general_information
+    environment:
+      main_name: quasi_exact_match
+      main_split: valid
+    taxonomy:
+      task: multimodal short answer question answering
+      what: Real-world images
+      who: Human experts
+      when: "2017"
+      language: English
+
+  - name: vqa_dialect
+    display_name: VQAv2 (AAE)
+    description: Open-ended questions about real-world images [(Goyal et al., 2017)](https://arxiv.org/abs/1612.00837).
+    metric_groups:
+      - fairness
+      - general_information
+    environment:
+      main_name: quasi_exact_match
+      main_split: valid
+    taxonomy:
+      task: multimodal short answer question answering
+      what: Real-world images
+      who: Human experts
+      when: "2017"
+      language: English
+
+  - name: vqa_robustness
+    display_name: VQAv2 (robustness)
+    description: Open-ended questions about real-world images [(Goyal et al., 2017)](https://arxiv.org/abs/1612.00837).
+    metric_groups:
+      - robustness
+      - general_information
+    environment:
+      main_name: quasi_exact_match
+      main_split: valid
+    taxonomy:
+      task: multimodal short answer question answering
+      what: Real-world images
+      who: Human experts
+      when: "2017"
+      language: English
+
+  - name: vqa_chinese
+    display_name: VQAv2 (chinese)
+    description: Open-ended questions about real-world images [(Goyal et al., 2017)](https://arxiv.org/abs/1612.00837).
+    metric_groups:
+      - translate
+      - general_information
+    environment:
+      main_name: quasi_exact_match
+      main_split: valid
+    taxonomy:
+      task: multimodal short answer question answering
+      what: Real-world images
+      who: Human experts
+      when: "2017"
+      language: Chinese
+
+  - name: vqa_hindi
+    display_name: VQAv2 (hindi)
+    description: Open-ended questions about real-world images [(Goyal et al., 2017)](https://arxiv.org/abs/1612.00837).
+    metric_groups:
+      - translate
+      - general_information
+    environment:
+      main_name: quasi_exact_match
+      main_split: valid
+    taxonomy:
+      task: multimodal short answer question answering
+      what: Real-world images
+      who: Human experts
+      when: "2017"
+      language: Hindi
+
+  - name: vqa_spanish
+    display_name: VQAv2 (spanish)
+    description: Open-ended questions about real-world images [(Goyal et al., 2017)](https://arxiv.org/abs/1612.00837).
+    metric_groups:
+      - translate
+      - general_information
+    environment:
+      main_name: quasi_exact_match
+      main_split: valid
+    taxonomy:
+      task: multimodal short answer question answering
+      what: Real-world images
+      who: Human experts
+      when: "2017"
+      language: Spanish
+
+  - name: math_vista
+    display_name: MathVista
+    description: Evaluating Math Reasoning in Visual Contexts
+    metric_groups:
+      - accuracy
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: multiple-choice question answering
+      what: Evaluating Math Reasoning in Visual Contexts
+      who: Human experts
+      when: "2024"
+      language: English
+
+  - name: mmmu
+    display_name: MMMU
+    description: A benchmark designed to evaluate multimodal models on massive multi-discipline tasks demanding college-level subject knowledge and deliberate reasoning [(Yue et al., 2023)](https://arxiv.org/abs/2311.16502).
+    metric_groups:
+      - accuracy
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: valid
+    taxonomy:
+      task: multimodal multiple-choice question answering
+      what: Art & Design, Business, Science, Health & Medicine, Humanities & Social Science, and Tech & Engineering
+      who: Human experts
+      when: "2023"
+      language: English
+  
+  - name: unicorn
+    display_name: Unicorn
+    description: Safety Evaluation Benchmark for Evaluating on Out-of-Distribution and Sketch Images
+    metric_groups:
+      - accuracy
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: short answer question answering
+      what: OOD images and sketch images
+      who: Human experts
+      when: "2023"
+      language: English
+  
+  - name: bingo
+    display_name: Bingo
+    description: Open-ended questions about biased images
+    metric_groups:
+      - accuracy
+      - general_information
+    environment:
+      main_name: prometheus_vision
+      main_split: test
+    taxonomy:
+      task: short answer question answering
+      what: Biased images about Region, OCR, Factual, Text-to-Image and Image-to-Image inference challenges
+      who: Human experts
+      when: "2023"
+      language: English, Chinese, Japanese, etc.
+
+  - name: pope
+    display_name: POPE
+    description: Open-ended questions about object appearance in real-world images for evaluating hallucination behaviour
+    metric_groups:
+      - accuracy
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: short answer question answering
+      what: Real-world images
+      who: Human experts
+      when: "2023"
+      language: English
+
+  - name: seed_bench
+    display_name: Seed Bench 
+    description: A massive multiple-choice question-answering benchmark that spans 9 evaluation aspects with the image input including the comprehension of both the image and video modality
+    metric_groups:
+      - accuracy
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: multiple-choice question answering
+      what: Real-world images
+      who: Human experts
+      when: "2023"
+      language: English
+
+  - name: mme
+    display_name: MME
+    description: A comprehensive MLLM Evaluation benchmark with perception and cognition evaluations on 14 subtasks
+    metric_groups:
+      - accuracy
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: multiple-choice question answering
+      what: Real-world images
+      who: Human experts
+      when: "2023"
+      language: English
+
+  - name: vibe_eval
+    display_name: Vibe Eval
+    description: hard evaluation suite for measuring progress of multimodal language models
+    metric_groups:
+      - accuracy
+      - general_information
+    environment:
+      main_name: prometheus_vision
+      main_split: test
+    taxonomy:
+      task: short answer question answering
+      what: Knowledge intensive
+      who: Human experts
+      when: "2024"
+      language: English
+
+  - name: mementos
+    display_name: Mementos
+    description: A Comprehensive Benchmark for Multimodal Large Language Model Reasoning over Image Sequences
+    metric_groups:
+      - accuracy
+      - general_information
+    environment:
+      main_name: prometheus_vision
+      main_split: test
+    taxonomy:
+      task: short answer question answering
+      what: Image sequences of comics, dailylife and robotics
+      who: Human experts
+      when: "2024"
+      language: English
+
+  - name: pairs
+    display_name: PAIRS
+    description: Examining Gender and Racial Bias in Large Vision-Language Models Using a Novel Dataset of Parallel Images.
+    metric_groups:
+      - accuracy
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: multiple-choice question answering
+      what: Bias
+      who: Human experts
+      when: "2024"
+      language: English