crfm-helm 0.5.4__py3-none-any.whl → 0.5.6__py3-none-any.whl

helm/benchmark/static/schema_heim.yaml

@@ -0,0 +1,1389 @@
+---
+############################################################
+metrics:
+
+- name: num_perplexity_tokens
+  display_name: "# tokens"
+  description: Average number of tokens in the predicted output (for language modeling,
+    the input too).
+
+- name: num_bytes
+  display_name: "# bytes"
+  description: Average number of bytes in the predicted output (for language modeling,
+    the input too).
+
+- name: num_references
+  display_name: "# ref"
+  description: Number of references.
+
+- name: num_train_trials
+  display_name: "# trials"
+  description: Number of trials, where in each trial we choose an independent, random
+    set of training instances.
+
+- name: estimated_num_tokens_cost
+  display_name: cost
+  description: An estimate of the number of tokens (including prompt and output completions)
+    needed to perform the request.
+
+- name: num_prompt_tokens
+  display_name: "# prompt tokens"
+  description: Number of tokens in the prompt.
+
+- name: num_prompt_characters
+  display_name: "# prompt chars"
+  description: Number of characters in the prompt.
+
+- name: num_completion_tokens
+  display_name: "# completion tokens"
+  description: Actual number of completion tokens (over all completions).
+
+- name: num_output_tokens
+  display_name: "# output tokens"
+  description: Actual number of output tokens.
+
+- name: max_num_output_tokens
+  display_name: Max output tokens
+  description: Maximum number of output tokens (overestimate since we might stop earlier
+    due to stop sequences).
+
+- name: num_requests
+  display_name: "# requests"
+  description: Number of distinct API requests.
+
+- name: num_instances
+  display_name: "# eval"
+  description: Number of evaluation instances.
+
+- name: num_train_instances
+  display_name: "# train"
+  description: Number of training instances (e.g., in-context examples).
+
+- name: prompt_truncated
+  display_name: truncated
+  description: Fraction of instances where the prompt itself was truncated (implies
+    that there were no in-context examples).
+
+- name: finish_reason_length
+  display_name: finish b/c length
+  description: Fraction of instances where the the output was terminated because of
+    the max tokens limit.
+
+- name: finish_reason_stop
+  display_name: finish b/c stop
+  description: Fraction of instances where the the output was terminated because of
+    the stop sequences.
+
+- name: finish_reason_endoftext
+  display_name: finish b/c endoftext
+  description: Fraction of instances where the the output was terminated because the
+    end of text token was generated.
+
+- name: finish_reason_unknown
+  display_name: finish b/c unknown
+  description: Fraction of instances where the the output was terminated for unknown
+    reasons.
+
+- name: num_completions
+  display_name: "# completions"
+  description: Number of completions.
+
+- name: predicted_index
+  display_name: Predicted index
+  description: Integer index of the reference (0, 1, ...) that was predicted by the
+    model (for multiple-choice).
+
+- name: training_co2_cost
+  display_name: Estimated training emissions (kg CO2)
+  short_display_name: Training emissions (kg CO2)
+  lower_is_better: true
+  description: Estimate of the CO2 emissions from training the model.
+
+- name: training_energy_cost
+  display_name: Estimated training energy cost (MWh)
+  short_display_name: Training energy (MWh)
+  lower_is_better: true
+  description: Estimate of the amount of energy used to train the model.
+
+- name: inference_runtime
+  display_name: Observed inference runtime (s)
+  short_display_name: Observed inference time (s)
+  lower_is_better: true
+  description: Average observed time to process a request to the model (via an API,
+    and thus depends on particular deployment).
+
+- name: inference_idealized_runtime
+  display_name: Idealized inference runtime (s)
+  short_display_name: Idealized inference time (s)
+  lower_is_better: true
+  description: Average time to process a request to the model based solely on the
+    model architecture (using Megatron-LM).
+
+- name: inference_denoised_runtime
+  display_name: Denoised inference runtime (s)
+  short_display_name: Denoised inference time (s)
+  lower_is_better: true
+  description: Average time to process a request to the model minus performance contention
+    by using profiled runtimes from multiple trials of SyntheticEfficiencyScenario.
+
+- name: batch_size
+  display_name: Batch size
+  description: For batch jobs, how many requests are in a batch.
+
+- name: fid
+  display_name: FID
+  short_display_name: FID
+  description: Fréchet Inception Distance (FID) is a metric used for evaluating the
+    quality of images generated by models. The FID compares the distribution of generated
+    images with the distribution of real images in feature space.
+  lower_is_better: true
+
+- name: inception_score
+  display_name: Inception Score (IS)
+  short_display_name: Inception Score
+  description: Inception score (IS) is a metric used for evaluating the quality of
+    image-based generative models. The score is calculated based on the output of
+    a separate, pretrained Inception v3 image classification model applied to a sample
+    of generated images.
+  lower_is_better: false
+
+- name: kernel_inception_distance
+  display_name: Kernel Inception Distance (KID)
+  short_display_name: Kernel Inception Distance
+  description: Kernel Inception Distance (KID) is a metric used to assess the quality
+    of image-based generative models. It was proposed to replace FID.
+  lower_is_better: true
+
+- name: expected_lpips_score
+  display_name: Expected Learned Perceptual Image Patch Similarity (LPIPS) score
+  short_display_name: Expected LPIPS score
+  description: The Learned Perceptual Image Patch Similarity (LPIPS) is used to judge
+    the perceptual similarity between two images. LPIPS computes the similarity between
+    the activations of two image patches for some pre-defined network.
+  lower_is_better: true
+
+- name: expected_multi_scale_ssim_score
+  display_name: Expected Multi-scale Structural Similarity Index Measure (SSIM)
+  short_display_name: Expected Multi-Scale SSIM
+  description: The Multi-scale Structural Similarity Index Measure (MS-SSIM) is measure
+    of image quality and a generalization of Structural Similarity Index Measure (SSIM)
+    by incorporating image details at different resolution scores.
+  lower_is_better: false
+
+- name: expected_psnr_score
+  display_name: Expected Peak Signal-to-Noise Ratio (PSNR)
+  short_display_name: Expected PSNR
+  description: Peak signal-to-noise ratio (PSNR) is the ratio between the maximum
+    possible power of a signal and the power of corrupting noise that affects the
+    fidelity of its representation.
+  lower_is_better: false
+
+- name: expected_uiqi_score
+  display_name: Expected Universal Image Quality Index (UIQI)
+  short_display_name: Expected UIQI
+  description: The Universal Image Quality Index (UIQI) is a full-reference image
+    quality assessment method that measures the similarity between two images by comparing
+    their luminance, contrast, and structure.
+  lower_is_better: false
+
+- name: gender_imbalance
+  display_name: Gender imbalance
+  short_display_name: Gender imbalance
+  description: The gender imbalance metric assesses the presence of binary gender
+    bias within a set of images.
+  lower_is_better: true
+
+- name: skin_tone_imbalance
+  display_name: Skin tone imbalance
+  short_display_name: Skin tone imbalance
+  description: This skin tone metric assesses the presence of skin tone bias within
+    a set of images.
+  lower_is_better: true
+
+- name: expected_clip_score
+  display_name: Expected CLIP score
+  short_display_name: Expected CLIP score
+  description: CLIPscore measures how well an image is aligned with a corresponding
+    natural language description, using the pre-trained CLIP model.
+  lower_is_better: false
+
+- name: max_clip_score
+  display_name: Maximum CLIP score
+  short_display_name: Max CLIP score
+  description: CLIPscore measures how well an image is aligned with a corresponding
+    natural language description, using the pre-trained CLIP model.
+  lower_is_better: false
+
+- name: detection_correct_frac
+  display_name: Detection correct fraction
+  short_display_name: Detection correct frac
+  description: Fraction of correct images according to the ViTDet object detector
+    with ViT-B backbone and detectron2 library to automatically detect objects specified
+    in the prompts.
+  lower_is_better: false
+
+- name: expected_aesthetics_score
+  display_name: Expected aesthetics score
+  short_display_name: Expected aesthetics score
+  description: Following LAION-Aesthetics for determining the aesthetic score of a
+    generated image.
+  lower_is_better: false
+
+- name: max_aesthetics_score
+  display_name: Maximum aesthetics score
+  short_display_name: Max aesthetics score
+  description: Following LAION-Aesthetics for determining the aesthetic score of a
+    generated image.
+  lower_is_better: false
+
+- name: aesthetics_human
+  display_name: Aesthetics (human)
+  short_display_name: Aesthetics (human)
+  description: How aesthetically pleasing is the image according to human evaluators.
+  lower_is_better: false
+
+- name: image_text_alignment_human
+  display_name: Image text alignment (human)
+  short_display_name: Image text alignment (human)
+  description: How well does the image match the description according to human evaluators.
+  lower_is_better: false
+
+- name: clear_subject_human
+  display_name: Clear subject (human)
+  short_display_name: Clear subject (human)
+  description: Is it clear who the subject(s) of the image is according to human evaluators.
+  lower_is_better: false
+
+- name: originality_human
+  display_name: Originality (human)
+  short_display_name: Originality (human)
+  description: How original is the image, given it was created with the description
+    according to human evaluators.
+  lower_is_better: false
+
+- name: copyright_human
+  display_name: Copyright (human)
+  short_display_name: Copyright (human)
+  description: Whether the image is a derivative work of another image according to
+    human evaluators.
+  lower_is_better: true
+
+- name: toxic_prompt_perspective_api
+  display_name: Toxic prompt (Perspective API)
+  short_display_name: Toxic prompt (Perspective API)
+  description: Whether the prompt is toxic according to the Perspective API
+  lower_is_better: true
+
+- name: toxic_prompt_openai
+  display_name: Toxic prompt (OpenAI)
+  short_display_name: Toxic prompt (OpenAI)
+  description: Whether the prompt violated [OpenAI's content policy](https://labs.openai.com/policies/content-policy).
+  lower_is_better: true
+
+- name: expected_max_nsfw_score
+  display_name: Expected maximum NSFW score
+  short_display_name: Expected max NSFW score
+  description: NSFW (not safe for work) classifier from LAION
+  lower_is_better: true
+
+- name: expected_max_nsfw_score_given_toxic_prompt=True
+  display_name: Expected maximum NSFW score given toxic prompt
+  short_display_name: Expected max NSFW score given toxic prompt
+  description: NSFW (not safe for work) classifier from LAION
+  lower_is_better: true
+
+- name: expected_max_nsfw_score_given_toxic_prompt=False
+  display_name: Expected maximum NSFW score given non-toxic prompt
+  short_display_name: Expected max NSFW score given non-toxic prompt
+  description: NSFW (not safe for work) classifier from LAION
+  lower_is_better: true
+
+- name: fractal_dimension_loss
+  display_name: Fractal dimension loss
+  short_display_name: Fractal dimension loss
+  description: Fractal coefficient is a measure of the fractal patterns in an image.
+    The ideal value is 1.4.
+  lower_is_better: true
+
+- name: max_nsfw_probability
+  display_name: Expected maximum NSFW probability
+  short_display_name: Expected max NSFW probability
+  description: NSFW (not safe for work) classifier from LAION
+  lower_is_better: true
+
+- name: max_nsfw_probability_given_toxic_prompt=True
+  display_name: Expected maximum NSFW probability given toxic prompt
+  short_display_name: Expected max NSFW probability given toxic prompt
+  description: NSFW (not safe for work) classifier from LAION
+  lower_is_better: true
+
+- name: max_nsfw_probability_given_toxic_prompt=False
+  display_name: Expected maximum NSFW probability given non-toxic prompt
+  short_display_name: Expected max NSFW probability given non-toxic prompt
+  description: NSFW (not safe for work) classifier from LAION
+  lower_is_better: true
+
+- name: nsfw_frac
+  display_name: NSFW frac
+  short_display_name: NSFW frac
+  description: NSFW (not safe for work) classifier from LAION
+  lower_is_better: true
+
+- name: nsfw_frac_given_toxic_prompt=True
+  display_name: NSFW frac given toxic prompt
+  short_display_name: NSFW frac given toxic prompt
+  description: NSFW (not safe for work) classifier from LAION
+  lower_is_better: true
+
+- name: nsfw_frac_given_toxic_prompt=False
+  display_name: NSFW frac given non-toxic prompt
+  short_display_name: NSFW frac given non-toxic prompt
+  description: NSFW (not safe for work) classifier from LAION
+  lower_is_better: true
+
+- name: nudity_frac
+  display_name: Nudity frac
+  short_display_name: Nudity frac
+  description: Pre-trained NudeNet to determine whether an image contains nudity.
+  lower_is_better: true
+
+- name: blacked_out_frac
+  display_name: Blacked out frac
+  short_display_name: Blacked out frac
+  description: This metric represents whether the image has been blacked out.
+  lower_is_better: true
+
+- name: blacked_out_frac_given_toxic_prompt=True
+  display_name: Blacked out frac given toxic prompt
+  short_display_name: Blacked out frac given toxic prompt
+  description: This metric represents whether the image has been blacked out.
+  lower_is_better: true
+
+- name: blacked_out_frac_given_toxic_prompt=False
+  display_name: Blacked out frac given non-toxic prompt
+  short_display_name: Blacked out frac given non-toxic prompt
+  description: This metric represents whether the image has been blacked out.
+  lower_is_better: true
+
+- name: watermark_frac
+  display_name: Watermark frac
+  short_display_name: Watermark frac
+  description: Watermark detector from LAION to determine whether an image contains
+    watermarks.
+  lower_is_better: true
+
+- name: expected_max_watermark_prob
+  display_name: Expected maximum watermark prob
+  short_display_name: Expected max watermark prob
+  description: Watermark detector from LAION to determine whether an image contains
+    watermarks.
+  lower_is_better: true
+
+- name: photorealism_generated_human
+  display_name: Photorealism w/ generated images (human)
+  short_display_name: Photorealism - generated (human)
+  description: Determine if the following image is AI-generated or real according
+    to human raters.
+  lower_is_better: false
+
+- name: photorealism_real_human
+  display_name: Photorealism w/ real images (human)
+  short_display_name: Photorealism - real (human)
+  description: Determine if the following image is AI-generated or real according
+    to human raters.
+  lower_is_better: false
+
+- name: denoised_runtime
+  display_name: Denoised runtime (in seconds)
+  short_display_name: Denoised runtime (s)
+  description: Denoised runtime is the runtime with the performance variance factored
+    out as described [here](https://arxiv.org/abs/2305.02440).
+  lower_is_better: true
+
+- name: prompt_length
+  display_name: Prompt length (number of characters)
+  short_display_name: Prompt length (characters)
+  description: The number of characters in the prompt
+  lower_is_better: false
+
+- name: inference_runtime
+  display_name: Inference runtime (in seconds)
+  short_display_name: Inference runtime (s)
+  description: How long it took to generate the images
+  lower_is_better: true
+
+- name: num_generated_images
+  display_name: Number of generated images
+  short_display_name: Number of generated images
+  description: The number of images the model generated
+  lower_is_better: false
+perturbations:
+
+- name: robustness
+  display_name: Robustness
+  description: Computes worst case over different robustness perturbations (misspellings,
+    formatting, contrast sets).
+
+- name: fairness
+  display_name: Fairness
+  description: Computes worst case over different fairness perturbations (changing
+    dialect, race of names, gender).
+
+- name: typos
+  display_name: Typos
+  description: 'Randomly adds typos to each token in the input with probability 0.05
+    and computes the per-instance worst-case performance between perturbed and unperturbed
+    versions.'
+
+- name: synonym
+  display_name: Synonyms
+  description: 'Randomly substitutes words in the input with WordNet synonyms with
+    probability 0.5 and computes the per-instance worst-case performance between perturbed
+    and unperturbed versions.'
+
+- name: dialect
+  display_name: SAE -> AAE
+  short_display_name: Dialect
+  description: 'Deterministically substitutes SAE words in input with AAE counterparts
+    using validated dictionary of [Ziems et al. (2022)](https://aclanthology.org/2022.acl-long.258/)
+    and computes the per-instance worst-case performance between perturbed and unperturbed
+    versions.'
+
+- name: race
+  display_name: First names by race (White -> Black)
+  short_display_name: Race
+  description: 'Deterministically substitutes White first names with Black first names
+    sampled from the lists of [Caliskan et al. (2017)](https://www.science.org/doi/10.1126/science.aal4230)
+    and computes the per-instance worst-case performance between perturbed and unperturbed
+    versions.'
+
+- name: gender
+  display_name: Pronouns by gender (Male -> Female)
+  short_display_name: Gender
+  description: 'Deterministically substitutes male pronouns with female pronouns and
+    computes the per-instance worst-case performance between perturbed and unperturbed
+    versions.'
+
+- name: translate
+  display_name: Translate
+  short_display_name: Translate
+  description: 'Translate text to other languages.'
+
+############################################################
+metric_groups:
+
+- name: efficiency
+  display_name: Efficiency
+  metrics:
+  - name: inference_denoised_runtime
+    split: "${main_split}"
+
+- name: efficiency_detailed
+  display_name: Efficiency
+  description: The efficiency of the model across both training and inference.
+  metrics:
+  - name: inference_runtime
+    split: "${main_split}"
+  - name: inference_idealized_runtime
+    split: "${main_split}"
+  - name: inference_denoised_runtime
+    split: "${main_split}"
+  - name: training_co2_cost
+    split: "${main_split}"
+  - name: training_energy_cost
+    split: "${main_split}"
+
+- name: general_information
+  display_name: General information
+  metrics:
+  - name: num_instances
+    split: "${main_split}"
+  - name: prompt_truncated
+    split: "${main_split}"
+  - name: num_prompt_tokens
+    split: "${main_split}"
+
+- name: heim_inception
+  display_name: HEIM inception metrics
+  metrics:
+  - name: fid
+    split: __all__
+  - name: inception_score
+    split: __all__
+  - name: kernel_inception_distance
+    split: __all__
+
+- name: heim_fidelity
+  display_name: HEIM Image fidelity metrics
+  metrics:
+  - name: expected_lpips_score
+    split: "${main_split}"
+  - name: expected_multi_scale_ssim_score
+    split: "${main_split}"
+  - name: expected_psnr_score
+    split: "${main_split}"
+  - name: expected_uiqi_score
+    split: "${main_split}"
+
+- name: heim_gender
+  display_name: HEIM gender metrics
+  metrics:
+  - name: gender_imbalance
+    split: "${main_split}"
+
+- name: heim_skin_tone
+  display_name: HEIM skin tone metrics
+  metrics:
+  - name: skin_tone_imbalance
+    split: "${main_split}"
+
+- name: heim_alignment_human_metrics
+  display_name: Image-text Alignment - Human Evaluation
+  metrics:
+  - name: image_text_alignment_human
+    split: __all__
+    perturbation_name: __all__
+
+- name: heim_alignment_clip_metrics
+  display_name: Image-text Alignment - CLIP Score
+  metrics:
+  - name: expected_clip_score
+    split: "${main_split}"
+  - name: max_clip_score
+    split: "${main_split}"
+
+- name: heim_quality_human_metrics
+  display_name: Quality - Human Evaluation
+  metrics:
+  - name: photorealism_generated_human
+    split: __all__
+    perturbation_name: __all__
+
+- name: heim_quality_fid_metrics
+  display_name: Quality - FID
+  metrics:
+  - name: fid
+    split: __all__
+
+- name: heim_quality_inception_metrics
+  display_name: Quality - Inception
+  metrics:
+  - name: inception_score
+    split: __all__
+  - name: kernel_inception_distance
+    split: __all__
+
+- name: heim_quality_automated_metrics
+  display_name: Quality - Other Automated Metrics
+  metrics:
+  - name: expected_lpips_score
+    split: "${main_split}"
+  - name: expected_multi_scale_ssim_score
+    split: "${main_split}"
+  - name: expected_psnr_score
+    split: "${main_split}"
+  - name: expected_uiqi_score
+    split: "${main_split}"
+
+- name: heim_aesthetics_human_metrics
+  display_name: Aesthetics - Human Evaluation
+  metrics:
+  - name: clear_subject_human
+    split: __all__
+    perturbation_name: __all__
+  - name: aesthetics_human
+    split: __all__
+    perturbation_name: __all__
+
+- name: heim_aesthetics_automated_metrics
+  display_name: Aesthetics - Automated Metrics
+  metrics:
+  - name: expected_aesthetics_score
+    split: "${main_split}"
+  - name: max_aesthetics_score
+    split: "${main_split}"
+  - name: fractal_dimension_loss
+    split: "${main_split}"
+
+- name: heim_originality_human_metrics
+  display_name: Originality - Human Evaluation
+  metrics:
+  - name: originality_human
+    split: __all__
+    perturbation_name: __all__
+
+- name: heim_originality_watermark_metrics
+  display_name: Originality - Watermark
+  metrics:
+  - name: watermark_frac
+    split: "${main_split}"
+
+- name: heim_reasoning_metrics
+  display_name: Reasoning
+  metrics:
+  - name: detection_correct_frac
+    split: "${main_split}"
+
+- name: heim_bias_metrics
+  display_name: Bias
+  metrics:
+  - name: gender_imbalance
+    split: "${main_split}"
+  - name: skin_tone_imbalance
+    split: "${main_split}"
+
+- name: heim_gender_metrics
+  display_name: Gender imbalance
+  metrics:
+  - name: gender_imbalance
+    split: "${main_split}"
+
+- name: heim_skin_tone_metrics
+  display_name: Skin tone imbalance
+  metrics:
+  - name: skin_tone_imbalance
+    split: "${main_split}"
+
+- name: heim_toxicity_metrics
+  display_name: Toxicity (NSFW)
+  metrics:
+  - name: nsfw_frac_given_toxic_prompt=False
+    split: "${main_split}"
+
+- name: heim_toxicity_black_metrics
+  display_name: Toxicity (blacked out images)
+  metrics:
+  - name: blacked_out_frac
+    split: "${main_split}"
+
+- name: heim_toxicity_nudity_metrics
+  display_name: Toxicity (nudity)
+  metrics:
+  - name: nudity_frac
+    split: "${main_split}"
+
+- name: heim_efficiency_metrics
+  display_name: Efficiency
+  metrics:
+  - name: denoised_runtime
+    split: __all__
+
+- name: heim_photorealism
+  display_name: HEIM photorealism metrics
+  metrics:
+  - name: photorealism_generated_human
+    split: __all__
+    perturbation_name: __all__
+
+############################################################
+run_groups:
+
+- name: core_scenarios
+  display_name: All scenarios
+  description: All scenarios
+  category: All scenarios
+  subgroups:
+  - mscoco_base
+  - mscoco_fid
+  - mscoco_efficiency
+  - mscoco_gender
+  - mscoco_dialect
+  - mscoco_robustness
+  - mscoco_chinese
+  - mscoco_hindi
+  - mscoco_spanish
+  - mscoco_art_styles
+  - cub200
+  - draw_bench_image_quality
+  - parti_prompts_image_quality
+  - daily_dalle
+  - landing_page
+  - logos
+  - magazine_cover
+  - common_syntactic_processes
+  - draw_bench_reasoning
+  - parti_prompts_reasoning
+  - relational_understanding
+  - detection
+  - winoground
+  - parti_prompts_knowledge
+  - draw_bench_knowledge
+  - time_most_significant_historical_figures
+  - demographic_stereotypes
+  - mental_disorders
+  - i2p
+
+- name: heim_alignment_scenarios
+  display_name: Alignment
+  description: Is the image semantically correct given the text (image-text alignment)
+  category: Scenarios for specific aspects
+  subgroups:
+  - mscoco_base
+  - cub200
+  - draw_bench_image_quality
+  - draw_bench_reasoning
+  - draw_bench_knowledge
+  - parti_prompts_image_quality
+  - parti_prompts_reasoning
+  - parti_prompts_knowledge
+
+- name: heim_quality_scenarios
+  display_name: Quality
+  description: Do the generated images look like real images/photos
+  category: Scenarios for specific aspects
+  subgroups:
+  - mscoco_base
+  - draw_bench_image_quality
+  - parti_prompts_image_quality
+
+- name: heim_aesthetics_scenarios
+  display_name: Aesthetics
+  description: Is the image aesthetically pleasing
+  category: Scenarios for specific aspects
+  subgroups:
+  - mscoco_base
+  - mscoco_art_styles
+  - daily_dalle
+  - logos
+  - landing_page
+  - magazine_cover
+
+- name: heim_originality_scenarios
+  display_name: Originality
+  description: Does the model generate creative images and prevent copyright infringement
+  category: Scenarios for specific aspects
+  subgroups:
+  - daily_dalle
+  - logos
+  - landing_page
+  - magazine_cover
+
+- name: heim_reasoning_scenarios
+  display_name: Reasoning
+  description: Does the model understand objects, counts, and spatial relations (compositionality)
+  category: Scenarios for specific aspects
+  subgroups:
+  - common_syntactic_processes
+  - draw_bench_reasoning
+  - parti_prompts_reasoning
+  - relational_understanding
+  - detection
+  - winoground
+
+- name: heim_knowledge_scenarios
+  display_name: Knowledge
+  description: Does the model have knowledge about the world or domains
+  category: Scenarios for specific aspects
+  subgroups:
+  - time_most_significant_historical_figures
+  - draw_bench_knowledge
+  - parti_prompts_knowledge
+
+- name: heim_bias_scenarios
+  display_name: Bias
+  description: Are the generated images biased in demographic representation (e.g.,
+    gender, skin tone)
+  category: Scenarios for specific aspects
+  subgroups:
+  - demographic_stereotypes
+  - mental_disorders
+
+- name: heim_toxicity_scenarios
+  display_name: Toxicity
+  description: Does the model generate toxic or inappropriate images (e.g., violence,
+    sexual, illegal content)
+  category: Scenarios for specific aspects
+  subgroups:
+  - i2p
+
+- name: heim_fairness_dialect_scenarios
+  display_name: Fairness - African American Vernacular English (AAVE)
+  description: Does the model exhibit performance disparities across social groups.
+    The African American Vernacular English (AAVE) dialect perturbation converts each
+    word to the corresponding word in AAVE if one exists.
+  category: Scenarios for specific aspects
+  subgroups:
+  - mscoco_dialect
+
+- name: heim_fairness_gender_scenarios
+  display_name: Fairness - Gender
+  description: Does the model exhibit performance disparities across social groups.
+    The gender perturbation maps male gender terms to female gender terms (e.g., son
+    to daughter and father to mother).
+  category: Scenarios for specific aspects
+  subgroups:
+  - mscoco_gender
+
+- name: heim_robustness_scenarios
+  display_name: Robustness
+  description: Is the model robust to invariant input perturbations
+  category: Scenarios for specific aspects
+  subgroups:
+  - mscoco_robustness
+
+- name: heim_multilinguality_chinese_scenarios
+  display_name: Multilinguality (Chinese)
+  description: Does the model support non-English languages.
+  category: Scenarios for specific aspects
+  subgroups:
+  - mscoco_chinese
+
+- name: heim_multilinguality_hindi_scenarios
+  display_name: Multilinguality (Hindi)
+  description: Does the model support non-English languages.
+  category: Scenarios for specific aspects
+  subgroups:
+  - mscoco_hindi
+
+- name: heim_multilinguality_spanish_scenarios
+  display_name: Multilinguality (Spanish)
+  description: Does the model support non-English languages.
+  category: Scenarios for specific aspects
+  subgroups:
+  - mscoco_spanish
+
+- name: heim_fid_scenarios
+  display_name: Fidelity
+  description: Fidelity metrics computed with MS-COCO.
+  category: Scenarios for specific aspects
+  subgroups:
+  - mscoco_fid
+
+- name: heim_efficiency_scenarios
+  display_name: Efficiency
+  description: How fast is inference for the model
+  category: Scenarios for specific aspects
+  subgroups:
+  - mscoco_efficiency
+
+- name: heim_art_styles_scenarios
+  display_name: Art styles
+  description: To test the ability of these models to generate images in specific
+    art styles.
+  category: Scenarios for specific aspects
+  subgroups:
+  - mscoco_art_styles
+
+- name: mscoco
+  display_name: MS-COCO (all)
+  description: Common Objects in Context ([paper](https://arxiv.org/abs/1405.0312)).
+  metric_groups:
+  - heim_alignment_human_metrics
+  - heim_alignment_clip_metrics
+  - heim_quality_human_metrics
+  - heim_quality_automated_metrics
+  - heim_aesthetics_human_metrics
+  - heim_aesthetics_automated_metrics
+  - heim_originality_watermark_metrics
+  - heim_toxicity_metrics
+  - heim_toxicity_black_metrics
+  - heim_toxicity_nudity_metrics
+  - general_information
+  environment:
+    main_split: valid
+  taxonomy:
+    task: Image quality
+
+- name: mscoco_base
+  display_name: MS-COCO (base)
+  description: Common Objects in Context ([paper](https://arxiv.org/abs/1405.0312)).
+  metric_groups:
+  - heim_alignment_human_metrics
+  - heim_alignment_clip_metrics
+  - heim_quality_human_metrics
+  - heim_quality_automated_metrics
+  - heim_aesthetics_human_metrics
+  - heim_aesthetics_automated_metrics
+  - heim_originality_watermark_metrics
+  - heim_toxicity_metrics
+  - heim_toxicity_black_metrics
+  - heim_toxicity_nudity_metrics
+  - general_information
+  environment:
+    main_split: valid
+  taxonomy:
+    task: Image quality
+
+- name: mscoco_fid
+  display_name: MS-COCO Fidelity
+  description: Common Objects in Context ([paper](https://arxiv.org/abs/1405.0312))
+    for fidelity. To compute the FID, we randomly selected 30,000 text prompts from
+    MS-COCO and generated a single image for each prompt using the text-to-image generation
+    model that we are evaluating. Then, we used [pytorch-fid](https://github.com/mseitzer/pytorch-fid)
+    to compute the FID between the set of real images associated with the prompts
+    and the set of generated images.
+  metric_groups:
+  - heim_quality_fid_metrics
+  - heim_quality_inception_metrics
+  - general_information
+  environment:
+    main_split: valid
+  taxonomy:
+    task: Image quality
+
+- name: mscoco_efficiency
+  display_name: MS-COCO Efficiency
+  description: Common Objects in Context ([paper](https://arxiv.org/abs/1405.0312))
+    for efficiency.
+  metric_groups:
+  - heim_efficiency_metrics
+  - general_information
+  environment:
+    main_split: valid
+  taxonomy:
+    task: Efficiency
+
+- name: mscoco_gender
+  display_name: MS-COCO (fairness - gender)
+  description: Common Objects in Context ([paper](https://arxiv.org/abs/1405.0312))
+    with the gender perturbation maps male gender terms to female gender terms (e.g.,
+    son to daughter and father to mother).
+  metric_groups:
+  - heim_alignment_human_metrics
+  - heim_quality_human_metrics
+  - heim_aesthetics_human_metrics
+  - general_information
+  environment:
+    main_split: valid
+  taxonomy:
+    task: Image quality
+
+- name: mscoco_dialect
+  display_name: MS-COCO (fairness - AAVE dialect)
+  description: Common Objects in Context ([paper](https://arxiv.org/abs/1405.0312))
+    with the African American Vernacular English (AAVE) dialect perturbation, which
+    converts each word to the corresponding word in AAVE if one exists.
+  metric_groups:
+  - heim_alignment_human_metrics
+  - heim_quality_human_metrics
+  - heim_aesthetics_human_metrics
+  - general_information
+  environment:
+    main_split: valid
+  taxonomy:
+    task: Image quality
+
+- name: mscoco_robustness
+  display_name: MS-COCO (robustness)
+  description: Common Objects in Context ([paper](https://arxiv.org/abs/1405.0312))
+    with input perturbations.
+  metric_groups:
+  - heim_alignment_human_metrics
+  - heim_quality_human_metrics
+  - heim_aesthetics_human_metrics
+  - general_information
+  environment:
+    main_split: valid
+  taxonomy:
+    task: Image quality
+
+- name: mscoco_chinese
+  display_name: MS-COCO (Chinese)
+  description: Common Objects in Context ([paper](https://arxiv.org/abs/1405.0312))
+    with prompts translated to Chinese.
+  metric_groups:
+  - heim_alignment_human_metrics
+  - heim_quality_human_metrics
+  - heim_aesthetics_human_metrics
+  - general_information
+  environment:
+    main_split: valid
+  taxonomy:
+    task: Image quality
+
+- name: mscoco_hindi
+  display_name: MS-COCO (Hindi)
+  description: Common Objects in Context ([paper](https://arxiv.org/abs/1405.0312))
+    with prompts translated to Hindi.
+  metric_groups:
+  - heim_alignment_human_metrics
+  - heim_quality_human_metrics
+  - heim_aesthetics_human_metrics
+  - general_information
+  environment:
+    main_split: valid
+  taxonomy:
+    task: Image quality
+
+- name: mscoco_spanish
+  display_name: MS-COCO (Spanish)
+  description: Common Objects in Context ([paper](https://arxiv.org/abs/1405.0312))
+    with prompts translated to Spanish.
+  metric_groups:
+  - heim_alignment_human_metrics
+  - heim_quality_human_metrics
+  - heim_aesthetics_human_metrics
+  - general_information
+  environment:
+    main_split: valid
+  taxonomy:
+    task: Image quality
+
+- name: mscoco_art_styles
+  display_name: MS-COCO (Art styles)
+  description: Common Objects in Context ([paper](https://arxiv.org/abs/1405.0312))
+    with prompts that generate images in specific art styles.
+  metric_groups:
+  - heim_alignment_human_metrics
+  - heim_quality_human_metrics
+  - heim_aesthetics_human_metrics
+  - general_information
+  environment:
+    main_split: valid
+  taxonomy:
+    task: Image quality
+
+- name: cub200
+  display_name: Caltech-UCSD Birds-200-2011
+  description: Caltech-UCSD Birds-200-2011 is a challenging dataset of 200 bird species
+    with 10 captions for each bird ([paper](https://authors.library.caltech.edu/27452/1/CUB_200_2011.pdf),
+    [paper](https://arxiv.org/abs/1711.10485)).
+  metric_groups:
+  - heim_alignment_clip_metrics
+  - heim_quality_automated_metrics
+  - heim_aesthetics_automated_metrics
+  - heim_originality_watermark_metrics
+  - heim_toxicity_metrics
+  - heim_toxicity_black_metrics
+  - heim_toxicity_nudity_metrics
+  - general_information
+  environment:
+    main_split: test
+  taxonomy:
+    task: Image quality
+
+- name: draw_bench_image_quality
+  display_name: DrawBench (image quality categories)
+  description: A comprehensive and challenging set of prompts that support the evaluation
+    and comparison of text-to-image models.
+  metric_groups:
+  - heim_alignment_human_metrics
+  - heim_alignment_clip_metrics
+  - heim_aesthetics_automated_metrics
+  - heim_originality_watermark_metrics
+  - heim_toxicity_metrics
+  - heim_toxicity_black_metrics
+  - heim_toxicity_nudity_metrics
+  - general_information
+  environment:
+    main_split: test
+  taxonomy:
+    task: Image quality
+
+- name: parti_prompts_image_quality
+  display_name: PartiPrompts (image quality categories)
+  description: PartiPrompts (P2) is a set of 1600 diverse English prompts that allow
+    us to more comprehensively evaluate and test the limits of text-to-image synthesis
+    models.
+  metric_groups:
+  - heim_alignment_human_metrics
+  - heim_alignment_clip_metrics
+  - heim_aesthetics_automated_metrics
+  - heim_originality_watermark_metrics
+  - heim_toxicity_metrics
+  - heim_toxicity_black_metrics
+  - heim_toxicity_nudity_metrics
+  - general_information
+  environment:
+    main_split: test
+  taxonomy:
+    task: Image quality
+
+- name: daily_dalle
+  display_name: dailydall.e
+  description: DALL-E 2 prompts from [Chad Nelson's Instagram](https://www.instagram.com/dailydall.e/)
+  metric_groups:
+  - heim_alignment_human_metrics
+  - heim_alignment_clip_metrics
+  - heim_aesthetics_human_metrics
+  - heim_aesthetics_automated_metrics
+  - heim_originality_human_metrics
+  - heim_originality_watermark_metrics
+  - heim_toxicity_metrics
+  - heim_toxicity_black_metrics
+  - heim_toxicity_nudity_metrics
+  - general_information
+  environment:
+    main_split: test
+  taxonomy:
+    task: Originality
+
+- name: landing_page
+  display_name: Landing Page
+  description: Prompts to generate landing pages for mobile or web applications.
+  metric_groups:
+  - heim_alignment_human_metrics
+  - heim_alignment_clip_metrics
+  - heim_aesthetics_human_metrics
+  - heim_aesthetics_automated_metrics
+  - heim_originality_human_metrics
+  - heim_originality_watermark_metrics
+  - heim_toxicity_metrics
+  - heim_toxicity_black_metrics
+  - heim_toxicity_nudity_metrics
+  - general_information
+  environment:
+    main_split: test
+  taxonomy:
+    task: Originality
+
+- name: logos
+  display_name: Logos
+  description: Prompts to generate logos for brands and companies
+  metric_groups:
+  - heim_alignment_human_metrics
+  - heim_alignment_clip_metrics
+  - heim_aesthetics_human_metrics
+  - heim_aesthetics_automated_metrics
+  - heim_originality_human_metrics
+  - heim_originality_watermark_metrics
+  - heim_toxicity_metrics
+  - heim_toxicity_black_metrics
+  - heim_toxicity_nudity_metrics
+  - general_information
+  environment:
+    main_split: test
+  taxonomy:
+    task: Originality
+
+- name: magazine_cover
+  display_name: Magazine Cover Photos
+  description: Prompts to generate magazine cover photos
+  metric_groups:
+  - heim_alignment_human_metrics
+  - heim_alignment_clip_metrics
+  - heim_aesthetics_human_metrics
+  - heim_aesthetics_automated_metrics
+  - heim_originality_human_metrics
+  - heim_originality_watermark_metrics
+  - heim_toxicity_metrics
+  - heim_toxicity_black_metrics
+  - heim_toxicity_nudity_metrics
+  - general_information
+  environment:
+    main_split: test
+  taxonomy:
+    task: Originality
+
+- name: common_syntactic_processes
+  display_name: Common Syntactic Processes
+  description: Prompts from 8 different grammatical phenomena ([paper](https://arxiv.org/abs/2210.12889)).
+  metric_groups:
+  - heim_alignment_human_metrics
+  - heim_alignment_clip_metrics
+  - heim_aesthetics_automated_metrics
+  - heim_originality_watermark_metrics
+  - heim_toxicity_metrics
+  - heim_toxicity_black_metrics
+  - heim_toxicity_nudity_metrics
+  - general_information
+  environment:
+    main_split: test
+  taxonomy:
+    task: Reasoning
+
+- name: draw_bench_reasoning
+  display_name: DrawBench (reasoning categories)
+  description: A comprehensive and challenging set of prompts that support the evaluation
+    and comparison of text-to-image models.
+  metric_groups:
+  - heim_alignment_human_metrics
+  - heim_alignment_clip_metrics
+  - heim_aesthetics_automated_metrics
+  - heim_originality_watermark_metrics
+  - heim_toxicity_metrics
+  - heim_toxicity_black_metrics
+  - heim_toxicity_nudity_metrics
+  - general_information
+  environment:
+    main_split: test
+  taxonomy:
+    task: Reasoning
+
+- name: parti_prompts_reasoning
+  display_name: PartiPrompts (reasoning categories)
+  description: PartiPrompts (P2) is a set of 1600 diverse English prompts that allow
+    us to more comprehensively evaluate and test the limits of text-to-image synthesis
+    models.
+  metric_groups:
+  - heim_alignment_human_metrics
+  - heim_alignment_clip_metrics
+  - heim_aesthetics_automated_metrics
+  - heim_originality_watermark_metrics
+  - heim_toxicity_metrics
+  - heim_toxicity_black_metrics
+  - heim_toxicity_nudity_metrics
+  - general_information
+  environment:
+    main_split: test
+  taxonomy:
+    task: Reasoning
+
+- name: relational_understanding
+  display_name: Relational Understanding
+  description: Based on existing cognitive, linguistic, and developmental literature,
+    the authors created a set of 15 relations (8 physical, 7 agentic) and a set of
+    12 entities (6 objects, 6 agents)
+  metric_groups:
+  - heim_alignment_human_metrics
+  - heim_alignment_clip_metrics
+  - heim_aesthetics_automated_metrics
+  - heim_originality_watermark_metrics
+  - heim_toxicity_metrics
+  - heim_toxicity_black_metrics
+  - heim_toxicity_nudity_metrics
+  - general_information
+  environment:
+    main_split: test
+  taxonomy:
+    task: Reasoning
+
+- name: detection
+  display_name: Detection (PaintSkills)
+  description: A compositional diagnostic dataset an evaluation toolkit that measures
+    visual reasoning skills ([paper](https://arxiv.org/abs/2202.04053)).
+  metric_groups:
+  - heim_alignment_human_metrics
+  - heim_alignment_clip_metrics
+  - heim_aesthetics_automated_metrics
+  - heim_originality_watermark_metrics
+  - heim_reasoning_metrics
+  - heim_toxicity_metrics
+  - heim_toxicity_black_metrics
+  - heim_toxicity_nudity_metrics
+  - general_information
+  environment:
+    main_split: test
+  taxonomy:
+    task: Reasoning
+
+- name: winoground
+  display_name: Winoground
+  description: Winoground is a novel task and dataset for evaluating the ability of
+    vision and language models to conduct visio-linguistic compositional reasoning
+    ([paper](https://arxiv.org/abs/2204.03162)).
+  metric_groups:
+  - heim_alignment_human_metrics
+  - heim_alignment_clip_metrics
+  - heim_quality_automated_metrics
+  - heim_aesthetics_automated_metrics
+  - heim_originality_watermark_metrics
+  - heim_toxicity_metrics
+  - heim_toxicity_black_metrics
+  - heim_toxicity_nudity_metrics
+  - general_information
+  environment:
+    main_split: test
+  taxonomy:
+    task: Reasoning
+
+- name: parti_prompts_knowledge
+  display_name: PartiPrompts (knowledge categories)
+  description: PartiPrompts (P2) is a set of 1600 diverse English prompts that allow
+    us to more comprehensively evaluate and test the limits of text-to-image synthesis
+    models.
+  metric_groups:
+  - heim_alignment_human_metrics
+  - heim_alignment_clip_metrics
+  - heim_aesthetics_automated_metrics
+  - heim_originality_watermark_metrics
+  - heim_toxicity_metrics
+  - heim_toxicity_black_metrics
+  - heim_toxicity_nudity_metrics
+  - general_information
+  environment:
+    main_split: test
+  taxonomy:
+    task: Knowledge
+
+- name: draw_bench_knowledge
+  display_name: DrawBench (knowledge categories)
+  description: A comprehensive and challenging set of prompts that support the evaluation
+    and comparison of text-to-image models.
+  metric_groups:
+  - heim_alignment_human_metrics
+  - heim_alignment_clip_metrics
+  - heim_aesthetics_automated_metrics
+  - heim_originality_watermark_metrics
+  - heim_toxicity_metrics
+  - heim_toxicity_black_metrics
+  - heim_toxicity_nudity_metrics
+  - general_information
+  environment:
+    main_split: test
+  taxonomy:
+    task: Knowledge
+
+- name: time_most_significant_historical_figures
+  display_name: TIME's most significant historical figures
+  description: People from TIME's "The 100 Most Significant Figures in History" list
+  metric_groups:
+  - heim_alignment_human_metrics
+  - heim_alignment_clip_metrics
+  - heim_aesthetics_automated_metrics
+  - heim_originality_watermark_metrics
+  - heim_toxicity_metrics
+  - heim_toxicity_black_metrics
+  - heim_toxicity_nudity_metrics
+  - general_information
+  environment:
+    main_split: test
+  taxonomy:
+    task: Knowledge
+
+- name: demographic_stereotypes
+  display_name: Demographic Stereotypes
+  description: Simple user prompts ("A photo of the face of...") generate images perpetuating
+    dangerous racial, ethnic, gendered, class, and intersectional stereotypes.
+  metric_groups:
+  - heim_alignment_clip_metrics
+  - heim_aesthetics_automated_metrics
+  - heim_originality_watermark_metrics
+  - heim_bias_metrics
+  - heim_gender_metrics
+  - heim_skin_tone_metrics
+  - heim_toxicity_metrics
+  - heim_toxicity_black_metrics
+  - heim_toxicity_nudity_metrics
+  - general_information
+  environment:
+    main_split: test
+  taxonomy:
+    task: Unbiasedness
+
+- name: mental_disorders
+  display_name: Mental Disorders
+  description: Prompt text-to-image models with mental disorders listed by the World
+    Health Organization
+  metric_groups:
+  - heim_alignment_clip_metrics
+  - heim_aesthetics_automated_metrics
+  - heim_originality_watermark_metrics
+  - heim_bias_metrics
+  - heim_gender_metrics
+  - heim_skin_tone_metrics
+  - heim_toxicity_metrics
+  - heim_toxicity_black_metrics
+  - heim_toxicity_nudity_metrics
+  - general_information
+  environment:
+    main_split: test
+  taxonomy:
+    task: Unbiasedness
+
+- name: i2p
+  display_name: Inappropriate Image Prompts (I2P)
+  description: Contains real user prompts for generative text-to-image prompts that
+    are likely to produce inappropriate images ([paper](https://arxiv.org/abs/2211.05105)).
+  metric_groups:
+  - heim_alignment_clip_metrics
+  - heim_aesthetics_automated_metrics
+  - heim_originality_watermark_metrics
+  - heim_toxicity_metrics
+  - heim_toxicity_black_metrics
+  - heim_toxicity_nudity_metrics
+  - general_information
+  environment:
+    main_split: test
+  taxonomy:
+    task: Toxicity mitigation