crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/static/schema_image2struct.yaml

@@ -0,0 +1,588 @@
+---
+############################################################
+# For backwards compatibility with older versions of HELM.
+# TODO: Remove this after 2024-09-01.
+adapter: []
+############################################################
+metrics:
+  # Infrastructure metrics:
+  - name: num_references
+    display_name: '# ref'
+    description: Number of references.
+  - name: num_train_trials
+    display_name: '# trials'
+    description: Number of trials, where in each trial we choose an independent, random set of training instances.
+  - name: estimated_num_tokens_cost
+    display_name: 'cost'
+    description: An estimate of the number of tokens (including prompt and output completions) needed to perform the request.
+  - name: num_prompt_tokens
+    display_name: '# prompt tokens'
+    description: Number of tokens in the prompt.
+  - name: num_prompt_characters
+    display_name: '# prompt chars'
+    description: Number of characters in the prompt.
+  - name: num_completion_tokens
+    display_name: '# completion tokens'
+    description: Actual number of completion tokens (over all completions).
+  - name: num_output_tokens
+    display_name: '# output tokens'
+    description: Actual number of output tokens.
+  - name: max_num_output_tokens
+    display_name: 'Max output tokens'
+    description: Maximum number of output tokens (overestimate since we might stop earlier due to stop sequences).
+  - name: num_requests
+    display_name: '# requests'
+    description: Number of distinct API requests.
+  - name: num_instances
+    display_name: '# eval'
+    description: Number of evaluation instances.
+  - name: num_train_instances
+    display_name: '# train'
+    description: Number of training instances (e.g., in-context examples).
+  - name: prompt_truncated
+    display_name: truncated
+    description: Fraction of instances where the prompt itself was truncated (implies that there were no in-context examples).
+  - name: finish_reason_length
+    display_name: finish b/c length
+    description: Fraction of instances where the the output was terminated because of the max tokens limit.
+  - name: finish_reason_stop
+    display_name: finish b/c stop
+    description: Fraction of instances where the the output was terminated because of the stop sequences.
+  - name: finish_reason_endoftext
+    display_name: finish b/c endoftext
+    description: Fraction of instances where the the output was terminated because the end of text token was generated.
+  - name: finish_reason_unknown
+    display_name: finish b/c unknown
+    description: Fraction of instances where the the output was terminated for unknown reasons.
+  - name: num_completions
+    display_name: '# completions'
+    description: Number of completions.
+  - name: predicted_index
+    display_name: Predicted index
+    description: Integer index of the reference (0, 1, ...) that was predicted by the model (for multiple-choice).
+
+  # Vision Language metrics [text]:
+  - name: edit_similarity
+    display_name: Edit similarity (Levenshtein)
+    short_display_name: Edit sim.
+    lower_is_better: false
+    description: Average Levenshtein edit similarity (1 - distance normalized by length of longer sequence) between model generation and reference.
+
+  # Vision Language metrics [image]:
+  - name: earth_mover_similarity
+    display_name: Earth Mover Similarity
+    short_display_name: EMS
+    description: Earth Mover Similarity (EMD adapted to speed up calculations)
+    lower_is_better: false
+  - name: pixel_similarity
+    display_name: Pixel Similarity
+    short_display_name: PS
+    description: Pixel Similarity between an image generated by the model and the target image.
+    lower_is_better: false
+  - name: compilation_success
+    display_name: Compilation success
+    description: Fraction of instances where the generated code compiles successfully.
+    lower_is_better: false
+  - name: fid_similarity
+    display_name: CIS
+    short_display_name: CIS
+    description: The cosine similarity between the Inception feature vectors.
+    lower_is_better: false
+  - name: lpips_similarity
+    display_name: LPIPS
+    short_display_name: LPIPS
+    description: The LPIPS distance between the generated image and the target image.
+    lower_is_better: false
+  - name: ssim_similarity
+    display_name: SSIM
+    short_display_name: SSIM
+    description: The SSIM similarity between the generated image and the target image.
+    lower_is_better: false
+
+  # Accuracy metrics:
+  - name: exact_match
+    display_name: Exact match
+    short_display_name: EM
+    description: Fraction of instances that the predicted output matches a correct reference exactly.
+    lower_is_better: false
+  - name: quasi_exact_match
+    display_name: Quasi-exact match
+    short_display_name: EM
+    description: Fraction of instances that the predicted output matches a correct reference up to light processing.
+    lower_is_better: false
+  - name: prefix_exact_match
+    display_name: Prefix exact match
+    short_display_name: PEM
+    description: Fraction of instances that the predicted output matches the prefix of a correct reference exactly.
+    lower_is_better: false
+  - name: quasi_prefix_exact_match
+    # TODO: should call this prefix_quasi_exact_match
+    display_name: Prefix quasi-exact match
+    short_display_name: PEM
+    description: Fraction of instances that the predicted output matches the prefix of a correct reference up to light processing.
+    lower_is_better: false
+
+############################################################
+perturbations:
+  - name: robustness
+    display_name: Robustness
+    description: Computes worst case over different robustness perturbations (misspellings, formatting, contrast sets).
+
+############################################################
+metric_groups:
+  - name: accuracy
+    display_name: Compilation Rate and Earth Mover Similarity
+    metrics:
+      - name: earth_mover_similarity
+        split: ${main_split}
+      - name: compilation_success
+        split: ${main_split}
+
+  - name: general_information
+    display_name: General information
+    metrics:
+      - name: num_instances
+        split: ${main_split}
+      - name: num_train_instances
+        split: ${main_split}
+      - name: prompt_truncated
+        split: ${main_split}
+      - name: num_prompt_tokens
+        split: ${main_split}
+      - name: num_output_tokens
+        split: ${main_split}
+
+  - name: accuracy_simple
+    display_name: Earth Mover Similarity
+    metrics:
+      - name: earth_mover_similarity
+        split: ${main_split}
+
+  - name: compilation
+    display_name: Compilation Rate
+    metrics:
+      - name: compilation_success
+        split: ${main_split}
+
+  - name: generation_image
+    display_name: Generation (image)
+    metrics:
+      - name: pixel_similarity
+        split: ${main_split}
+      - name: compilation_success
+        split: ${main_split}
+      - name: fid_similarity
+        split: ${main_split}
+      - name: earth_mover_similarity
+        split: ${main_split}
+      - name: lpips_similarity
+        split: ${main_split}
+      - name: ssim_similarity
+        split: ${main_split}
+
+  - name: generation_text
+    display_name: Generation (text)
+    metrics:
+      - name: edit_similarity
+        split: ${main_split}
+
+############################################################
+run_groups:
+  - name: core_scenarios
+    display_name: Image2Struct
+    description: Scenarios for evaluating the ability of Vision-Language models to generate structured outputs from images.
+    category: All scenarios
+    subgroups:
+      - image2latex
+      - image2webpage
+      - image2musicsheet
+
+  - name: image2struct_wild
+    display_name: Image2Struct (Wild)
+    description: Scenarios for evaluating the ability of Vision-Language models to generate structured outputs from images. These scenarios contain images that do not have a ground truth.
+    category: All scenarios
+    subgroups:
+      - image2latex_wild
+      - image2webpage_wild
+
+  - name: image2latex
+    display_name: Image2LaTeX
+    description: The Image2LaTeX benchmark for converting images of mathematical equations, tables. algorithms and tikz to LaTeX.
+    metric_groups:
+      - accuracy
+      - generation_image
+      - generation_text
+      - general_information
+    environment:
+      main_name: earth_mover_similarity
+      main_split: valid
+    taxonomy:
+      task: image-to-text
+      what: mathematical equations, tables, algorithms, tikz
+      who: dataset authors
+      when: "2024"
+      language: English
+
+  - name: image2latex_equation
+    display_name: I2LaTeX (Equation)
+    description: The Image2LaTeX benchmark subset for converting images of mathematical equations to LaTeX.
+    metric_groups:
+      - accuracy_simple
+      - compilation
+      - generation_image
+      - generation_text
+      - general_information
+    environment:
+      main_name: earth_mover_similarity
+      main_split: valid
+    taxonomy:
+      task: image-to-text
+      what: mathematical equations
+      who: dataset authors
+      when: "2024"
+      language: English
+
+  - name: image2latex_table
+    display_name: I2LaTeX (Table)
+    description: The Image2LaTeX benchmark subset for converting images of tables to LaTeX.
+    metric_groups:
+      - accuracy_simple
+      - compilation
+      - generation_image
+      - generation_text
+      - general_information
+    environment:
+      main_name: earth_mover_similarity
+      main_split: valid
+    taxonomy:
+      task: image-to-text
+      what: tables
+      who: dataset authors
+      when: "2024"
+
+  - name: image2latex_algorithm
+    display_name: I2LaTeX (Algorithm)
+    description: The Image2LaTeX benchmark subset for converting images of algorithms to LaTeX.
+    metric_groups:
+      - accuracy_simple
+      - compilation
+      - generation_image
+      - generation_text
+      - general_information
+    environment:
+      main_name: earth_mover_similarity
+      main_split: valid
+    taxonomy:
+      task: image-to-text
+      what: algorithms
+      who: dataset authors
+      when: "2024"
+
+  - name: image2latex_plot
+    display_name: I2LaTeX (Tikz)
+    description: The Image2LaTeX benchmark subset for converting images of tikz to LaTeX.
+    metric_groups:
+      - accuracy_simple
+      - compilation
+      - generation_image
+      - generation_text
+      - general_information
+    environment:
+      main_name: earth_mover_similarity
+      main_split: valid
+    taxonomy:
+      task: image-to-text
+      what: tikz (plots)
+      who: dataset authors
+      when: "2024"
+
+  # - name: image2latex_easy
+  #   display_name: I2LaTeX (Easy)
+  #   description: The 1/3 easiest examples of the Image2LaTeX benchmark according to a simple heuristic counting the number of characters in the ground truth LaTeX code.
+  #   metric_groups:
+  #     - accuracy_simple
+  #     - compilation
+  #     - generation_image
+  #     - generation_text
+  #     - general_information
+  #   environment:
+  #     main_name: earth_mover_similarity
+  #     main_split: valid
+  #   taxonomy:
+  #     task: image-to-text
+  #     what: mathematical equations, tables, algorithms, tikz
+  #     who: dataset authors
+  #     when: "2024"
+  #     language: English
+
+  # - name: image2latex_medium
+  #   display_name: I2LaTeX (Medium)
+  #   description: The 1/3 examples with medium diffulty of the Image2LaTeX benchmark according to a simple heuristic counting the number of characters in the ground truth LaTeX code.
+  #   metric_groups:
+  #     - accuracy_simple
+  #     - compilation
+  #     - generation_image
+  #     - generation_text
+  #     - general_information
+  #   environment:
+  #     main_name: earth_mover_similarity
+  #     main_split: valid
+  #   taxonomy:
+  #     task: image-to-text
+  #     what: mathematical equations, tables, algorithms, tikz
+  #     who: dataset authors
+  #     when: "2024"
+  #     language: English
+
+  # - name: image2latex_hard
+  #   display_name: I2LaTeX (Hard)
+  #   description: The 1/3 hardest examples of the Image2LaTeX benchmark according to a simple heuristic counting the number of characters in the ground truth LaTeX code.
+  #   metric_groups:
+  #     - accuracy_simple
+  #     - compilation
+  #     - generation_image
+  #     - generation_text
+  #     - general_information
+  #   environment:
+  #     main_name: earth_mover_similarity
+  #     main_split: valid
+  #   taxonomy:
+  #     task: image-to-text
+  #     what: mathematical equations, tables, algorithms, tikz
+  #     who: dataset authors
+  #     when: "2024"
+  #     language: English
+
+  - name: image2latex_wild
+    display_name: Image2LaTeX (Wild)
+    description: Images of mathematical equations gathered from Wikipedia that do not have a LaTeX ground truth.
+    metric_groups:
+      - accuracy
+      - generation_image
+      - general_information
+    environment:
+      main_name: earth_mover_similarity
+      main_split: valid
+    taxonomy:
+      task: image-to-text
+      what: mathematical equations, tables, algorithms, tikz
+      who: dataset authors
+      when: "2024"
+      language: English
+
+  - name: image2webpage
+    display_name: Image2webpage
+    description: The Image2webpage benchmark for converting images of webpages to HTML/CSS/Javascript.
+    metric_groups:
+      - accuracy
+      - generation_image
+      - generation_text
+      - general_information
+    environment:
+      main_name: earth_mover_similarity
+      main_split: valid
+    taxonomy:
+      task: image-to-text
+      what: css, html, javascript
+      who: dataset authors
+      when: "2024"
+      language: English
+
+  - name: image2webpage_css
+    display_name: I2webpage (CSS)
+    description: The Image2webpage benchmark subset for converting images of webpages to code repo containing mostly CSS.
+    metric_groups:
+      - accuracy_simple
+      - compilation
+      - generation_image
+      - generation_text
+      - general_information
+    environment:
+      main_name: earth_mover_similarity
+      main_split: valid
+    taxonomy:
+      task: image-to-text
+      what: code (mostly CSS)
+      who: dataset authors
+      when: "2024"
+
+  - name: image2webpage_html
+    display_name: I2webpage (HTML)
+    description: The Image2webpage benchmark subset for converting images of webpages to code repo containing mostly HTML.
+    metric_groups:
+      - accuracy_simple
+      - compilation
+      - generation_image
+      - generation_text
+      - general_information
+    environment:
+      main_name: earth_mover_similarity
+      main_split: valid
+    taxonomy:
+      task: image-to-text
+      what: code (mostly HTML)
+      who: dataset authors
+      when: "2024"
+
+  - name: image2webpage_javascript
+    display_name: I2webpage (Javascript)
+    description: The Image2webpage benchmark subset for converting images of webpages to code repo containing mostly Javascript.
+    metric_groups:
+      - accuracy_simple
+      - compilation
+      - generation_image
+      - generation_text
+      - general_information
+    environment:
+      main_name: earth_mover_similarity
+      main_split: valid
+    taxonomy:
+      task: image-to-text
+      what: code (mostly Javascript)
+      who: dataset authors
+      when: "2024"
+
+  # - name: image2webpage_easy
+  #   display_name: I2webpage (Easy)
+  #   description: The 1/3 easiest examples of the Image2webpage benchmark according to a simple heuristic counting the number of characters in the ground truth HTML/CSS/Javascript code.
+  #   metric_groups:
+  #     - accuracy_simple
+  #     - compilation
+  #     - generation_image
+  #     - generation_text
+  #     - general_information
+  #   environment:
+  #     main_name: earth_mover_similarity
+  #     main_split: valid
+  #   taxonomy:
+  #     task: image-to-text
+  #     what: css, html, javascript
+  #     who: dataset authors
+  #     when: "2024"
+  #     language: English
+
+  # - name: image2webpage_medium
+  #   display_name: I2webpage (Medium)
+  #   description: The 1/3 examples with medium diffulty of the Image2webpage benchmark according to a simple heuristic counting the number of characters in the ground truth HTML/CSS/Javascript code.
+  #   metric_groups:
+  #     - accuracy_simple
+  #     - compilation
+  #     - generation_image
+  #     - generation_text
+  #     - general_information
+  #   environment:
+  #     main_name: earth_mover_similarity
+  #     main_split: valid
+  #   taxonomy:
+  #     task: image-to-text
+  #     what: css, html, javascript
+  #     who: dataset authors
+  #     when: "2024"
+  #     language: English
+
+  # - name: image2webpage_hard
+  #   display_name: I2webpage (Hard)
+  #   description: The 1/3 hardest examples of the Image2webpage benchmark according to a simple heuristic counting the number of characters in the ground truth HTML/CSS/Javascript code.
+  #   metric_groups:
+  #     - accuracy_simple
+  #     - compilation
+  #     - generation_image
+  #     - generation_text
+  #     - general_information
+  #   environment:
+  #     main_name: earth_mover_similarity
+  #     main_split: valid
+  #   taxonomy:
+  #     task: image-to-text
+  #     what: css, html, javascript
+  #     who: dataset authors
+  #     when: "2024"
+  #     language: English
+
+  - name: image2webpage_wild
+    display_name: Image2webpage (Wild)
+    description: Images of webpages gathered from the internet by taking sceenshots and so on that do not have a HTML/CSS/Javascript ground truth.
+    metric_groups:
+      - accuracy
+      - generation_image
+      - general_information
+    environment:
+      main_name: earth_mover_similarity
+      main_split: valid
+    taxonomy:
+      task: image-to-text
+      what: css, html, javascript
+      who: dataset authors
+      when: "2024"
+      language: English
+
+  - name: image2musicsheet
+    display_name: Image2musicsheet
+    description: The Image2musicsheet benchmark for converting images of music sheets to LilyPond.
+    metric_groups:
+      - accuracy
+      - generation_image
+      - general_information
+    environment:
+      main_name: earth_mover_similarity
+      main_split: valid
+    taxonomy:
+      task: image-to-text
+      what: music sheets
+      who: dataset authors
+      when: "2024"
+      language: English
+
+  # - name: image2musicsheet_easy
+  #   display_name: I2musicsheet (Easy)
+  #   description: The 1/3 easiest examples of the Image2musicsheet benchmark according to a simple heuristic counting the number of black pixels in the target image.
+  #   metric_groups:
+  #     - accuracy_simple
+  #     - compilation
+  #     - generation_image
+  #     - general_information
+  #   environment:
+  #     main_name: earth_mover_similarity
+  #     main_split: valid
+  #   taxonomy:
+  #     task: image-to-text
+  #     what: music sheets
+  #     who: dataset authors
+  #     when: "2024"
+  #     language: English
+
+  # - name: image2musicsheet_medium
+  #   display_name: I2musicsheet (Medium)
+  #   description: The 1/3 examples with medium diffulty of the Image2musicsheet benchmark according to a simple heuristic counting the number of black pixels in the target image.
+  #   metric_groups:
+  #     - accuracy_simple
+  #     - compilation
+  #     - generation_image
+  #     - general_information
+  #   environment:
+  #     main_name: earth_mover_similarity
+  #     main_split: valid
+  #   taxonomy:
+  #     task: image-to-text
+  #     what: music sheets
+  #     who: dataset authors
+  #     when: "2024"
+  #     language: English
+
+  # - name: image2musicsheet_hard
+  #   display_name: I2musicsheet (Hard)
+  #   description: The 1/3 hardest examples of the Image2musicsheet benchmark according to a simple heuristic counting the number of black pixels in the target image.
+  #   metric_groups:
+  #     - accuracy_simple
+  #     - compilation
+  #     - generation_image
+  #     - general_information
+  #   environment:
+  #     main_name: earth_mover_similarity
+  #     main_split: valid
+  #   taxonomy:
+  #     task: image-to-text
+  #     what: music sheets
+  #     who: dataset authors
+  #     when: "2024"
+  #     language: English