wisent 0.7.379__py3-none-any.whl → 0.7.701__py3-none-any.whl

wisent/examples/scripts/results/failing_benchmarks.json

@@ -1,946 +0,0 @@
-{
-  "total_failures": 235,
-  "failures": [
-    {
-      "benchmark": "math500",
-      "error": "No extractor registered for task 'math500'. Known: 20_newsgroups, aclue, acp_bench, acp_bench_hard, advanced, aexams, afrimgsm, afrimmlu, afrixnli, ag, ag_news, agieval, ai2_arc, aime, anagrams1, anagrams2, anli, anli_r1, anli_r2, anli_r3, apps, arabculture, arabic, arabic_leaderboard_complete, arabic_leaderboard_light, arabicmmlu, aradice, arc, arc_challenge, arc_easy, argument, argument_topic, arithmetic, asdiv, assin, assin_entailment, atis, babi, babilong, bangla_mmlu, banking77, basque_benc..."
-    },
-    {
-      "benchmark": "aime2024",
-      "error": "No extractor registered for task 'aime2024'. Known: 20_newsgroups, aclue, acp_bench, acp_bench_hard, advanced, aexams, afrimgsm, afrimmlu, afrixnli, ag, ag_news, agieval, ai2_arc, aime, anagrams1, anagrams2, anli, anli_r1, anli_r2, anli_r3, apps, arabculture, arabic, arabic_leaderboard_complete, arabic_leaderboard_light, arabicmmlu, aradice, arc, arc_challenge, arc_easy, argument, argument_topic, arithmetic, asdiv, assin, assin_entailment, atis, babi, babilong, bangla_mmlu, banking77, basque_ben..."
-    },
-    {
-      "benchmark": "aime2025",
-      "error": "No extractor registered for task 'aime2025'. Known: 20_newsgroups, aclue, acp_bench, acp_bench_hard, advanced, aexams, afrimgsm, afrimmlu, afrixnli, ag, ag_news, agieval, ai2_arc, aime, anagrams1, anagrams2, anli, anli_r1, anli_r2, anli_r3, apps, arabculture, arabic, arabic_leaderboard_complete, arabic_leaderboard_light, arabicmmlu, aradice, arc, arc_challenge, arc_easy, argument, argument_topic, arithmetic, asdiv, assin, assin_entailment, atis, babi, babilong, bangla_mmlu, banking77, basque_ben..."
-    },
-    {
-      "benchmark": "hmmt",
-      "error": "No extractor registered for task 'hmmt'. Known: 20_newsgroups, aclue, acp_bench, acp_bench_hard, advanced, aexams, afrimgsm, afrimmlu, afrixnli, ag, ag_news, agieval, ai2_arc, aime, anagrams1, anagrams2, anli, anli_r1, anli_r2, anli_r3, apps, arabculture, arabic, arabic_leaderboard_complete, arabic_leaderboard_light, arabicmmlu, aradice, arc, arc_challenge, arc_easy, argument, argument_topic, arithmetic, asdiv, assin, assin_entailment, atis, babi, babilong, bangla_mmlu, banking77, basque_bench, ..."
-    },
-    {
-      "benchmark": "hmmt_feb_2025",
-      "error": "No extractor registered for task 'hmmt_feb_2025'. Known: 20_newsgroups, aclue, acp_bench, acp_bench_hard, advanced, aexams, afrimgsm, afrimmlu, afrixnli, ag, ag_news, agieval, ai2_arc, aime, anagrams1, anagrams2, anli, anli_r1, anli_r2, anli_r3, apps, arabculture, arabic, arabic_leaderboard_complete, arabic_leaderboard_light, arabicmmlu, aradice, arc, arc_challenge, arc_easy, argument, argument_topic, arithmetic, asdiv, assin, assin_entailment, atis, babi, babilong, bangla_mmlu, banking77, basqu..."
-    },
-    {
-      "benchmark": "polymath_en_medium",
-      "error": "No extractor registered for task 'polymath_en_medium'. Known: 20_newsgroups, aclue, acp_bench, acp_bench_hard, advanced, aexams, afrimgsm, afrimmlu, afrixnli, ag, ag_news, agieval, ai2_arc, aime, anagrams1, anagrams2, anli, anli_r1, anli_r2, anli_r3, apps, arabculture, arabic, arabic_leaderboard_complete, arabic_leaderboard_light, arabicmmlu, aradice, arc, arc_challenge, arc_easy, argument, argument_topic, arithmetic, asdiv, assin, assin_entailment, atis, babi, babilong, bangla_mmlu, banking77, ..."
-    },
-    {
-      "benchmark": "polymath_zh_medium",
-      "error": "No extractor registered for task 'polymath_zh_medium'. Known: 20_newsgroups, aclue, acp_bench, acp_bench_hard, advanced, aexams, afrimgsm, afrimmlu, afrixnli, ag, ag_news, agieval, ai2_arc, aime, anagrams1, anagrams2, anli, anli_r1, anli_r2, anli_r3, apps, arabculture, arabic, arabic_leaderboard_complete, arabic_leaderboard_light, arabicmmlu, aradice, arc, arc_challenge, arc_easy, argument, argument_topic, arithmetic, asdiv, assin, assin_entailment, atis, babi, babilong, bangla_mmlu, banking77, ..."
-    },
-    {
-      "benchmark": "polymath_en_high",
-      "error": "No extractor registered for task 'polymath_en_high'. Known: 20_newsgroups, aclue, acp_bench, acp_bench_hard, advanced, aexams, afrimgsm, afrimmlu, afrixnli, ag, ag_news, agieval, ai2_arc, aime, anagrams1, anagrams2, anli, anli_r1, anli_r2, anli_r3, apps, arabculture, arabic, arabic_leaderboard_complete, arabic_leaderboard_light, arabicmmlu, aradice, arc, arc_challenge, arc_easy, argument, argument_topic, arithmetic, asdiv, assin, assin_entailment, atis, babi, babilong, bangla_mmlu, banking77, ba..."
-    },
-    {
-      "benchmark": "polymath_zh_high",
-      "error": "No extractor registered for task 'polymath_zh_high'. Known: 20_newsgroups, aclue, acp_bench, acp_bench_hard, advanced, aexams, afrimgsm, afrimmlu, afrixnli, ag, ag_news, agieval, ai2_arc, aime, anagrams1, anagrams2, anli, anli_r1, anli_r2, anli_r3, apps, arabculture, arabic, arabic_leaderboard_complete, arabic_leaderboard_light, arabicmmlu, aradice, arc, arc_challenge, arc_easy, argument, argument_topic, arithmetic, asdiv, assin, assin_entailment, atis, babi, babilong, bangla_mmlu, banking77, ba..."
-    },
-    {
-      "benchmark": "livemathbench_cnmo_en",
-      "error": "No extractor registered for task 'livemathbench_cnmo_en'. Known: 20_newsgroups, aclue, acp_bench, acp_bench_hard, advanced, aexams, afrimgsm, afrimmlu, afrixnli, ag, ag_news, agieval, ai2_arc, aime, anagrams1, anagrams2, anli, anli_r1, anli_r2, anli_r3, apps, arabculture, arabic, arabic_leaderboard_complete, arabic_leaderboard_light, arabicmmlu, aradice, arc, arc_challenge, arc_easy, argument, argument_topic, arithmetic, asdiv, assin, assin_entailment, atis, babi, babilong, bangla_mmlu, banking7..."
-    },
-    {
-      "benchmark": "livemathbench_cnmo_zh",
-      "error": "Failed to extract pairs from 'livemathbench_cnmo_zh': Failed to load HuggingFace dataset 'opencompass/LiveMathBench'. Arguments were: config='v202412_CNMO_cn', split='test'. Underlying error: Feature type 'List' not found. Available feature types: ['Value', 'ClassLabel', 'Translation', 'TranslationVariableLanguages', 'LargeList', 'Sequence', 'Array2D', 'Array3D', 'Array4D', 'Array5D', 'Audio', 'Image', 'Video', 'Pdf']"
-    },
-    {
-      "benchmark": "instruct_humaneval",
-      "error": "No extractor registered for task 'instruct_humaneval'. Known: 20_newsgroups, aclue, acp_bench, acp_bench_hard, advanced, aexams, afrimgsm, afrimmlu, afrixnli, ag, ag_news, agieval, ai2_arc, aime, anagrams1, anagrams2, anli, anli_r1, anli_r2, anli_r3, apps, arabculture, arabic, arabic_leaderboard_complete, arabic_leaderboard_light, arabicmmlu, aradice, arc, arc_challenge, arc_easy, argument, argument_topic, arithmetic, asdiv, assin, assin_entailment, atis, babi, babilong, bangla_mmlu, banking77, ..."
-    },
-    {
-      "benchmark": "recode",
-      "error": "Failed to extract pairs from 'recode': Failed to load HuggingFace dataset 'code_x_glue_tc_nl_code_search_adv'. Arguments were: config='default', split='train'. Underlying error: Feature type 'List' not found. Available feature types: ['Value', 'ClassLabel', 'Translation', 'TranslationVariableLanguages', 'LargeList', 'Sequence', 'Array2D', 'Array3D', 'Array4D', 'Array5D', 'Audio', 'Image', 'Video', 'Pdf']"
-    },
-    {
-      "benchmark": "bbh",
-      "error": "(MaxRetryError('HTTPSConnectionPool(host=\\'huggingface.co\\', port=443): Max retries exceeded with url: /api/datasets/SaylorTwift/bbh/tree/b5306be6f827cfafbb545ff5a51f96916029b0fd/snarks?recursive=True&expand=False (Caused by NameResolutionError(\"<urllib3.connection.HTTPSConnection object at 0x358b1cf10>: Failed to resolve \\'huggingface.co\\' ([Errno 8] nodename nor servname provided, or not known)\"))'), '(Request ID: 191b7b73-810f-41c6-88b6-3206e1aefc69)')"
-    },
-    {
-      "benchmark": "siqa",
-      "error": "'siqa'"
-    },
-    {
-      "benchmark": "AraDICE",
-      "error": "'AraDICE'"
-    },
-    {
-      "benchmark": "ArabCulture",
-      "error": "'ArabCulture'"
-    },
-    {
-      "benchmark": "darija_bench",
-      "error": "'darija_bench'"
-    },
-    {
-      "benchmark": "ceval",
-      "error": "'ceval'"
-    },
-    {
-      "benchmark": "turkishmmlu",
-      "error": "Feature type 'List' not found. Available feature types: ['Value', 'ClassLabel', 'Translation', 'TranslationVariableLanguages', 'LargeList', 'Sequence', 'Array2D', 'Array3D', 'Array4D', 'Array5D', 'Audio', 'Image', 'Video', 'Pdf']"
-    },
-    {
-      "benchmark": "bangla_mmlu",
-      "error": "'bangla_mmlu'"
-    },
-    {
-      "benchmark": "japanese_leaderboard",
-      "error": "`fugashi` is required for `japanese_leaderboard`, please install `fugashi` via pip install lm_eval[japanese_leaderboard] or pip install -e .[japanese_leaderboard]"
-    },
-    {
-      "benchmark": "kormedmcqa",
-      "error": "No pairs could be loaded from any subtask of 'kormedmcqa'"
-    },
-    {
-      "benchmark": "basqueglue",
-      "error": "'basqueglue'"
-    },
-    {
-      "benchmark": "eus_exams",
-      "error": "'eus_exams'"
-    },
-    {
-      "benchmark": "catalan_bench",
-      "error": "(MaxRetryError('HTTPSConnectionPool(host=\\'huggingface.co\\', port=443): Max retries exceeded with url: /api/datasets/gplsi/cocoteros_va/revision/652b16cb3cd15c8a695e39ffdd483b552f233319 (Caused by NameResolutionError(\"<urllib3.connection.HTTPSConnection object at 0x41d1da6b0>: Failed to resolve \\'huggingface.co\\' ([Errno 8] nodename nor servname provided, or not known)\"))'), '(Request ID: 98ddb951-e4e5-48d9-8c75-a69a29fbdcd3)')"
-    },
-    {
-      "benchmark": "french_bench",
-      "error": "Feature type 'List' not found. Available feature types: ['Value', 'ClassLabel', 'Translation', 'TranslationVariableLanguages', 'LargeList', 'Sequence', 'Array2D', 'Array3D', 'Array4D', 'Array5D', 'Audio', 'Image', 'Video', 'Pdf']"
-    },
-    {
-      "benchmark": "icelandic_winogrande",
-      "error": "'icelandic_winogrande'"
-    },
-    {
-      "benchmark": "global_mmlu",
-      "error": "'global_mmlu'"
-    },
-    {
-      "benchmark": "global_piqa",
-      "error": "'global_piqa'"
-    },
-    {
-      "benchmark": "belebele",
-      "error": "No pairs could be loaded from any subtask of 'belebele'"
-    },
-    {
-      "benchmark": "mlqa",
-      "error": "'mlqa'"
-    },
-    {
-      "benchmark": "xquad",
-      "error": "No pairs could be loaded from any subtask of 'xquad'"
-    },
-    {
-      "benchmark": "xcopa",
-      "error": "No pairs could be loaded from any subtask of 'xcopa'"
-    },
-    {
-      "benchmark": "okapi/arc_multilingual",
-      "error": "'okapi/arc_multilingual'"
-    },
-    {
-      "benchmark": "okapi/hellaswag_multilingual",
-      "error": "'okapi/hellaswag_multilingual'"
-    },
-    {
-      "benchmark": "okapi/mmlu_multilingual",
-      "error": "'okapi/mmlu_multilingual'"
-    },
-    {
-      "benchmark": "okapi/truthfulqa_multilingual",
-      "error": "'okapi/truthfulqa_multilingual'"
-    },
-    {
-      "benchmark": "toxigen",
-      "error": "One of the splits is empty after splitting."
-    },
-    {
-      "benchmark": "moral_stories",
-      "error": "One of the splits is empty after splitting."
-    },
-    {
-      "benchmark": "realtoxicityprompts",
-      "error": "One of the splits is empty after splitting."
-    },
-    {
-      "benchmark": "discrim_eval",
-      "error": "'discrim_eval'"
-    },
-    {
-      "benchmark": "esbbq",
-      "error": "'esbbq'"
-    },
-    {
-      "benchmark": "cabbq",
-      "error": "'cabbq'"
-    },
-    {
-      "benchmark": "simple_cooccurrence_bias",
-      "error": "One of the splits is empty after splitting."
-    },
-    {
-      "benchmark": "winogender",
-      "error": "No pairs could be loaded from any subtask of 'winogender'"
-    },
-    {
-      "benchmark": "medmcqa",
-      "error": "One of the splits is empty after splitting."
-    },
-    {
-      "benchmark": "med_concepts_qa",
-      "error": "No pairs could be loaded from any subtask of 'med_concepts_qa'"
-    },
-    {
-      "benchmark": "meddialog",
-      "error": "'meddialog'"
-    },
-    {
-      "benchmark": "mediqa_qa2019",
-      "error": "Error loading evaluation metrics: Couldn't find a module script at /Users/lukaszbartoszcze/Documents/CodingProjects/Wisent/backends/wisent-open-source/wisent/examples/scripts/bleurt/bleurt.py. Module 'bleurt' doesn't exist on the Hugging Face Hub either.. Please check your installation."
-    },
-    {
-      "benchmark": "medtext",
-      "error": "Error loading evaluation metrics: Couldn't find a module script at /Users/lukaszbartoszcze/Documents/CodingProjects/Wisent/backends/wisent-open-source/wisent/examples/scripts/bleurt/bleurt.py. Module 'bleurt' doesn't exist on the Hugging Face Hub either.. Please check your installation."
-    },
-    {
-      "benchmark": "meqsum",
-      "error": "Error loading evaluation metrics: Couldn't find a module script at /Users/lukaszbartoszcze/Documents/CodingProjects/Wisent/backends/wisent-open-source/wisent/examples/scripts/bleurt/bleurt.py. Module 'bleurt' doesn't exist on the Hugging Face Hub either.. Please check your installation."
-    },
-    {
-      "benchmark": "mimic_repsum",
-      "error": "Please install evaluation metrics via pip install evaluate bert-score rouge_score>=0.1.2 nltk absl-py radgraphgit+https://github.com/google-research/bleurt.git"
-    },
-    {
-      "benchmark": "minerva_math",
-      "error": "Unsupported ANTLR version 4.7.2, only 4.9.3, 4.11.0, and 4.13.2 runtime versions are supported."
-    },
-    {
-      "benchmark": "mgsm",
-      "error": "'mgsm'"
-    },
-    {
-      "benchmark": "hrm8k",
-      "error": "No pairs could be loaded from any subtask of 'hrm8k'"
-    },
-    {
-      "benchmark": "babi",
-      "error": "'valid'"
-    },
-    {
-      "benchmark": "babilong",
-      "error": "'babilong'"
-    },
-    {
-      "benchmark": "longbench",
-      "error": "(MaxRetryError('HTTPSConnectionPool(host=\\'huggingface.co\\', port=443): Max retries exceeded with url: /api/datasets/THUDM/LongBench/revision/5e628be450b7e67fb7ae6e201bd6d8f7056f7672 (Caused by NameResolutionError(\"<urllib3.connection.HTTPSConnection object at 0x3741478b0>: Failed to resolve \\'huggingface.co\\' ([Errno 8] nodename nor servname provided, or not known)\"))'), '(Request ID: 0435569f-7b06-41dd-866e-d040c34fee01)')"
-    },
-    {
-      "benchmark": "longbenchv2",
-      "error": "'longbenchv2'"
-    },
-    {
-      "benchmark": "ruler",
-      "error": "HTTPConnectionPool(host='curtis.ml.cmu.edu', port=80): Max retries exceeded with url: /datasets/hotpot/hotpot_dev_distractor_v1.json (Caused by NameResolutionError(\"<urllib3.connection.HTTPConnection object at 0x364c5c7f0>: Failed to resolve 'curtis.ml.cmu.edu' ([Errno 8] nodename nor servname provided, or not known)\"))"
-    },
-    {
-      "benchmark": "scrolls",
-      "error": "'scrolls'"
-    },
-    {
-      "benchmark": "inverse_scaling",
-      "error": "'inverse_scaling'"
-    },
-    {
-      "benchmark": "storycloze",
-      "error": "The dataset story_cloze with config 2016 requires manual data."
-    },
-    {
-      "benchmark": "histoires_morales",
-      "error": "One of the splits is empty after splitting."
-    },
-    {
-      "benchmark": "groundcocoa",
-      "error": "(MaxRetryError('HTTPSConnectionPool(host=\\'huggingface.co\\', port=443): Max retries exceeded with url: /api/datasets/harsh147/GroundCocoa/revision/2f4bc9a10c17062001870e2e2d0518cb4fc9740e (Caused by NameResolutionError(\"<urllib3.connection.HTTPSConnection object at 0x3840a3c10>: Failed to resolve \\'huggingface.co\\' ([Errno 8] nodename nor servname provided, or not known)\"))'), '(Request ID: 175bd0e6-809e-459e-ba1a-9ba9408b36e2)')"
-    },
-    {
-      "benchmark": "blimp",
-      "error": "(ProtocolError('Connection aborted.', RemoteDisconnected('Remote end closed connection without response')), '(Request ID: df2181ba-3259-4ebc-ac3d-cb2fd6a67f01)')"
-    },
-    {
-      "benchmark": "blimp_nl",
-      "error": "'blimp_nl'"
-    },
-    {
-      "benchmark": "multiblimp",
-      "error": "(ReadTimeoutError(\"HTTPSConnectionPool(host='huggingface.co', port=443): Read timed out. (read timeout=10)\"), '(Request ID: 1808dfbf-bac5-491c-a884-e993184ee3ac)')"
-    },
-    {
-      "benchmark": "turblimp_core",
-      "error": "'turblimp_core'"
-    },
-    {
-      "benchmark": "zhoblimp",
-      "error": "'zhoblimp'"
-    },
-    {
-      "benchmark": "lambada_multilingual_stablelm",
-      "error": "'lambada_multilingual_stablelm'"
-    },
-    {
-      "benchmark": "paws-x",
-      "error": "'paws-x'"
-    },
-    {
-      "benchmark": "translation",
-      "error": "(ProtocolError('Connection aborted.', RemoteDisconnected('Remote end closed connection without response')), '(Request ID: 8c72b72b-e44f-4764-bb98-d2bb995dcf35)')"
-    },
-    {
-      "benchmark": "wmt2016",
-      "error": "'wmt2016'"
-    },
-    {
-      "benchmark": "code_x_glue",
-      "error": "'code_x_glue'"
-    },
-    {
-      "benchmark": "humaneval_infilling",
-      "error": "'humaneval_infilling'"
-    },
-    {
-      "benchmark": "bertaqa",
-      "error": "Feature type 'List' not found. Available feature types: ['Value', 'ClassLabel', 'Translation', 'TranslationVariableLanguages', 'LargeList', 'Sequence', 'Array2D', 'Array3D', 'Array4D', 'Array5D', 'Audio', 'Image', 'Video', 'Pdf']"
-    },
-    {
-      "benchmark": "careqa",
-      "error": "'careqa'"
-    },
-    {
-      "benchmark": "acp_bench",
-      "error": "No pairs could be loaded from any subtask of 'acp_bench'"
-    },
-    {
-      "benchmark": "acp_bench_hard",
-      "error": "No pairs could be loaded from any subtask of 'acp_bench_hard'"
-    },
-    {
-      "benchmark": "aexams",
-      "error": "Some evaluations incorrect"
-    },
-    {
-      "benchmark": "benchmarks",
-      "error": "'benchmarks'"
-    },
-    {
-      "benchmark": "bhs",
-      "error": "'bhs'"
-    },
-    {
-      "benchmark": "c4",
-      "error": "One of the splits is empty after splitting."
-    },
-    {
-      "benchmark": "chartqa",
-      "error": "One of the splits is empty after splitting."
-    },
-    {
-      "benchmark": "click",
-      "error": "'click'"
-    },
-    {
-      "benchmark": "eq-bench_ca",
-      "error": "'eq-bench_ca'"
-    },
-    {
-      "benchmark": "eq-bench_es",
-      "error": "'eq-bench_es'"
-    },
-    {
-      "benchmark": "evalita_LLM",
-      "error": "'evalita_LLM'"
-    },
-    {
-      "benchmark": "fda",
-      "error": "Some evaluations incorrect"
-    },
-    {
-      "benchmark": "fld",
-      "error": "'fld'"
-    },
-    {
-      "benchmark": "jsonschema_bench",
-      "error": "No pairs could be loaded from any subtask of 'jsonschema_bench'"
-    },
-    {
-      "benchmark": "kbl",
-      "error": "No pairs could be loaded from any subtask of 'kbl'"
-    },
-    {
-      "benchmark": "leaderboard",
-      "error": "Unsupported ANTLR version 4.7.2, only 4.9.3, 4.11.0, and 4.13.2 runtime versions are supported."
-    },
-    {
-      "benchmark": "libra",
-      "error": "No pairs could be loaded from any subtask of 'libra'"
-    },
-    {
-      "benchmark": "lingoly",
-      "error": "Dataset 'ambean/lingOly' is a gated dataset on the Hub. Visit the dataset page at https://huggingface.co/datasets/ambean/lingOly to ask for access."
-    },
-    {
-      "benchmark": "llama3",
-      "error": "'llama3'"
-    },
-    {
-      "benchmark": "lm_syneval",
-      "error": "'lm_syneval'"
-    },
-    {
-      "benchmark": "mastermind",
-      "error": "No pairs could be loaded from any subtask of 'mastermind'"
-    },
-    {
-      "benchmark": "mmlusr",
-      "error": "BuilderConfig 'question_and_answer_philosophy' not found. Available: ['answer_only', 'question_only', 'question_and_answer']"
-    },
-    {
-      "benchmark": "mmmu",
-      "error": "'mmmu'"
-    },
-    {
-      "benchmark": "model_written_evals",
-      "error": "'model_written_evals'"
-    },
-    {
-      "benchmark": "mts_dialog",
-      "error": "Please install evaluation metrics via pip install evaluate bert-score rouge_score>=0.1.2 nltk absl-py git+https://github.com/google-research/bleurt.git"
-    },
-    {
-      "benchmark": "multiblimp",
-      "error": "Couldn't find cache for jumelet/multiblimp for config 'gle'"
-    },
-    {
-      "benchmark": "noreval",
-      "error": "'noreval'"
-    },
-    {
-      "benchmark": "olaph",
-      "error": "Please install evaluation metrics via pip install evaluate bert-score rouge_score>=0.1.2 nltk absl-py git+https://github.com/google-research/bleurt.git"
-    },
-    {
-      "benchmark": "paloma",
-      "error": "Dataset 'allenai/paloma' is a gated dataset on the Hub. Visit the dataset page at https://huggingface.co/datasets/allenai/paloma to ask for access."
-    },
-    {
-      "benchmark": "pile",
-      "error": "'pile'"
-    },
-    {
-      "benchmark": "pile_10k",
-      "error": "One of the splits is empty after splitting."
-    },
-    {
-      "benchmark": "polemo2",
-      "error": "No pairs could be loaded from any subtask of 'polemo2'"
-    },
-    {
-      "benchmark": "score",
-      "error": "'score'"
-    },
-    {
-      "benchmark": "squad_completion",
-      "error": "One of the splits is empty after splitting."
-    },
-    {
-      "benchmark": "super_glue",
-      "error": "'super_glue'"
-    },
-    {
-      "benchmark": "swde",
-      "error": "One of the splits is empty after splitting."
-    },
-    {
-      "benchmark": "tinyBenchmarks",
-      "error": "`tinyBenchmarks` is required for tinyBenchmarks task metric calculation, install via `pip install git+https://github.com/felipemaiapolo/tinyBenchmarks`"
-    },
-    {
-      "benchmark": "truthfulqa-multi",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "unitxt",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "unscramble",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "wmdp",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "wsc273",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "global_mmlu_ar",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "arabic_exams",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "persona",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "afrixnli_en_direct_amh",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "evalita_mp",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "truthfulqa",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "eus_exams_es",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "flores",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "afrimgsm_direct_amh",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "ceval_valid",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "advanced_ai_risk",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "tmlu",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "arc_ar",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "afrimmlu_direct_amh",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "m_mmlu",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "non_greedy_robustness_agieval_aqua_rat",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "prompt_robustness_agieval_aqua_rat",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "inverse_scaling_hindsight_neglect_10shot",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "mela",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "paws_ca",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "ja_leaderboard_jaqket_v2",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "super_glue-boolq-t5-prompt",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "multiple_choice",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "option_order_robustness_agieval_aqua_rat",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "phrases_ca-va",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "code2text_go",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "ethics_cm",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "cabreu",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "sycophancy",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "evalita_sp_sum_task_fp-small_p1",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "glianorex",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "flan_held_in",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "assin_entailment",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "gsm_plus",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "mnli",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "tinyTruthfulQA",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "multimedqa",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "openllm",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "pythia",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "t0_eval",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "Tag",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "basque-glue",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "chain_of_thought",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "freebase",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "gpt3_translation_benchmarks",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "iwslt2017",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "llama",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "self_consistency",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "super-glue-lm-eval-v1",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "super-glue-lm-eval-v1-seq2seq",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "super-glue-t5-prompt",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "wmt14",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "wmt14_en_fr",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "wmt14_fr_en",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "wmt16_de_en",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "wmt16_en_de",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "wmt16_en_ro",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "wmt16_ro_en",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "20_newsgroups",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "ag_news",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "anagrams1",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "anagrams2",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "argument_topic",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "atis",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "banking77",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "bec2016eu",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "bhtc_v2",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "boolq-seq2seq",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "catalanqa",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "catcola",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "claim_stance_topic",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "cnn_dailymail",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "cocoteros_es",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "coedit_gec",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "cola",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "coqcat",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "cycle_letters",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "dbpedia_14",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "doc_vqa",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "epec_koref_bin",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "escola",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "ethos_binary",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "financial_tweets",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "galcola",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "iwslt2017-ar-en",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "iwslt2017-en-ar",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "law_stack_exchange",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "ledgar",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "logieval",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "medical_abstracts",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "noticia",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "parafraseja",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "parafrases_gl",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "qnlieu",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "random_insertion",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "reversed_words",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "sglue_rte",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "stsb",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "summarization_gl",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "teca",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "tinyArc",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "tinyGSM8k",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "tinyHellaswag",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "tinyMMLU",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "tinyWinogrande",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "unfair_tos",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "vaxx_stance",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "wiceu",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "wmt-ro-en-t5-prompt",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "xlsum_es",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "xsum",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "yahoo_answers_topics",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "instructhumaneval",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    },
-    {
-      "benchmark": "humanevalpack",
-      "error": "NOT RUN - Test suite stopped before reaching this benchmark"
-    }
-  ],
-  "note": "Includes failures from tests 1-198 and all untested benchmarks (tests 200-318). Test 199 (tmmluplus) got stuck on network I/O."
-}