wisent 0.7.379__py3-none-any.whl → 0.7.901__py3-none-any.whl

wisent/examples/scripts/results/benchmark_descriptions.json

@@ -1,1244 +0,0 @@
-{
-  ".": {
-    "name": "Tasks",
-    "description": "A list of supported tasks and task groupings can be viewed with `lm-eval --tasks list`.",
-    "paper": "",
-    "homepage": ""
-  },
-  "aclue": {
-    "name": "ACLUE",
-    "description": "Can Large Language Model Comprehend Ancient Chinese? A Preliminary Test on ACLUE\nhttps://arxiv.org/abs/2310.09550",
-    "paper": "",
-    "homepage": "Homepage: https://github.com/isen-zhang/ACLUE"
-  },
-  "acpbench": {
-    "name": "ACPBench",
-    "description": "**",
-    "paper": "",
-    "homepage": "Homepage:** https://ibm.github.io/ACPBench/"
-  },
-  "aexams": {
-    "name": "Arabic EXAMS",
-    "description": "EXAMS: a resource specialized in multilingual high school exam questions.\nThe original paper [EXAMS](https://aclanthology.org/2020.emnlp-main.438/)",
-    "paper": "paper [EXAMS](https://aclanthology.org/2020.emnlp-main.438/",
-    "homepage": "Homepage for Arabic EXAMS: [EXAMS Arabic Homepage](https://github.com/FreedomIntelligence/AceGPT/tree/main/eval/benchmark_eval/benchmarks/EXAMS_Arabic"
-  },
-  "afrimgsm": {
-    "name": "MathQA",
-    "description": "IrokoBench: A New Benchmark for African Languages in the Age of Large Language Models\nhttps://arxiv.org/pdf/2406.03368",
-    "paper": "",
-    "homepage": ""
-  },
-  "afrimmlu": {
-    "name": "MathQA",
-    "description": "IrokoBench: A New Benchmark for African Languages in the Age of Large Language Models\nhttps://arxiv.org/pdf/2406.03368",
-    "paper": "",
-    "homepage": ""
-  },
-  "afrixnli": {
-    "name": "IrokoBench",
-    "description": "IrokoBench: A New Benchmark for African Languages in the Age of Large Language Models\nhttps://arxiv.org/pdf/2406.03368",
-    "paper": "",
-    "homepage": ""
-  },
-  "afrobench": {
-    "name": "AfroBench",
-    "description": "> Large-scale multilingual evaluations, such as MEGA, often include only a handful of African languages due to the scarcity of high-quality evaluation data and the limited discoverability of existing African datasets. This lack of representation hinders comprehensive LLM evaluation across a diverse range of languages and tasks. To address these challenges, we introduce AfroBench -- a multi-task benchmark for evaluating the performance of LLMs across 64 African languages, 15 tasks and 22 datasets",
-    "paper": "Paper Link: https://arxiv.org/abs/2311.07978",
-    "homepage": "HomePage: https://mcgill-nlp.github.io/AfroBench/"
-  },
-  "afrobench_adr": {
-    "name": "Automatic Diacritics Restoration (ADR)",
-    "description": "Automatic Diacritics Restoration (ADR) is the task of restoring diacritical marks in text where they have been omitted or removed.\nThis process is essential for languages where diacritics alter pronunciation, meaning, or grammatical structure.\nADR requires the model to have a deep understanding of linguistic context, syntax, and semantics to accurately predict and reinsert the appropriate diacritics.",
-    "paper": "",
-    "homepage": ""
-  },
-  "afrobench_afriqa": {
-    "name": "## Paper",
-    "description": ">AfriQA is the first cross-lingual question answering (QA) dataset with a focus on African languages. The dataset includes over 12,000 XOR QA examples across 10 African languages, making it an invaluable resource for developing more equitable QA technology. African languages have historically been underserved in the digital landscape, with far less in-language content available online. This makes it difficult for QA systems to provide accurate information to users in their native language. Howev",
-    "paper": "Paper Link: https://arxiv.org/abs/2305.06897",
-    "homepage": "HomePage: https://github.com/masakhane-io/afriqa"
-  },
-  "afrobench_afrisenti": {
-    "name": "## Paper",
-    "description": ">Africa is home to over 2,000 languages from over six language families and has the highest linguistic diversity among all continents. This includes 75 languages with at least one million speakers each. Yet, there is little NLP research conducted on African languages. Crucial in enabling such research is the availability of high-quality annotated datasets. In this paper, we introduce AfriSenti, a sentiment analysis benchmark that contains a total of >110,000 tweets in 14 African languages (Amhar",
-    "paper": "Paper Link: https://aclanthology.org/2023.emnlp-main.862/",
-    "homepage": "HomePage: https://github.com/afrisenti-semeval/afrisent-semeval-2023"
-  },
-  "afrobench_belebele": {
-    "name": "## Paper",
-    "description": ">Belebele is a multiple-choice machine reading comprehension (MRC) dataset spanning 122 language variants. This dataset enables the evaluation of mono- and multi-lingual models in high-, medium-, and low-resource languages. Each question has four multiple-choice answers and is linked to a short passage from the FLORES-200 dataset. The human annotation procedure was carefully curated to create questions that discriminate between different levels of generalizable language comprehension and is rein",
-    "paper": "Paper Link: https://aclanthology.org/2023.emnlp-main.862/",
-    "homepage": "HomePage: https://github.com/facebookresearch/belebele"
-  },
-  "afrobench_flores": {
-    "name": "## Paper",
-    "description": "## Paper\nTitle: `The FLORES-200 Evaluation Benchmark for Low-Resource and Multilingual Machine Translation`",
-    "paper": "Paper Link: https://arxiv.org/abs/2207.04672",
-    "homepage": "HomePage: https://huggingface.co/datasets/facebook/flores"
-  },
-  "afrobench_injongointent": {
-    "name": "## Paper",
-    "description": ">Slot-filling and intent detection are well-established tasks in Conversational AI. However, current large-scale benchmarks for these tasks often exclude evaluations of low-resource languages and rely on translations from English benchmarks, thereby predominantly reflecting Western-centric concepts. In this paper, we introduce Injongo -- a multicultural, open-source benchmark dataset for 16 African languages with utterances generated by native speakers across diverse domains, including banking, ",
-    "paper": "Paper Link: https://arxiv.org/abs/2502.09814",
-    "homepage": ""
-  },
-  "afrobench_mafand": {
-    "name": "## Paper",
-    "description": ">Recent advances in the pre-training of language models leverage large-scale datasets to create multilingual models. However, low-resource languages are mostly left out in these datasets. This is primarily because many widely spoken languages are not well represented on the web and therefore excluded from the large-scale crawls used to create datasets. Furthermore, downstream users of these models are restricted to the selection of languages originally chosen for pre-training. This work investig",
-    "paper": "Paper Link: https://aclanthology.org/2022.naacl-main.223/",
-    "homepage": "HomePage: https://github.com/masakhane-io/lafand-mt"
-  },
-  "afrobench_masakhaner": {
-    "name": "## Paper",
-    "description": ">African languages are spoken by over a billion people, but they are under-represented in NLP research and development. Multiple challenges exist, including the limited availability of annotated training and evaluation datasets as well as the lack of understanding of which settings, languages, and recently proposed methods like cross-lingual transfer will be effective. In this paper, we aim to move towards solutions for these challenges, focusing on the task of named entity recognition (NER). We",
-    "paper": "Paper Link: https://aclanthology.org/2022.emnlp-main.298/",
-    "homepage": "HomePage: https://github.com/masakhane-io/masakhane-ner"
-  },
-  "afrobench_masakhanews": {
-    "name": "## Paper",
-    "description": ">African languages are severely under-represented in NLP research due to lack of datasets covering several NLP tasks. While there are individual language specific datasets that are being expanded to different tasks, only a handful of NLP tasks (e.g. named entity recognition and machine translation) have standardized benchmark datasets covering several geographical and typologically-diverse African languages. In this paper, we develop MasakhaNEWS -- a new benchmark dataset for news topic classifi",
-    "paper": "Paper Link: https://aclanthology.org/2023.ijcnlp-main.10/",
-    "homepage": "HomePage: https://github.com/masakhane-io/masakhane-news"
-  },
-  "afrobench_masakhapos": {
-    "name": "## Paper",
-    "description": ">In this paper, we present AfricaPOS, the largest part-of-speech (POS) dataset for 20 typologically diverse African languages. We discuss the challenges in annotating POS for these languages using the universal dependencies (UD) guidelines. We conducted extensive POS baseline experiments using both conditional random field and several multilingual pre-trained language models. We applied various cross-lingual transfer models trained with data available in the UD. Evaluating on the AfricaPOS datas",
-    "paper": "Paper Link: https://aclanthology.org/2023.acl-long.609/",
-    "homepage": "HomePage: https://github.com/masakhane-io/masakhane-pos"
-  },
-  "afrobench_naijarc": {
-    "name": "## Paper",
-    "description": ">In this paper, we create NaijaRC: a new multi-choice Reading Comprehension dataset for three native Nigeria languages that is based on high-school reading comprehension examination. We provide baseline results by performing cross-lingual transfer using existing English RACE and Belebele training dataset based on a pre-trained encoder-only model. Additionally, we provide results by prompting large language models (LLMs) like GPT-4.",
-    "paper": "Paper Link: https://arxiv.org/abs/2308.09768",
-    "homepage": "HomePage: https://huggingface.co/datasets/aremuadeolajr/NaijaRC"
-  },
-  "afrobench_nollysenti": {
-    "name": "## Paper",
-    "description": ">Africa has over 2000 indigenous languages but they are under-represented in NLP research due to lack of datasets. In recent years, there have been progress in developing labelled corpora for African languages. However, they are often available in a single domain and may not generalize to other domains. In this paper, we focus on the task of sentiment classification for cross-domain adaptation. We create a new dataset, Nollywood movie reviews for five languages widely spoken in Nigeria (English,",
-    "paper": "Paper Link: https://aclanthology.org/2023.acl-short.85/",
-    "homepage": "HomePage: https://github.com/IyanuSh/NollySenti"
-  },
-  "afrobench_ntrex": {
-    "name": "## Paper",
-    "description": ">We release NTREX-128, a data set for machine translation (MT) evaluation from English into a total of 128 target languages. The paper describes the data creation process and proposes a quality filtering method based on human evaluation. We show experimental results which confirm that the directionality of test sets translation indeed plays an important role wrt. the usefulness of the corresponding metrics\u2019 scores. Thus, we recommend that the NTREX-128 data set should be used for evaluation of E",
-    "paper": "Paper Link: https://aclanthology.org/2022.sumeval-1.4/",
-    "homepage": "HomePage: https://github.com/MicrosoftTranslator/NTREX"
-  },
-  "afrobench_openai_mmlu": {
-    "name": "## Paper",
-    "description": ">We propose a new test to measure a text model's multitask accuracy. The test covers 57 tasks including elementary mathematics, US history, computer science, law, and more. To attain high accuracy on this test, models must possess extensive world knowledge and problem solving ability. We find that while most recent models have near random-chance accuracy, the very largest GPT-3 model improves over random chance by almost 20 percentage points on average. However, on every one of the 57 tasks, the",
-    "paper": "Paper Link: https://arxiv.org/abs/2009.03300",
-    "homepage": "HomePage: https://huggingface.co/datasets/openai/MMMLU"
-  },
-  "afrobench_salt": {
-    "name": "## Paper",
-    "description": ">SALT is a multi-way parallel text and speech corpus of Engish and six languages widely spoken in Uganda and East Africa: Luganda, Lugbara, Acholi, Runyankole, Ateso and Swahili. The core of the dataset is a set of 25,000 sentences covering a range of topics of local relevance, such as agriculture, health and society. Each sentence is translated into all languages, to support machine translation, and speech recordings are made for approximately 5,000 of the sentences both by a variety of speaker",
-    "paper": "Paper Link: https://aclanthology.org/2023.emnlp-main.862/",
-    "homepage": "HomePage: https://github.com/SunbirdAI/salt"
-  },
-  "afrobench_sib": {
-    "name": "## Paper",
-    "description": ">Despite the progress in building multilingual language models, evaluation is often limited to a few languages with available datasets which excludes a large number of low-resource languages. In this paper, we create SIB-200\u2014a large-scale open-sourced benchmark dataset for topic classification in 205 languages and dialects to address the lack of evaluation dataset for Natural Language Understanding (NLU). For many of the languages covered in SIB-200, this is the first publicly available evaluati",
-    "paper": "Paper Link: https://aclanthology.org/2024.eacl-long.14/",
-    "homepage": "HomePage: https://github.com/dadelani/sib-200"
-  },
-  "afrobench_uhura-arc-easy": {
-    "name": "## Paper",
-    "description": ">Evaluations of Large Language Models (LLMs) on knowledge-intensive tasks and factual accuracy often focus on high-resource languages primarily because datasets for low-resource languages (LRLs) are scarce. In this paper, we present Uhura -- a new benchmark that focuses on two tasks in six typologically-diverse African languages, created via human translation of existing English benchmarks. The first dataset, Uhura-ARC-Easy, is composed of multiple-choice science questions. The second, Uhura-Tru",
-    "paper": "Paper Link: https://arxiv.org/abs/2412.00948",
-    "homepage": "HomePage: https://huggingface.co/datasets/masakhane/uhura-arc-easy"
-  },
-  "afrobench_xlsum": {
-    "name": "## Paper",
-    "description": "ive Summarization for 44 Languages`",
-    "paper": "Paper Link: https://aclanthology.org/2021.findings-acl.413/",
-    "homepage": "HomePage: https://github.com/csebuetnlp/xl-sum"
-  },
-  "agieval": {
-    "name": "AGIEval",
-    "description": "https://arxiv.org/abs/2304.06364.pdf",
-    "paper": "Abstract: https://arxiv.org/abs/2304.06364.pdf",
-    "homepage": "Homepage: https://github.com/ruixiangcui/AGIEval"
-  },
-  "alghafa_copa_ar": {
-    "name": "alghafa_copa_ar",
-    "description": "The Choice Of Plausible Alternatives (COPA) evaluation provides researchers with a tool for assessing progress in open-domain commonsense causal reasoning.",
-    "paper": "",
-    "homepage": "Homepage](https://people.ict.usc.edu/~gordon/copa.html"
-  },
-  "alghafa_piqa_ar": {
-    "name": "alghafa_piqa_ar",
-    "description": "Original Title: `PIQA: Reasoning about Physical Commonsense in Natural Language`",
-    "paper": "paper: [PICA](https://arxiv.org/abs/1911.11641",
-    "homepage": "Homepage](https://yonatanbisk.com/piqa"
-  },
-  "anli": {
-    "name": "ANLI",
-    "description": "Title: `Adversarial NLI: A New Benchmark for Natural Language Understanding`",
-    "paper": "Paper Link: https://arxiv.org/abs/1910.14599",
-    "homepage": "Homepage: https://github.com/facebookresearch/anli"
-  },
-  "arab_culture": {
-    "name": "Arab Culture",
-    "description": "https://arxiv.org/abs/2502.12788",
-    "paper": "Abstract: https://arxiv.org/abs/2502.12788",
-    "homepage": "Homepage: https://github.com/fajri91/ArabicCulture"
-  },
-  "arab_culture_completion": {
-    "name": "Arab Culture",
-    "description": "https://arxiv.org/abs/2502.12788",
-    "paper": "Abstract: https://arxiv.org/abs/2502.12788",
-    "homepage": "Homepage: https://github.com/fajri91/ArabicCulture"
-  },
-  "arabic_leaderboard_complete": {
-    "name": "Arabic Leaderboard",
-    "description": "= \"Recent advances in the space of Arabic large language models have opened up a wealth of potential practical applications. From optimal training strategies, large scale data acquisition and continuously increasing NLP resources, the Arabic LLM landscape has improved in a very short span of time, despite being plagued by training data scarcity and limited evaluation resources compared to English. In line with contributing towards this ever-growing field, we introduce AlGhafa, a new multiple-cho",
-    "paper": "Paper: https://aclanthology.org/2023.arabicnlp-1.21.pdf",
-    "homepage": "Homepage: https://huggingface.co/spaces/OALL/Open-Arabic-LLM-Leaderboard"
-  },
-  "arabic_leaderboard_light": {
-    "name": "Arabic Leaderboard Light",
-    "description": "This leaderboard follows all the details as in [`arabic_leaderboard_complete`](../arabic_leaderboard_complete), except that a light version - 10% random sample of the test set of each benchmark - is used to test the language models.",
-    "paper": "",
-    "homepage": ""
-  },
-  "arabicmmlu": {
-    "name": "ArabicMMLU",
-    "description": "https://arxiv.org/abs/2402.12840",
-    "paper": "Abstract: https://arxiv.org/abs/2402.12840",
-    "homepage": "Homepage: https://github.com/mbzuai-nlp/ArabicMMLU"
-  },
-  "aradice": {
-    "name": "AraDiCE",
-    "description": "** Arabic, with its rich diversity of dialects, remains significantly underrepresented in Large Language Models, particularly in dialectal variations. We address this gap by introducing seven synthetic datasets in dialects alongside Modern Standard Arabic (MSA), created using Machine Translation (MT) combined with human post-editing. We present AraDiCE, a benchmark for Arabic Dialect and Cultural Evaluation. We evaluate LLMs on dialect comprehension and generation, focusing specifically on low-r",
-    "paper": "",
-    "homepage": ""
-  },
-  "arc": {
-    "name": "ARC",
-    "description": "https://arxiv.org/abs/1803.05457",
-    "paper": "Abstract: https://arxiv.org/abs/1803.05457",
-    "homepage": "Homepage: https://allenai.org/data/arc"
-  },
-  "arc_mt": {
-    "name": "arc mt",
-    "description": "arc mt is an implementation of tasks to support machine translated arc\nchallenge evals, to improve eval support across a number of additional\nlanguages.",
-    "paper": "",
-    "homepage": ""
-  },
-  "arithmetic": {
-    "name": "Arithmetic",
-    "description": "https://arxiv.org/abs/2005.14165",
-    "paper": "Abstract: https://arxiv.org/abs/2005.14165",
-    "homepage": "Homepage: https://github.com/openai/gpt-3/tree/master/data"
-  },
-  "asdiv": {
-    "name": "ASDiv",
-    "description": "https://arxiv.org/abs/2106.15772",
-    "paper": "Abstract: https://arxiv.org/abs/2106.15772",
-    "homepage": "Homepage: https://github.com/chaochun/nlu-asdiv-dataset"
-  },
-  "babi": {
-    "name": "bAbI",
-    "description": "https://arxiv.org/abs/1502.05698",
-    "paper": "Abstract: https://arxiv.org/abs/1502.05698",
-    "homepage": "Homepage: https://github.com/facebookarchive/bAbI-tasks"
-  },
-  "basque_bench": {
-    "name": "BasqueBench",
-    "description": "BasqueBench is a benchmark for evaluating language models in Basque tasks. This is, it evaluates the ability of a language model to understand and generate Basque text. BasqueBench offers a combination of pre-existing, open datasets and datasets developed exclusivelly for this benchmark. All the details of BasqueBench will be published in a paper soon.",
-    "paper": "",
-    "homepage": ""
-  },
-  "basqueglue": {
-    "name": "BasqueGLUE",
-    "description": "`https://aclanthology.org/2022.lrec-1.172/`",
-    "paper": "Abstract: `https://aclanthology.org/2022.lrec-1.172/`",
-    "homepage": "Homepage: `https://github.com/orai-nlp/BasqueGLUE`"
-  },
-  "bbh": {
-    "name": "BigBenchHard",
-    "description": "https://arxiv.org/abs/2210.09261",
-    "paper": "Abstract: https://arxiv.org/abs/2210.09261",
-    "homepage": "Homepage: https://github.com/suzgunmirac/BIG-Bench-Hard"
-  },
-  "bbq": {
-    "name": "BBQ",
-    "description": "https://aclanthology.org/2022.findings-acl.165/",
-    "paper": "Abstract: https://aclanthology.org/2022.findings-acl.165/",
-    "homepage": "Homepage: https://github.com/nyu-mll/BBQ"
-  },
-  "belebele": {
-    "name": "Belebele",
-    "description": "The Belebele Benchmark for Massively Multilingual NLU Evaluation\nhttps://arxiv.org/abs/2308.16884",
-    "paper": "",
-    "homepage": "Homepage: https://github.com/facebookresearch/belebele"
-  },
-  "benchmarks": {
-    "name": "benchmarks",
-    "description": "### Changelog\n- 2025-Mar-17 OpenLLM v2: Fixed few-shot split to correctly use train set for arc_challenge.",
-    "paper": "",
-    "homepage": ""
-  },
-  "benchmarks_multimedqa": {
-    "name": "MultiMedQA (multiple-choice subset)",
-    "description": "https://arxiv.org/abs/2212.13138",
-    "paper": "Abstract: https://arxiv.org/abs/2212.13138",
-    "homepage": ""
-  },
-  "bertaqa": {
-    "name": "BertaQA",
-    "description": "https://arxiv.org/abs/2406.07302",
-    "paper": "Abstract: https://arxiv.org/abs/2406.07302",
-    "homepage": "Homepage: https://github.com/juletx/BertaQA"
-  },
-  "bigbench": {
-    "name": "BigBench",
-    "description": "https://arxiv.org/abs/2206.04615",
-    "paper": "Abstract: https://arxiv.org/abs/2206.04615",
-    "homepage": "Homepage: https://github.com/google/BIG-bench"
-  },
-  "blimp": {
-    "name": "Task-name",
-    "description": "`https://arxiv.org/abs/1912.00582`",
-    "paper": "Abstract: `https://arxiv.org/abs/1912.00582`",
-    "homepage": "Homepage: https://github.com/alexwarstadt/blimp"
-  },
-  "c4": {
-    "name": "Colossal Clean Crawled Corpus(C4)",
-    "description": "[Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/abs/1910.10683)",
-    "paper": "",
-    "homepage": "Homepage](https://huggingface.co/datasets/allenai/c4"
-  },
-  "careqa": {
-    "name": "CareQA",
-    "description": "[https://arxiv.org/abs/2502.06666](https://arxiv.org/abs/2502.06666)",
-    "paper": "Abstract: [https://arxiv.org/abs/2502.06666](https://arxiv.org/abs/2502.06666",
-    "homepage": ""
-  },
-  "catalan_bench": {
-    "name": "CatalanBench",
-    "description": "ive and extreme). - `phrases_va`: Two Phrases_va tasks for language adaptation between Catalan and Valencian.",
-    "paper": "",
-    "homepage": ""
-  },
-  "ceval": {
-    "name": "C-Eval (Validation)",
-    "description": "### Paper\nC-Eval: A Multi-Level Multi-Discipline Chinese Evaluation Suite for Foundation Models\nhttps://arxiv.org/pdf/2305.08322.pdf",
-    "paper": "",
-    "homepage": "Homepage: https://cevalbenchmark.com/"
-  },
-  "chartqa": {
-    "name": "Task-name",
-    "description": "`In this work, we present a large-scale benchmark covering 9.6K human-written questions as well as 23.1K questions generated from human-written chart summaries.`",
-    "paper": "",
-    "homepage": "Homepage: `https://github.com/vis-nlp/ChartQA`"
-  },
-  "cmmlu": {
-    "name": "CMMLU",
-    "description": "CMMLU: Measuring massive multitask language understanding in Chinese\nhttps://arxiv.org/abs/2306.09212",
-    "paper": "",
-    "homepage": "Homepage: https://github.com/haonan-li/CMMLU"
-  },
-  "code_x_glue": {
-    "name": "CodeXGLUE Code-to-Text",
-    "description": "- **Dataset:** CodeXGLUE (code-to-text tasks)\n- **Source:** https://github.com/microsoft/CodeXGLUE",
-    "paper": "",
-    "homepage": ""
-  },
-  "commonsense_qa": {
-    "name": "Task-name",
-    "description": "https://arxiv.org/pdf/1811.00937.pdf",
-    "paper": "Abstract: https://arxiv.org/pdf/1811.00937.pdf",
-    "homepage": "Homepage: https://www.tau-nlp.org/commonsenseqa"
-  },
-  "copal_id": {
-    "name": "COPAL",
-    "description": "`https://arxiv.org/abs/2311.01012`",
-    "paper": "Abstract: `https://arxiv.org/abs/2311.01012`",
-    "homepage": "Homepage: `https://github.com/haryoa/copal-id`"
-  },
-  "coqa": {
-    "name": "CoQA",
-    "description": "https://arxiv.org/pdf/1808.07042.pdf",
-    "paper": "Abstract: https://arxiv.org/pdf/1808.07042.pdf",
-    "homepage": "Homepage: https://stanfordnlp.github.io/coqa/"
-  },
-  "crows_pairs": {
-    "name": "CrowS-Pairs",
-    "description": "= \"Pretrained language models, especially masked language models (MLMs) have seen success across many NLP tasks. However, there is ample evidence that they use the cultural biases that are undoubtedly present in the corpora they are trained on, implicitly creating harm with biased representations. To measure some forms of social bias in language models against protected demographic groups in the US, we introduce the Crowdsourced Stereotype Pairs benchmark (CrowS-Pairs). CrowS-Pairs has 1508 exam",
-    "paper": "",
-    "homepage": "Homepage: https://github.com/nyu-mll/crows-pairs,"
-  },
-  "csatqa": {
-    "name": "CSATQA",
-    "description": "- **Dataset:** HAERAE-HUB/csatqa\n- **Source:** https://huggingface.co/datasets/HAERAE-HUB/csatqa",
-    "paper": "",
-    "homepage": ""
-  },
-  "darija_bench": {
-    "name": "DarijaBench",
-    "description": "[https://arxiv.org/abs/2409.17912](https://arxiv.org/abs/2409.17912)",
-    "paper": "Abstract: [https://arxiv.org/abs/2409.17912](https://arxiv.org/abs/2409.17912",
-    "homepage": "Homepage: [https://huggingface.co/datasets/MBZUAI-Paris/DarijaBench](https://huggingface.co/datasets/MBZUAI-Paris/DarijaBench"
-  },
-  "darija_bench_darija_sentiment": {
-    "name": "DarijaBench: Sentiment Analysis",
-    "description": "[https://arxiv.org/abs/2409.17912](https://arxiv.org/abs/2409.17912)",
-    "paper": "Abstract: [https://arxiv.org/abs/2409.17912](https://arxiv.org/abs/2409.17912",
-    "homepage": "Homepage: [https://huggingface.co/datasets/MBZUAI-Paris/DarijaBench](https://huggingface.co/datasets/MBZUAI-Paris/DarijaBench"
-  },
-  "darija_bench_darija_summarization": {
-    "name": "DarijaBench: Summarization",
-    "description": "[https://arxiv.org/abs/2409.17912](https://arxiv.org/abs/2409.17912)",
-    "paper": "Abstract: [https://arxiv.org/abs/2409.17912](https://arxiv.org/abs/2409.17912",
-    "homepage": "Homepage: [https://huggingface.co/datasets/MBZUAI-Paris/DarijaBench](https://huggingface.co/datasets/MBZUAI-Paris/DarijaBench"
-  },
-  "darija_bench_darija_translation": {
-    "name": "DarijaBench: Translation",
-    "description": "[https://arxiv.org/abs/2409.17912](https://arxiv.org/abs/2409.17912)",
-    "paper": "Abstract: [https://arxiv.org/abs/2409.17912](https://arxiv.org/abs/2409.17912",
-    "homepage": "Homepage: [https://huggingface.co/datasets/MBZUAI-Paris/DarijaBench](https://huggingface.co/datasets/MBZUAI-Paris/DarijaBench"
-  },
-  "darija_bench_darija_transliteration": {
-    "name": "DarijaBench: Transliteration",
-    "description": "[https://arxiv.org/abs/2409.17912](https://arxiv.org/abs/2409.17912)",
-    "paper": "Abstract: [https://arxiv.org/abs/2409.17912](https://arxiv.org/abs/2409.17912",
-    "homepage": "Homepage: [https://huggingface.co/datasets/MBZUAI-Paris/DarijaBench](https://huggingface.co/datasets/MBZUAI-Paris/DarijaBench"
-  },
-  "darijahellaswag": {
-    "name": "DarijaHellaSwag",
-    "description": "[https://arxiv.org/abs/2409.17912](https://arxiv.org/abs/2409.17912)",
-    "paper": "Abstract: [https://arxiv.org/abs/2409.17912](https://arxiv.org/abs/2409.17912",
-    "homepage": "Homepage: [https://huggingface.co/datasets/MBZUAI-Paris/DarijaHellaSwag](https://huggingface.co/datasets/MBZUAI-Paris/DarijaHellaSwag"
-  },
-  "darijammlu": {
-    "name": "DarijaMMLU",
-    "description": "[https://arxiv.org/abs/2409.17912](https://arxiv.org/abs/2409.17912)",
-    "paper": "Abstract: [https://arxiv.org/abs/2409.17912](https://arxiv.org/abs/2409.17912",
-    "homepage": "Homepage: [https://huggingface.co/datasets/MBZUAI-Paris/DarijaMMLU](https://huggingface.co/datasets/MBZUAI-Paris/DarijaMMLU"
-  },
-  "drop": {
-    "name": "DROP",
-    "description": "https://aclanthology.org/attachments/N19-1246.Supplementary.pdf",
-    "paper": "Abstract: https://aclanthology.org/attachments/N19-1246.Supplementary.pdf",
-    "homepage": "Homepage: https://allenai.org/data/drop"
-  },
-  "egyhellaswag": {
-    "name": "EgyHellaSwag",
-    "description": "[https://arxiv.org/abs/2505.18383](https://arxiv.org/abs/2505.18383)",
-    "paper": "Abstract: [https://arxiv.org/abs/2505.18383](https://arxiv.org/abs/2505.18383",
-    "homepage": "Homepage: [https://huggingface.co/datasets/UBC-NLP/EgyHellaSwag](https://huggingface.co/datasets/UBC-NLP/EgyHellaSwag"
-  },
-  "egymmlu": {
-    "name": "EgyMMLU",
-    "description": "[https://arxiv.org/abs/2505.18383](https://arxiv.org/abs/2505.18383)",
-    "paper": "Abstract: [https://arxiv.org/abs/2505.18383](https://arxiv.org/abs/2505.18383",
-    "homepage": "Homepage: [https://huggingface.co/datasets/UBC-NLP/EgyMMLU](https://huggingface.co/datasets/UBC-NLP/EgyMMLU"
-  },
-  "eq_bench": {
-    "name": "EQ-Bench",
-    "description": "https://arxiv.org/abs/2312.06281",
-    "paper": "Abstract: https://arxiv.org/abs/2312.06281",
-    "homepage": "Homepage: https://eqbench.com/"
-  },
-  "eus_exams": {
-    "name": "EusExams",
-    "description": "https://arxiv.org/abs/2403.20266",
-    "paper": "Abstract: https://arxiv.org/abs/2403.20266",
-    "homepage": "Homepage: https://github.com/hitz-zentroa/latxa"
-  },
-  "eus_proficiency": {
-    "name": "EusProficiency",
-    "description": "https://arxiv.org/abs/2403.20266",
-    "paper": "Abstract: https://arxiv.org/abs/2403.20266",
-    "homepage": "Homepage: https://github.com/hitz-zentroa/latxa"
-  },
-  "eus_reading": {
-    "name": "EusReading",
-    "description": "https://arxiv.org/abs/2403.20266",
-    "paper": "Abstract: https://arxiv.org/abs/2403.20266",
-    "homepage": "Homepage: https://github.com/hitz-zentroa/latxa"
-  },
-  "eus_trivia": {
-    "name": "EusTrivia",
-    "description": "https://arxiv.org/abs/2403.20266",
-    "paper": "Abstract: https://arxiv.org/abs/2403.20266",
-    "homepage": "Homepage: https://github.com/hitz-zentroa/latxa"
-  },
-  "evalita_llm": {
-    "name": "Evalita-LLM",
-    "description": "Evalita-LLM, a new benchmark designed to evaluate Large Language\nModels (LLMs) on Italian tasks. The distinguishing and innovative features of\nEvalita-LLM are the following: (i) all tasks are native Italian, avoiding issues of\ntranslating from Italian and potential cultural biases; (ii) in addition to well established multiple-choice tasks, the benchmark includes generative tasks, enabling more natural interaction with LLMs; (iii) all tasks are evaluated against multiple prompts, this way mitiga",
-    "paper": "",
-    "homepage": ""
-  },
-  "fda": {
-    "name": "FDA",
-    "description": "A long standing goal of the data management community is to develop general, automated systems that ingest semi-structured documents and output queryable tables without human effort or domain specific customization. Given the sheer variety of potential documents, state-of-the art systems make simplifying assumptions and use domain specific training. In this work, we ask whether we can maintain generality by using large language models (LLMs). LLMs, which are pretrained on broad data, can perform",
-    "paper": "",
-    "homepage": "Homepage: https://github.com/HazyResearch/based-evaluation-harness"
-  },
-  "fld": {
-    "name": "FLD",
-    "description": "https://arxiv.org/abs/2308.07336",
-    "paper": "Abstract: https://arxiv.org/abs/2308.07336",
-    "homepage": "Homepage: https://github.com/hitachi-nlp/FLD"
-  },
-  "french_bench": {
-    "name": "FrenchBench",
-    "description": "- french_bench_orangesum_title - french_bench_trivia - french_bench_hellaswag - french_bench_arc_challenge",
-    "paper": "",
-    "homepage": ""
-  },
-  "galician_bench": {
-    "name": "GalicianBench",
-    "description": "GalicianBench is a benchmark for evaluating language models in Galician tasks. This is, it evaluates the ability of a language model to understand and generate Galician text. GalicianBench offers a combination of pre-existing, open datasets and datasets developed exclusivelly for this benchmark. All the details of GalicianBench will be published in a paper soon.",
-    "paper": "",
-    "homepage": ""
-  },
-  "glianorex": {
-    "name": "Glianorex",
-    "description": "https://arxiv.org/abs/2406.02394",
-    "paper": "Abstract: https://arxiv.org/abs/2406.02394",
-    "homepage": ""
-  },
-  "global_mmlu": {
-    "name": "Global-MMLU",
-    "description": "[https://arxiv.org/abs/2412.03304](https://arxiv.org/abs/2412.03304)",
-    "paper": "Abstract: [https://arxiv.org/abs/2412.03304](https://arxiv.org/abs/2412.03304",
-    "homepage": ""
-  },
-  "glue": {
-    "name": "GLUE",
-    "description": "https://openreview.net/pdf?id=rJ4km2R5t7",
-    "paper": "Abstract: https://openreview.net/pdf?id=rJ4km2R5t7",
-    "homepage": "Homepage: https://gluebenchmark.com/"
-  },
-  "gpqa": {
-    "name": "GPQA",
-    "description": "https://arxiv.org/abs/2311.12022",
-    "paper": "Abstract: https://arxiv.org/abs/2311.12022",
-    "homepage": "Homepage: `https://github.com/idavidrein/gpqa/tree/main`"
-  },
-  "groundcocoa": {
-    "name": "GroundCocoa",
-    "description": "https://arxiv.org/abs/2404.04237",
-    "paper": "Abstract: https://arxiv.org/abs/2404.04237",
-    "homepage": "Homepage: `https://osu-nlp-group.github.io/GroundCocoa/`"
-  },
-  "gsm8k": {
-    "name": "GSM8k",
-    "description": "## Paper\nTraining Verifiers to Solve Math Word Problems\nhttps://arxiv.org/abs/2110.14168",
-    "paper": "",
-    "homepage": "Homepage: https://github.com/openai/grade-school-math"
-  },
-  "gsm8k_platinum": {
-    "name": "GSM8k Platinum",
-    "description": "GSM8K Platinum is a revised version of the full test set of GSM8K (Grade School Math 8K), a dataset of grade school math word problems. To revise this dataset, we ran a variety of frontier models each individual example and manually re-annotated any example for which at least one model made an error. We revise the labels of mislabeled examples, and remove any question that we determine to be poorly written (most often due to ambiguity in the problem statement). See our paper for further details ",
-    "paper": "",
-    "homepage": "Homepage: http://platinum-bench.csail.mit.edu/"
-  },
-  "gsm_plus": {
-    "name": "gsm_plus",
-    "description": "`Large language models (LLMs) have achieved impressive performance across various mathematical reasoning benchmarks. However, there are increasing debates regarding whether these models truly understand and apply mathematical knowledge or merely rely on shortcuts for mathematical reasoning. One essential and frequently occurring evidence is that when the math questions are slightly changed, LLMs can behave incorrectly. This motivates us to evaluate the robustness of LLMs\u2019 math reasoning capabili",
-    "paper": "",
-    "homepage": "Homepage: https://huggingface.co/datasets/qintongli/GSM-Plus"
-  },
-  "haerae": {
-    "name": "HAE-RAE BENCH",
-    "description": "`Large Language Models (LLMs) trained on massive corpora demonstrate impressive capabilities in a wide range of tasks. While there are ongoing efforts to adapt these models to languages beyond English, the attention given to their evaluation methodologies remains limited. Current multilingual benchmarks often rely on back translations or re-implementations of English tests, limiting their capacity to capture unique cultural and linguistic nuances. To bridge this gap for the Korean language, we i",
-    "paper": "",
-    "homepage": "Homepage: https://huggingface.co/datasets/HAERAE-HUB/HAE_RAE_BENCH"
-  },
-  "headqa": {
-    "name": "HEAD-QA",
-    "description": "= \"We present HEAD-QA, a multi-choice question answering testbed to encourage research on complex reasoning. The questions come from exams to access a specialized position in the Spanish healthcare system, and are challenging even for highly specialized humans. We then consider monolingual (Spanish) and cross-lingual (to English) experiments with information retrieval and neural techniques. We show that: (i) HEAD-QA challenges current methods, and (ii) the results lag well behind human performan",
-    "paper": "",
-    "homepage": "Homepage: https://aghie.github.io/head-qa/"
-  },
-  "hellaswag": {
-    "name": "HellaSwag",
-    "description": "https://arxiv.org/abs/1905.07830",
-    "paper": "Abstract: https://arxiv.org/abs/1905.07830",
-    "homepage": "Homepage: `https://rowanzellers.com/hellaswag/`"
-  },
-  "hendrycks_ethics": {
-    "name": "ETHICS Dataset",
-    "description": "Pointer Sentinel Mixture Models\nhttps://arxiv.org/pdf/1609.07843.pdf",
-    "paper": "",
-    "homepage": "Homepage: https://github.com/hendrycks/ethics"
-  },
-  "hendrycks_math": {
-    "name": "MATH",
-    "description": "## Paper\nMeasuring Mathematical Problem Solving With the MATH Dataset\nhttps://arxiv.org/abs/2103.03874",
-    "paper": "paper (https://arxiv.org/abs/2206.14858",
-    "homepage": "Homepage: https://github.com/hendrycks/math"
-  },
-  "histoires_morales": {
-    "name": "Histoires Morales",
-    "description": "`https://arxiv.org/pdf/2501.17117`",
-    "paper": "Abstract: `https://arxiv.org/pdf/2501.17117`",
-    "homepage": "Homepage: `https://huggingface.co/datasets/LabHC/histoires_morales`"
-  },
-  "hrm8k": {
-    "name": "HRM8K",
-    "description": "Title: [Understand, Solve and Translate: Bridging the Multilingual Mathematical Reasoning Gap](https://www.arxiv.org/abs/2501.02448)",
-    "paper": "",
-    "homepage": "Homepage: https://huggingface.co/datasets/HAERAE-HUB/HRM8K"
-  },
-  "humaneval": {
-    "name": "HumanEval",
-    "description": "## Paper\nEvaluating Large Language Models Trained on Code\nhttps://arxiv.org/abs/2107.03374",
-    "paper": "",
-    "homepage": "Homepage: https://github.com/openai/human-eval"
-  },
-  "ifeval": {
-    "name": "IFEval",
-    "description": "https://arxiv.org/abs/2311.07911",
-    "paper": "Abstract: https://arxiv.org/abs/2311.07911",
-    "homepage": "Homepage: https://github.com/google-research/google-research/tree/master/instruction_following_eval"
-  },
-  "include": {
-    "name": "INCLUDE",
-    "description": "[https://arxiv.org/abs/2411.19799](https://arxiv.org/abs/2411.19799)",
-    "paper": "Abstract: [https://arxiv.org/abs/2411.19799](https://arxiv.org/abs/2411.19799",
-    "homepage": ""
-  },
-  "inverse_scaling": {
-    "name": "inverse_scaling",
-    "description": "`Work on scaling laws has found that large language models (LMs) show predictable improvements to overall loss with increased scale (model size, training data, and compute). Here, we present evidence for the claim that LMs may show inverse scaling, or worse task performance with increased scale, e.g., due to flaws in the training objective and data. We present empirical evidence of inverse scaling on 11 datasets collected by running a public contest, the Inverse Scaling Prize, with a substantial",
-    "paper": "paper (but have been tested for equivalence to the OPT numbers reported at https://huggingface.co/inverse-scaling/opt-1.3b_eval",
-    "homepage": "Homepage: https://github.com/inverse-scaling/prize"
-  },
-  "japanese_leaderboard": {
-    "name": "Japanese Leaderboard",
-    "description": "ive Summarization for 44 Languages}, author = { Hasan, Tahmid and Bhattacharjee, Abhik and Islam, Md. Saiful and Mubasshir, Kazi and Li, Yuan-Fang and Kang, Yong-Bin and Rahman, M. Sohel and Shahriyar, Rifat }, year = 2021, month = aug, booktitle = {Findings of the Association for Computational Linguistics: ACL-IJCNLP 2021}, publisher = {Association for Computational Linguistics}, address = {Online}, pages = {4693--4703}, url = {https://aclanthology.org/2021.findings-acl.413} }",
-    "paper": "paper Language models are multilingual chain-of-thought reasoners. [Source](https://huggingface.co/datasets/juletxara/mgsm",
-    "homepage": ""
-  },
-  "jsonschema_bench": {
-    "name": "JSONSchema Bench",
-    "description": "- `jsonschema_bench_easy`, corresponding to the `github_easy` split of the original paper\n- `jsonschema_bench_medium`, corresponding to the `github_medium` split of the original paper\n- `jsonschema_bench_hard`, corresponding to the `github_hard` split of the original paper",
-    "paper": "",
-    "homepage": "Homepage: https://github.com/guidance-ai/jsonschemabench"
-  },
-  "kbl": {
-    "name": "kbl",
-    "description": "`Large language models (LLMs) have demonstrated remarkable performance in the legal domain, with GPT-4 even passing the Uniform Bar Exam in the U.S. However their efficacy remains limited for non-standardized tasks and tasks in languages other than English. This underscores the need for careful evaluation of LLMs within each legal system before application. Here, we introduce KBL, a benchmark for assessing the Korean legal language understanding of LLMs, consisting of (1) 7 legal knowledge tasks",
-    "paper": "",
-    "homepage": "Homepage: `https://github.com/lbox-kr/kbl`"
-  },
-  "kmmlu": {
-    "name": "k_mmlu",
-    "description": "`We propose KMMLU, a new Korean benchmark with 35,030 expert-level multiple-choice questions across 45 subjects ranging from humanities to STEM. Unlike previous Korean benchmarks that are translated from existing English benchmarks, KMMLU is collected from original Korean exams, capturing linguistic and cultural aspects of the Korean language. We test 26 publicly available and proprietary LLMs, identifying significant room for improvement. The best publicly available model achieves 50.54% on KMM",
-    "paper": "",
-    "homepage": "Homepage: https://huggingface.co/datasets/HAERAE-HUB/KMMLU"
-  },
-  "kobest": {
-    "name": "LAMBADA",
-    "description": "https://arxiv.org/abs/2204.04541",
-    "paper": "Abstract: https://arxiv.org/abs/2204.04541",
-    "homepage": "Homepage: https://huggingface.co/datasets/skt/kobest_v1"
-  },
-  "kormedmcqa": {
-    "name": "KorMedMCQA",
-    "description": "`We introduce KorMedMCQA, the first Korean multiple-choice question answering (MCQA) benchmark derived from Korean healthcare professional licensing examinations, covering from the year 2012 to year 2023. This dataset consists of a selection of questions from the license examinations for doctors, nurses, and pharmacists, featuring a diverse array of subjects. We conduct baseline experiments on various large language models, including proprietary/open-source, multilingual/Korean-additional pretra",
-    "paper": "Paper : https://arxiv.org/abs/2403.01469",
-    "homepage": "Homepage: https://huggingface.co/datasets/sean0042/KorMedMCQA"
-  },
-  "lambada": {
-    "name": "LAMBADA",
-    "description": "https://arxiv.org/pdf/1606.06031.pdf",
-    "paper": "Abstract: https://arxiv.org/pdf/1606.06031.pdf",
-    "homepage": "Homepage: https://zenodo.org/record/2630551#.X4Xzn5NKjUI"
-  },
-  "lambada_cloze": {
-    "name": "LAMBADA Cloze",
-    "description": "https://arxiv.org/abs/1606.06031",
-    "paper": "Abstract: https://arxiv.org/abs/1606.06031",
-    "homepage": "Homepage: https://zenodo.org/record/2630551#.X4Xzn5NKjUI"
-  },
-  "lambada_multilingual": {
-    "name": "LAMBADA",
-    "description": "### Paper\nThe LAMBADA dataset: Word prediction requiring a broad discourse context\nhttps://arxiv.org/pdf/1606.06031.pdf",
-    "paper": "",
-    "homepage": "Homepage: https://zenodo.org/record/2630551#.X4Xzn5NKjUI"
-  },
-  "lambada_multilingual_stablelm": {
-    "name": "LAMBADA",
-    "description": "### Paper\nThe LAMBADA dataset: Word prediction requiring a broad discourse context\nhttps://arxiv.org/pdf/1606.06031.pdf",
-    "paper": "",
-    "homepage": "Homepage: https://zenodo.org/record/2630551#.X4Xzn5NKjUI"
-  },
-  "leaderboard": {
-    "name": "Leaderboard evaluations",
-    "description": "# Leaderboard evaluations\nOur goal with this group is to create an unchanging through time version of\nevaluations that will power the Open LLM Leaderboard on HuggingFace.",
-    "paper": "paper: https://huggingface.co/papers/2210.09261",
-    "homepage": "Homepage: https://github.com/suzgunmirac/BIG-Bench-Hard"
-  },
-  "libra": {
-    "name": "Task-name",
-    "description": "`Datasets for proper evaluation of long-context understanding in Russian. For the Russian language LIBRA comprises 21 adapted datasets to study the LLM's abilities to understand long texts thoroughly. The tests are divided into four complexity groups and allow the evaluation of models across various context lengths ranging from 4k up to 128k tokens.`",
-    "paper": "",
-    "homepage": "Homepage: `https://huggingface.co/datasets/ai-forever/LIBRA`"
-  },
-  "lingoly": {
-    "name": "LingOly",
-    "description": "`https://arxiv.org/abs/2406.06196`",
-    "paper": "Abstract: `https://arxiv.org/abs/2406.06196`",
-    "homepage": "Homepage: `https://github.com/am-bean/lingOly`"
-  },
-  "llama3": {
-    "name": "Task-name",
-    "description": "Evals reproducing those provided by the LLAMA team in the Hugging Face repo.",
-    "paper": "",
-    "homepage": "Homepage: `https://huggingface.co/collections/meta-llama/llama-31-evals-66a2c5a14c2093e58298ac7f`"
-  },
-  "logiqa": {
-    "name": "LogiQA",
-    "description": "https://arxiv.org/abs/2007.08124",
-    "paper": "Abstract: https://arxiv.org/abs/2007.08124",
-    "homepage": "Homepage: https://github.com/lgw863/LogiQA-dataset"
-  },
-  "logiqa2": {
-    "name": "LogiQA 2.0",
-    "description": "LogiQA 2.0 \u2014 An Improved Dataset for Logical Reasoning in Natural Language Understanding https://ieeexplore.ieee.org/document/10174688",
-    "paper": "paper does not. There is another implementation of this task, but it designed for instruction tuned models: https://github.com/csitfun/LogiEval",
-    "homepage": "Homepage: https://github.com/csitfun/LogiQA2.0"
-  },
-  "longbench": {
-    "name": "LongBench",
-    "description": "`In this paper, we introduce LongBench, the first bilingual, multi-task benchmark for long context understanding, enabling a more rigorous evaluation of long context understanding. LongBench comprises 21 datasets across 6 task categories in both English and Chinese, with an average length of 6,711 words (English) and 13,386 characters (Chinese). These tasks cover key long-text application areas including single-doc QA, multi-doc QA, summarization, few-shot learning, synthetic tasks, and code com",
-    "paper": "",
-    "homepage": "Homepage: `https://github.com/THUDM/LongBench`"
-  },
-  "mastermind": {
-    "name": "MastermindEval",
-    "description": "https://arxiv.org/abs/2503.05891",
-    "paper": "Abstract: https://arxiv.org/abs/2503.05891",
-    "homepage": ""
-  },
-  "mathqa": {
-    "name": "MathQA",
-    "description": "MathQA: Towards Interpretable Math Word Problem Solving with Operation-Based Formalisms\nhttps://arxiv.org/pdf/1905.13319.pdf",
-    "paper": "",
-    "homepage": "Homepage: https://math-qa.github.io/math-QA/"
-  },
-  "mbpp": {
-    "name": "MBPP",
-    "description": "## Paper\nProgram Synthesis with Large Language Models\nhttps://arxiv.org/abs/2108.07732",
-    "paper": "",
-    "homepage": "Homepage: https://github.com/google-research/google-research/tree/master/mbpp"
-  },
-  "mc_taco": {
-    "name": "MC Taco",
-    "description": "https://arxiv.org/abs/1909.03065",
-    "paper": "Abstract: https://arxiv.org/abs/1909.03065",
-    "homepage": "Homepage: https://leaderboard.allenai.org/mctaco/submissions/public"
-  },
-  "med_concepts_qa": {
-    "name": "MedConceptsQA",
-    "description": "https://arxiv.org/abs/2405.07348",
-    "paper": "Abstract: https://arxiv.org/abs/2405.07348",
-    "homepage": ""
-  },
-  "meddialog": {
-    "name": "Meddialog",
-    "description": "[https://aclanthology.org/2020.emnlp-main.743/](https://aclanthology.org/2020.emnlp-main.743/)",
-    "paper": "Abstract: [https://aclanthology.org/2020.emnlp-main.743/](https://aclanthology.org/2020.emnlp-main.743/",
-    "homepage": ""
-  },
-  "mediqa_qa2019": {
-    "name": "MEDIQA_QA 2019",
-    "description": "[https://aclanthology.org/W19-5039/](https://aclanthology.org/W19-5039/)",
-    "paper": "Abstract: [https://aclanthology.org/W19-5039/](https://aclanthology.org/W19-5039/",
-    "homepage": ""
-  },
-  "medmcqa": {
-    "name": "MedMCQA",
-    "description": "- **Dataset:** medmcqa\n- **Source:** https://huggingface.co/datasets/medmcqa",
-    "paper": "",
-    "homepage": ""
-  },
-  "medqa": {
-    "name": "MedQA",
-    "description": "- **Dataset:** GBaker/MedQA-USMLE-4-options-hf\n- **Source:** https://huggingface.co/datasets/GBaker/MedQA-USMLE-4-options-hf",
-    "paper": "",
-    "homepage": ""
-  },
-  "medtext": {
-    "name": "MedText",
-    "description": "[https://arxiv.org/abs/1905.07002](https://arxiv.org/abs/1905.07002)",
-    "paper": "Abstract: [https://arxiv.org/abs/1905.07002](https://arxiv.org/abs/1905.07002",
-    "homepage": ""
-  },
-  "mela": {
-    "name": "Task-name",
-    "description": "**: In this work, we present the largest benchmark to date on linguistic acceptability: Multilingual Evaluation of Linguistic Acceptability -- MELA, with 46K samples covering 10 languages from a diverse set of language families. We establish LLM baselines on this benchmark, and investigate cross-lingual transfer in acceptability judgements with XLM-R. In pursuit of multilingual interpretability, we conduct probing experiments with fine-tuned XLM-R to explore the process of syntax capability acqu",
-    "paper": "",
-    "homepage": "Homepage: https://github.com/sjtu-compling/MELA"
-  },
-  "meqsum": {
-    "name": "MeqSum",
-    "description": "[https://aclanthology.org/P19-1215/](https://aclanthology.org/P19-1215/)",
-    "paper": "Abstract: [https://aclanthology.org/P19-1215/](https://aclanthology.org/P19-1215/",
-    "homepage": ""
-  },
-  "metabench": {
-    "name": "Metabench",
-    "description": "https://arxiv.org/abs/2407.12844",
-    "paper": "Abstract: https://arxiv.org/abs/2407.12844",
-    "homepage": "Homepage: https://github.com/adkipnis/metabench"
-  },
-  "mgsm": {
-    "name": "MGSM",
-    "description": "https://arxiv.org/abs/2210.03057",
-    "paper": "Abstract: https://arxiv.org/abs/2210.03057",
-    "homepage": "Homepage: https://github.com/google-research/url-nlp/tree/main/mgsm"
-  },
-  "mimic_repsum": {
-    "name": "MIMIC-III Report Summarization",
-    "description": "[https://www.nature.com/articles/sdata201635](https://www.nature.com/articles/sdata201635)",
-    "paper": "Abstract: [https://www.nature.com/articles/sdata201635](https://www.nature.com/articles/sdata201635",
-    "homepage": ""
-  },
-  "minerva_math": {
-    "name": "MATH",
-    "description": "# MATH\n\u2139\ufe0f This is the 4-shot variant!\n## Paper\nMeasuring Mathematical Problem Solving With the MATH Dataset\nhttps://arxiv.org/abs/2103.03874",
-    "paper": "paper is one where the model is first fine-tuned on the data. They do have a few-shot evaluation for GPT-3, however the few-shot context used here is sourced from [Lewkowycz et al](https://arxiv.org/abs/2206.14858",
-    "homepage": "Homepage: https://github.com/hendrycks/math"
-  },
-  "mlqa": {
-    "name": "MLQA",
-    "description": "`https://arxiv.org/abs/1910.07475`",
-    "paper": "Abstract: `https://arxiv.org/abs/1910.07475`",
-    "homepage": "Homepage: `https://github.com/facebookresearch/MLQA`"
-  },
-  "mmlu-pro-plus": {
-    "name": "mmlu_pro_plus",
-    "description": "`Existing benchmarks for large language models (LLMs) increasingly struggle to differentiate between top-performing models, underscoring the need for more challenging evaluation frameworks. We introduce MMLU-Pro+, an enhanced benchmark building upon MMLU-Pro to assess shortcut learning and higher-order reasoning in LLMs. By incorporating questions with multiple correct answers across diverse domains, MMLU-Pro+ tests LLMs' ability to engage in complex reasoning and resist simplistic problem-solvi",
-    "paper": "",
-    "homepage": "Homepage: https://github.com/asgsaeid/mmlu-pro-plus"
-  },
-  "mmlu": {
-    "name": "Task-name",
-    "description": "`https://arxiv.org/abs/2009.03300`",
-    "paper": "Abstract: `https://arxiv.org/abs/2009.03300`",
-    "homepage": "Homepage: `https://github.com/hendrycks/test`"
-  },
-  "mmlu_pro": {
-    "name": "mmlu_pro",
-    "description": "`In the age of large-scale language models, benchmarks like the Massive Multitask Language Understanding (MMLU) have been pivotal in pushing the boundaries of what AI can achieve in language comprehension and reasoning across diverse domains. However, as models continue to improve, their performance on these benchmarks has begun to plateau, making it increasingly difficult to discern differences in model capabilities. This paper introduces MMLU-Pro, an enhanced dataset designed to extend the mos",
-    "paper": "",
-    "homepage": "Homepage: https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro"
-  },
-  "mmlu_prox": {
-    "name": "MMLU-ProX",
-    "description": "`Traditional benchmarks like MMLU and MMLU-Pro focus primarily on single-language evaluation, limiting their ability to assess language models in multilingual and culturally diverse contexts. To address this gap, we introduce MMLU-ProX, a comprehensive multilingual benchmark that builds upon MMLU-Pro by covering multiple typologically diverse languages with approximately 11,829 questions per language.`",
-    "paper": "",
-    "homepage": "Homepage: https://mmluprox.github.io/"
-  },
-  "mmlusr": {
-    "name": "MMLU-SR",
-    "description": "_algebra`",
-    "paper": "paper is one where the model is first fine-tuned on the data. They do have a few-shot evaluation for GPT-3, however the few-shot context used here is sourced from [Lewkowycz et al](https://arxiv.org/abs/2206.14858",
-    "homepage": "Homepage: [https://github.com/Wang-ML-Lab/MMLU-SR](https://github.com/Wang-ML-Lab/MMLU-SR"
-  },
-  "mmmu": {
-    "name": "MMMU Benchmark",
-    "description": "`MMMU is a new benchmark designed to evaluate multimodal models on massive multi-discipline tasks demanding college-level subject knowledge and deliberate reasoning.`",
-    "paper": "",
-    "homepage": "Homepage: `https://github.com/MMMU-Benchmark/MMMU/tree/main/mmmu`"
-  },
-  "model_written_evals": {
-    "name": "Model Written Evals",
-    "description": "Model Written Evals are evaluation tasks auto-generated for evaluating a collection of AI Safety concerns. These tasks test language models on various safety-related dimensions including AI risk awareness, persona biases, and sycophancy.",
-    "paper": "",
-    "homepage": ""
-  },
-  "moral_stories": {
-    "name": "Moral Stories",
-    "description": "`https://aclanthology.org/2021.emnlp-main.54/`",
-    "paper": "Abstract: `https://aclanthology.org/2021.emnlp-main.54/`",
-    "homepage": "Homepage: `https://github.com/demelin/moral_stories`"
-  },
-  "mts_dialog": {
-    "name": "MTS-Dialog",
-    "description": "[https://aclanthology.org/2023.eacl-main.168/](https://aclanthology.org/2023.eacl-main.168/)",
-    "paper": "Abstract: [https://aclanthology.org/2023.eacl-main.168/](https://aclanthology.org/2023.eacl-main.168/",
-    "homepage": ""
-  },
-  "multiblimp": {
-    "name": "MultiBLiMP: A Massively Multilingual Benchmark of Linguistic Minimal Pairs",
-    "description": "# MultiBLiMP: A Massively Multilingual Benchmark of Linguistic Minimal Pairs",
-    "paper": "Paper: https://arxiv.org/abs/2504.02768",
-    "homepage": ""
-  },
-  "mutual": {
-    "name": "MuTual",
-    "description": "https://www.aclweb.org/anthology/2020.acl-main.130/",
-    "paper": "Abstract: https://www.aclweb.org/anthology/2020.acl-main.130/",
-    "homepage": "Homepage: https://github.com/Nealcly/MuTual"
-  },
-  "noreval": {
-    "name": "\ud83c\uddf3\ud83c\uddf4 NorEval",
-    "description": "[arxiv.org/abs/2504.07749](https://arxiv.org/abs/2504.07749) * Homepage: [github.com/ltgoslo/noreval](https://github.com/ltgoslo/noreval/tree/main)",
-    "paper": "Abstract: [arxiv.org/abs/2504.07749](https://arxiv.org/abs/2504.07749",
-    "homepage": "Homepage: [github.com/ltgoslo/noreval](https://github.com/ltgoslo/noreval/tree/main"
-  },
-  "noreval_ask_gec": {
-    "name": "noreval_ask_gec",
-    "description": "Here, we use the `--predict_only` argument and compute the performance metrics as described below.",
-    "paper": "",
-    "homepage": ""
-  },
-  "noticia": {
-    "name": "NoticIA",
-    "description": "https://arxiv.org/abs/2404.07611",
-    "paper": "Abstract: https://arxiv.org/abs/2404.07611",
-    "homepage": "Homepage: https://github.com/ikergarcia1996/NoticIA"
-  },
-  "nq_open": {
-    "name": "nq_open",
-    "description": "Question Answering dataset based on aggregated user queries from Google Search.",
-    "paper": "Paper: [aclanthology.org/P19-1612](https://aclanthology.org/P19-1612/",
-    "homepage": "Homepage: https://research.google/pubs/natural-questions-a-benchmark-for-question-answering-research/"
-  },
-  "okapi_arc_multilingual": {
-    "name": "Multilingual ARC",
-    "description": "https://arxiv.org/abs/2307.16039",
-    "paper": "Abstract: https://arxiv.org/abs/2307.16039",
-    "homepage": "Homepage: `https://github.com/nlp-uoregon/Okapi`"
-  },
-  "okapi_hellaswag_multilingual": {
-    "name": "Multilingual HellaSwag",
-    "description": "https://arxiv.org/abs/2307.16039",
-    "paper": "Abstract: https://arxiv.org/abs/2307.16039",
-    "homepage": "Homepage: `https://github.com/nlp-uoregon/Okapi`"
-  },
-  "okapi_mmlu_multilingual": {
-    "name": "OKAPI MMLU Multilingual",
-    "description": "- **Dataset:** alexandrainst/m_mmlu\n- **Source:** https://huggingface.co/datasets/alexandrainst/m_mmlu",
-    "paper": "",
-    "homepage": ""
-  },
-  "okapi_truthfulqa_multilingual": {
-    "name": "Multilingual TruthfulQA",
-    "description": "https://arxiv.org/abs/2307.16039",
-    "paper": "Abstract: https://arxiv.org/abs/2307.16039",
-    "homepage": "Homepage: `https://github.com/nlp-uoregon/Okapi`"
-  },
-  "olaph": {
-    "name": "OLAPH",
-    "description": "[https://arxiv.org/abs/2405.12701](https://arxiv.org/abs/2405.12701)",
-    "paper": "Abstract: [https://arxiv.org/abs/2405.12701](https://arxiv.org/abs/2405.12701",
-    "homepage": ""
-  },
-  "openbookqa": {
-    "name": "OpenBookQA",
-    "description": "https://arxiv.org/abs/1809.02789",
-    "paper": "Abstract: https://arxiv.org/abs/1809.02789",
-    "homepage": "Homepage: https://allenai.org/data/open-book-qa"
-  },
-  "paloma": {
-    "name": "Paloma",
-    "description": "https://arxiv.org/abs/2312.10523v1",
-    "paper": "Abstract: https://arxiv.org/abs/2312.10523v1",
-    "homepage": "Homepage: https://allenai.org/olmo"
-  },
-  "paws-x": {
-    "name": "PAWS-X",
-    "description": "https://arxiv.org/abs/1908.11828",
-    "paper": "Abstract: https://arxiv.org/abs/1908.11828",
-    "homepage": "Homepage: https://github.com/google-research-datasets/paws/tree/master/pawsx"
-  },
-  "pile": {
-    "name": "The Pile",
-    "description": "https://arxiv.org/abs/2101.00027",
-    "paper": "Abstract: https://arxiv.org/abs/2101.00027",
-    "homepage": "Homepage: https://pile.eleuther.ai/"
-  },
-  "pile_10k": {
-    "name": "Pile-10k",
-    "description": "The first 10K elements of [The Pile](https://pile.eleuther.ai/), useful for debugging models trained on it. See the [HuggingFace page for the full Pile](https://huggingface.co/datasets/the_pile) for more info. Inspired by [stas' great resource](https://huggingface.co/datasets/stas/openwebtext-10k) doing the same for OpenWebText",
-    "paper": "Abstract: The first 10K elements of [The Pile](https://pile.eleuther.ai/",
-    "homepage": "Homepage: [https://huggingface.co/datasets/NeelNanda/pile-10k](https://huggingface.co/datasets/NeelNanda/pile-10k"
-  },
-  "piqa": {
-    "name": "PIQA",
-    "description": "https://arxiv.org/abs/1911.11641",
-    "paper": "Abstract: https://arxiv.org/abs/1911.11641",
-    "homepage": "Homepage: https://yonatanbisk.com/piqa/"
-  },
-  "polemo2": {
-    "name": "PolEmo 2.0",
-    "description": "https://aclanthology.org/K19-1092/",
-    "paper": "Abstract: https://aclanthology.org/K19-1092/",
-    "homepage": "Homepage: https://clarin-pl.eu/dspace/handle/11321/710"
-  },
-  "portuguese_bench": {
-    "name": "PortugueseBench",
-    "description": "PortugueseBench is a benchmark for evaluating language models in Portuguese tasks. This is, it evaluates the ability of a language model to understand and generate Portuguese text. PortugueseBench offers a combination of pre-existing, open datasets. All the details of PortugueseBench will be published in a paper soon.",
-    "paper": "",
-    "homepage": ""
-  },
-  "prost": {
-    "name": "PROST",
-    "description": "https://arxiv.org/abs/2106.03634",
-    "paper": "Abstract: https://arxiv.org/abs/2106.03634",
-    "homepage": "Homepage: https://github.com/nala-cub/prost"
-  },
-  "pubmedqa": {
-    "name": "PubMedQA",
-    "description": "https://arxiv.org/abs/1909.06146",
-    "paper": "Abstract: https://arxiv.org/abs/1909.06146",
-    "homepage": "Homepage: https://pubmedqa.github.io/"
-  },
-  "qa4mre": {
-    "name": "QA4MRE",
-    "description": "https://www.cs.cmu.edu/~./hovy/papers/13CLEF-QA4MRE.pdf",
-    "paper": "Abstract: https://www.cs.cmu.edu/~./hovy/papers/13CLEF-QA4MRE.pdf",
-    "homepage": "Homepage: http://nlp.uned.es/clef-qa/repository/qa4mre.php"
-  },
-  "qasper": {
-    "name": "QASPER",
-    "description": "https://arxiv.org/abs/2105.03011",
-    "paper": "Abstract: https://arxiv.org/abs/2105.03011",
-    "homepage": "Homepage: https://allenai.org/data/qasper"
-  },
-  "race": {
-    "name": "RACE",
-    "description": "https://arxiv.org/abs/1704.04683",
-    "paper": "Abstract: https://arxiv.org/abs/1704.04683",
-    "homepage": "Homepage: https://www.cs.cmu.edu/~glai1/data/race/"
-  },
-  "realtoxicityprompts": {
-    "name": "RealToxicityPrompts",
-    "description": "- **Dataset:** allenai/real-toxicity-prompts\n- **Source:** https://huggingface.co/datasets/allenai/real-toxicity-prompts",
-    "paper": "",
-    "homepage": ""
-  },
-  "ruler": {
-    "name": "Task-name",
-    "description": "`https://arxiv.org/abs/2404.06654`",
-    "paper": "Abstract: `https://arxiv.org/abs/2404.06654`",
-    "homepage": "Homepage: `https://github.com/NVIDIA/RULER`"
-  },
-  "sciq": {
-    "name": "SciQ",
-    "description": "https://aclanthology.org/W17-4413.pdf",
-    "paper": "Abstract: https://aclanthology.org/W17-4413.pdf",
-    "homepage": "Homepage: https://allenai.org/data/sciq"
-  },
-  "score": {
-    "name": "SCORE: Systematic COnsistency and Robustness Evaluation for Large Language Models",
-    "description": "```\nCopyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.",
-    "paper": "",
-    "homepage": ""
-  },
-  "scrolls": {
-    "name": "scrolls",
-    "description": "\"\"\"\nSCROLLS: Standardized CompaRison Over Long Language Sequences\nhttps://arxiv.org/abs/2201.03533",
-    "paper": "",
-    "homepage": "Homepage: https://www.scrolls-benchmark.com/"
-  },
-  "simple_cooccurrence_bias": {
-    "name": "Simple Cooccurrence Bias",
-    "description": "https://arxiv.org/pdf/2005.14165.pdf",
-    "paper": "Abstract: https://arxiv.org/pdf/2005.14165.pdf",
-    "homepage": ""
-  },
-  "siqa": {
-    "name": "Social IQA",
-    "description": "https://arxiv.org/abs/1904.09728",
-    "paper": "Abstract: https://arxiv.org/abs/1904.09728",
-    "homepage": "Homepage: https://allenai.org/data/socialiqa"
-  },
-  "spanish_bench": {
-    "name": "SpanishBench",
-    "description": "ive Summarization for 44 Languages](https://aclanthology.org/2021.findings-acl.413/) | https://huggingface.co/datasets/csebuetnlp/xlsum | | XNLI_es | Natural Language Inference | [XNLI: Evaluating Cross-lingual Sentence Representations](https://aclanthology.org/D18-1269/) | https://huggingface.co/datasets/facebook/xnli | | XQuAD_es | Question Answering | [On the Cross-lingual Transferability of Monolingual Representations](https://aclanthology.org/2020.acl-main.421/) | https://huggingface.co/dat",
-    "paper": "paper04.pdf) | https://huggingface.co/datasets/gplsi/cocoteros",
-    "homepage": ""
-  },
-  "squad_completion": {
-    "name": "Squad-completion",
-    "description": "Title: Simple Linear Attention Language Models Balance The Recall-Throughput Tradeoff",
-    "paper": "",
-    "homepage": "Homepage: https://github.com/HazyResearch/based-evaluation-harness"
-  },
-  "squadv2": {
-    "name": "Task-name",
-    "description": "https://arxiv.org/abs/1806.03822",
-    "paper": "Abstract: https://arxiv.org/abs/1806.03822",
-    "homepage": "Homepage: https://rajpurkar.github.io/SQuAD-explorer/"
-  },
-  "storycloze": {
-    "name": "StoryCloze",
-    "description": "`https://arxiv.org/abs/1604.01696`",
-    "paper": "Abstract: `https://arxiv.org/abs/1604.01696`",
-    "homepage": "Homepage: https://cs.rochester.edu/nlp/rocstories/"
-  },
-  "super_glue": {
-    "name": "SuperGLUE",
-    "description": "`https://w4ngatang.github.io/static/papers/superglue.pdf`",
-    "paper": "Abstract: `https://w4ngatang.github.io/static/papers/superglue.pdf`",
-    "homepage": "Homepage: https://super.gluebenchmark.com/"
-  },
-  "swag": {
-    "name": "SWAG",
-    "description": "https://arxiv.org/pdf/1808.05326.pdf",
-    "paper": "Abstract: https://arxiv.org/pdf/1808.05326.pdf",
-    "homepage": "Homepage: https://rowanzellers.com/swag/"
-  },
-  "swde": {
-    "name": "SWDE",
-    "description": "A long standing goal of the data management community is to develop general, automated systems that ingest semi-structured documents and output queryable tables without human effort or domain specific customization. Given the sheer variety of potential documents, state-of-the art systems make simplifying assumptions and use domain specific training. In this work, we ask whether we can maintain generality by using large language models (LLMs). LLMs, which are pretrained on broad data, can perform",
-    "paper": "",
-    "homepage": "Homepage: https://github.com/HazyResearch/based-evaluation-harness"
-  },
-  "tinyBenchmarks": {
-    "name": "tinyBenchmarks",
-    "description": "https://arxiv.org/abs/2402.14992",
-    "paper": "Abstract: https://arxiv.org/abs/2402.14992",
-    "homepage": ""
-  },
-  "tmlu": {
-    "name": "TMLU",
-    "description": "`The evaluation of large language models (LLMs) has drawn substantial attention in the field recently. This work focuses on evaluating LLMs in a Chinese context, specifically, for Traditional Chinese which has been largely underrepresented in existing benchmarks. We present TMLU, a holistic evaluation suit tailored for assessing the advanced knowledge and reasoning capability in LLMs, under the context of Taiwanese Mandarin. TMLU consists of an array of 37 subjects across social science, STEM, h",
-    "paper": "",
-    "homepage": "Homepage: [TMLU Huggingface Dataset](https://huggingface.co/datasets/miulab/tmlu"
-  },
-  "tmmluplus": {
-    "name": "TMMLU+",
-    "description": "`We present TMMLU+, a comprehensive dataset designed for the Traditional Chinese massive multitask language understanding dataset. TMMLU+ is a multiple-choice question-answering dataset with 66 subjects from elementary to professional level. Compared to its predecessor, TMMLU, TMMLU+ is six times larger and boasts a more balanced subject distribution. We included benchmark results in TMMLU+ from closed-source models and 24 open-weight Chinese large language models of parameters ranging from 1.8B",
-    "paper": "",
-    "homepage": "Homepage: [https://huggingface.co/datasets/ikala/tmmluplus](https://huggingface.co/datasets/ikala/tmmluplus"
-  },
-  "toxigen": {
-    "name": "ToxiGen",
-    "description": "https://arxiv.org/abs/2203.09509",
-    "paper": "Abstract: https://arxiv.org/abs/2203.09509",
-    "homepage": "Homepage: https://github.com/microsoft/TOXIGEN"
-  },
-  "translation": {
-    "name": "Translation Tasks",
-    "description": "* `gpt3_translation_tasks`\n* `wmt14`\n* `wmt16`\n* `wmt20`\n* `iwslt2017`",
-    "paper": "",
-    "homepage": ""
-  },
-  "triviaqa": {
-    "name": "Trivia QA",
-    "description": "https://arxiv.org/abs/1705.03551",
-    "paper": "Abstract: https://arxiv.org/abs/1705.03551",
-    "homepage": "Homepage: https://nlp.cs.washington.edu/triviaqa/"
-  },
-  "truthfulqa-multi": {
-    "name": "TruthfulQA-Multi",
-    "description": "`[https://arxiv.org/abs/2502.09387v1](https://arxiv.org/abs/2502.09387v1)`",
-    "paper": "Abstract: `[https://arxiv.org/abs/2502.09387v1](https://arxiv.org/abs/2502.09387v1",
-    "homepage": ""
-  },
-  "truthfulqa": {
-    "name": "TruthfulQA",
-    "description": "`https://arxiv.org/abs/2109.07958`",
-    "paper": "Abstract: `https://arxiv.org/abs/2109.07958`",
-    "homepage": "Homepage: `https://github.com/sylinrl/TruthfulQA`"
-  },
-  "turkishmmlu": {
-    "name": "TurkishMMLU",
-    "description": "Multiple choice question answering tasks evaluate the reasoning, comprehension, and mathematical abilities of Large Language Models (LLMs). While existing benchmarks employ automatic translation for multilingual evaluation, this approach is error-prone and potentially introduces culturally biased questions, especially in social sciences. We introduce the first multitask, multiple-choice Turkish QA benchmark, TurkishMMLU, to evaluate LLMs' understanding of the Turkish language. TurkishMMLU includ",
-    "paper": "",
-    "homepage": ""
-  },
-  "unitxt": {
-    "name": "Unitxt",
-    "description": "[link](https://arxiv.org/abs/2401.14019)",
-    "paper": "Abstract: [link](https://arxiv.org/abs/2401.14019",
-    "homepage": ""
-  },
-  "unscramble": {
-    "name": "Unscramble",
-    "description": "Language Models are Few-Shot Learners\nhttps://arxiv.org/pdf/2005.14165.pdf",
-    "paper": "",
-    "homepage": "Homepage: https://github.com/openai/gpt-3/tree/master/data"
-  },
-  "webqs": {
-    "name": "WEBQs",
-    "description": "`https://cs.stanford.edu/~pliang/papers/freebase-emnlp2013.pdf`",
-    "paper": "Abstract: `https://cs.stanford.edu/~pliang/papers/freebase-emnlp2013.pdf`",
-    "homepage": "Homepage: `https://worksheets.codalab.org/worksheets/0xba659fe363cb46e7a505c5b6a774dc8a`"
-  },
-  "wikitext": {
-    "name": "Wikitext",
-    "description": "Pointer Sentinel Mixture Models\nhttps://arxiv.org/pdf/1609.07843.pdf",
-    "paper": "",
-    "homepage": "Homepage: https://www.salesforce.com/products/einstein/ai-research/the-wikitext-dependency-language-modeling-dataset/"
-  },
-  "winogender": {
-    "name": "Winogender",
-    "description": "https://aclanthology.org/N18-2002.pdf",
-    "paper": "Abstract: https://aclanthology.org/N18-2002.pdf",
-    "homepage": "Homepage: https://github.com/rudinger/winogender-schemas"
-  },
-  "winogrande": {
-    "name": "WinoGrande",
-    "description": "https://arxiv.org/abs/1907.10641",
-    "paper": "Abstract: https://arxiv.org/abs/1907.10641",
-    "homepage": "Homepage: https://leaderboard.allenai.org/winogrande/submissions/public"
-  },
-  "wmdp": {
-    "name": "WMDP",
-    "description": "`https://arxiv.org/abs/2403.03218`",
-    "paper": "Abstract: `https://arxiv.org/abs/2403.03218`",
-    "homepage": "Homepage: https://wmdp.ai"
-  },
-  "wmt2016": {
-    "name": "WMT16",
-    "description": "http://www.aclweb.org/anthology/W/W16/W16-2301",
-    "paper": "Abstract: http://www.aclweb.org/anthology/W/W16/W16-2301",
-    "homepage": "Homepage: https://huggingface.co/datasets/wmt16"
-  },
-  "wsc273": {
-    "name": "WSC273",
-    "description": "http://commonsensereasoning.org/2011/papers/Levesque.pdf",
-    "paper": "Abstract: http://commonsensereasoning.org/2011/papers/Levesque.pdf",
-    "homepage": "Homepage: https://cs.nyu.edu/~davise/papers/WinogradSchemas/WS.html"
-  },
-  "xcopa": {
-    "name": "XCOPA",
-    "description": "https://ducdauge.github.io/files/xcopa.pdf",
-    "paper": "Abstract: https://ducdauge.github.io/files/xcopa.pdf",
-    "homepage": "Homepage: https://github.com/cambridgeltl/xcopa"
-  },
-  "xnli": {
-    "name": "XNLI",
-    "description": "https://arxiv.org/abs/1809.05053",
-    "paper": "Abstract: https://arxiv.org/abs/1809.05053",
-    "homepage": "Homepage: https://github.com/facebookresearch/XNLI"
-  },
-  "xnli_eu": {
-    "name": "XNLIeu",
-    "description": "https://arxiv.org/abs/2404.06996",
-    "paper": "Abstract: https://arxiv.org/abs/2404.06996",
-    "homepage": "Homepage: https://github.com/hitz-zentroa/xnli-eu"
-  },
-  "xquad": {
-    "name": "XQuAD",
-    "description": "https://aclanthology.org/2020.acl-main.421.pdf",
-    "paper": "Abstract: https://aclanthology.org/2020.acl-main.421.pdf",
-    "homepage": "Homepage: https://github.com/deepmind/xquad"
-  },
-  "xstorycloze": {
-    "name": "XStoryCloze",
-    "description": "https://arxiv.org/abs/2112.10668",
-    "paper": "Abstract: https://arxiv.org/abs/2112.10668",
-    "homepage": "Homepage: https://github.com/facebookresearch/fairseq/pull/4820"
-  },
-  "xwinograd": {
-    "name": "Task-name",
-    "description": "`https://arxiv.org/abs/2106.12066`",
-    "paper": "Abstract: `https://arxiv.org/abs/2106.12066`",
-    "homepage": "Homepage: `https://huggingface.co/datasets/Muennighoff/xwinograd`"
-  }
-}