opencompass 0.5.2__tar.gz → 0.5.4__tar.gz

opencompass-0.5.4/PKG-INFO

@@ -0,0 +1,801 @@
+Metadata-Version: 2.4
+Name: opencompass
+Version: 0.5.4
+Summary: A comprehensive toolkit for large model evaluation
+Home-page: https://github.com/open-compass/opencompass
+Author: OpenCompass Contributors
+Maintainer: OpenCompass Authors
+License: Apache License 2.0
+Keywords: AI,NLP,in-context learning,large language model,evaluation,benchmark,llm
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Education
+Classifier: Intended Audience :: Science/Research
+Requires-Python: >=3.8.0
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: absl-py
+Requires-Dist: accelerate>=0.19.0
+Requires-Dist: cpm_kernels
+Requires-Dist: datasets<4.0.0,>=2.12.0
+Requires-Dist: einops>=0.5.0
+Requires-Dist: evaluate>=0.3.0
+Requires-Dist: func_timeout
+Requires-Dist: fuzzywuzzy
+Requires-Dist: gradio-client
+Requires-Dist: h5py
+Requires-Dist: httpx==0.27.2
+Requires-Dist: huggingface_hub
+Requires-Dist: immutabledict
+Requires-Dist: importlib-metadata
+Requires-Dist: jieba
+Requires-Dist: json5
+Requires-Dist: jsonlines
+Requires-Dist: mmengine-lite==0.11.0rc3
+Requires-Dist: nltk>=3.7
+Requires-Dist: numpy<2.0.0,>=1.23.4
+Requires-Dist: openai
+Requires-Dist: opencv-python-headless
+Requires-Dist: pandas
+Requires-Dist: prettytable
+Requires-Dist: protobuf
+Requires-Dist: python-Levenshtein
+Requires-Dist: rank_bm25==0.2.2
+Requires-Dist: rapidfuzz
+Requires-Dist: requests>=2.31.0
+Requires-Dist: retrying
+Requires-Dist: rich
+Requires-Dist: rouge
+Requires-Dist: rouge_chinese
+Requires-Dist: rouge_score
+Requires-Dist: sacrebleu
+Requires-Dist: scikit_learn==1.5.0
+Requires-Dist: seaborn
+Requires-Dist: sentence_transformers
+Requires-Dist: tabulate
+Requires-Dist: tiktoken
+Requires-Dist: timeout_decorator
+Requires-Dist: tokenizers>=0.13.3
+Requires-Dist: torch>=1.13.1
+Requires-Dist: tqdm>=4.64.1
+Requires-Dist: transformers>=4.29.1
+Requires-Dist: tree-sitter==0.21.3
+Requires-Dist: tree_sitter_languages>=1.10.2
+Requires-Dist: typer
+Provides-Extra: lmdeploy
+Requires-Dist: lmdeploy; extra == "lmdeploy"
+Requires-Dist: absl-py; extra == "lmdeploy"
+Requires-Dist: accelerate>=0.19.0; extra == "lmdeploy"
+Requires-Dist: cpm_kernels; extra == "lmdeploy"
+Requires-Dist: datasets<4.0.0,>=2.12.0; extra == "lmdeploy"
+Requires-Dist: einops>=0.5.0; extra == "lmdeploy"
+Requires-Dist: evaluate>=0.3.0; extra == "lmdeploy"
+Requires-Dist: func_timeout; extra == "lmdeploy"
+Requires-Dist: fuzzywuzzy; extra == "lmdeploy"
+Requires-Dist: gradio-client; extra == "lmdeploy"
+Requires-Dist: h5py; extra == "lmdeploy"
+Requires-Dist: httpx==0.27.2; extra == "lmdeploy"
+Requires-Dist: huggingface_hub; extra == "lmdeploy"
+Requires-Dist: immutabledict; extra == "lmdeploy"
+Requires-Dist: importlib-metadata; extra == "lmdeploy"
+Requires-Dist: jieba; extra == "lmdeploy"
+Requires-Dist: json5; extra == "lmdeploy"
+Requires-Dist: jsonlines; extra == "lmdeploy"
+Requires-Dist: mmengine-lite==0.11.0rc3; extra == "lmdeploy"
+Requires-Dist: nltk>=3.7; extra == "lmdeploy"
+Requires-Dist: numpy<2.0.0,>=1.23.4; extra == "lmdeploy"
+Requires-Dist: openai; extra == "lmdeploy"
+Requires-Dist: opencv-python-headless; extra == "lmdeploy"
+Requires-Dist: pandas; extra == "lmdeploy"
+Requires-Dist: prettytable; extra == "lmdeploy"
+Requires-Dist: protobuf; extra == "lmdeploy"
+Requires-Dist: python-Levenshtein; extra == "lmdeploy"
+Requires-Dist: rank_bm25==0.2.2; extra == "lmdeploy"
+Requires-Dist: rapidfuzz; extra == "lmdeploy"
+Requires-Dist: requests>=2.31.0; extra == "lmdeploy"
+Requires-Dist: retrying; extra == "lmdeploy"
+Requires-Dist: rich; extra == "lmdeploy"
+Requires-Dist: rouge; extra == "lmdeploy"
+Requires-Dist: rouge_chinese; extra == "lmdeploy"
+Requires-Dist: rouge_score; extra == "lmdeploy"
+Requires-Dist: sacrebleu; extra == "lmdeploy"
+Requires-Dist: scikit_learn==1.5.0; extra == "lmdeploy"
+Requires-Dist: seaborn; extra == "lmdeploy"
+Requires-Dist: sentence_transformers; extra == "lmdeploy"
+Requires-Dist: tabulate; extra == "lmdeploy"
+Requires-Dist: tiktoken; extra == "lmdeploy"
+Requires-Dist: timeout_decorator; extra == "lmdeploy"
+Requires-Dist: tokenizers>=0.13.3; extra == "lmdeploy"
+Requires-Dist: torch>=1.13.1; extra == "lmdeploy"
+Requires-Dist: tqdm>=4.64.1; extra == "lmdeploy"
+Requires-Dist: transformers>=4.29.1; extra == "lmdeploy"
+Requires-Dist: tree-sitter==0.21.3; extra == "lmdeploy"
+Requires-Dist: tree_sitter_languages>=1.10.2; extra == "lmdeploy"
+Requires-Dist: typer; extra == "lmdeploy"
+Provides-Extra: vllm
+Requires-Dist: vllm; extra == "vllm"
+Requires-Dist: absl-py; extra == "vllm"
+Requires-Dist: accelerate>=0.19.0; extra == "vllm"
+Requires-Dist: cpm_kernels; extra == "vllm"
+Requires-Dist: datasets<4.0.0,>=2.12.0; extra == "vllm"
+Requires-Dist: einops>=0.5.0; extra == "vllm"
+Requires-Dist: evaluate>=0.3.0; extra == "vllm"
+Requires-Dist: func_timeout; extra == "vllm"
+Requires-Dist: fuzzywuzzy; extra == "vllm"
+Requires-Dist: gradio-client; extra == "vllm"
+Requires-Dist: h5py; extra == "vllm"
+Requires-Dist: httpx==0.27.2; extra == "vllm"
+Requires-Dist: huggingface_hub; extra == "vllm"
+Requires-Dist: immutabledict; extra == "vllm"
+Requires-Dist: importlib-metadata; extra == "vllm"
+Requires-Dist: jieba; extra == "vllm"
+Requires-Dist: json5; extra == "vllm"
+Requires-Dist: jsonlines; extra == "vllm"
+Requires-Dist: mmengine-lite==0.11.0rc3; extra == "vllm"
+Requires-Dist: nltk>=3.7; extra == "vllm"
+Requires-Dist: numpy<2.0.0,>=1.23.4; extra == "vllm"
+Requires-Dist: openai; extra == "vllm"
+Requires-Dist: opencv-python-headless; extra == "vllm"
+Requires-Dist: pandas; extra == "vllm"
+Requires-Dist: prettytable; extra == "vllm"
+Requires-Dist: protobuf; extra == "vllm"
+Requires-Dist: python-Levenshtein; extra == "vllm"
+Requires-Dist: rank_bm25==0.2.2; extra == "vllm"
+Requires-Dist: rapidfuzz; extra == "vllm"
+Requires-Dist: requests>=2.31.0; extra == "vllm"
+Requires-Dist: retrying; extra == "vllm"
+Requires-Dist: rich; extra == "vllm"
+Requires-Dist: rouge; extra == "vllm"
+Requires-Dist: rouge_chinese; extra == "vllm"
+Requires-Dist: rouge_score; extra == "vllm"
+Requires-Dist: sacrebleu; extra == "vllm"
+Requires-Dist: scikit_learn==1.5.0; extra == "vllm"
+Requires-Dist: seaborn; extra == "vllm"
+Requires-Dist: sentence_transformers; extra == "vllm"
+Requires-Dist: tabulate; extra == "vllm"
+Requires-Dist: tiktoken; extra == "vllm"
+Requires-Dist: timeout_decorator; extra == "vllm"
+Requires-Dist: tokenizers>=0.13.3; extra == "vllm"
+Requires-Dist: torch>=1.13.1; extra == "vllm"
+Requires-Dist: tqdm>=4.64.1; extra == "vllm"
+Requires-Dist: transformers>=4.29.1; extra == "vllm"
+Requires-Dist: tree-sitter==0.21.3; extra == "vllm"
+Requires-Dist: tree_sitter_languages>=1.10.2; extra == "vllm"
+Requires-Dist: typer; extra == "vllm"
+Provides-Extra: api
+Requires-Dist: anthropic; extra == "api"
+Requires-Dist: dashscope; extra == "api"
+Requires-Dist: openai; extra == "api"
+Requires-Dist: spark_ai_python; extra == "api"
+Requires-Dist: sseclient-py==1.7.2; extra == "api"
+Requires-Dist: tencentcloud-sdk-python; extra == "api"
+Requires-Dist: volcengine; extra == "api"
+Requires-Dist: volcengine-python-sdk; extra == "api"
+Requires-Dist: websocket-client; extra == "api"
+Requires-Dist: zhipuai; extra == "api"
+Requires-Dist: absl-py; extra == "api"
+Requires-Dist: accelerate>=0.19.0; extra == "api"
+Requires-Dist: cpm_kernels; extra == "api"
+Requires-Dist: datasets<4.0.0,>=2.12.0; extra == "api"
+Requires-Dist: einops>=0.5.0; extra == "api"
+Requires-Dist: evaluate>=0.3.0; extra == "api"
+Requires-Dist: func_timeout; extra == "api"
+Requires-Dist: fuzzywuzzy; extra == "api"
+Requires-Dist: gradio-client; extra == "api"
+Requires-Dist: h5py; extra == "api"
+Requires-Dist: httpx==0.27.2; extra == "api"
+Requires-Dist: huggingface_hub; extra == "api"
+Requires-Dist: immutabledict; extra == "api"
+Requires-Dist: importlib-metadata; extra == "api"
+Requires-Dist: jieba; extra == "api"
+Requires-Dist: json5; extra == "api"
+Requires-Dist: jsonlines; extra == "api"
+Requires-Dist: mmengine-lite==0.11.0rc3; extra == "api"
+Requires-Dist: nltk>=3.7; extra == "api"
+Requires-Dist: numpy<2.0.0,>=1.23.4; extra == "api"
+Requires-Dist: openai; extra == "api"
+Requires-Dist: opencv-python-headless; extra == "api"
+Requires-Dist: pandas; extra == "api"
+Requires-Dist: prettytable; extra == "api"
+Requires-Dist: protobuf; extra == "api"
+Requires-Dist: python-Levenshtein; extra == "api"
+Requires-Dist: rank_bm25==0.2.2; extra == "api"
+Requires-Dist: rapidfuzz; extra == "api"
+Requires-Dist: requests>=2.31.0; extra == "api"
+Requires-Dist: retrying; extra == "api"
+Requires-Dist: rich; extra == "api"
+Requires-Dist: rouge; extra == "api"
+Requires-Dist: rouge_chinese; extra == "api"
+Requires-Dist: rouge_score; extra == "api"
+Requires-Dist: sacrebleu; extra == "api"
+Requires-Dist: scikit_learn==1.5.0; extra == "api"
+Requires-Dist: seaborn; extra == "api"
+Requires-Dist: sentence_transformers; extra == "api"
+Requires-Dist: tabulate; extra == "api"
+Requires-Dist: tiktoken; extra == "api"
+Requires-Dist: timeout_decorator; extra == "api"
+Requires-Dist: tokenizers>=0.13.3; extra == "api"
+Requires-Dist: torch>=1.13.1; extra == "api"
+Requires-Dist: tqdm>=4.64.1; extra == "api"
+Requires-Dist: transformers>=4.29.1; extra == "api"
+Requires-Dist: tree-sitter==0.21.3; extra == "api"
+Requires-Dist: tree_sitter_languages>=1.10.2; extra == "api"
+Requires-Dist: typer; extra == "api"
+Provides-Extra: vlm
+Requires-Dist: google-genai==1.2.0; extra == "vlm"
+Requires-Dist: litellm==1.83.0; extra == "vlm"
+Requires-Dist: absl-py; extra == "vlm"
+Requires-Dist: accelerate>=0.19.0; extra == "vlm"
+Requires-Dist: cpm_kernels; extra == "vlm"
+Requires-Dist: datasets<4.0.0,>=2.12.0; extra == "vlm"
+Requires-Dist: einops>=0.5.0; extra == "vlm"
+Requires-Dist: evaluate>=0.3.0; extra == "vlm"
+Requires-Dist: func_timeout; extra == "vlm"
+Requires-Dist: fuzzywuzzy; extra == "vlm"
+Requires-Dist: gradio-client; extra == "vlm"
+Requires-Dist: h5py; extra == "vlm"
+Requires-Dist: httpx==0.27.2; extra == "vlm"
+Requires-Dist: huggingface_hub; extra == "vlm"
+Requires-Dist: immutabledict; extra == "vlm"
+Requires-Dist: importlib-metadata; extra == "vlm"
+Requires-Dist: jieba; extra == "vlm"
+Requires-Dist: json5; extra == "vlm"
+Requires-Dist: jsonlines; extra == "vlm"
+Requires-Dist: mmengine-lite==0.11.0rc3; extra == "vlm"
+Requires-Dist: nltk>=3.7; extra == "vlm"
+Requires-Dist: numpy<2.0.0,>=1.23.4; extra == "vlm"
+Requires-Dist: openai; extra == "vlm"
+Requires-Dist: opencv-python-headless; extra == "vlm"
+Requires-Dist: pandas; extra == "vlm"
+Requires-Dist: prettytable; extra == "vlm"
+Requires-Dist: protobuf; extra == "vlm"
+Requires-Dist: python-Levenshtein; extra == "vlm"
+Requires-Dist: rank_bm25==0.2.2; extra == "vlm"
+Requires-Dist: rapidfuzz; extra == "vlm"
+Requires-Dist: requests>=2.31.0; extra == "vlm"
+Requires-Dist: retrying; extra == "vlm"
+Requires-Dist: rich; extra == "vlm"
+Requires-Dist: rouge; extra == "vlm"
+Requires-Dist: rouge_chinese; extra == "vlm"
+Requires-Dist: rouge_score; extra == "vlm"
+Requires-Dist: sacrebleu; extra == "vlm"
+Requires-Dist: scikit_learn==1.5.0; extra == "vlm"
+Requires-Dist: seaborn; extra == "vlm"
+Requires-Dist: sentence_transformers; extra == "vlm"
+Requires-Dist: tabulate; extra == "vlm"
+Requires-Dist: tiktoken; extra == "vlm"
+Requires-Dist: timeout_decorator; extra == "vlm"
+Requires-Dist: tokenizers>=0.13.3; extra == "vlm"
+Requires-Dist: torch>=1.13.1; extra == "vlm"
+Requires-Dist: tqdm>=4.64.1; extra == "vlm"
+Requires-Dist: transformers>=4.29.1; extra == "vlm"
+Requires-Dist: tree-sitter==0.21.3; extra == "vlm"
+Requires-Dist: tree_sitter_languages>=1.10.2; extra == "vlm"
+Requires-Dist: typer; extra == "vlm"
+Provides-Extra: full
+Requires-Dist: alpaca-eval==0.6; extra == "full"
+Requires-Dist: antlr4-python3-runtime==4.11; extra == "full"
+Requires-Dist: bert_score; extra == "full"
+Requires-Dist: Bio; extra == "full"
+Requires-Dist: cn2an; extra == "full"
+Requires-Dist: dingo-python==1.11.1; extra == "full"
+Requires-Dist: emoji; extra == "full"
+Requires-Dist: evalplus==0.2.1; extra == "full"
+Requires-Dist: faiss-gpu==1.14.3; python_version >= "3.10" and extra == "full"
+Requires-Dist: faiss_gpu==1.7.2; python_version < "3.10" and extra == "full"
+Requires-Dist: human-eval; extra == "full"
+Requires-Dist: langdetect; extra == "full"
+Requires-Dist: ltp; extra == "full"
+Requires-Dist: math-verify[antlr4_11_0]; extra == "full"
+Requires-Dist: OpenCC; extra == "full"
+Requires-Dist: pint; extra == "full"
+Requires-Dist: pyext==0.7; python_version < "3.11" and extra == "full"
+Requires-Dist: pypinyin; extra == "full"
+Requires-Dist: rdchiral; extra == "full"
+Requires-Dist: rdkit; extra == "full"
+Requires-Dist: selfies; extra == "full"
+Requires-Dist: smact==3.0.2; extra == "full"
+Requires-Dist: syllapy; extra == "full"
+Requires-Dist: wonderwords; extra == "full"
+Requires-Dist: absl-py; extra == "full"
+Requires-Dist: accelerate>=0.19.0; extra == "full"
+Requires-Dist: cpm_kernels; extra == "full"
+Requires-Dist: datasets<4.0.0,>=2.12.0; extra == "full"
+Requires-Dist: einops>=0.5.0; extra == "full"
+Requires-Dist: evaluate>=0.3.0; extra == "full"
+Requires-Dist: func_timeout; extra == "full"
+Requires-Dist: fuzzywuzzy; extra == "full"
+Requires-Dist: gradio-client; extra == "full"
+Requires-Dist: h5py; extra == "full"
+Requires-Dist: httpx==0.27.2; extra == "full"
+Requires-Dist: huggingface_hub; extra == "full"
+Requires-Dist: immutabledict; extra == "full"
+Requires-Dist: importlib-metadata; extra == "full"
+Requires-Dist: jieba; extra == "full"
+Requires-Dist: json5; extra == "full"
+Requires-Dist: jsonlines; extra == "full"
+Requires-Dist: mmengine-lite==0.11.0rc3; extra == "full"
+Requires-Dist: nltk>=3.7; extra == "full"
+Requires-Dist: numpy<2.0.0,>=1.23.4; extra == "full"
+Requires-Dist: openai; extra == "full"
+Requires-Dist: opencv-python-headless; extra == "full"
+Requires-Dist: pandas; extra == "full"
+Requires-Dist: prettytable; extra == "full"
+Requires-Dist: protobuf; extra == "full"
+Requires-Dist: python-Levenshtein; extra == "full"
+Requires-Dist: rank_bm25==0.2.2; extra == "full"
+Requires-Dist: rapidfuzz; extra == "full"
+Requires-Dist: requests>=2.31.0; extra == "full"
+Requires-Dist: retrying; extra == "full"
+Requires-Dist: rich; extra == "full"
+Requires-Dist: rouge; extra == "full"
+Requires-Dist: rouge_chinese; extra == "full"
+Requires-Dist: rouge_score; extra == "full"
+Requires-Dist: sacrebleu; extra == "full"
+Requires-Dist: scikit_learn==1.5.0; extra == "full"
+Requires-Dist: seaborn; extra == "full"
+Requires-Dist: sentence_transformers; extra == "full"
+Requires-Dist: tabulate; extra == "full"
+Requires-Dist: tiktoken; extra == "full"
+Requires-Dist: timeout_decorator; extra == "full"
+Requires-Dist: tokenizers>=0.13.3; extra == "full"
+Requires-Dist: torch>=1.13.1; extra == "full"
+Requires-Dist: tqdm>=4.64.1; extra == "full"
+Requires-Dist: transformers>=4.29.1; extra == "full"
+Requires-Dist: tree-sitter==0.21.3; extra == "full"
+Requires-Dist: tree_sitter_languages>=1.10.2; extra == "full"
+Requires-Dist: typer; extra == "full"
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: maintainer
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+<div align="center">
+  <img src="docs/en/_static/image/logo.svg" width="500px"/>
+  <br />
+  <br />
+
+[![][github-release-shield]][github-release-link]
+[![][github-releasedate-shield]][github-releasedate-link]
+[![][github-contributors-shield]][github-contributors-link]<br>
+[![][github-forks-shield]][github-forks-link]
+[![][github-stars-shield]][github-stars-link]
+[![][github-issues-shield]][github-issues-link]
+[![][github-license-shield]][github-license-link]
+
+<!-- [![PyPI](https://badge.fury.io/py/opencompass.svg)](https://pypi.org/project/opencompass/) -->
+
+[🌐Website](https://opencompass.org.cn/) |
+[📖CompassHub](https://hub.opencompass.org.cn/home) |
+[📊CompassRank](https://rank.opencompass.org.cn/home) |
+[📘Documentation](https://opencompass.readthedocs.io/en/latest/) |
+[🛠️Installation](https://opencompass.readthedocs.io/en/latest/get_started/installation.html) |
+[🤔Reporting Issues](https://github.com/open-compass/opencompass/issues/new/choose)
+
+English | [简体中文](README_zh-CN.md)
+
+[![][github-trending-shield]][github-trending-url]
+
+</div>
+
+<p align="center">
+    👋 join us on <a href="https://discord.gg/KKwfEbFj7U" target="_blank">Discord</a> and <a href="https://r.vansin.top/?r=opencompass" target="_blank">WeChat</a>
+</p>
+
+> \[!IMPORTANT\]
+>
+> **Star Us**, You will receive all release notifications from GitHub without any delay ~ ⭐️
+
+<details>
+  <summary><kbd>Star History</kbd></summary>
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=open-compass%2Fopencompass&theme=dark&type=Date">
+    <img width="100%" src="https://api.star-history.com/svg?repos=open-compass%2Fopencompass&type=Date">
+  </picture>
+</details>
+
+## 🧭	Welcome
+
+to **OpenCompass**!
+
+Just like a compass guides us on our journey, OpenCompass will guide you through the complex landscape of evaluating large language models. With its powerful algorithms and intuitive interface, OpenCompass makes it easy to assess the quality and effectiveness of your NLP models.
+
+🚩🚩🚩 Explore opportunities at OpenCompass! We're currently **hiring full-time researchers/engineers and interns**. If you're passionate about LLM and OpenCompass, don't hesitate to reach out to us via [email](mailto:zhangsongyang@pjlab.org.cn). We'd love to hear from you!
+
+🔥🔥🔥 We are delighted to announce that **the OpenCompass has been recommended by the Meta AI**, click [Get Started](https://ai.meta.com/llama/get-started/#validation) of Llama for more information.
+
+> **Attention**<br />
+> Breaking Change Notice: In version 0.4.0, we are consolidating all AMOTIC configuration files (previously located in ./configs/datasets, ./configs/models, and ./configs/summarizers) into the opencompass package. Users are advised to update their configuration references to reflect this structural change.
+
+## 🚀 What's New <a><img width="35" height="20" src="https://user-images.githubusercontent.com/12782558/212848161-5e783dd6-11e8-4fe0-bbba-39ffb77730be.png"></a>
+
+- **\[2026.08.25\]** OpenCompass now integrates with [VLMEvalKit](https://github.com/open-compass/VLMEvalKit), enabling native multimodal dataset loading, inference through OpenAI-compatible APIs, and evaluation with official VLMEvalKit metrics. Check out the [MMBench example](examples/eval_mmbench_vlmevalkit.py) and [MMMU-Pro example](examples/eval_mmmu_pro_vlmevalkit.py) for details! 🔥🔥🔥
+- **\[2026.07.28\]** OpenCompass has expanded its API model ecosystem with support for the OpenAI Responses API and LiteLLM AI Gateway, while updating the Gemini and Anthropic integrations to their latest SDK interfaces. Check out the [OpenAI Responses API implementation](opencompass/models/openai_response.py), [LiteLLM AI Gateway implementation](opencompass/models/litellm_api.py), [Gemini SDK implementation](opencompass/models/gemini_sdk_api.py), and [Anthropic SDK implementation](opencompass/models/claude_sdk_api.py) for details!
+- **\[2026.07.27\]** OpenCompass now supports multi-round inference in `GenInferencer` and adds support for the Multi-IF dataset to evaluate multi-turn instruction-following capabilities. Check out the [Multi-IF evaluation configuration](opencompass/configs/datasets/MultiIF/MultiIF_gen.py) for details! 🔥🔥🔥
+- **\[2026.05.25\]** OpenCompass now provides repeat analysis tools for detecting repetitive content and looping model outputs in current evaluation tasks or existing evaluation results. Check out the [repeat analysis tool](tools/analyze_repeat.py) for details!
+- **\[2026.03.20\]** OpenCompass now supports concurrent inference across tasks together with evaluation watching, enabling completed inference tasks to be monitored and subsequent evaluations to be triggered in a coordinated pipeline. Parallel inferencers, task monitoring, and heartbeat mechanisms further improve large-scale evaluation efficiency. Check out the [concurrent inference implementation](opencompass/tasks/openicl_infer_concurrent.py) and [evaluation watcher implementation](opencompass/tasks/openicl_eval_watch.py) for details!
+- **\[2026.03.17\]** OpenCompass introduces `RawPromptTemplate`, allowing original benchmark prompts and structured conversations to be passed to models without unintended formatting transformations. It supports API models, ChatML datasets, and appending additional prompt content on the model side. Check out the [RawPromptTemplate guide](docs/en/notes/rawprompttemplate.md) for details!
+- **\[2026.02.05\]** OpenCompass now supports Intern-S1-Pro related general and scientific evaluation benchmarks. Please check [Example for Evaluating Intern-S1-Pro](examples/eval_intern_s1_pro.py) and [Model Card](https://huggingface.co/internlm/Intern-S1-Pro) for more details! 🔥🔥🔥
+- **\[2025.12.08\]** OpenCompass now supports evaluation for SciReasoner. Please check [Example for Evaluating SciReasoner](examples/eval_scireasoner.py) and [Project GitHub Repo](https://github.com/InternScience/SciReason) for more details! 🔥🔥🔥
+- **\[2025.07.26\]** OpenCompass now supports Intern-S1 related general and scientific evaluation benchmarks. Please check [Tutorial for Evaluating Intern-S1](https://opencompass.readthedocs.io/en/latest/user_guides/interns1.html) for more details! 🔥🔥🔥
+- **\[2025.04.01\]** OpenCompass now supports `CascadeEvaluator`, a flexible evaluation mechanism that allows multiple evaluators to work in sequence. This enables creating customized evaluation pipelines for complex assessment scenarios. Check out the [documentation](docs/en/advanced_guides/llm_judge.md) for more details! 🔥🔥🔥
+- **\[2025.03.11\]** We have supported evaluation for `SuperGPQA` which is a great benchmark for measuring LLM knowledge ability 🔥🔥🔥
+- **\[2025.02.28\]** We have added a tutorial for `DeepSeek-R1` series model, please check [Evaluating Reasoning Model](docs/en/user_guides/deepseek_r1.md) for more details! 🔥🔥🔥
+- **\[2025.02.15\]** We have added two powerful evaluation tools: `GenericLLMEvaluator` for LLM-as-judge evaluations and `MATHVerifyEvaluator` for mathematical reasoning assessments. Check out the documentation for [LLM Judge](docs/en/advanced_guides/llm_judge.md) and [Math Evaluation](docs/en/advanced_guides/math_verify.md) for more details! 🔥🔥🔥
+
+> [More](docs/en/notes/news.md)
+
+## 📊 Leaderboard
+
+We provide [OpenCompass Leaderboard](https://rank.opencompass.org.cn/home) for the community to rank all public models and API models. If you would like to join the evaluation, please provide the model repository URL or a standard API interface to the email address `opencompass@pjlab.org.cn`.
+
+You can also refer to [Guide to Reproducing CompassAcademic Leaderboard Results](https://opencompass.readthedocs.io/zh-cn/latest/academic.html) to quickly reproduce the leaderboard results.
+
+<p align="right"><a href="#top">🔝Back to top</a></p>
+
+## 🛠️ Installation
+
+Below are the steps for quick installation and datasets preparation.
+
+### 💻 Environment Setup
+
+We highly recommend using conda to manage your python environment.
+OpenCompass supports Python 3.12 for regular and full installation. If
+your evaluation depends on `pyext`-backed code execution datasets, use
+Python 3.10 instead: `pyext==0.7` is skipped on Python >=3.11 because it
+relies on `inspect.getargspec`, which was removed in Python 3.11.
+Without `pyext`, APPS (`apps`, `apps_mini`), TACO, and LiveCodeBench Code
+Generation are unavailable.
+
+- #### Create your virtual environment
+
+  ```bash
+  conda create --name opencompass python=3.12 -y
+  conda activate opencompass
+  ```
+
+- #### Install OpenCompass via pip
+
+  ```bash
+    pip install -U opencompass
+
+    ## Full installation (with support for more datasets)
+    # pip install "opencompass[full]"
+
+    ## Environment with model acceleration frameworks
+    ## Manage different acceleration frameworks using virtual environments
+    ## since they usually have dependency conflicts with each other.
+    # pip install "opencompass[lmdeploy]"
+    # pip install "opencompass[vllm]"
+
+    ## API evaluation (i.e. Openai, Qwen)
+    # pip install "opencompass[api]"
+  ```
+
+- #### Install OpenCompass from source
+
+  If you want to use opencompass's latest features, or develop new features, you can also build it from source
+
+  ```bash
+    git clone https://github.com/open-compass/opencompass opencompass
+    cd opencompass
+    pip install -e .
+    # pip install -e ".[full]"
+    # pip install -e ".[vllm]"
+  ```
+
+### 📂 Data Preparation
+
+You can choose one for the following method to prepare datasets.
+
+#### Offline Preparation
+
+You can download and extract the datasets with the following commands:
+
+```bash
+# Download dataset to data/ folder
+wget https://github.com/open-compass/opencompass/releases/download/0.2.2.rc1/OpenCompassData-core-20240207.zip
+unzip OpenCompassData-core-20240207.zip
+```
+
+#### Automatic Download from OpenCompass
+
+We have supported download datasets automatic from the OpenCompass storage server. You can run the evaluation with extra `--dry-run` to download these datasets.
+Currently, the supported datasets are listed in [here](https://github.com/open-compass/opencompass/blob/main/opencompass/utils/datasets_info.py#L259). More datasets will be uploaded recently.
+
+#### (Optional) Automatic Download with ModelScope
+
+Also you can use the [ModelScope](www.modelscope.cn) to load the datasets on demand.
+
+Installation:
+
+```bash
+pip install modelscope[framework]
+export DATASET_SOURCE=ModelScope
+```
+
+Then submit the evaluation task without downloading all the data to your local disk. Available datasets include:
+
+```bash
+humaneval, triviaqa, commonsenseqa, tydiqa, strategyqa, cmmlu, lambada, piqa, ceval, math, LCSTS, Xsum, winogrande, openbookqa, AGIEval, gsm8k, nq, race, siqa, mbpp, mmlu, hellaswag, ARC, BBH, xstory_cloze, summedits, GAOKAO-BENCH, OCNLI, cmnli
+```
+
+Some third-party features, like Humaneval and Llama, may require additional steps to work properly, for detailed steps please refer to the [Installation Guide](https://opencompass.readthedocs.io/en/latest/get_started/installation.html).
+
+<p align="right"><a href="#top">🔝Back to top</a></p>
+
+## 🏗️ ️Evaluation
+
+After ensuring that OpenCompass is installed correctly according to the above steps and the datasets are prepared. Now you can start your first evaluation using OpenCompass!
+
+### Your first evaluation with OpenCompass!
+
+OpenCompass support setting your configs via CLI or a python script. For simple evaluation settings we recommend using CLI, for more complex evaluation, it is suggested using the script way. You can find more example scripts under the configs folder.
+
+```bash
+# CLI
+opencompass --models hf_internlm2_5_1_8b_chat --datasets demo_gsm8k_chat_gen
+
+# Python scripts
+opencompass examples/eval_chat_demo.py
+```
+
+You can find more script examples under [examples](./examples) folder.
+
+### API evaluation
+
+OpenCompass, by its design, does not really discriminate between open-source models and API models. You can evaluate both model types in the same way or even in one settings.
+
+```bash
+export OPENAI_API_KEY="YOUR_OPEN_API_KEY"
+# CLI
+opencompass --models gpt_4o_2024_05_13 --datasets demo_gsm8k_chat_gen
+
+# Python scripts
+opencompass examples/eval_api_demo.py
+
+# You can use o1_mini_2024_09_12/o1_preview_2024_09_12  for o1 models, we set max_completion_tokens=8192 as default.
+```
+
+### Accelerated Evaluation
+
+Additionally, if you want to use an inference backend other than HuggingFace for accelerated evaluation, such as LMDeploy or vLLM, you can do so with the command below. Please ensure that you have installed the necessary packages for the chosen backend and that your model supports accelerated inference with it. For more information, see the documentation on inference acceleration backends [here](docs/en/advanced_guides/accelerator_intro.md). Below is an example using LMDeploy:
+
+```bash
+# CLI
+opencompass --models hf_internlm2_5_1_8b_chat --datasets demo_gsm8k_chat_gen -a lmdeploy
+
+# Python scripts
+opencompass examples/eval_lmdeploy_demo.py
+```
+
+### Supported Models and Datasets
+
+OpenCompass has predefined configurations for many models and datasets. You can list all available model and dataset configurations using the [tools](./docs/en/tools.md#list-configs).
+
+```bash
+# List all configurations
+python tools/list_configs.py
+# List all configurations related to llama and mmlu
+python tools/list_configs.py llama mmlu
+```
+
+#### Supported Models
+
+If the model is not on the list but supported by Huggingface AutoModel class or encapsulation of inference engine based on OpenAI interface (see [docs](https://opencompass.readthedocs.io/en/latest/advanced_guides/new_model.html) for details), you can also evaluate it with OpenCompass. You are welcome to contribute to the maintenance of the OpenCompass supported model and dataset lists.
+
+```bash
+opencompass --datasets demo_gsm8k_chat_gen --hf-type chat --hf-path internlm/internlm2_5-1_8b-chat
+```
+
+#### Supported Datasets
+
+Currently, OpenCompass have provided standard recommended configurations for datasets. Generally, config files ending with `_gen.py` or `_llm_judge_gen.py` will point to the recommended config we provide for this dataset. You can refer to [docs](https://opencompass.readthedocs.io/en/latest/dataset_statistics.html) for more details.
+
+```bash
+# Recommended Evaluation Config based on Rules
+opencompass --datasets aime2024_gen --models hf_internlm2_5_1_8b_chat
+
+# Recommended Evaluation Config based on LLM Judge
+opencompass --datasets aime2024_llmjudge_gen --models hf_internlm2_5_1_8b_chat
+```
+
+If you want to use multiple GPUs to evaluate the model in data parallel, you can use `--max-num-worker`.
+
+```bash
+CUDA_VISIBLE_DEVICES=0,1 opencompass --datasets demo_gsm8k_chat_gen --hf-type chat --hf-path internlm/internlm2_5-1_8b-chat --max-num-worker 2
+```
+
+> \[!TIP\]
+>
+> `--hf-num-gpus` is used for model parallel(huggingface format), `--max-num-worker` is used for data parallel.
+
+> \[!TIP\]
+>
+> configuration with `_ppl` is designed for base model typically.
+> configuration with `_gen` can be used for both base model and chat model.
+
+Through the command line or configuration files, OpenCompass also supports evaluating APIs or custom models, as well as more diversified evaluation strategies. Please read the [Quick Start](https://opencompass.readthedocs.io/en/latest/get_started/quick_start.html) to learn how to run an evaluation task.
+
+<p align="right"><a href="#top">🔝Back to top</a></p>
+
+## 📣 OpenCompass 2.0
+
+We are thrilled to introduce OpenCompass 2.0, an advanced suite featuring three key components: [CompassKit](https://github.com/open-compass), [CompassHub](https://hub.opencompass.org.cn/home), and [CompassRank](https://rank.opencompass.org.cn/home).
+![oc20](https://github.com/tonysy/opencompass/assets/7881589/90dbe1c0-c323-470a-991e-2b37ab5350b2)
+
+**CompassRank** has been significantly enhanced into the leaderboards that now incorporates both open-source benchmarks and proprietary benchmarks. This upgrade allows for a more comprehensive evaluation of models across the industry.
+
+**CompassHub** presents a pioneering benchmark browser interface, designed to simplify and expedite the exploration and utilization of an extensive array of benchmarks for researchers and practitioners alike. To enhance the visibility of your own benchmark within the community, we warmly invite you to contribute it to CompassHub. You may initiate the submission process by clicking [here](https://hub.opencompass.org.cn/dataset-submit).
+
+**CompassKit** is a powerful collection of evaluation toolkits specifically tailored for Large Language Models and Large Vision-language Models. It provides an extensive set of tools to assess and measure the performance of these complex models effectively. Welcome to try our toolkits for in your research and products.
+
+## ✨ Introduction
+
+![image](https://github.com/open-compass/opencompass/assets/22607038/f45fe125-4aed-4f8c-8fe8-df4efb41a8ea)
+
+OpenCompass is a one-stop platform for large model evaluation, aiming to provide a fair, open, and reproducible benchmark for large model evaluation. Its main features include:
+
+- **Comprehensive support for models and datasets**: Pre-support for 20+ HuggingFace and API models, a model evaluation scheme of 70+ datasets with about 400,000 questions, comprehensively evaluating the capabilities of the models in five dimensions.
+
+- **Efficient distributed evaluation**: One line command to implement task division and distributed evaluation, completing the full evaluation of billion-scale models in just a few hours.
+
+- **Diversified evaluation paradigms**: Support for zero-shot, few-shot, and chain-of-thought evaluations, combined with standard or dialogue-type prompt templates, to easily stimulate the maximum performance of various models.
+
+- **Modular design with high extensibility**: Want to add new models or datasets, customize an advanced task division strategy, or even support a new cluster management system? Everything about OpenCompass can be easily expanded!
+
+- **Experiment management and reporting mechanism**: Use config files to fully record each experiment, and support real-time reporting of results.
+
+## 📖 Dataset Support
+
+We have supported a statistical list of all datasets that can be used on this platform in the documentation on the OpenCompass website.
+
+You can quickly find the dataset you need from the list through sorting, filtering, and searching functions.
+
+In addition, we provide a recommended configuration for each dataset, and some datasets also support LLM Judge-based configurations.
+
+Please refer to the dataset statistics chapter of [docs](https://opencompass.readthedocs.io/en/latest/dataset_statistics.html) for details.
+
+<p align="right"><a href="#top">🔝Back to top</a></p>
+
+## 📖 Model Support
+
+<table align="center">
+  <tbody>
+    <tr align="center" valign="bottom">
+      <td>
+        <b>Open-source Models</b>
+      </td>
+      <td>
+        <b>API Models</b>
+      </td>
+      <!-- <td>
+        <b>Custom Models</b>
+      </td> -->
+    </tr>
+    <tr valign="top">
+      <td>
+
+- [Alpaca](https://github.com/tatsu-lab/stanford_alpaca)
+- [Baichuan](https://github.com/baichuan-inc)
+- [BlueLM](https://github.com/vivo-ai-lab/BlueLM)
+- [ChatGLM2](https://github.com/THUDM/ChatGLM2-6B)
+- [ChatGLM3](https://github.com/THUDM/ChatGLM3-6B)
+- [Gemma](https://huggingface.co/google/gemma-7b)
+- [InternLM](https://github.com/InternLM/InternLM)
+- [LLaMA](https://github.com/facebookresearch/llama)
+- [LLaMA3](https://github.com/meta-llama/llama3)
+- [Qwen](https://github.com/QwenLM/Qwen)
+- [TigerBot](https://github.com/TigerResearch/TigerBot)
+- [Vicuna](https://github.com/lm-sys/FastChat)
+- [WizardLM](https://github.com/nlpxucan/WizardLM)
+- [Yi](https://github.com/01-ai/Yi)
+- ……
+
+</td>
+<td>
+
+- OpenAI
+- Gemini
+- Claude
+- ZhipuAI(ChatGLM)
+- Baichuan
+- ByteDance(YunQue)
+- Huawei(PanGu)
+- 360
+- Baidu(ERNIEBot)
+- MiniMax(ABAB-Chat)
+- SenseTime(nova)
+- Xunfei(Spark)
+- ……
+
+</td>
+
+</tr>
+  </tbody>
+</table>
+
+<p align="right"><a href="#top">🔝Back to top</a></p>
+
+## 🔜 Roadmap
+
+- [x] Subjective Evaluation
+  - [x] Release CompassAreana.
+  - [x] Subjective evaluation.
+- [x] Long-context
+  - [x] Long-context evaluation with extensive datasets.
+  - [ ] Long-context leaderboard.
+- [x] Coding
+  - [ ] Coding evaluation leaderboard.
+  - [x] Non-python language evaluation service.
+- [x] Agent
+  - [ ] Support various agent frameworks.
+  - [x] Evaluation of tool use of the LLMs.
+- [x] Robustness
+  - [x] Support various attack methods.
+
+## 👷‍♂️ Contributing
+
+We appreciate all contributions to improving OpenCompass. Please refer to the [contributing guideline](https://opencompass.readthedocs.io/en/latest/notes/contribution_guide.html) for the best practice.
+
+<!-- Copy-paste in your Readme.md file -->
+
+<!-- Made with [OSS Insight](https://ossinsight.io/) -->
+
+<a href="https://github.com/open-compass/opencompass/graphs/contributors" target="_blank">
+  <table>
+    <tr>
+      <th colspan="2">
+        <br><img src="https://contrib.rocks/image?repo=open-compass/opencompass"><br><br>
+      </th>
+    </tr>
+  </table>
+</a>
+
+## 🤝 Acknowledgements
+
+Some code in this project is cited and modified from [OpenICL](https://github.com/Shark-NLP/OpenICL).
+
+Some datasets and prompt implementations are modified from [chain-of-thought-hub](https://github.com/FranxYao/chain-of-thought-hub) and [instruct-eval](https://github.com/declare-lab/instruct-eval).
+
+## 🖊️ Citation
+
+```bibtex
+@misc{2023opencompass,
+    title={OpenCompass: A Universal Evaluation Platform for Foundation Models},
+    author={OpenCompass Contributors},
+    howpublished = {\url{https://github.com/open-compass/opencompass}},
+    year={2023}
+}
+```
+
+<p align="right"><a href="#top">🔝Back to top</a></p>
+
+[github-contributors-link]: https://github.com/open-compass/opencompass/graphs/contributors
+[github-contributors-shield]: https://img.shields.io/github/contributors/open-compass/opencompass?color=c4f042&labelColor=black&style=flat-square
+[github-forks-link]: https://github.com/open-compass/opencompass/network/members
+[github-forks-shield]: https://img.shields.io/github/forks/open-compass/opencompass?color=8ae8ff&labelColor=black&style=flat-square
+[github-issues-link]: https://github.com/open-compass/opencompass/issues
+[github-issues-shield]: https://img.shields.io/github/issues/open-compass/opencompass?color=ff80eb&labelColor=black&style=flat-square
+[github-license-link]: https://github.com/open-compass/opencompass/blob/main/LICENSE
+[github-license-shield]: https://img.shields.io/github/license/open-compass/opencompass?color=white&labelColor=black&style=flat-square
+[github-release-link]: https://github.com/open-compass/opencompass/releases
+[github-release-shield]: https://img.shields.io/github/v/release/open-compass/opencompass?color=369eff&labelColor=black&logo=github&style=flat-square
+[github-releasedate-link]: https://github.com/open-compass/opencompass/releases
+[github-releasedate-shield]: https://img.shields.io/github/release-date/open-compass/opencompass?labelColor=black&style=flat-square
+[github-stars-link]: https://github.com/open-compass/opencompass/stargazers
+[github-stars-shield]: https://img.shields.io/github/stars/open-compass/opencompass?color=ffcb47&labelColor=black&style=flat-square
+[github-trending-shield]: https://trendshift.io/api/badge/repositories/6630
+[github-trending-url]: https://trendshift.io/repositories/6630