evalscope 0.5.3__tar.gz → 0.5.4__tar.gz

{evalscope-0.5.3 → evalscope-0.5.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: evalscope
-Version: 0.5.3
+Version: 0.5.4
 Summary: EvalScope: Lightweight LLMs Evaluation Framework
 Home-page: https://github.com/modelscope/evalscope
 Author: ModelScope team
@@ -19,6 +19,7 @@ Requires-Dist: torch
 Requires-Dist: absl-py
 Requires-Dist: accelerate
 Requires-Dist: cachetools
+Requires-Dist: datasets<3.0.0,>=2.18.0
 Requires-Dist: editdistance
 Requires-Dist: jsonlines
 Requires-Dist: matplotlib
@@ -42,7 +43,7 @@ Requires-Dist: simple-ddl-parser
 Requires-Dist: tabulate
 Requires-Dist: tiktoken
 Requires-Dist: tqdm
-Requires-Dist: transformers<4.43,>=4.33
+Requires-Dist: transformers>=4.33
 Requires-Dist: transformers_stream_generator
 Requires-Dist: jieba
 Requires-Dist: rouge-chinese
@@ -82,6 +83,7 @@ Requires-Dist: torch; extra == "all"
 Requires-Dist: absl-py; extra == "all"
 Requires-Dist: accelerate; extra == "all"
 Requires-Dist: cachetools; extra == "all"
+Requires-Dist: datasets<3.0.0,>=2.18.0; extra == "all"
 Requires-Dist: editdistance; extra == "all"
 Requires-Dist: jsonlines; extra == "all"
 Requires-Dist: matplotlib; extra == "all"
@@ -105,7 +107,7 @@ Requires-Dist: simple-ddl-parser; extra == "all"
 Requires-Dist: tabulate; extra == "all"
 Requires-Dist: tiktoken; extra == "all"
 Requires-Dist: tqdm; extra == "all"
-Requires-Dist: transformers<4.43,>=4.33; extra == "all"
+Requires-Dist: transformers>=4.33; extra == "all"
 Requires-Dist: transformers_stream_generator; extra == "all"
 Requires-Dist: jieba; extra == "all"
 Requires-Dist: rouge-chinese; extra == "all"
@@ -161,7 +163,9 @@ Large Model (including Large Language Models, Multi-modal Large Language Models)
 - **VLMEvalKit Integration**: Supports VLMEvalKit as the evaluation backend, facilitating the initiation of multi-modal evaluation tasks, supporting various multi-modal models and datasets.
 - **Full-Link Support**: Through seamless integration with the [ms-swift](https://github.com/modelscope/ms-swift) training framework, provides a one-stop development process for model training, model deployment, model evaluation, and report viewing, enhancing user development efficiency.
 
-### Overall Architecture
+
+<details><summary>Overall Architecture</summary>
+
 <p align="center">
   <img src="docs/en/_static/images/evalscope_framework.png" width="70%">
   <br>Fig 1. EvalScope Framework.
@@ -178,14 +182,20 @@ The architecture includes the following modules:
 4. **Performance Evaluator**: Model performance evaluation, responsible for measuring model inference service performance, including performance testing, stress testing, performance report generation, and visualization.
 5. **Evaluation Report**: The final generated evaluation report summarizes the model's performance, which can be used for decision-making and further model optimization.
 6. **Visualization**: Visualization results help users intuitively understand evaluation results, facilitating analysis and comparison of different model performances.
+</details>
+
 
 ## 🎉 News
-- **[2024.08.09]** Simplified installation process, supporting PyPI installation for vlmeval dependencies; Optimized multi-modal models evaluation experience with pipeline that based on OpenAI API, achieving up to 10x acceleration 🚀🚀🚀
-- **[2024.07.31]** Breaking change: The sdk name has been changed from `llmuses` to `evalscope`, please update the sdk name in your code.
-- **[2024.07.26]** Supports **VLMEvalKit** as a third-party evaluation framework, initiating multimodal model evaluation tasks. 🔥🔥🔥
-- **[2024.06.29]** Supports **OpenCompass** as a third-party evaluation framework. We have provided a high-level wrapper, supporting installation via pip and simplifying the evaluation task configuration. 🔥🔥🔥
-- **[2024.06.13]** EvalScope has been updated to version 0.3.x, which supports the ModelScope SWIFT framework for LLMs evaluation. 🚀🚀🚀
-- **[2024.06.13]** We have supported the ToolBench as a third-party evaluation backend for Agents evaluation. 🚀🚀🚀
+- 🔥 **[2024.09.18]** Our documentation has been updated to include a blog module, featuring some technical research and discussions related to evaluations. We invite you to [📖 read it](https://evalscope.readthedocs.io/en/refact_readme/blog/index.html).
+- 🔥 **[2024.09.12]** Support for LongWriter evaluation, which supports 10,000+ word generation. You can use the benchmark [LongBench-Write](evalscope/third_party/longbench_write/README.md) to measure the long output quality as well as the output length.
+- 🔥 **[2024.08.30]** Support for custom dataset evaluations, including text datasets and multimodal image-text datasets.
+- 🔥 **[2024.08.20]** Updated the official documentation, including getting started guides, best practices, and FAQs. Feel free to [📖read it here](https://evalscope.readthedocs.io/en/latest/)!
+- 🔥 **[2024.08.09]** Simplified the installation process, allowing for pypi installation of vlmeval dependencies; optimized the multimodal model evaluation experience, achieving up to 10x acceleration based on the OpenAI API evaluation chain.
+- 🔥 **[2024.07.31]** Important change: The package name `llmuses` has been changed to `evalscope`. Please update your code accordingly.
+- 🔥 **[2024.07.26]** Support for **VLMEvalKit** as a third-party evaluation framework to initiate multimodal model evaluation tasks.
+- 🔥 **[2024.06.29]** Support for **OpenCompass** as a third-party evaluation framework, which we have encapsulated at a higher level, supporting pip installation and simplifying evaluation task configuration.
+- 🔥 **[2024.06.13]** EvalScope seamlessly integrates with the fine-tuning framework SWIFT, providing full-chain support from LLM training to evaluation.
+- 🔥 **[2024.06.13]** Integrated the Agent evaluation dataset ToolBench.
 
 
 
@@ -265,8 +275,8 @@ If prompted with `Do you wish to run the custom code? [y/N]`, please type `y`.
 
 #### Basic Parameter Descriptions
 - `--model`: Specifies the `model_id` of the model on [ModelScope](https://modelscope.cn/), allowing automatic download. For example, see the [Qwen2-0.5B-Instruct model link](https://modelscope.cn/models/qwen/Qwen2-0.5B-Instruct/summary); you can also use a local path, such as `/path/to/model`.
-- `--template-type`: Specifies the template type corresponding to the model. Refer to the `Default Template` field in the [template table](https://swift.readthedocs.io/en/latest/LLM/Supported-models-datasets.html) for filling in this field.
-- `--datasets`: The dataset name, allowing multiple datasets to be specified, separated by spaces; these datasets will be automatically downloaded. Refer to the [supported datasets list](#supported-datasets-list) for available options.
+- `--template-type`: Specifies the template type corresponding to the model. Refer to the `Default Template` field in the [template table](https://swift.readthedocs.io/en/latest/Instruction/Supported-models-datasets.html#llm) for filling in this field.
+- `--datasets`: The dataset name, allowing multiple datasets to be specified, separated by spaces; these datasets will be automatically downloaded. Refer to the [supported datasets list](https://evalscope.readthedocs.io/en/latest/get_started/supported_dataset.html) for available options.
 
 ### 2. Parameterized Evaluation
 If you wish to conduct a more customized evaluation, such as modifying model parameters or dataset parameters, you can use the following commands:
@@ -276,8 +286,8 @@ If you wish to conduct a more customized evaluation, such as modifying model par
 python evalscope/run.py \
  --model qwen/Qwen2-0.5B-Instruct \
  --template-type qwen \
- --model-args revision=v1.0.2,precision=torch.float16,device_map=auto \
- --datasets mmlu ceval \
+ --model-args revision=master,precision=torch.float16,device_map=auto \
+ --datasets gsm8k ceval \
  --use-cache true \
  --limit 10
 ```
@@ -342,24 +352,6 @@ from evalscope.run import run_task
 run_task(task_cfg=your_task_cfg)
 ```
 
-### Supported Datasets List
-> [!NOTE]
-> The framework currently supports the following datasets. If the dataset you need is not in the list, please submit an issue, or use the [OpenCompass backend](https://evalscope.readthedocs.io/en/latest/user_guides/opencompass_backend.html) for evaluation, or use the [VLMEvalKit backend](https://evalscope.readthedocs.io/en/latest/user_guides/vlmevalkit_backend.html) for multi-modal model evaluation.
-
-| Dataset Name       | Link                                                                                   | Status | Note |
-|--------------------|----------------------------------------------------------------------------------------|--------|------|
-| `mmlu`             | [mmlu](https://modelscope.cn/datasets/modelscope/mmlu/summary)                         | Active |      |
-| `ceval`            | [ceval](https://modelscope.cn/datasets/modelscope/ceval-exam/summary)                  | Active |      |
-| `gsm8k`            | [gsm8k](https://modelscope.cn/datasets/modelscope/gsm8k/summary)                       | Active |      |
-| `arc`              | [arc](https://modelscope.cn/datasets/modelscope/ai2_arc/summary)                       | Active |      |
-| `hellaswag`        | [hellaswag](https://modelscope.cn/datasets/modelscope/hellaswag/summary)               | Active |      |
-| `truthful_qa`      | [truthful_qa](https://modelscope.cn/datasets/modelscope/truthful_qa/summary)           | Active |      |
-| `competition_math` | [competition_math](https://modelscope.cn/datasets/modelscope/competition_math/summary) | Active |      |
-| `humaneval`        | [humaneval](https://modelscope.cn/datasets/modelscope/humaneval/summary)               | Active |      |
-| `bbh`              | [bbh](https://modelscope.cn/datasets/modelscope/bbh/summary)                           | Active |      |
-| `race`             | [race](https://modelscope.cn/datasets/modelscope/race/summary)                         | Active |      |
-| `trivia_qa`        | [trivia_qa](https://modelscope.cn/datasets/modelscope/trivia_qa/summary)               | To be integrated |      |
-
 
 ## Evaluation Backend
 EvalScope supports using third-party evaluation frameworks to initiate evaluation tasks, which we call Evaluation Backend. Currently supported Evaluation Backend includes:

{evalscope-0.5.3 → evalscope-0.5.4}/README.md

@@ -47,7 +47,9 @@ Large Model (including Large Language Models, Multi-modal Large Language Models)
 - **VLMEvalKit Integration**: Supports VLMEvalKit as the evaluation backend, facilitating the initiation of multi-modal evaluation tasks, supporting various multi-modal models and datasets.
 - **Full-Link Support**: Through seamless integration with the [ms-swift](https://github.com/modelscope/ms-swift) training framework, provides a one-stop development process for model training, model deployment, model evaluation, and report viewing, enhancing user development efficiency.
 
-### Overall Architecture
+
+<details><summary>Overall Architecture</summary>
+
 <p align="center">
   <img src="docs/en/_static/images/evalscope_framework.png" width="70%">
   <br>Fig 1. EvalScope Framework.
@@ -64,14 +66,20 @@ The architecture includes the following modules:
 4. **Performance Evaluator**: Model performance evaluation, responsible for measuring model inference service performance, including performance testing, stress testing, performance report generation, and visualization.
 5. **Evaluation Report**: The final generated evaluation report summarizes the model's performance, which can be used for decision-making and further model optimization.
 6. **Visualization**: Visualization results help users intuitively understand evaluation results, facilitating analysis and comparison of different model performances.
+</details>
+
 
 ## 🎉 News
-- **[2024.08.09]** Simplified installation process, supporting PyPI installation for vlmeval dependencies; Optimized multi-modal models evaluation experience with pipeline that based on OpenAI API, achieving up to 10x acceleration 🚀🚀🚀
-- **[2024.07.31]** Breaking change: The sdk name has been changed from `llmuses` to `evalscope`, please update the sdk name in your code.
-- **[2024.07.26]** Supports **VLMEvalKit** as a third-party evaluation framework, initiating multimodal model evaluation tasks. 🔥🔥🔥
-- **[2024.06.29]** Supports **OpenCompass** as a third-party evaluation framework. We have provided a high-level wrapper, supporting installation via pip and simplifying the evaluation task configuration. 🔥🔥🔥
-- **[2024.06.13]** EvalScope has been updated to version 0.3.x, which supports the ModelScope SWIFT framework for LLMs evaluation. 🚀🚀🚀
-- **[2024.06.13]** We have supported the ToolBench as a third-party evaluation backend for Agents evaluation. 🚀🚀🚀
+- 🔥 **[2024.09.18]** Our documentation has been updated to include a blog module, featuring some technical research and discussions related to evaluations. We invite you to [📖 read it](https://evalscope.readthedocs.io/en/refact_readme/blog/index.html).
+- 🔥 **[2024.09.12]** Support for LongWriter evaluation, which supports 10,000+ word generation. You can use the benchmark [LongBench-Write](evalscope/third_party/longbench_write/README.md) to measure the long output quality as well as the output length.
+- 🔥 **[2024.08.30]** Support for custom dataset evaluations, including text datasets and multimodal image-text datasets.
+- 🔥 **[2024.08.20]** Updated the official documentation, including getting started guides, best practices, and FAQs. Feel free to [📖read it here](https://evalscope.readthedocs.io/en/latest/)!
+- 🔥 **[2024.08.09]** Simplified the installation process, allowing for pypi installation of vlmeval dependencies; optimized the multimodal model evaluation experience, achieving up to 10x acceleration based on the OpenAI API evaluation chain.
+- 🔥 **[2024.07.31]** Important change: The package name `llmuses` has been changed to `evalscope`. Please update your code accordingly.
+- 🔥 **[2024.07.26]** Support for **VLMEvalKit** as a third-party evaluation framework to initiate multimodal model evaluation tasks.
+- 🔥 **[2024.06.29]** Support for **OpenCompass** as a third-party evaluation framework, which we have encapsulated at a higher level, supporting pip installation and simplifying evaluation task configuration.
+- 🔥 **[2024.06.13]** EvalScope seamlessly integrates with the fine-tuning framework SWIFT, providing full-chain support from LLM training to evaluation.
+- 🔥 **[2024.06.13]** Integrated the Agent evaluation dataset ToolBench.
 
 
 
@@ -151,8 +159,8 @@ If prompted with `Do you wish to run the custom code? [y/N]`, please type `y`.
 
 #### Basic Parameter Descriptions
 - `--model`: Specifies the `model_id` of the model on [ModelScope](https://modelscope.cn/), allowing automatic download. For example, see the [Qwen2-0.5B-Instruct model link](https://modelscope.cn/models/qwen/Qwen2-0.5B-Instruct/summary); you can also use a local path, such as `/path/to/model`.
-- `--template-type`: Specifies the template type corresponding to the model. Refer to the `Default Template` field in the [template table](https://swift.readthedocs.io/en/latest/LLM/Supported-models-datasets.html) for filling in this field.
-- `--datasets`: The dataset name, allowing multiple datasets to be specified, separated by spaces; these datasets will be automatically downloaded. Refer to the [supported datasets list](#supported-datasets-list) for available options.
+- `--template-type`: Specifies the template type corresponding to the model. Refer to the `Default Template` field in the [template table](https://swift.readthedocs.io/en/latest/Instruction/Supported-models-datasets.html#llm) for filling in this field.
+- `--datasets`: The dataset name, allowing multiple datasets to be specified, separated by spaces; these datasets will be automatically downloaded. Refer to the [supported datasets list](https://evalscope.readthedocs.io/en/latest/get_started/supported_dataset.html) for available options.
 
 ### 2. Parameterized Evaluation
 If you wish to conduct a more customized evaluation, such as modifying model parameters or dataset parameters, you can use the following commands:
@@ -162,8 +170,8 @@ If you wish to conduct a more customized evaluation, such as modifying model par
 python evalscope/run.py \
  --model qwen/Qwen2-0.5B-Instruct \
  --template-type qwen \
- --model-args revision=v1.0.2,precision=torch.float16,device_map=auto \
- --datasets mmlu ceval \
+ --model-args revision=master,precision=torch.float16,device_map=auto \
+ --datasets gsm8k ceval \
  --use-cache true \
  --limit 10
 ```
@@ -228,24 +236,6 @@ from evalscope.run import run_task
 run_task(task_cfg=your_task_cfg)
 ```
 
-### Supported Datasets List
-> [!NOTE]
-> The framework currently supports the following datasets. If the dataset you need is not in the list, please submit an issue, or use the [OpenCompass backend](https://evalscope.readthedocs.io/en/latest/user_guides/opencompass_backend.html) for evaluation, or use the [VLMEvalKit backend](https://evalscope.readthedocs.io/en/latest/user_guides/vlmevalkit_backend.html) for multi-modal model evaluation.
-
-| Dataset Name       | Link                                                                                   | Status | Note |
-|--------------------|----------------------------------------------------------------------------------------|--------|------|
-| `mmlu`             | [mmlu](https://modelscope.cn/datasets/modelscope/mmlu/summary)                         | Active |      |
-| `ceval`            | [ceval](https://modelscope.cn/datasets/modelscope/ceval-exam/summary)                  | Active |      |
-| `gsm8k`            | [gsm8k](https://modelscope.cn/datasets/modelscope/gsm8k/summary)                       | Active |      |
-| `arc`              | [arc](https://modelscope.cn/datasets/modelscope/ai2_arc/summary)                       | Active |      |
-| `hellaswag`        | [hellaswag](https://modelscope.cn/datasets/modelscope/hellaswag/summary)               | Active |      |
-| `truthful_qa`      | [truthful_qa](https://modelscope.cn/datasets/modelscope/truthful_qa/summary)           | Active |      |
-| `competition_math` | [competition_math](https://modelscope.cn/datasets/modelscope/competition_math/summary) | Active |      |
-| `humaneval`        | [humaneval](https://modelscope.cn/datasets/modelscope/humaneval/summary)               | Active |      |
-| `bbh`              | [bbh](https://modelscope.cn/datasets/modelscope/bbh/summary)                           | Active |      |
-| `race`             | [race](https://modelscope.cn/datasets/modelscope/race/summary)                         | Active |      |
-| `trivia_qa`        | [trivia_qa](https://modelscope.cn/datasets/modelscope/trivia_qa/summary)               | To be integrated |      |
-
 
 ## Evaluation Backend
 EvalScope supports using third-party evaluation frameworks to initiate evaluation tasks, which we call Evaluation Backend. Currently supported Evaluation Backend includes:

{evalscope-0.5.3 → evalscope-0.5.4}/evalscope/backend/opencompass/backend_manager.py

@@ -242,4 +242,6 @@ if __name__ == '__main__':
                 'limit': 5
                 }
     )
+    all_datasets = OpenCompassBackendManager.list_datasets()
+    print(f'all_datasets: {all_datasets}')
     oc_backend_manager.run()

{evalscope-0.5.3 → evalscope-0.5.4}/evalscope/backend/vlm_eval_kit/backend_manager.py

@@ -1,5 +1,5 @@
 from typing import Optional, Union
-from evalscope.utils import is_module_installed, get_module_path, get_valid_list, yaml_to_dict, json_to_dict
+from evalscope.utils import is_module_installed, get_valid_list
 from evalscope.backend.base import BackendManager
 from evalscope.utils.logger import get_logger
 from functools import partial

{evalscope-0.5.3 → evalscope-0.5.4}/evalscope/benchmarks/benchmark.py

@@ -46,7 +46,7 @@ class Benchmark(object):
 
             dataset.dataset_name = dataset_name.split('/')[-1]
             dataset.subset_name = subset
-            dataset.split = split
+            # dataset.split = split
             return dataset
         elif hub == 'HuggingFace':
             # TODO: implement this by xingjun.wxj@alibaba-inc.com

{evalscope-0.5.3 → evalscope-0.5.4}/evalscope/evaluator/evaluator.py

@@ -244,8 +244,8 @@ class Evaluator(object):
                 answer_d[AnswerKeys.ORIGIN_PROMPT] = input_prompt
 
                 if debug:
-                    logger.debug(f'**input_prompt: {json.dumps(input_prompt, ensure_ascii=False)} \n')
-                    logger.debug(f'**predicted ans: {json.dumps(answer_d, ensure_ascii=False)} \n')
+                    logger.info(f'**input_prompt: {json.dumps(input_prompt, ensure_ascii=False)} \n')
+                    logger.info(f'**predicted ans: {json.dumps(answer_d, ensure_ascii=False)} \n')
 
                 answers_list.append(answer_d)
 
@@ -349,7 +349,7 @@ class Evaluator(object):
             review_d = self._get_review(answer_d=answer_d, review_id=review_id, reviewer_spec=reviewer_spec)
 
             if debug:
-                logger.debug(review_d)
+                logger.info(review_d)
 
             reviews_list.append(review_d)
 

evalscope-0.5.4/evalscope/models/api/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from evalscope.models.api.openai_api import OpenaiApi

evalscope-0.5.4/evalscope/models/api/openai_api.py

@@ -0,0 +1,228 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import json
+import threading
+import time
+from asyncio import Queue
+
+import requests
+from typing import Union, List, Optional, Dict
+from concurrent.futures import ThreadPoolExecutor
+from modelscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+class OpenaiApi:
+
+    def __init__(self,
+                 model: str,
+                 openai_api_key,
+                 openai_api_base,
+                 logprobs: Optional[bool] = False,
+                 top_logprobs: Optional[int] = None,
+                 max_new_tokens: int = 4096,
+                 temperature: Optional[float] = 0.0,
+                 repetition_penalty: Optional[float] = 1.0,
+                 is_chat: bool = True,
+                 verbose: bool = True,
+                 retry: int = 3,
+                 query_per_second: int = 10,     # TODO
+                 **kwargs):
+
+        self.temperature = temperature
+        self.repetition_penalty = repetition_penalty
+        self.max_tokens = max_new_tokens
+        self.logprobs = logprobs
+        self.top_logprobs = top_logprobs
+
+        self.openai_api_key = openai_api_key
+        self.url = openai_api_base
+        self.model = model
+        self.is_chat = is_chat
+        self.retry = retry
+        self.verbose = verbose
+
+        self.token_bucket = TokenBucket(query_per_second, verbose)
+
+    def generate_simple(self, inputs: Union[List[str]]):
+
+        def process_one(in_data: str):
+
+            if self.is_chat:
+                data = dict(
+                    model=self.model,
+                    messages=[{'role': 'user', 'content': in_data}],
+                    max_tokens=self.max_tokens,
+                    n=1,
+                    logprobs=self.logprobs,
+                    top_logprobs=self.top_logprobs,
+                    stop=None,
+                    temperature=self.temperature,
+                    repetition_penalty=self.repetition_penalty,
+                )
+            else:
+                data = dict(
+                    model=self.model,
+                    prompt=in_data,
+                    max_tokens=self.max_tokens,
+                    temperature=self.temperature,
+                    repetition_penalty=self.repetition_penalty,
+                )
+
+            # todo
+            openai_api_key = self.openai_api_key or ''
+            header = {'Authorization': f'Bearer ', 'content-type': 'application/json', }
+            data = json.dumps(data, ensure_ascii=False)
+
+            if self.verbose:
+                print(f'>>data in generate_simple: {data}')
+
+            resp = requests.post(self.url, headers=header, data=data)
+            resp = resp.json()
+            if self.verbose:
+                print(f'>>resp in generate_simple: {resp}')
+
+            if self.logprobs:
+                return resp['choices']
+            else:
+                if self.is_chat:
+                    return resp['choices'][0]['message']['content'].strip()
+                else:
+                    return resp['choices'][0]['text'].strip()
+
+        with ThreadPoolExecutor() as executor:
+            results = list(executor.map(process_one, inputs))
+
+        return results
+
+    def generate(self,
+                 inputs: Union[List[str], List[List]],
+                 **kwargs) -> List[str]:
+        """
+        Generate responses from OpenAI API.
+
+        Args:
+            inputs: The input messages for the model. It can be a string or a list of messages.
+                e.g. ['who are you ?', 'what is your name ?']
+                e.g. [[{'role': 'user', 'content': 'who are you ?'}], ...]
+            kwargs: The optional arguments for the model.
+        """
+        results = []
+        # with ThreadPoolExecutor() as executor:
+        #     results = list(executor.map(self._generate, inputs))
+
+        for input in inputs:
+            results.append(self._generate(input))
+
+        return results
+
+    def _generate(self, messages: Union[str, List[Dict]]) -> str:
+
+        if isinstance(messages, str):
+            messages = [{'role': 'user', 'content': messages}]
+
+        max_num_retries = 0
+        while max_num_retries < self.retry:
+            # self.wait()
+
+            header = {
+                'Authorization': f'Bearer {self.openai_api_key}',
+                'content-type': 'application/json',
+            }
+
+            try:
+                if self.is_chat:
+                    data = dict(
+                        model=self.model,
+                        messages=messages,
+                        max_tokens=self.max_tokens,
+                        n=1,
+                        logprobs=self.logprobs,
+                        top_logprobs=self.top_logprobs,
+                        stop=None,
+                        temperature=self.temperature,
+                        repetition_penalty=self.repetition_penalty,
+                    )
+                else:
+                    # TODO: This is a temporary solution for non-chat models.
+                    input_prompts = []
+                    for msg in messages:
+                        input_prompts.append(msg['content'])
+
+                    data = dict(
+                        model=self.model,
+                        prompt='\n'.join(input_prompts),
+                        max_tokens=self.max_tokens,
+                        temperature=self.temperature,
+                        repetition_penalty=self.repetition_penalty,
+                    )
+
+                def remove_none_val(input_d: dict):
+                    return {k: v for k, v in input_d.items() if v is not None}
+                data = remove_none_val(data)
+
+                if self.verbose:
+                    logger.info(f'>> Post data: {json.dumps(data, ensure_ascii=False)}')
+                raw_response = requests.post(self.url,
+                                             headers=header,
+                                             data=json.dumps(data, ensure_ascii=False))
+
+                response = raw_response.json()
+                if self.verbose:
+                    logger.info(f'>> response: {response}')
+
+                if self.logprobs:
+                    return response['choices']
+                else:
+                    if self.is_chat:
+                        return response['choices'][0]['message']['content'].strip()
+                    else:
+                        return response['choices'][0]['text'].strip()
+
+            except Exception as e:
+                logger.error(f'Error occurs: {str(e)}')
+                max_num_retries += 1
+                continue
+
+    def wait(self):
+        return self.token_bucket.get_token()
+
+
+class TokenBucket:
+    """A token bucket for rate limiting.
+
+    Args:
+        query_per_second (float): The rate of the token bucket.
+    """
+
+    def __init__(self, rate, verbose=False):
+        self._rate = rate
+        self._tokens = threading.Semaphore(0)
+        self.started = False
+        self._request_queue = Queue()
+        self.logger = get_logger()
+        self.verbose = verbose
+
+    def _add_tokens(self):
+        """Add tokens to the bucket."""
+        while True:
+            if self._tokens._value < self._rate:
+                self._tokens.release()
+            time.sleep(1 / self._rate)
+
+    def get_token(self):
+        """Get a token from the bucket."""
+        if not self.started:
+            self.started = True
+            threading.Thread(target=self._add_tokens, daemon=True).start()
+        self._tokens.acquire()
+        if self.verbose:
+            cur_time = time.time()
+            while not self._request_queue.empty():
+                if cur_time - self._request_queue.queue[0] > 60:
+                    self._request_queue.get()
+                else:
+                    break
+            self._request_queue.put(cur_time)
+            self.logger.info(f'Current RPM {self._request_queue.qsize()}.')

{evalscope-0.5.3 → evalscope-0.5.4}/evalscope/perf/http_client.py

@@ -51,15 +51,15 @@ UNLIMITED_RATE = -1
 
 
 async def on_request_start(session, context, params):
-    logger.debug(f'Starting request: <{params}>')
+    logger.info(f'Starting request: <{params}>')
 
 
 async def on_request_chunk_sent(session, context, params):
-    logger.debug(f'Request body: {params}')
+    logger.info(f'Request body: {params}')
 
 
 async def on_response_chunk_received(session, context, params):
-    logger.debug(f'Response info: <{params}>')
+    logger.info(f'Response info: <{params}>')
 
 
 class AioHttpClient:
@@ -116,7 +116,7 @@ class AioHttpClient:
                 line = line.decode("utf8")
                 line = line.rstrip("\n").rstrip("\r")
                 if self.debug:
-                    logger.debug(line)
+                    logger.info(line)
                 sse_msg = ServerSentEvent.decode(line)
                 if not sse_msg:
                     continue
@@ -567,7 +567,7 @@ async def send_requests_worker(task_id, request_queue: asyncio.Queue, benchmark_
                     else:
                         if response_data:
                             collected_messages.append(response_data)  # save the message
-                            logger.debug(response_data)
+                            logger.info(response_data)
                             benchmark_data["chunk_times"].append(time.perf_counter())
 
                 benchmark_data["response_messages"] = collected_messages

evalscope-0.5.4/evalscope/third_party/longbench_write/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from evalscope.third_party.longbench_write.longbench_write import run_task