evalscope 0.8.1__py3-none-any.whl → 0.8.2__py3-none-any.whl

evalscope/config.py

@@ -114,7 +114,9 @@ class TaskConfig:
     def from_args(args: Namespace):
         # Convert Namespace to a dictionary and filter out None values
         args_dict = {k: v for k, v in vars(args).items() if v is not None}
-        del args_dict['func']  # Note: compat CLI arguments
+
+        if 'func' in args_dict:
+            del args_dict['func']  # Note: compat CLI arguments
 
         return TaskConfig.from_dict(args_dict)
 

evalscope/evaluator/evaluator.py

@@ -86,6 +86,7 @@ class Evaluator(object):
             **kwargs)
 
         # Get prompts from dataset
+        # TODO: support sampler
         self.prompts = self.data_adapter.gen_prompts(data_dict=self.dataset)
         del self.dataset
 

evalscope/models/model_adapter.py

@@ -429,7 +429,7 @@ class ChatGenerationModelAdapter(BaseModelAdapter):
         fix_do_sample_warning(self.generation_config)
 
         # Run inference
-        output_ids = self.model.generate(**inputs, generation_config=self.generation_config)
+        output_ids = self.model.generate(input_ids, generation_config=self.generation_config)
 
         response = self.tokenizer.decode(output_ids[0, len(input_ids[0]):], skip_special_tokens=True)
         return response

evalscope/perf/arguments.py

@@ -68,6 +68,7 @@ class Arguments:
             model=args.model,
             attn_implementation=args.attn_implementation,
             url=args.url,
+            port=args.port,
             api_key=args.api_key,
             connect_timeout=args.connect_timeout,
             read_timeout=args.read_timeout,

evalscope/perf/benchmark.py

@@ -157,7 +157,7 @@ async def statistic_benchmark_metric_worker(benchmark_data_queue: asyncio.Queue,
             while not (data_process_completed_event.is_set() and benchmark_data_queue.empty()):
                 try:
                     # Attempt to get benchmark data from the queue with a timeout
-                    benchmark_data = await asyncio.wait_for(benchmark_data_queue.get(), timeout=1)
+                    benchmark_data = await asyncio.wait_for(benchmark_data_queue.get(), timeout=0.01)
                     benchmark_data_queue.task_done()
                 except asyncio.TimeoutError:
                     # If timeout, continue to the next iteration

evalscope/perf/main.py

@@ -19,7 +19,9 @@ def run_perf_benchmark(args):
         args = Arguments(**args)
     elif isinstance(args, Namespace):
         args = Arguments.from_args(args)
-    seed_everything(args.seed)
+
+    if args.seed is not None:
+        seed_everything(args.seed)
 
     # Setup logger and output
     args.outputs_dir = get_output_path(args)

evalscope/perf/plugin/api/openai_api.py

@@ -96,60 +96,64 @@ class OpenaiPlugin(ApiPluginBase):
 
     def parse_responses(self, responses, request: Any = None, **kwargs) -> Dict:
         """Parser responses and return number of request and response tokens.
-           sample of the output delta:
-           {"id":"4","object":"chat.completion.chunk","created":1714030870,"model":"llama3","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}]}
+        Only one response for non-stream, multiple responses for stream.
+        """
 
+        # when stream, the last response is the full usage
+        # when non-stream, the last response is the first response
+        last_response_js = json.loads(responses[-1])
+        if 'usage' in last_response_js and last_response_js['usage']:
+            input_tokens = last_response_js['usage']['prompt_tokens']
+            output_tokens = last_response_js['usage']['completion_tokens']
+            return input_tokens, output_tokens
 
-        Args:
-            responses (List[bytes]): List of http response body, for stream output,
-                there are multiple responses, for general only one.
-            kwargs: (Any): The command line --parameter content.
-        Returns:
-            Tuple: Return number of prompt token and number of completion tokens.
-        """
-        full_response_content = ''
+        # no usage information in the response, parse the response to get the tokens
         delta_contents = {}
-        input_tokens = None
-        output_tokens = None
         for response in responses:
             js = json.loads(response)
-            if js['object'] == 'chat.completion':
-                for choice in js['choices']:
-                    delta_contents[choice['index']] = [choice['message']['content']]
-                input_tokens = js['usage']['prompt_tokens']
-                output_tokens = js['usage']['completion_tokens']
-            elif js['object'] == 'text_completion':
-                for choice in js['choices']:
-                    delta_contents[choice['index']] = [choice['text']]
-                input_tokens = js['usage']['prompt_tokens']
-                output_tokens = js['usage']['completion_tokens']
-            elif js['object'] == 'chat.completion.chunk':
-                if 'choices' in js:
-                    for choice in js['choices']:
-                        if 'delta' in choice and 'index' in choice:
-                            delta = choice['delta']
-                            idx = choice['index']
-                            if 'content' in delta:
-                                delta_content = delta['content']
-                                if idx in delta_contents:
-                                    delta_contents[idx].append(delta_content)
-                                else:
-                                    delta_contents[idx] = [delta_content]
-                # usage in chunk: {"id":"","object":"chat.completion.chunk","created":1718269986,"model":"llama3",
-                # "choices":[],"usage":{"prompt_tokens":32,"total_tokens":384,"completion_tokens":352}}
-                if 'usage' in js and js['usage']:
-                    input_tokens = js['usage']['prompt_tokens']
-                    output_tokens = js['usage']['completion_tokens']
-        if (input_tokens is None and output_tokens is None and self.tokenizer is not None):
-            input_tokens = 0
-            output_tokens = 0
+            if 'object' in js:
+                self.__process_response_object(js, delta_contents)
+            else:
+                self.__process_no_object(js, delta_contents)
+
+        input_tokens, output_tokens = self.__calculate_tokens_from_content(request, delta_contents)
+        return input_tokens, output_tokens
+
+    def __process_response_object(self, js, delta_contents):
+        if js['object'] == 'chat.completion':
+            for choice in js['choices']:
+                delta_contents[choice['index']] = [choice['message']['content']]
+        elif js['object'] == 'text_completion':
+            for choice in js['choices']:
+                delta_contents[choice['index']] = [choice['text']]
+        elif js['object'] == 'chat.completion.chunk':
+            for choice in js.get('choices', []):
+                if 'delta' in choice and 'index' in choice:
+                    delta = choice['delta']
+                    idx = choice['index']
+                    if 'content' in delta:
+                        delta_content = delta['content']
+                        delta_contents.setdefault(idx, []).append(delta_content)
+
+    def __process_no_object(self, js, delta_contents):
+        #  assume the response is a single choice
+        for choice in js['choices']:
+            if 'delta' in choice:
+                delta = choice['delta']
+                idx = choice['index']
+                if 'content' in delta:
+                    delta_content = delta['content']
+                    delta_contents.setdefault(idx, []).append(delta_content)
+            else:
+                delta_contents[choice['index']] = [choice['message']['content']]
+
+    def __calculate_tokens_from_content(self, request, delta_contents):
+        input_tokens = output_tokens = 0
+        if self.tokenizer is not None:
             for idx, choice_contents in delta_contents.items():
-                full_response_content = ''.join([m for m in choice_contents])
+                full_response_content = ''.join(choice_contents)
                 input_tokens += len(self.tokenizer.encode(request['messages'][0]['content']))
                 output_tokens += len(self.tokenizer.encode(full_response_content))
-        elif input_tokens is None and output_tokens is None:  # no usage info get.
-            input_tokens = 0
-            output_tokens = 0
+        else:
             logger.warning('No usage information found. Please specify `--tokenizer-path` to generate usage details.')
-
         return input_tokens, output_tokens

evalscope/perf/utils/local_server.py

@@ -103,6 +103,7 @@ def start_app(args: Arguments):
     elif args.api == 'local_vllm':
         os.environ['VLLM_USE_MODELSCOPE'] = 'True'
         os.environ['VLLM_ALLOW_LONG_MAX_MODEL_LEN'] = '1'
+        os.environ['VLLM_WORKER_MULTIPROC_METHOD'] = 'spawn'
         # yapf: disable
         proc = subprocess.Popen([
             'python', '-m', 'vllm.entrypoints.openai.api_server',

evalscope/version.py

@@ -1,4 +1,4 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-__version__ = '0.8.1'
-__release_datetime__ = '2024-12-17 20:00:00'
+__version__ = '0.8.2'
+__release_datetime__ = '2024-12-26 20:00:00'

{evalscope-0.8.1.dist-info → evalscope-0.8.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: evalscope
-Version: 0.8.1
+Version: 0.8.2
 Summary: EvalScope: Lightweight LLMs Evaluation Framework
 Home-page: https://github.com/modelscope/evalscope
 Author: ModelScope team
@@ -84,7 +84,7 @@ Requires-Dist: transformers-stream-generator; extra == "all"
 Requires-Dist: ms-opencompass>=0.1.4; extra == "all"
 Requires-Dist: ms-vlmeval>=0.0.9; extra == "all"
 Requires-Dist: mteb==1.19.4; extra == "all"
-Requires-Dist: ragas==0.2.7; extra == "all"
+Requires-Dist: ragas==0.2.9; extra == "all"
 Requires-Dist: webdataset>0.2.0; extra == "all"
 Requires-Dist: aiohttp; extra == "all"
 Requires-Dist: fastapi; extra == "all"
@@ -129,7 +129,7 @@ Requires-Dist: transformers; extra == "perf"
 Requires-Dist: unicorn; extra == "perf"
 Provides-Extra: rag
 Requires-Dist: mteb==1.19.4; extra == "rag"
-Requires-Dist: ragas==0.2.7; extra == "rag"
+Requires-Dist: ragas==0.2.9; extra == "rag"
 Requires-Dist: webdataset>0.2.0; extra == "rag"
 Provides-Extra: vlmeval
 Requires-Dist: ms-vlmeval>=0.0.9; extra == "vlmeval"
@@ -181,6 +181,8 @@ The framework accommodates multiple evaluation scenarios such as end-to-end RAG
   <br>EvalScope Framework.
 </p>
 
+<details><summary>Framework Description</summary>
+
 The architecture includes the following modules:
 1. **Model Adapter**: The model adapter is used to convert the outputs of specific models into the format required by the framework, supporting both API call models and locally run models.
 2. **Data Adapter**: The data adapter is responsible for converting and processing input data to meet various evaluation needs and formats.
@@ -194,6 +196,16 @@ The architecture includes the following modules:
 5. **Evaluation Report**: The final generated evaluation report summarizes the model's performance, which can be used for decision-making and further model optimization.
 6. **Visualization**: Visualization results help users intuitively understand evaluation results, facilitating analysis and comparison of different model performances.
 
+</details>
+
+## ☎ User Groups
+
+Please scan the QR code below to join our community groups:
+
+[Discord Group](https://discord.com/invite/D27yfEFVz5)              |  WeChat Group | DingTalk Group
+:-------------------------:|:-------------------------:|:-------------------------:
+<img src="docs/asset/discord_qr.jpg" width="160" height="160">  |  <img src="docs/asset/wechat.png" width="160" height="160"> | <img src="docs/asset/dingding.png" width="160" height="160">
+
 
 ## 🎉 News
 - 🔥 **[2024.12.13]** Model evaluation optimization: no need to pass the `--template-type` parameter anymore; supports starting evaluation with `evalscope eval --args`. Refer to the [📖 User Guide](https://evalscope.readthedocs.io/en/latest/get_started/basic_usage.html) for more details.

{evalscope-0.8.1.dist-info → evalscope-0.8.2.dist-info}/RECORD

@@ -1,11 +1,11 @@
 evalscope/__init__.py,sha256=RY0EjssSquqqsysRobElYm9Ix6E41uTXeaeh7lI7kqs,106
 evalscope/arguments.py,sha256=nozBnog45l77jxTFH_lyyJkj04ER3yyIpICepc2tC1Y,3783
-evalscope/config.py,sha256=ZDN0XVCCXMSSD675Smzm57fNDOx-cZTsNvPboMtYVow,8407
+evalscope/config.py,sha256=_4IRpoAssdHEg75UKPKVw6FVaCu2NaP2aOMA5DRsuGU,8444
 evalscope/constants.py,sha256=M5qJ8b7kp-RF52IwBjx5EMjeuiH1e1jdollCsbIT-c4,3753
 evalscope/run.py,sha256=s_qE1ukrt4HBfRVAPJjC1XiqD9k7rSH7lX8yysyf5do,7279
 evalscope/run_arena.py,sha256=6nc_S8KL7B3V4SsnpIexfvczHN9kQwHR9R1GXb2sqgI,8586
 evalscope/summarizer.py,sha256=FgdYz7LlNs5XpDMlj2ULkVQGIg5XVeeWdWJ1_OMweq0,5882
-evalscope/version.py,sha256=OXwZDg6ML1mbsIw-CBhWRf4zVz2ArW2PFzzLK9FVAZk,118
+evalscope/version.py,sha256=uvEbCM3fC0oZ2Rt82Q0oErXsM-iYBNxJtPPLXPwscAU,118
 evalscope/backend/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 evalscope/backend/base.py,sha256=qYu8Shokrtrx-N6T_BAJk_6OCpovUBYuN0p3wngt-dw,1030
 evalscope/backend/opencompass/__init__.py,sha256=UP_TW5KBq6V_Nvqkeb7PGvGGX3rVYussT43npwCwDgE,135
@@ -42,26 +42,6 @@ evalscope/backend/rag_eval/ragas/__init__.py,sha256=D0yJkN9SuNGIAL3niZw4BI08Yh3H
 evalscope/backend/rag_eval/ragas/arguments.py,sha256=8SYCV15d25ocdDHRqmGMQzd9zR6gwfOrVSFBe4T-KCo,1806
 evalscope/backend/rag_eval/ragas/task_template.py,sha256=a_3bWfLx0j2zJkWgEWNStO0XXAeUFdnFpeukpoGfxLg,1669
 evalscope/backend/rag_eval/ragas/prompts/persona_prompt.py,sha256=fX9sCci787ViGiL3BhGsykx0bnWfOWWEFueaJKyR8g4,793
-evalscope/backend/rag_eval/ragas/prompts/chinese/AnswerCorrectness/correctness_prompt_chinese.json,sha256=4wPfjNh-OVFQdvho3CAJ66_B2TZuRZVm6-xUIXokKcY,3935
-evalscope/backend/rag_eval/ragas/prompts/chinese/AnswerCorrectness/long_form_answer_prompt_chinese.json,sha256=wWidnp8726hf6-fY31ZoqCt9zhZgVM260o8MwdBI0d8,1737
-evalscope/backend/rag_eval/ragas/prompts/chinese/AnswerRelevancy/question_generation_chinese.json,sha256=o5RXPz-O1JM8gFRCLCY2iobh0uLc4mznT_zLCpWaPFE,968
-evalscope/backend/rag_eval/ragas/prompts/chinese/ContextPrecision/context_precision_prompt_chinese.json,sha256=eEs6gdAKuYfDohCz9EzM1o0ykIEUbvwoRu1Pd2dL92E,3168
-evalscope/backend/rag_eval/ragas/prompts/chinese/CustomNodeFilter/scoring_prompt_chinese.json,sha256=qZhHR9Ki374Ykb6V8dmptE1whXmPKRvAJ0Gl2akoaX0,216
-evalscope/backend/rag_eval/ragas/prompts/chinese/Faithfulness/nli_statements_message_chinese.json,sha256=k5LjoxcIDM9Yvj0h5bje6ANXEOgFbioRs1i23259Md8,2486
-evalscope/backend/rag_eval/ragas/prompts/chinese/Faithfulness/statement_prompt_chinese.json,sha256=Pn1rGIjfyIeY6BZQEOeR4v-QC5xcmTN6aIh0G2E2Xuo,1740
-evalscope/backend/rag_eval/ragas/prompts/chinese/HeadlinesExtractor/prompt_chinese.json,sha256=p7RrFdNWY1Wo5s03SvtXQSZ-CEn96NkPZ3EHsJ3UIFE,1137
-evalscope/backend/rag_eval/ragas/prompts/chinese/MultiHopAbstractQuerySynthesizer/concept_combination_prompt_chinese.json,sha256=s2mlf9BTWnmnCZ9H3yLZgPvPUPWnPgIIDtRtH0qStMM,991
-evalscope/backend/rag_eval/ragas/prompts/chinese/MultiHopAbstractQuerySynthesizer/generate_query_reference_prompt_chinese.json,sha256=s_6K6surhTGpr5efryHjW-PFDKlYJTTpgXDlC_TbzVw,1943
-evalscope/backend/rag_eval/ragas/prompts/chinese/MultiHopAbstractQuerySynthesizer/theme_persona_matching_prompt_chinese.json,sha256=XmHkc-bj7PFdLxGKoM3UDeOv2FO0X2Pc9Wpd6JOkdns,919
-evalscope/backend/rag_eval/ragas/prompts/chinese/MultiHopSpecificQuerySynthesizer/generate_query_reference_prompt_chinese.json,sha256=p-vCyibNNezGcuID2kGvBDZJGdPXm3NvTTVvH6ij7N4,1973
-evalscope/backend/rag_eval/ragas/prompts/chinese/MultiHopSpecificQuerySynthesizer/theme_persona_matching_prompt_chinese.json,sha256=XmHkc-bj7PFdLxGKoM3UDeOv2FO0X2Pc9Wpd6JOkdns,919
-evalscope/backend/rag_eval/ragas/prompts/chinese/MultiModalFaithfulness/faithfulness_prompt_chinese.json,sha256=yayuzrNO2EO9eIqSv5mthNTVXnw_7D_HOJZ_tse-qw0,1374
-evalscope/backend/rag_eval/ragas/prompts/chinese/MultiModalRelevance/relevance_prompt_chinese.json,sha256=-rOBZuhZGbVrlti3PycavxAoInEry3dMYt9VN3Qvo-E,1475
-evalscope/backend/rag_eval/ragas/prompts/chinese/NERExtractor/prompt_chinese.json,sha256=svZ_xzfQp3KMzdVJoqTVPGnwgls2JjXXplTcUj1jVFo,767
-evalscope/backend/rag_eval/ragas/prompts/chinese/SingleHopSpecificQuerySynthesizer/generate_query_reference_prompt_chinese.json,sha256=VRO9Hy-e5Dba1AkLqxj2R-Ezwoby3BvipM9zNlZJ4GY,1328
-evalscope/backend/rag_eval/ragas/prompts/chinese/SingleHopSpecificQuerySynthesizer/theme_persona_matching_prompt_chinese.json,sha256=XmHkc-bj7PFdLxGKoM3UDeOv2FO0X2Pc9Wpd6JOkdns,919
-evalscope/backend/rag_eval/ragas/prompts/chinese/SummaryExtractor/prompt_chinese.json,sha256=1YVcklCc4otS0mkO0aiNNFx7Zecc1L3wB6ol3NPxTt0,697
-evalscope/backend/rag_eval/ragas/prompts/chinese/ThemesExtractor/prompt_chinese.json,sha256=c70_FGepQm3_dZngdjNudX_iCmu39tvZncyBqNxMrfg,658
 evalscope/backend/rag_eval/ragas/tasks/__init__.py,sha256=hErdWKbvV9aRqOpQTzdFHw1tcYoDbnttmic7GpZzKx8,173
 evalscope/backend/rag_eval/ragas/tasks/build_distribution.py,sha256=vFfemiqtPx22u5pwwZxEQJKYf3B9efYmwbpWDI5hY30,1491
 evalscope/backend/rag_eval/ragas/tasks/build_transform.py,sha256=GtAYqdVOy7BxIGyC4rSZ_UfXagKYzE6eEtXbaOI_g-k,5425
@@ -154,7 +134,7 @@ evalscope/cli/start_eval.py,sha256=2lyD2WSQ0DnP6T31VvTimQ-6POnwxeEP9GLPFnT7Tfo,7
 evalscope/cli/start_perf.py,sha256=lEHJBSpzNsO4KGlWfQc-EfZGXq1M_FpOwtRxRdb4fso,813
 evalscope/cli/start_server.py,sha256=DQRIfbsHaOAsVcLGF6iRyJnxmd5Sf_tgytpJNfiWCeE,3662
 evalscope/evaluator/__init__.py,sha256=S6MU1O_iiNAaKxNIhO9MEmdW-BSNf_YH2l6NQ9lxVNo,103
-evalscope/evaluator/evaluator.py,sha256=nRR6aaa9J8nRfB8QPZwexSrfKDvPkPSGQpFVpbWLeW0,18380
+evalscope/evaluator/evaluator.py,sha256=wrTWyvyD1eqSvsZRwDRV1UVBxXv7y-2A29UCD9F-5qI,18412
 evalscope/evaluator/rating_eval.py,sha256=uo0uj9z_TDsxdYlT8WIfNZhFLAfRkW9zn_wlu-F72O0,5575
 evalscope/evaluator/reviewer/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
 evalscope/evaluator/reviewer/auto_reviewer.py,sha256=nL8k-i92L1iMwjPOnNxzQyZICfukZKJul4ZBvOWkHGw,16414
@@ -170,24 +150,24 @@ evalscope/metrics/resources/gpt2-zhcn3-v4.json,sha256=WkM4J_FDPPNQwYi0kj5sM5SVjk
 evalscope/models/__init__.py,sha256=b-jXJ2Cj6dH8notAU7lvCVKbGrcEaf8Gfr5w79qNHAk,111
 evalscope/models/dummy_chat_model.py,sha256=aG3yolnnIN_-gsfF9FsyjyGMewQteEnUfOxTGScROSE,1272
 evalscope/models/model.py,sha256=ZzzVzZHVzuzdt5F1r-rEBT44ZfW9B7R1spsrV-T8nSw,3020
-evalscope/models/model_adapter.py,sha256=XBeSFTR9pXmnhFWRRddcobnITC5T4JKooeFUeWEtUVI,19006
+evalscope/models/model_adapter.py,sha256=5jzDXpFp24ZZ25tjpIMJeDTz-lDSD_EHp040gJOZACc,19007
 evalscope/models/openai_model.py,sha256=-tPBu6v0Ogf_flmG88tFuu66QNKrOyxv3AjYwVtuR44,3313
 evalscope/models/api/__init__.py,sha256=0c75K78O1KaV02BqqtEp-hhtSSClXLawb8E0c2iqN_A,105
 evalscope/models/api/openai_api.py,sha256=PiIvvDYJkn041SJkLoroXwl1B8TtwpB7licVfqNSeuQ,8168
 evalscope/models/custom/__init__.py,sha256=wb6f_Bi39s5sj-VO7EXRDXB2WhyFb49BUtEMk77ksNQ,102
 evalscope/models/custom/custom_model.py,sha256=rBQLAuPEw_OPUtRSCEmxEfpcA8jPj8bAdsmtKs4ygus,1566
 evalscope/perf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-evalscope/perf/arguments.py,sha256=J067vNJF-RObJNZ0oE2RBIBNjliCYcflWtt6aGAt40g,9205
-evalscope/perf/benchmark.py,sha256=h151QXsVbg7lMe09aH_mxUdPRALIl1A35I9VO2zryEo,9615
+evalscope/perf/arguments.py,sha256=8KiD4u51B_twEaIiI0_kw4Jknk3YG4S6XN-vgvutChA,9233
+evalscope/perf/benchmark.py,sha256=qNgDNseW8N0beuAB_4-JVtTdHs7ZaJEHK5XnkMU9vRU,9618
 evalscope/perf/http_client.py,sha256=TfnQT9OaBlUCpGwi4ifSJBaaGsn3P2KVBPMGuw-Rqkk,7073
-evalscope/perf/main.py,sha256=2GrE9wHibprzaw4gmcovdc5ods_EHwoSwwmkFDLTUjQ,1257
+evalscope/perf/main.py,sha256=Qg99KhGUjnVAMkNofbDsvMGFxijewH8ri3QoW1y1U7U,1292
 evalscope/perf/plugin/__init__.py,sha256=1sl5s-csrwKb_LVTnpF3HqArz06TRD5LYJ0hpqvokUA,85
 evalscope/perf/plugin/registry.py,sha256=w1IAt6GDdluzSYK5i-yrntvx3_EvIIqJamEL0xZv3zA,1323
 evalscope/perf/plugin/api/__init__.py,sha256=Ckzbq4CkSMVQTedQcDHCYlRd6FTwQAElt2mHB-VXJac,195
 evalscope/perf/plugin/api/base.py,sha256=B_H04qKx7eRTn155rnDrbTYur7PK1mvxfQKYcqYbndU,2118
 evalscope/perf/plugin/api/custom_api.py,sha256=IplmkCu8v9yQrY5CeqBEQDWdOfOp3vRkiDYUcvhw2yY,3775
 evalscope/perf/plugin/api/dashscope_api.py,sha256=V5fwn-p_fLH0dWKzhN9TvYSHRgla4INfXC4NDaIjoQ8,3825
-evalscope/perf/plugin/api/openai_api.py,sha256=raa4SaatEphNfWuK6_3ecfe49Vg4yftD6C-enhufJuE,7020
+evalscope/perf/plugin/api/openai_api.py,sha256=WV2EUIl1PTg-Dj7HMSxJrAE7OUxJZqQmZLJZLHffcJo,6805
 evalscope/perf/plugin/datasets/__init__.py,sha256=9mz2TnVHhxbEKAS9pLbKMQuIoShNlZpGiRo9e2RQLUs,490
 evalscope/perf/plugin/datasets/base.py,sha256=Z-INWueeYjfEZhP4lbTlBMVwIa6BcXZKWx-w7Pop3mA,1786
 evalscope/perf/plugin/datasets/custom.py,sha256=_GSC5yR90_BjcRjdJqrJT2vHQAzskz5XxYOxngUM2Pg,815
@@ -201,7 +181,7 @@ evalscope/perf/utils/analysis_result.py,sha256=ig0zPwbUODGh1GUr3GmnNF4lJJp9SQvW0
 evalscope/perf/utils/benchmark_util.py,sha256=T_pXpSCwCNLJgfzgv3IO7kG61ghTLthVMsXZhBCGP_4,5541
 evalscope/perf/utils/db_util.py,sha256=PSBq16uWyzXx0zyoEE4wazWKN19UAA8_GjobS7rTPso,9001
 evalscope/perf/utils/handler.py,sha256=HyKIxbzC0XCyQanlbb7UEY7yaeqjJTePNea8kMV3Sdc,1192
-evalscope/perf/utils/local_server.py,sha256=A26gqBbxsnZA8CqQospyO50x3prVnD9XiT2l--ERxK0,4566
+evalscope/perf/utils/local_server.py,sha256=clF8i0UFmaxBBB6gX05KvVCyzSv0xzsAidz0_sLLlAk,4627
 evalscope/registry/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
 evalscope/registry/config/cfg_arena.yaml,sha256=rub6ceaQxxB1mbSjdoFf0IaVgGfbOonV2nYRebv2OKo,3292
 evalscope/registry/config/cfg_arena_zhihu.yaml,sha256=tvvihBwvoTjoezwTSaZwoGOB44ysofpnin4pNyY9TfQ,2755
@@ -266,7 +246,7 @@ tests/test_run_all.py,sha256=YcMTlWoFpvWY8jevWyIf2G_tz8hgDD1cAwSvmyZt96M,429
 tests/cli/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
 tests/cli/test_run.py,sha256=pMZvI3b0Vs-UFfciDoPwCYFAaYJzocQjxEaMLFTxYSo,4289
 tests/perf/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
-tests/perf/test_perf.py,sha256=AQB2QuMwJ1TnenHFPBF4YAtifbR0D0pSobP6xmDysqw,3023
+tests/perf/test_perf.py,sha256=iB8Mg565SfwPsObdAByHYfZNqN71kUtPW7ucmyiOWo8,3025
 tests/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/rag/test_clip_benchmark.py,sha256=Ar8Br2CoAFYT2h4zCv_JKMKCGJKbKGYZgNwJ410ZaoU,2597
 tests/rag/test_mteb.py,sha256=CaEJ0f1M06Z90c72FQb9z23IC_KZtkURWsc_oRMgQn8,4609
@@ -277,9 +257,9 @@ tests/swift/test_run_swift_vlm_eval.py,sha256=C8DftjewnZaerQWfERI70bU3sQLWQ-ejZU
 tests/swift/test_run_swift_vlm_jugde_eval.py,sha256=THZEXUOSqm9rWslwJHmZyh-Ytv5c_QKpgRW5J2s_69E,6017
 tests/vlm/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
 tests/vlm/test_vlmeval.py,sha256=nzWXjw49SlxXgDnYS9N5JSFtcUp8xPOW2YNNzupvtt4,1806
-evalscope-0.8.1.dist-info/LICENSE,sha256=K_2M03pN0PxVMyx9IQUKsHGhhDMkw5ryQ02rlMvzj3I,11416
-evalscope-0.8.1.dist-info/METADATA,sha256=HydrEYb1OxbvVUMl11oLekV2sjvlgQQvtEpkcNAiW5A,23190
-evalscope-0.8.1.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-evalscope-0.8.1.dist-info/entry_points.txt,sha256=Qr4oTgGhg_K-iUtKwVH6lWUhFHDUiH9trIqydHGTEug,56
-evalscope-0.8.1.dist-info/top_level.txt,sha256=Yv0iprOqZQ4rfUO-AWJp7Ni6m0Twxny1yvZwO-8hUDM,16
-evalscope-0.8.1.dist-info/RECORD,,
+evalscope-0.8.2.dist-info/LICENSE,sha256=K_2M03pN0PxVMyx9IQUKsHGhhDMkw5ryQ02rlMvzj3I,11416
+evalscope-0.8.2.dist-info/METADATA,sha256=Fk1p0gh2RycQ7yOBj7fMYym7G-SYj8sL-IZX8cgGxVQ,23709
+evalscope-0.8.2.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+evalscope-0.8.2.dist-info/entry_points.txt,sha256=Qr4oTgGhg_K-iUtKwVH6lWUhFHDUiH9trIqydHGTEug,56
+evalscope-0.8.2.dist-info/top_level.txt,sha256=Yv0iprOqZQ4rfUO-AWJp7Ni6m0Twxny1yvZwO-8hUDM,16
+evalscope-0.8.2.dist-info/RECORD,,

tests/perf/test_perf.py

@@ -19,13 +19,13 @@ class TestPerf(unittest.TestCase):
     @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
     def test_run_perf(self):
         task_cfg = {
-            'url': 'http://127.0.0.1:8000/v1/chat/completions',
+            'url': 'http://127.0.0.1:8001/v1/chat/completions',
             'parallel': 1,
             'model': 'qwen2.5',
             'number': 15,
             'api': 'openai',
             'dataset': 'openqa',
-            'stream': True,
+            # 'stream': True,
             'debug': True,
         }
         run_perf_benchmark(task_cfg)
@@ -47,7 +47,7 @@ class TestPerf(unittest.TestCase):
     @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
     def test_run_perf_speed_benchmark(self):
         task_cfg = {
-            'url': 'http://127.0.0.1:8801/v1/completions',
+            'url': 'http://127.0.0.1:8001/v1/completions',
             'parallel': 1,
             'model': 'qwen2.5',
             'api': 'openai',

evalscope/backend/rag_eval/ragas/prompts/chinese/AnswerCorrectness/correctness_prompt_chinese.json

@@ -1,87 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": -492257975294377194,
-  "language": "chinese",
-  "instruction": "给定一个真实情况和一个答案陈述，分析每个陈述并将其分类为以下类别之一：TP（真正）：答案中存在的陈述也直接由一个或多个真实情况中的陈述支持，FP（假正）：答案中存在的陈述但没有被任何真实情况中的陈述直接支持，FN（假负）：在真实情况中发现但在答案中不存在的陈述。每个陈述只能属于其中一个类别。为每个分类提供理由。",
-  "examples": [
-    {
-      "input": {
-        "question": "是什么为太阳提供能量，它的主要功能是什么？",
-        "answer": [
-          "太阳的能量来自核裂变，类似于地球上的核反应堆。",
-          "太阳的主要功能是为太阳系提供光。"
-        ],
-        "ground_truth": [
-          "太阳的能量来自核聚变，其中氢原子融合形成氦。",
-          "太阳核心的这种聚变过程释放出巨大的能量。",
-          "来自太阳的能量提供热量和光，这对地球上的生命至关重要。",
-          "太阳的光在地球的气候系统中起着关键作用。",
-          "阳光有助于驱动天气和海洋洋流。"
-        ]
-      },
-      "output": {
-        "TP": [
-          {
-            "statement": "太阳的主要功能是为太阳系提供光。",
-            "reason": "这一说法在某种程度上得到了地面事实的支持，提到太阳提供光和它的作用，尽管它更广泛地关注太阳的能量。"
-          }
-        ],
-        "FP": [
-          {
-            "statement": "太阳的能量来自核裂变，类似于地球上的核反应堆。",
-            "reason": "这一说法是不正确的，与地面事实相矛盾，地面事实指出太阳的能量来自核聚变。"
-          }
-        ],
-        "FN": [
-          {
-            "statement": "太阳的能量来自核聚变，其中氢原子融合形成氦。",
-            "reason": "这种对太阳能量来源的准确描述没有包含在答案中。"
-          },
-          {
-            "statement": "太阳核心的这种聚变过程释放出巨大的能量。",
-            "reason": "这个过程及其重要性没有在答案中提到。"
-          },
-          {
-            "statement": "来自太阳的能量提供热量和光，这对地球上的生命至关重要。",
-            "reason": "答案中只提到了光，忽略了热量及其对生命的必要性，这些在地面事实中都有涵盖。"
-          },
-          {
-            "statement": "太阳的光在地球的气候系统中起着关键作用。",
-            "reason": "太阳光对地球气候系统的这种更广泛的影响没有在答案中提到。"
-          },
-          {
-            "statement": "阳光有助于驱动天气和海洋洋流。",
-            "reason": "答案中省略了阳光对天气模式和海洋洋流的影响。"
-          }
-        ]
-      }
-    },
-    {
-      "input": {
-        "question": "水的沸点是多少？",
-        "answer": [
-          "水的沸点在海平面上是100摄氏度。"
-        ],
-        "ground_truth": [
-          "水的沸点在海平面上是100摄氏度（212华氏度）。",
-          "水的沸点会随着海拔的变化而变化。"
-        ]
-      },
-      "output": {
-        "TP": [
-          {
-            "statement": "水的沸点在海平面上是100摄氏度。",
-            "reason": "这一说法直接得到了地面事实的支持，地面事实具体说明了水的沸点在海平面上是100摄氏度。"
-          }
-        ],
-        "FP": [],
-        "FN": [
-          {
-            "statement": "水的沸点会随着海拔的变化而变化。",
-            "reason": "关于水的沸点如何随海拔变化的额外信息没有在答案中提到。"
-          }
-        ]
-      }
-    }
-  ]
-}

evalscope/backend/rag_eval/ragas/prompts/chinese/AnswerCorrectness/long_form_answer_prompt_chinese.json

@@ -1,36 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": -8546983388246528139,
-  "language": "chinese",
-  "instruction": "给定一个问题、一个答案和答案中的句子，分析在“句子”下给出的每个句子的复杂性，并将每个句子分解为一个或多个完全可理解的陈述，同时确保每个陈述中不使用代词。将输出格式化为JSON。",
-  "examples": [
-    {
-      "input": {
-        "question": "阿尔伯特·爱因斯坦是谁，他以什么而闻名？",
-        "answer": "他是一位出生于德国的理论物理学家，被广泛认为是有史以来最伟大和最有影响力的物理学家之一。他最著名的是发展了相对论，他还对量子力学理论的发展做出了重要贡献。",
-        "sentences": {
-          "0": "他是一位出生于德国的理论物理学家，被广泛认为是有史以来最伟大和最有影响力的物理学家之一。",
-          "1": "他最著名的是发展了相对论，他还对量子力学理论的发展做出了重要贡献。"
-        }
-      },
-      "output": {
-        "sentences": [
-          {
-            "sentence_index": 0,
-            "simpler_statements": [
-              "阿尔伯特·爱因斯坦是一位出生于德国的理论物理学家。",
-              "阿尔伯特·爱因斯坦被认为是有史以来最伟大和最有影响力的物理学家之一。"
-            ]
-          },
-          {
-            "sentence_index": 1,
-            "simpler_statements": [
-              "阿尔伯特·爱因斯坦最著名的是发展了相对论。",
-              "阿尔伯特·爱因斯坦还对量子力学理论的发展做出了重要贡献。"
-            ]
-          }
-        ]
-      }
-    }
-  ]
-}

evalscope/backend/rag_eval/ragas/prompts/chinese/AnswerRelevancy/question_generation_chinese.json

@@ -1,26 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": 7951911230338252816,
-  "language": "chinese",
-  "instruction": "为给定的答案生成一个问题，并识别答案是否含糊不清。如果答案含糊不清，则给出1；如果答案明确，则给出0。含糊不清的答案是指那些回避的、模糊的或不明确的答案。例如，“我不知道”或“我不确定”是含糊不清的答案。",
-  "examples": [
-    {
-      "input": {
-        "response": "阿尔伯特·爱因斯坦出生在德国。"
-      },
-      "output": {
-        "question": "阿尔伯特·爱因斯坦出生在哪里？",
-        "noncommittal": 0
-      }
-    },
-    {
-      "input": {
-        "response": "我不知道2023年发明的智能手机的突破性功能，因为我对2022年以后的信息不了解。"
-      },
-      "output": {
-        "question": "2023年发明的智能手机的突破性功能是什么？",
-        "noncommittal": 1
-      }
-    }
-  ]
-}

evalscope/backend/rag_eval/ragas/prompts/chinese/ContextPrecision/context_precision_prompt_chinese.json

@@ -1,41 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": -5318808809674890018,
-  "language": "chinese",
-  "instruction": "给定问题、答案和背景，验证背景在得出给定答案时是否有用。如果有用，判定为“1”，如果没有用，判定为“0”，并以json格式输出。",
-  "examples": [
-    {
-      "input": {
-        "question": "你能告诉我关于阿尔伯特·爱因斯坦的什么？",
-        "context": "阿尔伯特·爱因斯坦（1879年3月14日－1955年4月18日）是一位德国出生的理论物理学家，被广泛认为是有史以来最伟大和最有影响力的科学家之一。他因发展相对论而闻名，同时也对量子力学做出了重要贡献，因此在20世纪前几十年现代物理学对自然科学理解的革命性重塑中起到了核心作用。他的质能等价公式E=mc²，源于相对论，被称为“世界上最著名的方程”。他因“对理论物理学的贡献，特别是发现光电效应定律”而获得1921年诺贝尔物理学奖，这是量子理论发展的关键一步。他的工作也因其对科学哲学的影响而闻名。在1999年由英国《物理世界》杂志对全球130位顶尖物理学家的调查中，爱因斯坦被评为有史以来最伟大的物理学家。他的智力成就和原创性使爱因斯坦成为天才的代名词。",
-        "answer": "阿尔伯特·爱因斯坦，生于1879年3月14日，是一位德国出生的理论物理学家，被广泛认为是有史以来最伟大和最有影响力的科学家之一。他因对理论物理学的贡献而获得1921年诺贝尔物理学奖。"
-      },
-      "output": {
-        "reason": "提供的背景确实有助于得出给定的答案。背景包括关于阿尔伯特·爱因斯坦的生活和贡献的关键信息，这些信息在答案中得到了反映。",
-        "verdict": 1
-      }
-    },
-    {
-      "input": {
-        "question": "谁赢得了2020年ICC世界杯？",
-        "context": "2022年ICC男子T20世界杯于2022年10月16日至11月13日在澳大利亚举行，是该赛事的第八届。原定于2020年举行，但因COVID-19大流行而推迟。英格兰在决赛中以五个小门击败巴基斯坦，赢得了他们的第二个ICC男子T20世界杯冠军。",
-        "answer": "英格兰"
-      },
-      "output": {
-        "reason": "背景有助于澄清关于2020年ICC世界杯的情况，并指出英格兰是原定于2020年举行但实际上在2022年举行的比赛的获胜者。",
-        "verdict": 1
-      }
-    },
-    {
-      "input": {
-        "question": "世界上最高的山是什么？",
-        "context": "安第斯山脉是世界上最长的大陆山脉，位于南美洲。它横跨七个国家，拥有西半球许多最高的山峰。该山脉以其多样的生态系统而闻名，包括高海拔的安第斯高原和亚马逊雨林。",
-        "answer": "珠穆朗玛峰。"
-      },
-      "output": {
-        "reason": "提供的背景讨论了安第斯山脉，虽然令人印象深刻，但不包括珠穆朗玛峰，也与关于世界最高山的问题没有直接关系。",
-        "verdict": 0
-      }
-    }
-  ]
-}

evalscope/backend/rag_eval/ragas/prompts/chinese/CustomNodeFilter/scoring_prompt_chinese.json

@@ -1,7 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": -1333942410710431097,
-  "language": "chinese",
-  "instruction": "给定文档摘要和节点内容，将节点内容评分在1到5的范围内。",
-  "examples": []
-}

evalscope/backend/rag_eval/ragas/prompts/chinese/Faithfulness/nli_statements_message_chinese.json

@@ -1,60 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": 5296785184599215999,
-  "language": "chinese",
-  "instruction": "您的任务是根据给定的上下文判断一系列陈述的真实性。对于每个陈述，如果可以根据上下文直接推断出该陈述，则必须返回判决为1；如果不能根据上下文直接推断出该陈述，则返回判决为0。",
-  "examples": [
-    {
-      "input": {
-        "context": "约翰是XYZ大学的学生。他正在攻读计算机科学学位。本学期他注册了几门课程，包括数据结构、算法和数据库管理。约翰是一个勤奋的学生，花费大量时间学习和完成作业。他经常在图书馆待到很晚以完成他的项目。",
-        "statements": [
-          "约翰主修生物学。",
-          "约翰正在学习一门人工智能课程。",
-          "约翰是一个勤奋的学生。",
-          "约翰有一份兼职工作。"
-        ]
-      },
-      "output": {
-        "statements": [
-          {
-            "statement": "约翰主修生物学。",
-            "reason": "约翰的专业明确提到是计算机科学。没有信息表明他主修生物学。",
-            "verdict": 0
-          },
-          {
-            "statement": "约翰正在学习一门人工智能课程。",
-            "reason": "上下文中提到约翰目前注册的课程，并未提到人工智能。因此，不能推断出约翰正在学习人工智能课程。",
-            "verdict": 0
-          },
-          {
-            "statement": "约翰是一个勤奋的学生。",
-            "reason": "上下文中提到他花费大量时间学习和完成作业。此外，还提到他经常在图书馆待到很晚以完成他的项目，这意味着他很勤奋。",
-            "verdict": 1
-          },
-          {
-            "statement": "约翰有一份兼职工作。",
-            "reason": "上下文中没有给出约翰有兼职工作的信息。",
-            "verdict": 0
-          }
-        ]
-      }
-    },
-    {
-      "input": {
-        "context": "光合作用是植物、藻类和某些细菌用来将光能转化为化学能的过程。",
-        "statements": [
-          "阿尔伯特·爱因斯坦是个天才。"
-        ]
-      },
-      "output": {
-        "statements": [
-          {
-            "statement": "阿尔伯特·爱因斯坦是个天才。",
-            "reason": "上下文和陈述无关",
-            "verdict": 0
-          }
-        ]
-      }
-    }
-  ]
-}

evalscope/backend/rag_eval/ragas/prompts/chinese/Faithfulness/statement_prompt_chinese.json

@@ -1,36 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": -8546983388246528139,
-  "language": "chinese",
-  "instruction": "给定一个问题、一个答案和答案中的句子，分析在“句子”下给出的每个句子的复杂性，并将每个句子分解为一个或多个完全可理解的陈述，同时确保每个陈述中不使用代词。将输出格式化为JSON。",
-  "examples": [
-    {
-      "input": {
-        "question": "阿尔伯特·爱因斯坦是谁，他最出名的是什么？",
-        "answer": "他是一位出生于德国的理论物理学家，被广泛认为是有史以来最伟大和最有影响力的物理学家之一。他最出名的是发展了相对论，他还为量子力学理论的发展做出了重要贡献。",
-        "sentences": {
-          "0": "他是一位出生于德国的理论物理学家，被广泛认为是有史以来最伟大和最有影响力的物理学家之一。",
-          "1": "他最出名的是发展了相对论，他还为量子力学理论的发展做出了重要贡献。"
-        }
-      },
-      "output": {
-        "sentences": [
-          {
-            "sentence_index": 0,
-            "simpler_statements": [
-              "阿尔伯特·爱因斯坦是一位出生于德国的理论物理学家。",
-              "阿尔伯特·爱因斯坦被认为是有史以来最伟大和最有影响力的物理学家之一。"
-            ]
-          },
-          {
-            "sentence_index": 1,
-            "simpler_statements": [
-              "阿尔伯特·爱因斯坦最出名的是发展了相对论。",
-              "阿尔伯特·爱因斯坦还为量子力学理论的发展做出了重要贡献。"
-            ]
-          }
-        ]
-      }
-    }
-  ]
-}

evalscope/backend/rag_eval/ragas/prompts/chinese/HeadlinesExtractor/prompt_chinese.json

@@ -1,24 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": 7972318980248949928,
-  "language": "chinese",
-  "instruction": "从给定文本中提取最重要的max_num标题，这些标题可用于将文本分成独立的部分。重点关注第2级和第3级标题。",
-  "examples": [
-    {
-      "input": {
-        "text": "                介绍\n                主题概述...\n\n                主要概念\n                核心思想的解释...\n\n                详细分析\n                分析的技术和方法...\n\n                小节：专业技术\n                专业技术的进一步细节...\n\n                未来方向\n                对即将到来的趋势的见解...\n\n                小节：研究的下一步\n                新研究领域的讨论...\n\n                结论\n                最后的评论和总结。\n                ",
-        "max_num": 6
-      },
-      "output": {
-        "headlines": [
-          "介绍",
-          "主要概念",
-          "详细分析",
-          "小节：专业技术",
-          "未来方向",
-          "结论"
-        ]
-      }
-    }
-  ]
-}

evalscope/backend/rag_eval/ragas/prompts/chinese/MultiHopAbstractQuerySynthesizer/concept_combination_prompt_chinese.json

@@ -1,35 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": 5035835898922847346,
-  "language": "chinese",
-  "instruction": "通过将来自至少两个不同列表的概念配对来形成组合。\n**说明：**\n- 查看每个节点的概念。\n- 确定可以逻辑连接或对比的概念。\n- 形成涉及来自不同节点的概念的组合。\n- 每个组合应至少包括来自两个或多个节点的一个概念。\n- 清晰简洁地列出组合。\n- 不要重复相同的组合。",
-  "examples": [
-    {
-      "input": {
-        "lists_of_concepts": [
-          [
-            "人工智能",
-            "自动化"
-          ],
-          [
-            "医疗保健",
-            "数据隐私"
-          ]
-        ],
-        "max_combinations": 2
-      },
-      "output": {
-        "combinations": [
-          [
-            "人工智能",
-            "医疗保健"
-          ],
-          [
-            "自动化",
-            "数据隐私"
-          ]
-        ]
-      }
-    }
-  ]
-}

evalscope/backend/rag_eval/ragas/prompts/chinese/MultiHopAbstractQuerySynthesizer/generate_query_reference_prompt_chinese.json

@@ -1,30 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": 5691378570114822729,
-  "language": "chinese",
-  "instruction": "根据指定的条件（角色、主题、风格、长度）和提供的上下文生成一个多跳查询和答案。主题代表从上下文中提取或生成的一组短语，这些短语突出显示了所选上下文在创建多跳查询时的适用性。确保查询明确包含这些主题。### 指导：\n1. **生成多跳查询**：使用提供的上下文片段和主题形成一个需要结合多个片段信息的查询（例如，`<1-hop>` 和 `<2-hop>`）。确保查询明确包含一个或多个主题，并反映其与上下文的相关性。\n2. **生成答案**：仅使用提供的上下文中的内容来创建查询的详细和忠实的答案。避免添加不直接存在或无法从给定上下文推断的信息。\n3. **多跳上下文标签**：\n   - 每个上下文片段标记为 `<1-hop>`、`<2-hop>` 等。\n   - 确保查询使用至少两个片段的信息并有意义地连接它们。",
-  "examples": [
-    {
-      "input": {
-        "persona": {
-          "name": "历史学家",
-          "role_description": "专注于主要科学里程碑及其全球影响。"
-        },
-        "themes": [
-          "相对论",
-          "实验验证"
-        ],
-        "query_style": "正式",
-        "query_length": "中等",
-        "context": [
-          "<1-hop> 阿尔伯特·爱因斯坦发展了相对论，引入了时空的概念。",
-          "<2-hop> 在1919年的日食期间，光线被重力弯曲的现象得到了证实，支持了爱因斯坦的理论。"
-        ]
-      },
-      "output": {
-        "query": "在1919年日食期间，相对论的实验验证是如何实现的？",
-        "answer": "在1919年日食期间，通过确认光线被重力弯曲，实现了相对论的实验验证，这支持了爱因斯坦在理论中提出的时空概念。"
-      }
-    }
-  ]
-}

evalscope/backend/rag_eval/ragas/prompts/chinese/MultiHopAbstractQuerySynthesizer/theme_persona_matching_prompt_chinese.json

@@ -1,39 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": 4608101540215877909,
-  "language": "chinese",
-  "instruction": "给定一个主题和角色列表，根据角色描述将每个角色与相关主题关联起来。",
-  "examples": [
-    {
-      "input": {
-        "themes": [
-          "同理心",
-          "包容性",
-          "远程工作"
-        ],
-        "personas": [
-          {
-            "name": "人力资源经理",
-            "role_description": "专注于包容性和员工支持。"
-          },
-          {
-            "name": "远程团队负责人",
-            "role_description": "管理远程团队沟通。"
-          }
-        ]
-      },
-      "output": {
-        "mapping": {
-          "HR Manager": [
-            "包容性",
-            "同理心"
-          ],
-          "Remote Team Lead": [
-            "远程工作",
-            "同理心"
-          ]
-        }
-      }
-    }
-  ]
-}

evalscope/backend/rag_eval/ragas/prompts/chinese/MultiHopSpecificQuerySynthesizer/generate_query_reference_prompt_chinese.json

@@ -1,30 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": 5691378570114822729,
-  "language": "chinese",
-  "instruction": "根据指定的条件（角色、主题、风格、长度）和提供的上下文生成一个多跳查询和答案。主题代表从上下文中提取或生成的一组短语，这些短语突出显示了所选上下文适合多跳查询创建的适用性。确保查询明确包含这些主题。### 指导：\n1. **生成多跳查询**：使用提供的上下文片段和主题形成一个需要结合多个片段信息的查询（例如，`<1-hop>` 和 `<2-hop>`）。确保查询明确包含一个或多个主题，并反映其与上下文的相关性。\n2. **生成答案**：仅使用提供的上下文中的内容来创建对查询的详细和忠实的答案。避免添加不直接存在或无法从给定上下文推断的信息。\n3. **多跳上下文标签**：\n   - 每个上下文片段标记为 `<1-hop>`、`<2-hop>` 等。\n   - 确保查询使用至少两个片段的信息并有意义地连接它们。",
-  "examples": [
-    {
-      "input": {
-        "persona": {
-          "name": "历史学家",
-          "role_description": "专注于重大的科学里程碑及其全球影响。"
-        },
-        "themes": [
-          "相对论",
-          "实验验证"
-        ],
-        "query_style": "正式",
-        "query_length": "中等",
-        "context": [
-          "<1-hop> 阿尔伯特·爱因斯坦发展了相对论，引入了时空的概念。",
-          "<2-hop> 在1919年的日全食期间，光线被重力弯曲的现象得到了证实，支持了爱因斯坦的理论。"
-        ]
-      },
-      "output": {
-        "query": "在1919年的日全食期间，相对论的实验验证是如何实现的？",
-        "answer": "在1919年的日全食期间，通过确认光线被重力弯曲的现象，实现了相对论的实验验证，这支持了爱因斯坦在理论中提出的时空概念。"
-      }
-    }
-  ]
-}

evalscope/backend/rag_eval/ragas/prompts/chinese/MultiHopSpecificQuerySynthesizer/theme_persona_matching_prompt_chinese.json

@@ -1,39 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": 4608101540215877909,
-  "language": "chinese",
-  "instruction": "给定一个主题和角色列表，根据角色描述将每个角色与相关主题关联起来。",
-  "examples": [
-    {
-      "input": {
-        "themes": [
-          "同理心",
-          "包容性",
-          "远程工作"
-        ],
-        "personas": [
-          {
-            "name": "人力资源经理",
-            "role_description": "专注于包容性和员工支持。"
-          },
-          {
-            "name": "远程团队负责人",
-            "role_description": "管理远程团队沟通。"
-          }
-        ]
-      },
-      "output": {
-        "mapping": {
-          "HR Manager": [
-            "包容性",
-            "同理心"
-          ],
-          "Remote Team Lead": [
-            "远程工作",
-            "同理心"
-          ]
-        }
-      }
-    }
-  ]
-}

evalscope/backend/rag_eval/ragas/prompts/chinese/MultiModalFaithfulness/faithfulness_prompt_chinese.json

@@ -1,34 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": -1903496084584659501,
-  "language": "chinese",
-  "instruction": "请说明给定的信息是否得到视觉和文本上下文信息的支持。您需要回答“是”或“否”。如果任何图像和文本上下文支持该信息，请回答“是”。",
-  "examples": [
-    {
-      "input": {
-        "response": "苹果派通常是双层皮的。",
-        "retrieved_contexts": [
-          "苹果派是一种水果派，其主要馅料成分是苹果。",
-          "苹果派通常与奶油、冰淇淋（“苹果派 à la mode”）、蛋奶沙司或切达干酪一起食用。",
-          "它通常是双层皮的，上下都有糕点；上层皮可以是实心的或格状的（交叉条纹编织而成）。"
-        ]
-      },
-      "output": {
-        "faithful": true
-      }
-    },
-    {
-      "input": {
-        "response": "苹果派味道不好。",
-        "retrieved_contexts": [
-          "苹果派是一种水果派，其主要馅料成分是苹果。",
-          "苹果派通常与奶油、冰淇淋（“苹果派 à la mode”）、蛋奶沙司或切达干酪一起食用。",
-          "它通常是双层皮的，上下都有糕点；上层皮可以是实心的或格状的（交叉条纹编织而成）。"
-        ]
-      },
-      "output": {
-        "faithful": false
-      }
-    }
-  ]
-}

evalscope/backend/rag_eval/ragas/prompts/chinese/MultiModalRelevance/relevance_prompt_chinese.json

@@ -1,36 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": -2067734205359291291,
-  "language": "chinese",
-  "instruction": "\n您的任务是评估查询的响应是否与提供的图像和文本上下文信息一致。\n您有两个选项可以回答。要么是 True / False。\n如果查询的响应与上下文信息一致，则回答 - True，否则为 False。\n",
-  "examples": [
-    {
-      "input": {
-        "user_input": "传统玛格丽塔披萨的主要成分是什么？",
-        "response": "玛格丽塔披萨的主要成分是番茄、马苏里拉奶酪和新鲜罗勒。",
-        "retrieved_contexts": [
-          "传统的玛格丽塔披萨由薄饼皮组成。",
-          "主要的配料包括番茄、马苏里拉奶酪、新鲜罗勒、盐和橄榄油。",
-          "它是最简单和最经典的披萨类型之一。"
-        ]
-      },
-      "output": {
-        "relevance": true
-      }
-    },
-    {
-      "input": {
-        "user_input": "谁在2021年获得了奥斯卡最佳男演员奖？",
-        "response": "2021年的最佳男演员奖由莱昂纳多·迪卡普里奥获得。",
-        "retrieved_contexts": [
-          "第93届奥斯卡颁奖典礼于2021年举行。",
-          "安东尼·霍普金斯凭借在《困在时间里的父亲》中的角色获得了最佳男演员奖。",
-          "由于COVID-19的限制，这次活动具有独特性。"
-        ]
-      },
-      "output": {
-        "relevance": false
-      }
-    }
-  ]
-}

evalscope/backend/rag_eval/ragas/prompts/chinese/NERExtractor/prompt_chinese.json

@@ -1,25 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": -677862064343016555,
-  "language": "chinese",
-  "instruction": "从给定文本中提取命名实体，限制输出为顶级实体。确保实体数量不超过指定的最大值。",
-  "examples": [
-    {
-      "input": {
-        "text": "特斯拉和SpaceX的首席执行官埃隆·马斯克宣布计划将业务扩展到欧洲和亚洲的新地点。\n                此次扩展预计将创造数千个就业机会，特别是在柏林和上海等城市。",
-        "max_num": 10
-      },
-      "output": {
-        "entities": [
-          "埃隆·马斯克",
-          "特斯拉",
-          "SpaceX",
-          "欧洲",
-          "亚洲",
-          "柏林",
-          "上海"
-        ]
-      }
-    }
-  ]
-}

evalscope/backend/rag_eval/ragas/prompts/chinese/SingleHopSpecificQuerySynthesizer/generate_query_reference_prompt_chinese.json

@@ -1,24 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": 3079700511467088808,
-  "language": "chinese",
-  "instruction": "根据指定的条件（角色、术语、风格、长度）和提供的上下文生成一个单跳查询和答案。确保答案完全忠实于上下文，仅使用提供的上下文中的信息。### 指导：\n1. **生成查询**：根据上下文、角色、术语、风格和长度，创建一个与角色视角一致并包含术语的问题。\n2. **生成答案**：仅使用提供的上下文中的内容，构建对查询的详细答案。不要添加上下文中未包含或无法推断的信息。\n",
-  "examples": [
-    {
-      "input": {
-        "persona": {
-          "name": "软件工程师",
-          "role_description": "专注于编码最佳实践和系统设计。"
-        },
-        "term": "微服务",
-        "query_style": "正式",
-        "query_length": "中等",
-        "context": "微服务是一种架构风格，其中应用程序被构建为一组松散耦合的服务。每个服务都是细粒度的，并专注于单一功能。"
-      },
-      "output": {
-        "query": "微服务在软件架构中的目的是什么？",
-        "answer": "微服务旨在将应用程序结构化为一组松散耦合的服务，每个服务专注于单一功能。"
-      }
-    }
-  ]
-}

evalscope/backend/rag_eval/ragas/prompts/chinese/SingleHopSpecificQuerySynthesizer/theme_persona_matching_prompt_chinese.json

@@ -1,39 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": 4608101540215877909,
-  "language": "chinese",
-  "instruction": "给定一个主题和角色列表，根据角色描述将每个角色与相关主题关联起来。",
-  "examples": [
-    {
-      "input": {
-        "themes": [
-          "同理心",
-          "包容性",
-          "远程工作"
-        ],
-        "personas": [
-          {
-            "name": "人力资源经理",
-            "role_description": "专注于包容性和员工支持。"
-          },
-          {
-            "name": "远程团队负责人",
-            "role_description": "管理远程团队沟通。"
-          }
-        ]
-      },
-      "output": {
-        "mapping": {
-          "HR Manager": [
-            "包容性",
-            "同理心"
-          ],
-          "Remote Team Lead": [
-            "远程工作",
-            "同理心"
-          ]
-        }
-      }
-    }
-  ]
-}

evalscope/backend/rag_eval/ragas/prompts/chinese/SummaryExtractor/prompt_chinese.json

@@ -1,16 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": -2203889341293275650,
-  "language": "chinese",
-  "instruction": "将给定文本总结为不超过10个句子。",
-  "examples": [
-    {
-      "input": {
-        "text": "人工智能\n\n人工智能正在通过自动化以前需要人类智能的任务来改变各个行业。从医疗保健到金融，人工智能正在被用来快速准确地分析大量数据。这项技术还推动了自动驾驶汽车和个性化推荐等领域的创新。"
-      },
-      "output": {
-        "text": "人工智能通过自动化任务、分析数据和推动自动驾驶汽车和个性化推荐等创新，正在革新各个行业。"
-      }
-    }
-  ]
-}

evalscope/backend/rag_eval/ragas/prompts/chinese/ThemesExtractor/prompt_chinese.json

@@ -1,24 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": -7344189172470926110,
-  "language": "chinese",
-  "instruction": "从给定的文本中提取主要主题和概念。",
-  "examples": [
-    {
-      "input": {
-        "text": "人工智能通过自动化需要人类智能的任务来改变行业。人工智能快速准确地分析大量数据，推动了自动驾驶汽车和个性化推荐等创新。",
-        "max_num": 10
-      },
-      "output": {
-        "output": [
-          "人工智能",
-          "自动化",
-          "数据分析",
-          "创新",
-          "自动驾驶汽车",
-          "个性化推荐"
-        ]
-      }
-    }
-  ]
-}