evalscope 0.8.1__tar.gz → 0.8.2__tar.gz

{evalscope-0.8.1/evalscope.egg-info → evalscope-0.8.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: evalscope
-Version: 0.8.1
+Version: 0.8.2
 Summary: EvalScope: Lightweight LLMs Evaluation Framework
 Home-page: https://github.com/modelscope/evalscope
 Author: ModelScope team
@@ -54,7 +54,7 @@ Provides-Extra: vlmeval
 Requires-Dist: ms-vlmeval>=0.0.9; extra == "vlmeval"
 Provides-Extra: rag
 Requires-Dist: mteb==1.19.4; extra == "rag"
-Requires-Dist: ragas==0.2.7; extra == "rag"
+Requires-Dist: ragas==0.2.9; extra == "rag"
 Requires-Dist: webdataset>0.2.0; extra == "rag"
 Provides-Extra: perf
 Requires-Dist: aiohttp; extra == "perf"
@@ -125,7 +125,7 @@ Requires-Dist: transformers_stream_generator; extra == "all"
 Requires-Dist: ms-opencompass>=0.1.4; extra == "all"
 Requires-Dist: ms-vlmeval>=0.0.9; extra == "all"
 Requires-Dist: mteb==1.19.4; extra == "all"
-Requires-Dist: ragas==0.2.7; extra == "all"
+Requires-Dist: ragas==0.2.9; extra == "all"
 Requires-Dist: webdataset>0.2.0; extra == "all"
 Requires-Dist: aiohttp; extra == "all"
 Requires-Dist: fastapi; extra == "all"
@@ -181,6 +181,8 @@ The framework accommodates multiple evaluation scenarios such as end-to-end RAG
   <br>EvalScope Framework.
 </p>
 
+<details><summary>Framework Description</summary>
+
 The architecture includes the following modules:
 1. **Model Adapter**: The model adapter is used to convert the outputs of specific models into the format required by the framework, supporting both API call models and locally run models.
 2. **Data Adapter**: The data adapter is responsible for converting and processing input data to meet various evaluation needs and formats.
@@ -194,6 +196,16 @@ The architecture includes the following modules:
 5. **Evaluation Report**: The final generated evaluation report summarizes the model's performance, which can be used for decision-making and further model optimization.
 6. **Visualization**: Visualization results help users intuitively understand evaluation results, facilitating analysis and comparison of different model performances.
 
+</details>
+
+## ☎ User Groups
+
+Please scan the QR code below to join our community groups:
+
+[Discord Group](https://discord.com/invite/D27yfEFVz5)              |  WeChat Group | DingTalk Group
+:-------------------------:|:-------------------------:|:-------------------------:
+<img src="docs/asset/discord_qr.jpg" width="160" height="160">  |  <img src="docs/asset/wechat.png" width="160" height="160"> | <img src="docs/asset/dingding.png" width="160" height="160">
+
 
 ## 🎉 News
 - 🔥 **[2024.12.13]** Model evaluation optimization: no need to pass the `--template-type` parameter anymore; supports starting evaluation with `evalscope eval --args`. Refer to the [📖 User Guide](https://evalscope.readthedocs.io/en/latest/get_started/basic_usage.html) for more details.

{evalscope-0.8.1 → evalscope-0.8.2}/README.md

@@ -45,6 +45,8 @@ The framework accommodates multiple evaluation scenarios such as end-to-end RAG
   <br>EvalScope Framework.
 </p>
 
+<details><summary>Framework Description</summary>
+
 The architecture includes the following modules:
 1. **Model Adapter**: The model adapter is used to convert the outputs of specific models into the format required by the framework, supporting both API call models and locally run models.
 2. **Data Adapter**: The data adapter is responsible for converting and processing input data to meet various evaluation needs and formats.
@@ -58,6 +60,16 @@ The architecture includes the following modules:
 5. **Evaluation Report**: The final generated evaluation report summarizes the model's performance, which can be used for decision-making and further model optimization.
 6. **Visualization**: Visualization results help users intuitively understand evaluation results, facilitating analysis and comparison of different model performances.
 
+</details>
+
+## ☎ User Groups
+
+Please scan the QR code below to join our community groups:
+
+[Discord Group](https://discord.com/invite/D27yfEFVz5)              |  WeChat Group | DingTalk Group
+:-------------------------:|:-------------------------:|:-------------------------:
+<img src="docs/asset/discord_qr.jpg" width="160" height="160">  |  <img src="docs/asset/wechat.png" width="160" height="160"> | <img src="docs/asset/dingding.png" width="160" height="160">
+
 
 ## 🎉 News
 - 🔥 **[2024.12.13]** Model evaluation optimization: no need to pass the `--template-type` parameter anymore; supports starting evaluation with `evalscope eval --args`. Refer to the [📖 User Guide](https://evalscope.readthedocs.io/en/latest/get_started/basic_usage.html) for more details.

{evalscope-0.8.1 → evalscope-0.8.2}/evalscope/config.py

@@ -114,7 +114,9 @@ class TaskConfig:
     def from_args(args: Namespace):
         # Convert Namespace to a dictionary and filter out None values
         args_dict = {k: v for k, v in vars(args).items() if v is not None}
-        del args_dict['func']  # Note: compat CLI arguments
+
+        if 'func' in args_dict:
+            del args_dict['func']  # Note: compat CLI arguments
 
         return TaskConfig.from_dict(args_dict)
 

{evalscope-0.8.1 → evalscope-0.8.2}/evalscope/evaluator/evaluator.py

@@ -86,6 +86,7 @@ class Evaluator(object):
             **kwargs)
 
         # Get prompts from dataset
+        # TODO: support sampler
         self.prompts = self.data_adapter.gen_prompts(data_dict=self.dataset)
         del self.dataset
 

{evalscope-0.8.1 → evalscope-0.8.2}/evalscope/models/model_adapter.py

@@ -429,7 +429,7 @@ class ChatGenerationModelAdapter(BaseModelAdapter):
         fix_do_sample_warning(self.generation_config)
 
         # Run inference
-        output_ids = self.model.generate(**inputs, generation_config=self.generation_config)
+        output_ids = self.model.generate(input_ids, generation_config=self.generation_config)
 
         response = self.tokenizer.decode(output_ids[0, len(input_ids[0]):], skip_special_tokens=True)
         return response

{evalscope-0.8.1 → evalscope-0.8.2}/evalscope/perf/arguments.py

@@ -68,6 +68,7 @@ class Arguments:
             model=args.model,
             attn_implementation=args.attn_implementation,
             url=args.url,
+            port=args.port,
             api_key=args.api_key,
             connect_timeout=args.connect_timeout,
             read_timeout=args.read_timeout,

{evalscope-0.8.1 → evalscope-0.8.2}/evalscope/perf/benchmark.py

@@ -157,7 +157,7 @@ async def statistic_benchmark_metric_worker(benchmark_data_queue: asyncio.Queue,
             while not (data_process_completed_event.is_set() and benchmark_data_queue.empty()):
                 try:
                     # Attempt to get benchmark data from the queue with a timeout
-                    benchmark_data = await asyncio.wait_for(benchmark_data_queue.get(), timeout=1)
+                    benchmark_data = await asyncio.wait_for(benchmark_data_queue.get(), timeout=0.01)
                     benchmark_data_queue.task_done()
                 except asyncio.TimeoutError:
                     # If timeout, continue to the next iteration

{evalscope-0.8.1 → evalscope-0.8.2}/evalscope/perf/main.py

@@ -19,7 +19,9 @@ def run_perf_benchmark(args):
         args = Arguments(**args)
     elif isinstance(args, Namespace):
         args = Arguments.from_args(args)
-    seed_everything(args.seed)
+
+    if args.seed is not None:
+        seed_everything(args.seed)
 
     # Setup logger and output
     args.outputs_dir = get_output_path(args)

{evalscope-0.8.1 → evalscope-0.8.2}/evalscope/perf/plugin/api/openai_api.py

@@ -96,60 +96,64 @@ class OpenaiPlugin(ApiPluginBase):
 
     def parse_responses(self, responses, request: Any = None, **kwargs) -> Dict:
         """Parser responses and return number of request and response tokens.
-           sample of the output delta:
-           {"id":"4","object":"chat.completion.chunk","created":1714030870,"model":"llama3","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}]}
+        Only one response for non-stream, multiple responses for stream.
+        """
 
+        # when stream, the last response is the full usage
+        # when non-stream, the last response is the first response
+        last_response_js = json.loads(responses[-1])
+        if 'usage' in last_response_js and last_response_js['usage']:
+            input_tokens = last_response_js['usage']['prompt_tokens']
+            output_tokens = last_response_js['usage']['completion_tokens']
+            return input_tokens, output_tokens
 
-        Args:
-            responses (List[bytes]): List of http response body, for stream output,
-                there are multiple responses, for general only one.
-            kwargs: (Any): The command line --parameter content.
-        Returns:
-            Tuple: Return number of prompt token and number of completion tokens.
-        """
-        full_response_content = ''
+        # no usage information in the response, parse the response to get the tokens
         delta_contents = {}
-        input_tokens = None
-        output_tokens = None
         for response in responses:
             js = json.loads(response)
-            if js['object'] == 'chat.completion':
-                for choice in js['choices']:
-                    delta_contents[choice['index']] = [choice['message']['content']]
-                input_tokens = js['usage']['prompt_tokens']
-                output_tokens = js['usage']['completion_tokens']
-            elif js['object'] == 'text_completion':
-                for choice in js['choices']:
-                    delta_contents[choice['index']] = [choice['text']]
-                input_tokens = js['usage']['prompt_tokens']
-                output_tokens = js['usage']['completion_tokens']
-            elif js['object'] == 'chat.completion.chunk':
-                if 'choices' in js:
-                    for choice in js['choices']:
-                        if 'delta' in choice and 'index' in choice:
-                            delta = choice['delta']
-                            idx = choice['index']
-                            if 'content' in delta:
-                                delta_content = delta['content']
-                                if idx in delta_contents:
-                                    delta_contents[idx].append(delta_content)
-                                else:
-                                    delta_contents[idx] = [delta_content]
-                # usage in chunk: {"id":"","object":"chat.completion.chunk","created":1718269986,"model":"llama3",
-                # "choices":[],"usage":{"prompt_tokens":32,"total_tokens":384,"completion_tokens":352}}
-                if 'usage' in js and js['usage']:
-                    input_tokens = js['usage']['prompt_tokens']
-                    output_tokens = js['usage']['completion_tokens']
-        if (input_tokens is None and output_tokens is None and self.tokenizer is not None):
-            input_tokens = 0
-            output_tokens = 0
+            if 'object' in js:
+                self.__process_response_object(js, delta_contents)
+            else:
+                self.__process_no_object(js, delta_contents)
+
+        input_tokens, output_tokens = self.__calculate_tokens_from_content(request, delta_contents)
+        return input_tokens, output_tokens
+
+    def __process_response_object(self, js, delta_contents):
+        if js['object'] == 'chat.completion':
+            for choice in js['choices']:
+                delta_contents[choice['index']] = [choice['message']['content']]
+        elif js['object'] == 'text_completion':
+            for choice in js['choices']:
+                delta_contents[choice['index']] = [choice['text']]
+        elif js['object'] == 'chat.completion.chunk':
+            for choice in js.get('choices', []):
+                if 'delta' in choice and 'index' in choice:
+                    delta = choice['delta']
+                    idx = choice['index']
+                    if 'content' in delta:
+                        delta_content = delta['content']
+                        delta_contents.setdefault(idx, []).append(delta_content)
+
+    def __process_no_object(self, js, delta_contents):
+        #  assume the response is a single choice
+        for choice in js['choices']:
+            if 'delta' in choice:
+                delta = choice['delta']
+                idx = choice['index']
+                if 'content' in delta:
+                    delta_content = delta['content']
+                    delta_contents.setdefault(idx, []).append(delta_content)
+            else:
+                delta_contents[choice['index']] = [choice['message']['content']]
+
+    def __calculate_tokens_from_content(self, request, delta_contents):
+        input_tokens = output_tokens = 0
+        if self.tokenizer is not None:
             for idx, choice_contents in delta_contents.items():
-                full_response_content = ''.join([m for m in choice_contents])
+                full_response_content = ''.join(choice_contents)
                 input_tokens += len(self.tokenizer.encode(request['messages'][0]['content']))
                 output_tokens += len(self.tokenizer.encode(full_response_content))
-        elif input_tokens is None and output_tokens is None:  # no usage info get.
-            input_tokens = 0
-            output_tokens = 0
+        else:
             logger.warning('No usage information found. Please specify `--tokenizer-path` to generate usage details.')
-
         return input_tokens, output_tokens

{evalscope-0.8.1 → evalscope-0.8.2}/evalscope/perf/utils/local_server.py

@@ -103,6 +103,7 @@ def start_app(args: Arguments):
     elif args.api == 'local_vllm':
         os.environ['VLLM_USE_MODELSCOPE'] = 'True'
         os.environ['VLLM_ALLOW_LONG_MAX_MODEL_LEN'] = '1'
+        os.environ['VLLM_WORKER_MULTIPROC_METHOD'] = 'spawn'
         # yapf: disable
         proc = subprocess.Popen([
             'python', '-m', 'vllm.entrypoints.openai.api_server',

evalscope-0.8.2/evalscope/version.py

@@ -0,0 +1,4 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+__version__ = '0.8.2'
+__release_datetime__ = '2024-12-26 20:00:00'

{evalscope-0.8.1 → evalscope-0.8.2/evalscope.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: evalscope
-Version: 0.8.1
+Version: 0.8.2
 Summary: EvalScope: Lightweight LLMs Evaluation Framework
 Home-page: https://github.com/modelscope/evalscope
 Author: ModelScope team
@@ -54,7 +54,7 @@ Provides-Extra: vlmeval
 Requires-Dist: ms-vlmeval>=0.0.9; extra == "vlmeval"
 Provides-Extra: rag
 Requires-Dist: mteb==1.19.4; extra == "rag"
-Requires-Dist: ragas==0.2.7; extra == "rag"
+Requires-Dist: ragas==0.2.9; extra == "rag"
 Requires-Dist: webdataset>0.2.0; extra == "rag"
 Provides-Extra: perf
 Requires-Dist: aiohttp; extra == "perf"
@@ -125,7 +125,7 @@ Requires-Dist: transformers_stream_generator; extra == "all"
 Requires-Dist: ms-opencompass>=0.1.4; extra == "all"
 Requires-Dist: ms-vlmeval>=0.0.9; extra == "all"
 Requires-Dist: mteb==1.19.4; extra == "all"
-Requires-Dist: ragas==0.2.7; extra == "all"
+Requires-Dist: ragas==0.2.9; extra == "all"
 Requires-Dist: webdataset>0.2.0; extra == "all"
 Requires-Dist: aiohttp; extra == "all"
 Requires-Dist: fastapi; extra == "all"
@@ -181,6 +181,8 @@ The framework accommodates multiple evaluation scenarios such as end-to-end RAG
   <br>EvalScope Framework.
 </p>
 
+<details><summary>Framework Description</summary>
+
 The architecture includes the following modules:
 1. **Model Adapter**: The model adapter is used to convert the outputs of specific models into the format required by the framework, supporting both API call models and locally run models.
 2. **Data Adapter**: The data adapter is responsible for converting and processing input data to meet various evaluation needs and formats.
@@ -194,6 +196,16 @@ The architecture includes the following modules:
 5. **Evaluation Report**: The final generated evaluation report summarizes the model's performance, which can be used for decision-making and further model optimization.
 6. **Visualization**: Visualization results help users intuitively understand evaluation results, facilitating analysis and comparison of different model performances.
 
+</details>
+
+## ☎ User Groups
+
+Please scan the QR code below to join our community groups:
+
+[Discord Group](https://discord.com/invite/D27yfEFVz5)              |  WeChat Group | DingTalk Group
+:-------------------------:|:-------------------------:|:-------------------------:
+<img src="docs/asset/discord_qr.jpg" width="160" height="160">  |  <img src="docs/asset/wechat.png" width="160" height="160"> | <img src="docs/asset/dingding.png" width="160" height="160">
+
 
 ## 🎉 News
 - 🔥 **[2024.12.13]** Model evaluation optimization: no need to pass the `--template-type` parameter anymore; supports starting evaluation with `evalscope eval --args`. Refer to the [📖 User Guide](https://evalscope.readthedocs.io/en/latest/get_started/basic_usage.html) for more details.

{evalscope-0.8.1 → evalscope-0.8.2}/evalscope.egg-info/SOURCES.txt

@@ -55,26 +55,6 @@ evalscope/backend/rag_eval/ragas/__init__.py
 evalscope/backend/rag_eval/ragas/arguments.py
 evalscope/backend/rag_eval/ragas/task_template.py
 evalscope/backend/rag_eval/ragas/prompts/persona_prompt.py
-evalscope/backend/rag_eval/ragas/prompts/chinese/AnswerCorrectness/correctness_prompt_chinese.json
-evalscope/backend/rag_eval/ragas/prompts/chinese/AnswerCorrectness/long_form_answer_prompt_chinese.json
-evalscope/backend/rag_eval/ragas/prompts/chinese/AnswerRelevancy/question_generation_chinese.json
-evalscope/backend/rag_eval/ragas/prompts/chinese/ContextPrecision/context_precision_prompt_chinese.json
-evalscope/backend/rag_eval/ragas/prompts/chinese/CustomNodeFilter/scoring_prompt_chinese.json
-evalscope/backend/rag_eval/ragas/prompts/chinese/Faithfulness/nli_statements_message_chinese.json
-evalscope/backend/rag_eval/ragas/prompts/chinese/Faithfulness/statement_prompt_chinese.json
-evalscope/backend/rag_eval/ragas/prompts/chinese/HeadlinesExtractor/prompt_chinese.json
-evalscope/backend/rag_eval/ragas/prompts/chinese/MultiHopAbstractQuerySynthesizer/concept_combination_prompt_chinese.json
-evalscope/backend/rag_eval/ragas/prompts/chinese/MultiHopAbstractQuerySynthesizer/generate_query_reference_prompt_chinese.json
-evalscope/backend/rag_eval/ragas/prompts/chinese/MultiHopAbstractQuerySynthesizer/theme_persona_matching_prompt_chinese.json
-evalscope/backend/rag_eval/ragas/prompts/chinese/MultiHopSpecificQuerySynthesizer/generate_query_reference_prompt_chinese.json
-evalscope/backend/rag_eval/ragas/prompts/chinese/MultiHopSpecificQuerySynthesizer/theme_persona_matching_prompt_chinese.json
-evalscope/backend/rag_eval/ragas/prompts/chinese/MultiModalFaithfulness/faithfulness_prompt_chinese.json
-evalscope/backend/rag_eval/ragas/prompts/chinese/MultiModalRelevance/relevance_prompt_chinese.json
-evalscope/backend/rag_eval/ragas/prompts/chinese/NERExtractor/prompt_chinese.json
-evalscope/backend/rag_eval/ragas/prompts/chinese/SingleHopSpecificQuerySynthesizer/generate_query_reference_prompt_chinese.json
-evalscope/backend/rag_eval/ragas/prompts/chinese/SingleHopSpecificQuerySynthesizer/theme_persona_matching_prompt_chinese.json
-evalscope/backend/rag_eval/ragas/prompts/chinese/SummaryExtractor/prompt_chinese.json
-evalscope/backend/rag_eval/ragas/prompts/chinese/ThemesExtractor/prompt_chinese.json
 evalscope/backend/rag_eval/ragas/tasks/__init__.py
 evalscope/backend/rag_eval/ragas/tasks/build_distribution.py
 evalscope/backend/rag_eval/ragas/tasks/build_transform.py

{evalscope-0.8.1 → evalscope-0.8.2}/evalscope.egg-info/requires.txt

@@ -67,7 +67,7 @@ transformers_stream_generator
 ms-opencompass>=0.1.4
 ms-vlmeval>=0.0.9
 mteb==1.19.4
-ragas==0.2.7
+ragas==0.2.9
 webdataset>0.2.0
 aiohttp
 fastapi
@@ -116,7 +116,7 @@ unicorn
 
 [rag]
 mteb==1.19.4
-ragas==0.2.7
+ragas==0.2.9
 webdataset>0.2.0
 
 [vlmeval]

{evalscope-0.8.1 → evalscope-0.8.2}/requirements/rag.txt

@@ -1,3 +1,3 @@
 mteb==1.19.4
-ragas==0.2.7
+ragas==0.2.9
 webdataset>0.2.0

{evalscope-0.8.1 → evalscope-0.8.2}/tests/perf/test_perf.py

@@ -19,13 +19,13 @@ class TestPerf(unittest.TestCase):
     @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
     def test_run_perf(self):
         task_cfg = {
-            'url': 'http://127.0.0.1:8000/v1/chat/completions',
+            'url': 'http://127.0.0.1:8001/v1/chat/completions',
             'parallel': 1,
             'model': 'qwen2.5',
             'number': 15,
             'api': 'openai',
             'dataset': 'openqa',
-            'stream': True,
+            # 'stream': True,
             'debug': True,
         }
         run_perf_benchmark(task_cfg)
@@ -47,7 +47,7 @@ class TestPerf(unittest.TestCase):
     @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
     def test_run_perf_speed_benchmark(self):
         task_cfg = {
-            'url': 'http://127.0.0.1:8801/v1/completions',
+            'url': 'http://127.0.0.1:8001/v1/completions',
             'parallel': 1,
             'model': 'qwen2.5',
             'api': 'openai',

evalscope-0.8.1/evalscope/backend/rag_eval/ragas/prompts/chinese/AnswerCorrectness/correctness_prompt_chinese.json

@@ -1,87 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": -492257975294377194,
-  "language": "chinese",
-  "instruction": "给定一个真实情况和一个答案陈述，分析每个陈述并将其分类为以下类别之一：TP（真正）：答案中存在的陈述也直接由一个或多个真实情况中的陈述支持，FP（假正）：答案中存在的陈述但没有被任何真实情况中的陈述直接支持，FN（假负）：在真实情况中发现但在答案中不存在的陈述。每个陈述只能属于其中一个类别。为每个分类提供理由。",
-  "examples": [
-    {
-      "input": {
-        "question": "是什么为太阳提供能量，它的主要功能是什么？",
-        "answer": [
-          "太阳的能量来自核裂变，类似于地球上的核反应堆。",
-          "太阳的主要功能是为太阳系提供光。"
-        ],
-        "ground_truth": [
-          "太阳的能量来自核聚变，其中氢原子融合形成氦。",
-          "太阳核心的这种聚变过程释放出巨大的能量。",
-          "来自太阳的能量提供热量和光，这对地球上的生命至关重要。",
-          "太阳的光在地球的气候系统中起着关键作用。",
-          "阳光有助于驱动天气和海洋洋流。"
-        ]
-      },
-      "output": {
-        "TP": [
-          {
-            "statement": "太阳的主要功能是为太阳系提供光。",
-            "reason": "这一说法在某种程度上得到了地面事实的支持，提到太阳提供光和它的作用，尽管它更广泛地关注太阳的能量。"
-          }
-        ],
-        "FP": [
-          {
-            "statement": "太阳的能量来自核裂变，类似于地球上的核反应堆。",
-            "reason": "这一说法是不正确的，与地面事实相矛盾，地面事实指出太阳的能量来自核聚变。"
-          }
-        ],
-        "FN": [
-          {
-            "statement": "太阳的能量来自核聚变，其中氢原子融合形成氦。",
-            "reason": "这种对太阳能量来源的准确描述没有包含在答案中。"
-          },
-          {
-            "statement": "太阳核心的这种聚变过程释放出巨大的能量。",
-            "reason": "这个过程及其重要性没有在答案中提到。"
-          },
-          {
-            "statement": "来自太阳的能量提供热量和光，这对地球上的生命至关重要。",
-            "reason": "答案中只提到了光，忽略了热量及其对生命的必要性，这些在地面事实中都有涵盖。"
-          },
-          {
-            "statement": "太阳的光在地球的气候系统中起着关键作用。",
-            "reason": "太阳光对地球气候系统的这种更广泛的影响没有在答案中提到。"
-          },
-          {
-            "statement": "阳光有助于驱动天气和海洋洋流。",
-            "reason": "答案中省略了阳光对天气模式和海洋洋流的影响。"
-          }
-        ]
-      }
-    },
-    {
-      "input": {
-        "question": "水的沸点是多少？",
-        "answer": [
-          "水的沸点在海平面上是100摄氏度。"
-        ],
-        "ground_truth": [
-          "水的沸点在海平面上是100摄氏度（212华氏度）。",
-          "水的沸点会随着海拔的变化而变化。"
-        ]
-      },
-      "output": {
-        "TP": [
-          {
-            "statement": "水的沸点在海平面上是100摄氏度。",
-            "reason": "这一说法直接得到了地面事实的支持，地面事实具体说明了水的沸点在海平面上是100摄氏度。"
-          }
-        ],
-        "FP": [],
-        "FN": [
-          {
-            "statement": "水的沸点会随着海拔的变化而变化。",
-            "reason": "关于水的沸点如何随海拔变化的额外信息没有在答案中提到。"
-          }
-        ]
-      }
-    }
-  ]
-}

evalscope-0.8.1/evalscope/backend/rag_eval/ragas/prompts/chinese/AnswerCorrectness/long_form_answer_prompt_chinese.json

@@ -1,36 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": -8546983388246528139,
-  "language": "chinese",
-  "instruction": "给定一个问题、一个答案和答案中的句子，分析在“句子”下给出的每个句子的复杂性，并将每个句子分解为一个或多个完全可理解的陈述，同时确保每个陈述中不使用代词。将输出格式化为JSON。",
-  "examples": [
-    {
-      "input": {
-        "question": "阿尔伯特·爱因斯坦是谁，他以什么而闻名？",
-        "answer": "他是一位出生于德国的理论物理学家，被广泛认为是有史以来最伟大和最有影响力的物理学家之一。他最著名的是发展了相对论，他还对量子力学理论的发展做出了重要贡献。",
-        "sentences": {
-          "0": "他是一位出生于德国的理论物理学家，被广泛认为是有史以来最伟大和最有影响力的物理学家之一。",
-          "1": "他最著名的是发展了相对论，他还对量子力学理论的发展做出了重要贡献。"
-        }
-      },
-      "output": {
-        "sentences": [
-          {
-            "sentence_index": 0,
-            "simpler_statements": [
-              "阿尔伯特·爱因斯坦是一位出生于德国的理论物理学家。",
-              "阿尔伯特·爱因斯坦被认为是有史以来最伟大和最有影响力的物理学家之一。"
-            ]
-          },
-          {
-            "sentence_index": 1,
-            "simpler_statements": [
-              "阿尔伯特·爱因斯坦最著名的是发展了相对论。",
-              "阿尔伯特·爱因斯坦还对量子力学理论的发展做出了重要贡献。"
-            ]
-          }
-        ]
-      }
-    }
-  ]
-}

evalscope-0.8.1/evalscope/backend/rag_eval/ragas/prompts/chinese/AnswerRelevancy/question_generation_chinese.json

@@ -1,26 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": 7951911230338252816,
-  "language": "chinese",
-  "instruction": "为给定的答案生成一个问题，并识别答案是否含糊不清。如果答案含糊不清，则给出1；如果答案明确，则给出0。含糊不清的答案是指那些回避的、模糊的或不明确的答案。例如，“我不知道”或“我不确定”是含糊不清的答案。",
-  "examples": [
-    {
-      "input": {
-        "response": "阿尔伯特·爱因斯坦出生在德国。"
-      },
-      "output": {
-        "question": "阿尔伯特·爱因斯坦出生在哪里？",
-        "noncommittal": 0
-      }
-    },
-    {
-      "input": {
-        "response": "我不知道2023年发明的智能手机的突破性功能，因为我对2022年以后的信息不了解。"
-      },
-      "output": {
-        "question": "2023年发明的智能手机的突破性功能是什么？",
-        "noncommittal": 1
-      }
-    }
-  ]
-}

evalscope-0.8.1/evalscope/backend/rag_eval/ragas/prompts/chinese/ContextPrecision/context_precision_prompt_chinese.json

@@ -1,41 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": -5318808809674890018,
-  "language": "chinese",
-  "instruction": "给定问题、答案和背景，验证背景在得出给定答案时是否有用。如果有用，判定为“1”，如果没有用，判定为“0”，并以json格式输出。",
-  "examples": [
-    {
-      "input": {
-        "question": "你能告诉我关于阿尔伯特·爱因斯坦的什么？",
-        "context": "阿尔伯特·爱因斯坦（1879年3月14日－1955年4月18日）是一位德国出生的理论物理学家，被广泛认为是有史以来最伟大和最有影响力的科学家之一。他因发展相对论而闻名，同时也对量子力学做出了重要贡献，因此在20世纪前几十年现代物理学对自然科学理解的革命性重塑中起到了核心作用。他的质能等价公式E=mc²，源于相对论，被称为“世界上最著名的方程”。他因“对理论物理学的贡献，特别是发现光电效应定律”而获得1921年诺贝尔物理学奖，这是量子理论发展的关键一步。他的工作也因其对科学哲学的影响而闻名。在1999年由英国《物理世界》杂志对全球130位顶尖物理学家的调查中，爱因斯坦被评为有史以来最伟大的物理学家。他的智力成就和原创性使爱因斯坦成为天才的代名词。",
-        "answer": "阿尔伯特·爱因斯坦，生于1879年3月14日，是一位德国出生的理论物理学家，被广泛认为是有史以来最伟大和最有影响力的科学家之一。他因对理论物理学的贡献而获得1921年诺贝尔物理学奖。"
-      },
-      "output": {
-        "reason": "提供的背景确实有助于得出给定的答案。背景包括关于阿尔伯特·爱因斯坦的生活和贡献的关键信息，这些信息在答案中得到了反映。",
-        "verdict": 1
-      }
-    },
-    {
-      "input": {
-        "question": "谁赢得了2020年ICC世界杯？",
-        "context": "2022年ICC男子T20世界杯于2022年10月16日至11月13日在澳大利亚举行，是该赛事的第八届。原定于2020年举行，但因COVID-19大流行而推迟。英格兰在决赛中以五个小门击败巴基斯坦，赢得了他们的第二个ICC男子T20世界杯冠军。",
-        "answer": "英格兰"
-      },
-      "output": {
-        "reason": "背景有助于澄清关于2020年ICC世界杯的情况，并指出英格兰是原定于2020年举行但实际上在2022年举行的比赛的获胜者。",
-        "verdict": 1
-      }
-    },
-    {
-      "input": {
-        "question": "世界上最高的山是什么？",
-        "context": "安第斯山脉是世界上最长的大陆山脉，位于南美洲。它横跨七个国家，拥有西半球许多最高的山峰。该山脉以其多样的生态系统而闻名，包括高海拔的安第斯高原和亚马逊雨林。",
-        "answer": "珠穆朗玛峰。"
-      },
-      "output": {
-        "reason": "提供的背景讨论了安第斯山脉，虽然令人印象深刻，但不包括珠穆朗玛峰，也与关于世界最高山的问题没有直接关系。",
-        "verdict": 0
-      }
-    }
-  ]
-}

evalscope-0.8.1/evalscope/backend/rag_eval/ragas/prompts/chinese/CustomNodeFilter/scoring_prompt_chinese.json

@@ -1,7 +0,0 @@
-{
-  "ragas_version": "0.2.7",
-  "original_hash": -1333942410710431097,
-  "language": "chinese",
-  "instruction": "给定文档摘要和节点内容，将节点内容评分在1到5的范围内。",
-  "examples": []
-}