evalscope 0.7.2__py3-none-any.whl → 0.8.0__py3-none-any.whl

evalscope/third_party/longbench_write/infer.py

@@ -1,18 +1,15 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 # Copyright (c) ZhipuAI, Inc. and its affiliates.
 
-import os
 import json
-from typing import List
-
-import torch
 import numpy as np
+import os
 import random
-from modelscope import AutoTokenizer, AutoModelForCausalLM
-from tqdm import tqdm
+import torch
+from typing import List
 
-from evalscope.third_party.longbench_write.utils import count_words
 from evalscope.models.api import OpenaiApi
+from evalscope.third_party.longbench_write.utils import count_words
 from evalscope.utils import get_logger
 
 logger = get_logger()
@@ -25,39 +22,6 @@ Refer to https://github.com/THUDM/LongWriter for more details.
 """
 
 
-def get_pred(rank, world_size, data, path, max_new_tokens, temperature, tokenizer, fout):
-    device = torch.device(f'cuda:{rank}')
-    model = AutoModelForCausalLM.from_pretrained(path, trust_remote_code=True, torch_dtype=torch.bfloat16).to(device)
-    model = model.eval()
-
-    for dt in tqdm(data, total=len(data), desc=f'Infer on rank-{rank}: '):
-        prompt = dt['prompt']
-        if "llama" in path.lower():
-            prompt = f"[INST]{prompt}[/INST]"
-            input = tokenizer(prompt, truncation=False, return_tensors="pt").to(device)
-            context_length = input.input_ids.shape[-1]
-            output = model.generate(
-                **input,
-                max_new_tokens=max_new_tokens,
-                num_beams=1,
-                do_sample=True,
-                temperature=temperature,
-            )[0]
-            response = tokenizer.decode(output[context_length:], skip_special_tokens=True)
-        else:
-            response, history = model.chat(tokenizer, prompt, history=[], max_new_tokens=max_new_tokens,
-                                           temperature=temperature)
-        dt["response_length"], _ = count_words(response)
-        dt["response"] = response
-
-        logger.info(dt)
-
-        fout.write(json.dumps(dt, ensure_ascii=False) + '\n')
-        fout.flush()
-
-    logger.info(f'Successfully generated predictions for {len(data)} samples.')
-
-
 def seed_everything(seed):
     torch.manual_seed(seed)
     torch.cuda.manual_seed(seed)
@@ -68,69 +32,13 @@ def seed_everything(seed):
     torch.cuda.manual_seed_all(seed)
 
 
-# def run_infer(model: str,
-#               data_path: str,
-#               output_dir: str,
-#               generation_kwargs: dict = None,
-#               enable: bool = True, ):
-#     """
-#     Process inference for LongWriter model.
-#
-#     Args:
-#         model: The model id of the LongWriter model on ModelScope, or local model path.
-#         data_path: The path to the data file.
-#         output_dir: The output directory for the predictions.
-#         generation_kwargs: The generation arguments for the model.
-#             Attributes: `max_new_tokens`: The maximum number of tokens to generate. `temperature`: The temperature
-#         enable: Whether to run infer process.
-#     """
-#     model_id_path: str = os.path.join(output_dir, model.strip(os.sep).replace(os.sep, '__'))
-#
-#     if not enable:
-#         logger.warning('*** Skip `infer` stage ***')
-#         return f'{model_id_path}/pred.jsonl'
-#
-#     seed_everything(42)
-#
-#     os.makedirs(model_id_path, exist_ok=True)
-#     fout = open(f'{model_id_path}/pred.jsonl', 'w', encoding='utf-8')
-#
-#     if generation_kwargs is None:
-#         generation_kwargs = dict({
-#             'max_new_tokens': 32768,
-#             'temperature': 0.5
-#         })
-#
-#     tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
-#     world_size = torch.cuda.device_count()
-#
-#     logger.info(f'>>Input data path: {data_path}')
-#     with open(data_path, encoding='utf-8') as f:
-#         data = [json.loads(line) for line in f]
-#
-#     data_subsets = [data[i::world_size] for i in range(world_size)]
-#     processes = []
-#     for rank in range(world_size):
-#         p = mp.Process(target=get_pred,
-#                        args=(rank, world_size, data_subsets[rank], model, generation_kwargs.get('max_new_tokens'), generation_kwargs.get('temperature'), tokenizer, fout))
-#         p.start()
-#         processes.append(p)
-#
-#     for p in processes:
-#         p.join()
-#
-#     logger.info(f'Finish generating predictions for {model}.')
-#     logger.info(f'Predictions are saved in {model_id_path}/pred.jsonl.')
-#
-#     return f'{model_id_path}/pred.jsonl'
-
-
 def run_infer(model: str,
               data_path: str,
               output_dir: str,
               api_config: dict,
               generation_kwargs: dict = None,
-              enable: bool = True, ):
+              enable: bool = True,
+              proc_num: int = DEFAULT_PROC_NUM):
     """
     Process inference for LongWriter model.
 
@@ -147,6 +55,7 @@ def run_infer(model: str,
         generation_kwargs: The generation arguments for the model.
             Attributes: `max_new_tokens`: The maximum number of tokens to generate. `temperature`: The temperature
         enable: Whether to run infer process.
+        proc_num: calling OpenAI api service with proc_num
     """
     model_id_path: str = os.path.join(output_dir, model.strip(os.sep).replace(os.sep, '__'))
 
@@ -173,7 +82,8 @@ def run_infer(model: str,
 
     api_client = OpenaiApi(model=model,
                            openai_api_key=None,
-                           openai_api_base=api_config.get('openai_api_base', 'http://127.0.0.1:8000/v1/chat/completions'),
+                           openai_api_base=api_config.get('openai_api_base',
+                                                          'http://127.0.0.1:8000/v1/chat/completions'),
                            max_new_tokens=generation_kwargs.get('max_new_tokens', 4096),
                            temperature=generation_kwargs.get('temperature', 0.0),
                            repetition_penalty=generation_kwargs.get('repetition_penalty', 1.0),
@@ -181,9 +91,11 @@ def run_infer(model: str,
                            verbose=api_config.get('verbose', False),
                            )
 
-    # TODO: ONLY FOR TEST  generate_simple
-    results: List[str] = api_client.generate_simple(inputs=[example['prompt'] for example in data_list])
-    assert len(results) == len(data_list), f'Error: The number of predictions {len(results)} is not equal to the number of inputs {len(data_list)}.'
+    # TODO: refine generate_simple
+    results: List[str] = api_client.generate_simple(inputs=[example['prompt'] for example in data_list],
+                                                    num_proc=proc_num)
+    assert len(results) == len(data_list), \
+        f'Error: The number of predictions {len(results)} is not equal to the number of inputs {len(data_list)}.'
     logger.info(f'Finish generating predictions with {len(data_list)} samples for {model}')
 
     # Outputs
@@ -191,8 +103,8 @@ def run_infer(model: str,
     output_pred_file: str = f'{model_id_path}/pred.jsonl'
     with open(output_pred_file, 'w', encoding='utf-8') as f:
         for dt, res in zip(data_list, results):
-            dt["response_length"], _ = count_words(res)
-            dt["response"] = res
+            dt['response_length'], _ = count_words(res)
+            dt['response'] = res
             f.write(json.dumps(dt, ensure_ascii=False) + '\n')
 
     logger.info(f'Predictions are saved in {output_pred_file}')

evalscope/third_party/longbench_write/longbench_write.py

@@ -2,10 +2,9 @@
 import os
 from typing import Union
 
-from evalscope.third_party.longbench_write.infer import run_infer
 from evalscope.third_party.longbench_write.eval import run_eval
-from evalscope.utils import yaml_to_dict, json_to_dict
-from evalscope.utils import get_logger
+from evalscope.third_party.longbench_write.infer import run_infer
+from evalscope.utils import get_logger, json_to_dict, yaml_to_dict
 
 logger = get_logger()
 
@@ -45,7 +44,8 @@ def run_task(task_cfg: Union[str, dict]):
                                   verbose=infer_config.get('verbose', False),
                               ),
                               generation_kwargs=infer_config.get('generation_kwargs'),
-                              enable='infer' in stage)
+                              enable='infer' in stage,
+                              proc_num=infer_config.get('proc_num', 16))
 
     # Run eval process
     run_eval(model=model,
@@ -77,7 +77,7 @@ if __name__ == '__main__':
                     },
 
                     eval_config={
-                        'openai_api_key': 'YOUR_OPENAI_API_KEY',
+                        'openai_api_key': None,
                         'openai_api_base': 'https://api.openai.com/v1/chat/completions',
                         'openai_gpt_model': 'gpt-4o-2024-05-13',
                         'generation_kwargs': {'max_new_tokens': 1024, 'temperature': 0.5, 'stop': None},

evalscope/third_party/longbench_write/resources/judge.txt

@@ -28,4 +28,4 @@ $RESPONSE$
 
 </Response>
 
-Please evaluate the quality of the response. You must first provide a brief analysis of its quality, then give a comprehensive analysis with scores for each dimension. The output must strictly follow the JSON format: {"Analysis": ..., "Relevance": ..., "Accuracy": ..., "Coherence": ..., "Clarity": ..., "Breadth and Depth": ..., "Reading Experience": ...}. You do not need to consider whether the response meets the user's length requirements in your evaluation. Ensure that only one integer between 1 and 5 is output for each dimension score.
+Please evaluate the quality of the response. You must first provide a brief analysis of its quality, then give a comprehensive analysis with scores for each dimension. The output must strictly follow the JSON format: {"Analysis": ..., "Relevance": ..., "Accuracy": ..., "Coherence": ..., "Clarity": ..., "Breadth and Depth": ..., "Reading Experience": ...}. You do not need to consider whether the response meets the user's length requirements in your evaluation. Ensure that only one integer between 1 and 5 is output for each dimension score.

evalscope/third_party/longbench_write/tools/data_etl.py

@@ -1,16 +1,15 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
+import json
 import os.path
-from typing import List
 import re
-import json
+from typing import List
 
 from evalscope.third_party.longbench_write.eval import EvalLength
-from evalscope.third_party.longbench_write.utils import count_words, chinese_to_arabic
+from evalscope.third_party.longbench_write.utils import chinese_to_arabic, count_words
 from evalscope.utils import jsonl_to_list
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
-
 """
 This script is used to preprocess the dataset for the LongWriter.
 """
@@ -141,7 +140,7 @@ class DataETL:
         return out_file
 
 
-if __name__ == "__main__":
+if __name__ == '__main__':
     # run `no_required_length`: got 1748 exampels left
 
     # Refer to: https://modelscope.cn/datasets/ZhipuAI/LongWriter-6k/files

evalscope/third_party/longbench_write/utils.py

@@ -29,7 +29,6 @@ def chinese_to_arabic(chinese_number: str) -> int:
         '七': 7,
         '八': 8,
         '九': 9,
-
         '俩': 2,
         '两': 2,
     }

evalscope/third_party/toolbench_static/eval.py

@@ -1,10 +1,9 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
 import json
+import os
 from dataclasses import dataclass
-
 from rouge import Rouge
-import os
 
 
 @dataclass
@@ -24,7 +23,7 @@ def run_eval(args: EvalArgs):
             return 0
         rouge = Rouge()
         rouge_score = rouge.get_scores(hyps=cand_list, refs=ref_list, avg=True)
-        rougel = rouge_score["rouge-l"]["f"]
+        rougel = rouge_score['rouge-l']['f']
         return rougel
 
     def evaluate_action_em(cand_list: list, ref_list: list):
@@ -97,8 +96,8 @@ def run_eval(args: EvalArgs):
         data = json.load(f)
 
     def parse_action(text):
-        action = "None"
-        action_input = "{}"
+        action = 'None'
+        action_input = '{}'
         if 'Action Input:' in text:
             input_idx = text.rindex('Action Input:')
             action_input = text[input_idx + len('Action Input:'):].strip()
@@ -117,24 +116,24 @@ def run_eval(args: EvalArgs):
 
     def parse_output(text):
         action, action_input = parse_action(text)
-        if action == "Finish":
+        if action == 'Finish':
             try:
                 action_input = json.loads(action_input)
                 # print(action_input)
                 # print(json.dumps(action_input,indent=2))
-                return_type = action_input["return_type"]
-                if return_type == "give_answer":
-                    if "final_answer" in action_input.keys():
+                return_type = action_input['return_type']
+                if return_type == 'give_answer':
+                    if 'final_answer' in action_input.keys():
                         answer = str(action_input['final_answer'])
                         if answer.strip() in ['', '.', ',']:
-                            answer = "None"
+                            answer = 'None'
                     else:
-                        answer = "None"
-                    return "finish", action, action_input, answer
+                        answer = 'None'
+                    return 'finish', action, action_input, answer
                 else:
-                    return "give up", None, None, None
+                    return 'give up', None, None, None
             except:
-                return "give up", None, None, None
+                return 'give up', None, None, None
         else:
             plan = 'call'
             answer = None
@@ -163,7 +162,7 @@ def run_eval(args: EvalArgs):
         # ref_ans: None
 
         pred_plan, pred_action, pred_input, pred_ans = parse_output(prediction)
-        if ref_action is not None and ref_action == "invalid_hallucination_function_name":
+        if ref_action is not None and ref_action == 'invalid_hallucination_function_name':
             continue
         if pred_action is not None and ref_action != 'none' and ref_action not in [t['name'] for t in d['tools']]:
             continue

evalscope/third_party/toolbench_static/infer.py

@@ -16,13 +16,13 @@
 
 # Make it more memory efficient by monkey patching the LLaMA model with FlashAttn.
 
-from dataclasses import dataclass, field
 import json
 import os
-from rouge import Rouge
+import requests
 import time
+from dataclasses import dataclass, field
+from rouge import Rouge
 from urllib3.exceptions import MaxRetryError, NewConnectionError
-import requests
 
 
 def evaluate_rouge_l(cand_list: list, ref_list: list):
@@ -30,7 +30,7 @@ def evaluate_rouge_l(cand_list: list, ref_list: list):
         return 0
     rouge = Rouge()
     rouge_score = rouge.get_scores(hyps=cand_list, refs=ref_list, avg=True)
-    rougel = rouge_score["rouge-l"]["f"]
+    rougel = rouge_score['rouge-l']['f']
     return rougel
 
 
@@ -42,8 +42,8 @@ def nested_load_test_data(data_path):
             test_raw_data += temp_test
         return test_raw_data
     elif os.path.isfile(data_path) and data_path.endswith('.json'):
-        print("Load data from", data_path)
-        temp_data = json.load(open(data_path, "r"))
+        print('Load data from', data_path)
+        temp_data = json.load(open(data_path, 'r'))
         test_raw_data = temp_data
         return test_raw_data
     else:
@@ -51,39 +51,24 @@ def nested_load_test_data(data_path):
 
 
 def baichuan_call(context: list, system: str):
-    url = "https://api.baichuan-ai.com/v1/chat/completions"
-    api_key = "sk-xxx"
+    url = 'https://api.baichuan-ai.com/v1/chat/completions'
+    api_key = 'sk-xxx'
 
     new_msg = []
-    new_msg.append({
-        "role": 'system',
-        'content': system})
+    new_msg.append({'role': 'system', 'content': system})
     for m in context:
-        if m['role'] == "user":
-            new_msg.append({
-                'role': 'user', 'content': m['content']
-            })
-        elif m['role'] == "function":
-            new_msg.append({
-                'role': 'user', 'content': m['content']
-            })
+        if m['role'] == 'user':
+            new_msg.append({'role': 'user', 'content': m['content']})
+        elif m['role'] == 'function':
+            new_msg.append({'role': 'user', 'content': m['content']})
         elif m['role'] == 'assistant':
-            new_msg.append({
-                'role': 'assistant', 'content': m['content']
-            })
+            new_msg.append({'role': 'assistant', 'content': m['content']})
     # print(json.dumps(new_msg, indent=2))
-    data = {
-        "model": "Baichuan2-Turbo",
-        "messages": new_msg,
-        "stream": False
-    }
+    data = {'model': 'Baichuan2-Turbo', 'messages': new_msg, 'stream': False}
 
     json_data = json.dumps(data)
 
-    headers = {
-        "Content-Type": "application/json",
-        "Authorization": "Bearer " + api_key
-    }
+    headers = {'Content-Type': 'application/json', 'Authorization': 'Bearer ' + api_key}
 
     for i in range(5):
         res = None
@@ -91,7 +76,7 @@ def baichuan_call(context: list, system: str):
             res = requests.post(url, data=json_data, headers=headers, timeout=60)
             res = res._content.decode('utf-8')
             res = json.loads(res)
-            return res["choices"][0]["message"]["content"]
+            return res['choices'][0]['message']['content']
         except KeyError:
             print(res)
             time.sleep(1)
@@ -105,57 +90,52 @@ def baichuan_call(context: list, system: str):
         except NewConnectionError:
             time.sleep(5)
             continue
-    return ""
+    return ''
 
 
 def minimax_call(context: list, system: str):
-    group_id = "your-id"
-    api_key = "your-xxx"
+    group_id = 'your-id'
+    api_key = 'your-xxx'
 
-    url = f"https://api.minimax.chat/v1/text/chatcompletion_pro?GroupId={group_id}"
-    headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
+    url = f'https://api.minimax.chat/v1/text/chatcompletion_pro?GroupId={group_id}'
+    headers = {'Authorization': f'Bearer {api_key}', 'Content-Type': 'application/json'}
 
     # construct message
-    system_prompt = "MM智能助理是一款由MiniMax自研的，没有调用其他产品的接口的大型语言模型。" \
-                    "MiniMax是一家中国科技公司，一直致力于进行大模型相关的研究。"
+    system_prompt = 'MM智能助理是一款由MiniMax自研的，没有调用其他产品的接口的大型语言模型。' \
+                    'MiniMax是一家中国科技公司，一直致力于进行大模型相关的研究。'
     system_prompt += ('\n' + system)
 
     new_msg = []
     for m in context:
-        if m['role'] == "user":
-            new_msg.append({
-                'sender_type': 'USER', 'sender_name': 'user', 'text': m['content']
-            })
-        elif m['role'] == "function":
-            new_msg.append({
-                'sender_type': 'USER', 'sender_name': 'funtion', 'text': m['content']
-            })
+        if m['role'] == 'user':
+            new_msg.append({'sender_type': 'USER', 'sender_name': 'user', 'text': m['content']})
+        elif m['role'] == 'function':
+            new_msg.append({'sender_type': 'USER', 'sender_name': 'funtion', 'text': m['content']})
         elif m['role'] == 'assistant':
-            new_msg.append({
-                'sender_type': 'BOT', 'sender_name': 'MM智能助理', 'text': m['content']
-            })
+            new_msg.append({'sender_type': 'BOT', 'sender_name': 'MM智能助理', 'text': m['content']})
 
     request_body = {
-        "model": "abab6-chat",
+        'model': 'abab6-chat',
         # "model": "abab5.5s-chat",
-        "tokens_to_generate": 8192,
-        "reply_constraints": {"sender_type": "BOT", "sender_name": "MM智能助理"},
-        "messages": new_msg,
-        "bot_setting": [
-            {
-                "bot_name": "MM智能助理",
-                "content": system_prompt,
-            }
-        ],
+        'tokens_to_generate': 8192,
+        'reply_constraints': {
+            'sender_type': 'BOT',
+            'sender_name': 'MM智能助理'
+        },
+        'messages': new_msg,
+        'bot_setting': [{
+            'bot_name': 'MM智能助理',
+            'content': system_prompt,
+        }],
     }
     response = requests.post(url, headers=headers, json=request_body)
     status_code = response.status_code
     for i in range(5):
         try:
             if status_code == 200:
-                reply = response.json()["reply"]
+                reply = response.json()['reply']
                 if len(reply) == 0:
-                    print("limit rate")
+                    print('limit rate')
                     time.sleep(8)
                     continue
                 print(f'>>return: {reply}')
@@ -167,12 +147,12 @@ def minimax_call(context: list, system: str):
             print(response)
             time.sleep(5)
             continue
-    return ""
+    return ''
 
 
 def swift_call(context: list, system: str, swift_infer_obj):
     query_d: dict = context[-1]
-    history_list = context[: -1]
+    history_list = context[:-1]
 
     query: str = query_d['content']
     history_msg = []
@@ -211,9 +191,8 @@ def run_infer(args: InferArgs):
 
     if args.deploy_type == 'swift':
         from evalscope.third_party.toolbench_static.llm.swift_infer import SwiftInfer, SwiftInferArgs
-        swift_infer_args = SwiftInferArgs(model_id_or_path=args.model_name_or_path,
-                                          model_type=args.model_type,
-                                          max_new_tokens=args.max_new_tokens)
+        swift_infer_args = SwiftInferArgs(
+            model_id_or_path=args.model_name_or_path, model_type=args.model_type, max_new_tokens=args.max_new_tokens)
         swift_infer = SwiftInfer(args=swift_infer_args)
     else:
         swift_infer = None
@@ -232,7 +211,7 @@ def run_infer(args: InferArgs):
     preds = []
     refs = []
     for i, o in enumerate(infer_samples):
-        if i < len(processed_samples) and "predictions" in processed_samples[i].keys():
+        if i < len(processed_samples) and 'predictions' in processed_samples[i].keys():
             infer_samples[i]['predictions'] = processed_samples[i]['predictions']
             refs.append(processed_samples[i]['target'])
             preds.append(processed_samples[i]['predictions'])
@@ -267,7 +246,7 @@ def run_infer(args: InferArgs):
         reference = infer_samples[i]['target']
         infer_samples[i]['predictions'] = candidate
         if reference.strip() in ['', '.', ',']:
-            reference = "none"
+            reference = 'none'
         refs.append(reference)
         preds.append(candidate)
 

evalscope/third_party/toolbench_static/llm/swift_infer.py

@@ -1,9 +1,6 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 from dataclasses import dataclass
-
-from swift.llm import (
-    get_model_tokenizer, get_template, inference, get_default_template_type,
-)
+from swift.llm import get_default_template_type, get_model_tokenizer, get_template, inference
 from swift.utils import seed_everything
 
 # TODO: Support custom model for swift infer
@@ -21,9 +18,8 @@ class SwiftInfer:
     def __init__(self, args: SwiftInferArgs):
         model_type = args.model_type
         template_type = get_default_template_type(model_type)
-        model, tokenizer = get_model_tokenizer(model_type,
-                                               model_id_or_path=args.model_id_or_path,
-                                               model_kwargs={'device_map': 'auto'})
+        model, tokenizer = get_model_tokenizer(
+            model_type, model_id_or_path=args.model_id_or_path, model_kwargs={'device_map': 'auto'})
         model.generation_config.max_new_tokens = args.max_new_tokens
         print(f'** Generation config: {model.generation_config}')
 
@@ -36,10 +32,6 @@ class SwiftInfer:
 
     def predict(self, system: str, query: str, history: list):
 
-        response, history = inference(self.model,
-                                      self.template,
-                                      query=query,
-                                      system=system,
-                                      history=history)
+        response, history = inference(self.model, self.template, query=query, system=system, history=history)
 
         return response

evalscope/third_party/toolbench_static/requirements.txt

@@ -1,2 +1,2 @@
 ms-swift>=2.1.0
-rouge
+rouge

evalscope/third_party/toolbench_static/toolbench_static.py

@@ -1,11 +1,11 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 import os
-from typing import Union
 from copy import deepcopy
+from typing import Union
 
-from evalscope.third_party.toolbench_static.infer import InferArgs, run_infer
 from evalscope.third_party.toolbench_static.eval import EvalArgs, run_eval
-from evalscope.utils import yaml_to_dict, get_logger, json_to_dict
+from evalscope.third_party.toolbench_static.infer import InferArgs, run_infer
+from evalscope.utils import get_logger, json_to_dict, yaml_to_dict
 
 logger = get_logger()