evalscope 0.5.0__py3-none-any.whl

evalscope/third_party/toolbench_static/eval.py

@@ -0,0 +1,219 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import json
+from dataclasses import dataclass
+
+from rouge import Rouge
+import os
+
+
+@dataclass
+class EvalArgs:
+    input_path: str
+    output_path: str
+
+
+def run_eval(args: EvalArgs):
+    print(f'*** Start evaluation with eval args: {args}\n')
+
+    args.input_path = os.path.join(args.input_path, 'predictions.json')
+    args.output_path = os.path.join(args.output_path, 'metrics.json')
+
+    def evaluate_rougel(cand_list: list, ref_list: list):
+        if len(ref_list) == 0:
+            return 0
+        rouge = Rouge()
+        rouge_score = rouge.get_scores(hyps=cand_list, refs=ref_list, avg=True)
+        rougel = rouge_score["rouge-l"]["f"]
+        return rougel
+
+    def evaluate_action_em(cand_list: list, ref_list: list):
+        if len(ref_list) == 0:
+            return 0
+        em = 0
+        for cand, ref in zip(cand_list, ref_list):
+            em += (1 if cand == ref else 0)
+        return em / len(cand_list)
+
+    def evaluate_action_input_f1(action_pred: list, action_ref: list, cand_list: list, ref_list: list):
+        easy_f1 = []
+        hard_f1 = []
+        f1 = []
+        for i in range(len(action_pred)):
+            ref_action = action_ref[i]
+            pred_action = action_pred[i]
+
+            ref_input = ref_list[i]
+            cand_input = cand_list[i]
+
+            if ref_action != pred_action:
+                easy_f1.append(0)
+                hard_f1.append(0)
+                f1.append(0)
+            else:
+                try:
+                    ref_input_json = json.loads(ref_input)
+                    try:
+                        cand_input_json = json.loads(cand_input)
+                        half_match = 0
+                        full_match = 0
+                        if ref_input_json == {}:
+                            if cand_input_json == {}:
+                                easy_f1.append(1)
+                                f1.append(1)
+                            else:
+                                easy_f1.append(0)
+                                f1.append(0)
+                        else:
+                            for k, v in ref_input_json.items():
+                                if k in cand_input_json.keys():
+                                    if cand_input_json[k] == v:
+                                        full_match += 1
+                                    else:
+                                        half_match += 1
+
+                            recall = (0.5 * half_match + full_match) / (len(ref_input_json) + 1e-30)
+                            precision = (0.5 * half_match + full_match) / (len(cand_input_json) + 1e-30)
+                            hard_f1.append((2 * recall * precision) / (recall + precision))
+                            f1.append((2 * recall * precision) / (recall + precision))
+                    except:
+                        # cand_input = cand_input.replace("\n","").replace("\"","")
+                        # ref_input = cand_input.replace("\n","").replace("\"","")
+                        # rouge = Rouge()
+                        # rouge_score = rouge.get_scores(hyps=[cand_input], refs=[ref_input], avg=True)
+                        if ref_input_json == {}:
+                            easy_f1.append(0)
+                        else:
+                            hard_f1.append(0)
+                        # hard_f1.append(rouge_score["rouge-l"]["f"])
+                        # f1.append(rouge_score["rouge-l"]["f"])
+                        f1.append(0)
+                except:
+                    pass
+
+        return sum(easy_f1) / len(easy_f1) + 1e-30, sum(hard_f1) / len(hard_f1) + 1e-30, sum(f1) / len(f1) + 1e-30
+
+    with open(args.input_path, encoding='utf-8') as f:
+        data = json.load(f)
+
+    def parse_action(text):
+        action = "None"
+        action_input = "{}"
+        if 'Action Input:' in text:
+            input_idx = text.rindex('Action Input:')
+            action_input = text[input_idx + len('Action Input:'):].strip()
+        else:
+            action_input = '{}'
+
+        if 'Action:' in text:
+            action_idx = text.rindex('Action:')
+            action = text[action_idx + len('Action:'):].strip()
+            if 'Action Input:' in action:
+                input_idx = action.index('Action Input:')
+                action = action[:input_idx].strip()
+        else:
+            action = 'none'
+        return action, action_input
+
+    def parse_output(text):
+        action, action_input = parse_action(text)
+        if action == "Finish":
+            try:
+                action_input = json.loads(action_input)
+                # print(action_input)
+                # print(json.dumps(action_input,indent=2))
+                return_type = action_input["return_type"]
+                if return_type == "give_answer":
+                    if "final_answer" in action_input.keys():
+                        answer = str(action_input['final_answer'])
+                        if answer.strip() in ['', '.', ',']:
+                            answer = "None"
+                    else:
+                        answer = "None"
+                    return "finish", action, action_input, answer
+                else:
+                    return "give up", None, None, None
+            except:
+                return "give up", None, None, None
+        else:
+            plan = 'call'
+            answer = None
+            return plan, action, action_input, answer
+
+    plan_ref = []
+    plan_pred = []
+    hallu_cases = []
+    error_cases = []
+    new_data = []
+    answer_ref = []
+    action_ref = []
+    action_input_ref = []
+    hallu_ref = 0
+    answer_pred = []
+    action_pred = []
+    action_input_pred = []
+    hallu_pred = 0
+    for d in data:
+        reference = d['target']
+        prediction = d['predictions']
+        ref_plan, ref_action, ref_input, ref_ans = parse_output(reference)
+        # ref_plan: call
+        # ref_action: spott
+        # ref_input: {"is_id": "city center" }
+        # ref_ans: None
+
+        pred_plan, pred_action, pred_input, pred_ans = parse_output(prediction)
+        if ref_action is not None and ref_action == "invalid_hallucination_function_name":
+            continue
+        if pred_action is not None and ref_action != 'none' and ref_action not in [t['name'] for t in d['tools']]:
+            continue
+
+        if pred_action is not None and pred_action != 'none' and pred_action not in [t['name'] for t in d['tools']]:
+            hallu_pred += 1
+            hallu_cases.append(d)
+
+        plan_ref.append(ref_plan)
+        plan_pred.append(pred_plan)
+        if ref_plan == 'give up':
+            pass
+        elif ref_plan == 'finish':
+            answer_ref.append(ref_ans)
+            if pred_ans is None:
+                answer_pred.append('none')
+            else:
+                answer_pred.append(pred_ans)
+        else:
+            action_ref.append(ref_action)
+            action_input_ref.append(ref_input)
+            if pred_action is None:
+                action_pred.append('none')
+            else:
+                action_pred.append(pred_action)
+
+            if pred_input is None:
+                action_input_pred.append('{}')
+            else:
+                action_input_pred.append(pred_input)
+
+    metric = {}
+    rouge = evaluate_rougel(answer_pred, answer_ref)
+    plan_em = evaluate_action_em(cand_list=plan_pred, ref_list=plan_ref)
+    action_em = evaluate_action_em(cand_list=action_pred, ref_list=action_ref)
+    easy_f1, hard_f1, f1 = evaluate_action_input_f1(action_pred, action_ref, action_input_pred, action_input_ref)
+    hallu_rate = hallu_pred / len(data)
+    metric['rouge'] = rouge
+    metric['plan_em'] = plan_em
+    metric['action_em'] = action_em
+    metric['easy_f1'] = easy_f1
+    metric['hard_f1'] = hard_f1
+    metric['f1'] = f1
+    metric['hallu_rate'] = hallu_rate
+
+    if not os.path.exists(os.path.dirname(args.output_path)):
+        os.makedirs(os.path.dirname(args.output_path))
+    print(metric)
+    with open(args.output_path, 'w', encoding='utf-8') as f:
+        json.dump(metric, f, indent=2)
+
+    with open(args.output_path.replace('metrics.json', 'hallu_cases.json'), 'w', encoding='utf-8') as f:
+        json.dump(hallu_cases, f, indent=2)

evalscope/third_party/toolbench_static/infer.py

@@ -0,0 +1,278 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+# Adopted from tatsu-lab@stanford_alpaca. Below is the original copyright:
+#    Copyright 2023 Rohan Taori, Ishaan Gulrajani, Tianyi Zhang, Yann Dubois, Xuechen Li
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+# Make it more memory efficient by monkey patching the LLaMA model with FlashAttn.
+
+from dataclasses import dataclass, field
+import json
+import os
+from rouge import Rouge
+import time
+from urllib3.exceptions import MaxRetryError, NewConnectionError
+import requests
+
+
+def evaluate_rouge_l(cand_list: list, ref_list: list):
+    if len(ref_list) == 0:
+        return 0
+    rouge = Rouge()
+    rouge_score = rouge.get_scores(hyps=cand_list, refs=ref_list, avg=True)
+    rougel = rouge_score["rouge-l"]["f"]
+    return rougel
+
+
+def nested_load_test_data(data_path):
+    test_raw_data = []
+    if os.path.isdir(data_path):
+        for f in os.listdir(data_path):
+            temp_test = nested_load_test_data(os.path.join(data_path, f))
+            test_raw_data += temp_test
+        return test_raw_data
+    elif os.path.isfile(data_path) and data_path.endswith('.json'):
+        print("Load data from", data_path)
+        temp_data = json.load(open(data_path, "r"))
+        test_raw_data = temp_data
+        return test_raw_data
+    else:
+        return []
+
+
+def baichuan_call(context: list, system: str):
+    url = "https://api.baichuan-ai.com/v1/chat/completions"
+    api_key = "sk-xxx"
+
+    new_msg = []
+    new_msg.append({
+        "role": 'system',
+        'content': system})
+    for m in context:
+        if m['role'] == "user":
+            new_msg.append({
+                'role': 'user', 'content': m['content']
+            })
+        elif m['role'] == "function":
+            new_msg.append({
+                'role': 'user', 'content': m['content']
+            })
+        elif m['role'] == 'assistant':
+            new_msg.append({
+                'role': 'assistant', 'content': m['content']
+            })
+    # print(json.dumps(new_msg, indent=2))
+    data = {
+        "model": "Baichuan2-Turbo",
+        "messages": new_msg,
+        "stream": False
+    }
+
+    json_data = json.dumps(data)
+
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": "Bearer " + api_key
+    }
+
+    for i in range(5):
+        res = None
+        try:
+            res = requests.post(url, data=json_data, headers=headers, timeout=60)
+            res = res._content.decode('utf-8')
+            res = json.loads(res)
+            return res["choices"][0]["message"]["content"]
+        except KeyError:
+            print(res)
+            time.sleep(1)
+            continue
+        except ConnectionError:
+            time.sleep(5)
+            continue
+        except MaxRetryError:
+            time.sleep(5)
+            continue
+        except NewConnectionError:
+            time.sleep(5)
+            continue
+    return ""
+
+
+def minimax_call(context: list, system: str):
+    group_id = "your-id"
+    api_key = "your-xxx"
+
+    url = f"https://api.minimax.chat/v1/text/chatcompletion_pro?GroupId={group_id}"
+    headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
+
+    # construct message
+    system_prompt = "MM智能助理是一款由MiniMax自研的，没有调用其他产品的接口的大型语言模型。" \
+                    "MiniMax是一家中国科技公司，一直致力于进行大模型相关的研究。"
+    system_prompt += ('\n' + system)
+
+    new_msg = []
+    for m in context:
+        if m['role'] == "user":
+            new_msg.append({
+                'sender_type': 'USER', 'sender_name': 'user', 'text': m['content']
+            })
+        elif m['role'] == "function":
+            new_msg.append({
+                'sender_type': 'USER', 'sender_name': 'funtion', 'text': m['content']
+            })
+        elif m['role'] == 'assistant':
+            new_msg.append({
+                'sender_type': 'BOT', 'sender_name': 'MM智能助理', 'text': m['content']
+            })
+
+    request_body = {
+        "model": "abab6-chat",
+        # "model": "abab5.5s-chat",
+        "tokens_to_generate": 8192,
+        "reply_constraints": {"sender_type": "BOT", "sender_name": "MM智能助理"},
+        "messages": new_msg,
+        "bot_setting": [
+            {
+                "bot_name": "MM智能助理",
+                "content": system_prompt,
+            }
+        ],
+    }
+    response = requests.post(url, headers=headers, json=request_body)
+    status_code = response.status_code
+    for i in range(5):
+        try:
+            if status_code == 200:
+                reply = response.json()["reply"]
+                if len(reply) == 0:
+                    print("limit rate")
+                    time.sleep(8)
+                    continue
+                print(f'>>return: {reply}')
+                return reply
+            else:
+                print(response._content)
+                time.sleep(5)
+        except KeyError:
+            print(response)
+            time.sleep(5)
+            continue
+    return ""
+
+
+def swift_call(context: list, system: str, swift_infer_obj):
+    query_d: dict = context[-1]
+    history_list = context[: -1]
+
+    query: str = query_d['content']
+    history_msg = []
+
+    tmp_list = []
+    for idx, item in enumerate(history_list):
+
+        if idx % 2 == 0:
+            tmp_list.append(item['content'])
+        else:
+            tmp_list.append(item['content'])
+            history_msg.append(tuple(tmp_list))
+            tmp_list = []
+
+    try:
+        resp_str: str = swift_infer_obj.predict(system=system, query=query, history=history_msg)
+    except Exception as e:
+        print(e)
+        resp_str = ''
+
+    return resp_str
+
+
+@dataclass
+class InferArgs:
+    model_name_or_path: str
+    model_type: str
+    data_path: str
+    output_dir: str
+    deploy_type: str
+    max_new_tokens: int = 2048
+    num_infer_samples: int = None
+
+
+def run_infer(args: InferArgs):
+
+    if args.deploy_type == 'swift':
+        from evalscope.third_party.toolbench_static.llm.swift_infer import SwiftInfer, SwiftInferArgs
+        swift_infer_args = SwiftInferArgs(model_id_or_path=args.model_name_or_path,
+                                          model_type=args.model_type,
+                                          max_new_tokens=args.max_new_tokens)
+        swift_infer = SwiftInfer(args=swift_infer_args)
+    else:
+        swift_infer = None
+
+    # load data
+    infer_samples = nested_load_test_data(args.data_path)
+    if args.num_infer_samples is not None:
+        infer_samples = infer_samples[:args.num_infer_samples]
+
+    os.makedirs(args.output_dir, exist_ok=True)
+    if os.path.exists(os.path.join(args.output_dir, 'predictions.json')):
+        with open(os.path.join(args.output_dir, 'predictions.json')) as f:
+            processed_samples = json.load(f)
+    else:
+        processed_samples = []
+    preds = []
+    refs = []
+    for i, o in enumerate(infer_samples):
+        if i < len(processed_samples) and "predictions" in processed_samples[i].keys():
+            infer_samples[i]['predictions'] = processed_samples[i]['predictions']
+            refs.append(processed_samples[i]['target'])
+            preds.append(processed_samples[i]['predictions'])
+            continue
+
+        system = o['messages'][0]['content']
+        new_msg = o['messages'][1:]
+
+        print('================================')
+        print('case', str(i))
+
+        if args.deploy_type == 'minimax':
+            response_text = minimax_call(new_msg, system)
+        # elif model_args.model_type == 'xingchen':
+        #     response_text = spark_call(new_msg, system)
+        # elif model_args.model_type == 'xingchen_v2':
+        #     response_text = spark_call_v2(new_msg, system, model_args)
+        elif args.deploy_type == 'baichuan':
+            response_text = baichuan_call(new_msg, system)
+        elif args.deploy_type == 'swift':
+            assert swift_infer is not None, 'ModelScope Swift infer process is not initialized.'
+            response_text = swift_call(new_msg, system, swift_infer)
+        else:
+            raise NotImplementedError
+
+        candidate = response_text
+        print(candidate)
+        if candidate.startswith(': '):
+            candidate = candidate[2:]
+        if candidate.strip() in ['', '.', ',']:
+            candidate = 'none'
+        reference = infer_samples[i]['target']
+        infer_samples[i]['predictions'] = candidate
+        if reference.strip() in ['', '.', ',']:
+            reference = "none"
+        refs.append(reference)
+        preds.append(candidate)
+
+        with open(os.path.join(args.output_dir, 'predictions.json'), 'w') as f:
+            json.dump(infer_samples[:i + 1], f, indent=4)
+
+    rouge_l = round(evaluate_rouge_l(preds, refs), 2)
+    print('\n*** Overall rouge:', rouge_l)

evalscope/third_party/toolbench_static/llm/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.

evalscope/third_party/toolbench_static/llm/swift_infer.py

@@ -0,0 +1,45 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from dataclasses import dataclass
+
+from swift.llm import (
+    get_model_tokenizer, get_template, inference, get_default_template_type,
+)
+from swift.utils import seed_everything
+
+# TODO: Support custom model for swift infer
+
+
+@dataclass
+class SwiftInferArgs:
+    model_id_or_path: str
+    model_type: str
+    max_new_tokens: int = 2048
+
+
+class SwiftInfer:
+
+    def __init__(self, args: SwiftInferArgs):
+        model_type = args.model_type
+        template_type = get_default_template_type(model_type)
+        model, tokenizer = get_model_tokenizer(model_type,
+                                               model_id_or_path=args.model_id_or_path,
+                                               model_kwargs={'device_map': 'auto'})
+        model.generation_config.max_new_tokens = args.max_new_tokens
+        print(f'** Generation config: {model.generation_config}')
+
+        template = get_template(template_type, tokenizer)
+        seed_everything(42)
+
+        self.tokenizer = tokenizer
+        self.model = model
+        self.template = template
+
+    def predict(self, system: str, query: str, history: list):
+
+        response, history = inference(self.model,
+                                      self.template,
+                                      query=query,
+                                      system=system,
+                                      history=history)
+
+        return response

evalscope/third_party/toolbench_static/toolbench_static.py

@@ -0,0 +1,50 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+from typing import Union
+from copy import deepcopy
+
+from evalscope.third_party.toolbench_static.infer import InferArgs, run_infer
+from evalscope.third_party.toolbench_static.eval import EvalArgs, run_eval
+from evalscope.utils import yaml_to_dict, get_logger, json_to_dict
+
+logger = get_logger()
+
+
+def run_task(task_cfg: Union[str, dict]):
+
+    if isinstance(task_cfg, str):
+        if task_cfg.endswith('.yaml'):
+            task_cfg: dict = yaml_to_dict(task_cfg)
+        elif task_cfg.endswith('.json'):
+            task_cfg: dict = json_to_dict(task_cfg)
+        else:
+            raise ValueError(f'Unsupported file format: {task_cfg}, should be yaml or json file.')
+
+    # Run inference for each domain
+    infer_args: dict = task_cfg['infer_args']
+    for domain in ['in_domain', 'out_of_domain']:
+        domain_infer_args = deepcopy(infer_args)
+        domain_infer_args.update({'data_path': os.path.join(infer_args['data_path'], f'{domain}.json')})
+        domain_infer_args.update({'output_dir': os.path.join(infer_args['output_dir'], domain)})
+
+        task_infer_args = InferArgs(**domain_infer_args)
+        print(f'**Run infer config: {task_infer_args}')
+        run_infer(task_infer_args)
+
+    # Run evaluation for each domain
+    eval_args: dict = task_cfg['eval_args']
+    for domain in ['in_domain', 'out_of_domain']:
+        domain_eval_args = deepcopy(eval_args)
+        domain_eval_args.update({'input_path': os.path.join(eval_args['input_path'], domain)})
+        domain_eval_args.update({'output_path': os.path.join(eval_args['output_path'], domain)})
+
+        task_eval_args = EvalArgs(**domain_eval_args)
+        print(f'**Run eval config: {task_eval_args}')
+        run_eval(task_eval_args)
+
+
+if __name__ == '__main__':
+    # task_cfg_file = 'config_default.yaml'
+    task_cfg_file = 'config_default.json'
+
+    run_task(task_cfg=task_cfg_file)

evalscope/tools/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.