evalscope 0.15.0__py3-none-any.whl → 0.16.0__py3-none-any.whl

evalscope/arguments.py

@@ -9,6 +9,15 @@ class ParseStrArgsAction(argparse.Action):
     def __call__(self, parser, namespace, values, option_string=None):
         assert isinstance(values, str), 'args should be a string.'
 
+        # try json load first
+        try:
+            arg_dict = json.loads(values)
+            setattr(namespace, self.dest, arg_dict)
+            return
+        except (json.JSONDecodeError, ValueError):
+            pass
+
+        # If JSON load fails, fall back to parsing as key=value pairs
         arg_dict = {}
         for arg in values.strip().split(','):
             key, value = map(str.strip, arg.split('=', 1))  # Use maxsplit=1 to handle multiple '='
@@ -67,6 +76,7 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--work-dir', type=str, help='The root cache dir.')
 
     # Debug and runtime mode arguments
+    parser.add_argument('--ignore-errors', action='store_true', default=False, help='Ignore errors during evaluation.')
     parser.add_argument('--debug', action='store_true', default=False, help='Debug mode, will print information for debugging.')  # noqa: E501
     parser.add_argument('--dry-run', action='store_true', default=False, help='Dry run in single processing mode.')
     parser.add_argument('--seed', type=int, default=42, help='Random seed for reproducibility.')

evalscope/backend/rag_eval/utils/llm.py

@@ -52,7 +52,7 @@ class LocalLLM(BaseLLM):
         """Run the LLM on the given input."""
         infer_cfg = {'stop': stop}
 
-        response, _ = self.model._model_generate([prompt], infer_cfg=infer_cfg)
+        response, _ = self.model.predict([{'data': [prompt]}], infer_cfg=infer_cfg)
         return response[0][0]
 
     @property

evalscope/benchmarks/aigc/t2i/general_t2i_adapter.py

@@ -34,7 +34,7 @@ class GeneralT2IAdapter(T2IBaseAdapter):
         subset_list = subset_list or self.subset_list
 
         data_file_dict = defaultdict(str)
-        data_list = []
+        data_item_dict = defaultdict(list)
 
         # get data file path and subset name
         if os.path.isdir(dataset_name_or_path):
@@ -49,10 +49,10 @@ class GeneralT2IAdapter(T2IBaseAdapter):
         # load data from local disk
         try:
             for subset_name, file_path in data_file_dict.items():
-                data_list.extend(jsonl_to_list(file_path))
+                data_item_dict[subset_name] = jsonl_to_list(file_path)
         except Exception as e:
             raise ValueError(f'Failed to load data from {self.dataset_id}, got error: {e}')
 
-        data_dict = {subset_name: {'test': data_list} for subset_name in data_file_dict.keys()}
+        data_dict = {subset_name: {'test': data_item_dict[subset_name]} for subset_name in data_file_dict.keys()}
 
         return data_dict

evalscope/benchmarks/alpaca_eval/alpaca_eval_adapter.py

@@ -96,12 +96,6 @@ class AlpacaEvalAdapter(DataAdapter):
             return None
 
     def compute_metric(self, review_res_list: List[bool], **kwargs) -> List[dict]:
-        """
-        compute weighted mean of the bleu score of all samples
-
-        Args:
-            review_res_list: [{'is_correct': 1, 'is_incorrect': 0, 'is_not_attempted': 0}, ...]
-        """
         # zip dict answers
         res_list = [res for res in review_res_list if res is not None]
 

evalscope/benchmarks/chinese_simple_qa/csimple_qa_adapter.py

@@ -148,6 +148,7 @@ class ChineseSimpleQAAdapter(DataAdapter):
             'is_correct': 1 if res == 'A' else 0,
             'is_incorrect': 1 if res == 'B' else 0,
             'is_not_attempted': 1 if res == 'C' else 0,
+            'judge_response': grading_response,
         }
 
     def compute_metric(self, review_res_list: List[dict], **kwargs) -> List[dict]:

evalscope/benchmarks/data_adapter.py

@@ -258,7 +258,7 @@ class DataAdapter(ABC):
             avg_res: Dict[str, List[float]]
 
         """
-        if isinstance(review_res_list[0], list):
+        if len(review_res_list) > 0 and isinstance(review_res_list[0], list):
             review_res_list = [item for sublist in review_res_list for item in sublist]
 
         items = defaultdict(list)
@@ -322,6 +322,7 @@ class DataAdapter(ABC):
                         choices: Optional[List[str]] = None,
                         index: Optional[Union[int, str]] = None,
                         id: Optional[Union[int, str]] = None,
+                        messages: Optional[List[dict]] = None,
                         **kwargs) -> dict:
         data = [prompt] if not isinstance(prompt, list) else prompt
         prompt_data = PromptData(
@@ -329,7 +330,8 @@ class DataAdapter(ABC):
             multi_choices=choices or self.choices,
             system_prompt=system_prompt or self.system_prompt,
             index=index or 0,
-            id=id)
+            id=id,
+            messages=messages)
         return prompt_data.to_dict()
 
     def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> Any:

evalscope/benchmarks/drop/drop_adapter.py

@@ -0,0 +1,133 @@
+import re
+from typing import List
+
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import EvalType
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+DROP_EXAMPLES = '''Some examples of passages and Q&A are provided below.
+
+# Examples
+---
+Passage: Trunajaya rebellion  or Trunajaya War was the ultimately unsuccessful rebellion waged by the Madurese prince Trunajaya and fighters from Makassar against the Mataram Sultanate and its Dutch East India Company  supporters in Java  during the 1670s. The rebellion was initially successful: the rebels defeated the royal army at Gegodog , captured most of the Javanese north coast, and took the Mataram capital Plered . King Amangkurat I died during the retreat of the royal court. His son and successor, Amangkurat II, requested help from the VOC in exchange for financial remuneration and geopolitical concessions. The VOC\'s subsequent involvement turned the tide of the war. VOC and Mataram forces recovered lost territories and overran Trunajaya\'s new capital at Kediri . However, the rebellion continued until the capture of Trunajaya at the end of 1679, and the defeat, death, or surrender of the other rebel leaders . Trunajaya was killed by Amangkurat II personally in 1680 while a prisoner of the VOC. After his father\'s death in 1677, Amangkurat II also faced rival claims to the throne. The most serious rival was his brother Pangeran Puger, who took the capital Plered in 1677 and did not surrender until 1681.
+Question: How many years was it between Trunajaya\'s capture and his death while prisoner of the VOC?
+Answer:  1
+
+---
+Passage: Led by former Giant Kurt Warner, the defending NFC champions took the field at Giants Stadium against a Giants team still reeling from their bad loss in New Orleans. The Giants scored first, sending Jacobs in for a 4-yard touchdown run following a Terrell Thomas interception. Later, Arizona running back Beanie Wells scored his first career touchdown on a 13-yard rush. Manning responded by throwing a 62-yard touchdown to Nicks for his longest reception of the year. In the second half, the Cardinals\' Tim Hightower and Jason Wright scored touchdowns. But it was turnovers that decided this game; Manning\'s 3 interceptions were as many as he had thrown all season. The Giants scored only 3 points in the second half, ending the game on an interception to Antrel Rolle. The Giants notable streak of 38 consecutive starts by the same offensive line unit was ended here, as offensive tackle Kareem McKenzie missed the game with a groin injury. McKenzie returned the following week.
+Question: Which player made the first score of the game?
+Answer:  Jacobs
+
+---
+Passage: Hoping to rebound from their road loss to the Bills, the Chargers flew to Wembley Stadium for the 2008 International Series game with the New Orleans Saints. In the first quarter, San Diego trailed early as kicker Taylor Mehlhaff got a 23-yard field goal.  The \'Bolts would respond with kicker Nate Kaeding getting a 33-yard field goal.  In the second quarter, New Orleans regained the lead as QB Drew Brees (a former Charger) completed a 12-yard TD pass to WR Devery Henderson (with a failed PAT) and RB Deuce McAllister getting a 1-yard TD run.  San Diego answered as QB Philip Rivers completed a 12-yard TD pass to RB LaDainian Tomlinson, but the Saints replied with Brees completing a 30-yard TD pass to WR Lance Moore.  The Chargers closed out the half with Rivers completing a 12-yard TD pass to TE Antonio Gates. In the third quarter, New Orleans increased its lead Brees completing a 1-yard TD pass to TE Mark Campbell, after a very controversial Pass interference call on cornerback Cletis Gordon put the Saints on the 1-yard line.  The \'Bolts would answer with Kaeding getting a 24-yard field goal.  In the fourth quarter, the Saints continued to build its lead as FB Mike Karney got a 1-yard TD run.  San Diego tried to rally as Kaeding nailed a 31-yard field goal, Rivers completed a 14-yard TD pass to WR Vincent Jackson, and Brees giving the \'Bolts a safety via an incomplete pass thrown into the back of his own endzone.  However, New Orleans\' defense stiffened for the win. With the loss, the Chargers went into their bye week at 3-5.
+Question: How many total yards of touchdown passes did Drew Brees make?
+Answer:  43
+
+'''  # noqa: E501
+
+
+@Benchmark.register(
+    name='drop',
+    pretty_name='DROP',
+    dataset_id='AI-ModelScope/DROP',
+    metric_list=['AverageAccuracy'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='validation',
+    prompt_template=
+    'You will be asked to read a passage and answer a question.{drop_examples}# Your Task\n\n---\n{query}\n\nThink step by step, then write a line of the form "Answer: $ANSWER" at the end of your response.',  # noqa: E501
+)
+class DROPAdapter(DataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        few_shot_num = kwargs.get('few_shot_num', 0)
+        if few_shot_num != 0:
+            self.few_shot_num = 3
+            logger.info(f'Few shot num is set to {self.few_shot_num} for DROP dataset by system.')
+        else:
+            self.few_shot_num = 0
+
+    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
+        """
+        Generate model prompt from input data.
+        """
+        drop_examples = '' if self.few_shot_num == 0 else DROP_EXAMPLES
+        query = f"Passage: {input_d['passage']}\nQuestion: {input_d['question']}"
+        prompt = self.prompt_template.format(
+            drop_examples=drop_examples,
+            query=query,
+        )
+        return self.gen_prompt_data(prompt)
+
+    def get_gold_answer(self, input_d: dict) -> List[str]:
+        """
+        Parse the raw input labels (gold).
+        """
+
+        def _flatten_validated_answers(validated_answers):
+            """Flattens a dict of lists of validated answers.
+            {"number": ['1', '8'], ...}
+            -> [{"number": ['1'], ...}, {"number": ['8'], ...}]
+            """
+            valid_answers = []
+            for i in range(len(validated_answers['number'])):
+                valid_answers.append({
+                    'number': validated_answers['number'][i],
+                    'date': validated_answers['date'][i],
+                    'spans': validated_answers['spans'][i],
+                })
+            return valid_answers
+
+        answers = []
+        answers_set = set()
+        candidates = [input_d['answer']] + _flatten_validated_answers(input_d['validated_answers'])
+        for candidate in candidates:
+            answer = DROPAdapter.parse_answer(candidate)
+            if answer in answers_set:
+                continue
+            answers_set.add(answer)
+            answers.append(answer)
+        return answers
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+        """
+        Parse the predicted result and extract proper answer.
+        """
+        match = re.search(r'(?i)Answer\s*:\s*([^\n]+)', result)
+        extracted_answer = match.group(1) if match else result
+        return extracted_answer
+
+    def match(self, gold: List[str], pred: str) -> float:
+        """
+        Match the gold answer and the predicted answer.
+        """
+        from .utils import _answer_to_bags
+
+        max_em = 0
+        for gold_answer in gold:
+            # Convert the answers to bags of answers
+            predicted_bags = _answer_to_bags(pred)
+            gold_bags = _answer_to_bags(gold_answer)
+
+            if set(predicted_bags[0]) == set(gold_bags[0]) and len(predicted_bags[0]) == len(gold_bags[0]):
+                exact_match = 1.0
+            else:
+                exact_match = 0.0
+            # Check if the answer is empty
+            if gold_answer[0].strip():
+                max_em = max(max_em, exact_match)
+
+        return max_em
+
+    @staticmethod
+    def parse_answer(answer):
+        # NOTE: Everything is returned as a tuple for uniformity and hashability.
+        if answer['number'] != '':
+            return (str(answer['number']), )
+        if answer['spans'] != []:
+            return tuple(answer['spans'])
+        return (' '.join([answer['date']['day'], answer['date']['month'], answer['date']['year']]).strip(), )

evalscope/benchmarks/drop/utils.py

@@ -0,0 +1,59 @@
+import re
+import string
+
+_ARTICLES = re.compile(r'\b(a|an|the)\b', re.UNICODE)
+
+
+def _answer_to_bags(answer):
+    if isinstance(answer, (list, tuple)):
+        raw_spans = answer
+    else:
+        raw_spans = [answer]
+    normalized_spans = []
+    token_bags = []
+    for raw_span in raw_spans:
+        normalized_span = _normalize(raw_span)
+        normalized_spans.append(normalized_span)
+        token_bags.append(set(normalized_span.split()))
+    return normalized_spans, token_bags
+
+
+def _is_number(text):
+    try:
+        float(text)
+        return True
+    except ValueError:
+        return False
+
+
+def _remove_articles(text):
+    return _ARTICLES.sub(' ', text)
+
+
+def _white_space_fix(text):
+    return ' '.join(text.split())
+
+
+def _remove_punc(text):
+    exclude = set(string.punctuation)
+    if not _is_number(text):
+        return ''.join(ch for ch in text if ch not in exclude)
+    else:
+        return text
+
+
+def _fix_number(text):
+    return str(float(text)) if _is_number(text) else text
+
+
+def _tokenize(text):
+    return re.split(' |-', text)
+
+
+def _normalize(answer):
+    tokens = [
+        _white_space_fix(_remove_articles(_fix_number(_remove_punc(token.lower())))) for token in _tokenize(answer)
+    ]
+    tokens = [token for token in tokens if token.strip()]
+    normalized = ' '.join(tokens).strip()
+    return normalized

evalscope/benchmarks/general_qa/general_qa_adapter.py

@@ -4,7 +4,7 @@ from collections import defaultdict
 from typing import List, Optional, Union
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import bleu_ngram_one_sample, compute_rouge_score_one_sample_zh, mean
+from evalscope.metrics import mean
 from evalscope.utils.io_utils import jsonl_to_list
 from evalscope.utils.logger import get_logger
 
@@ -33,7 +33,7 @@ class GeneralQAAdapter(DataAdapter):
         subset_list = subset_list or self.subset_list
 
         data_file_dict = defaultdict(str)
-        data_list = []
+        data_item_dict = defaultdict(list)
 
         # get data file path and subset name
         if os.path.isdir(dataset_name_or_path):
@@ -48,11 +48,11 @@ class GeneralQAAdapter(DataAdapter):
         # load data from local disk
         try:
             for subset_name, file_path in data_file_dict.items():
-                data_list.extend(jsonl_to_list(file_path))
+                data_item_dict[subset_name] = jsonl_to_list(file_path)
         except Exception as e:
             raise ValueError(f'Failed to load data from {self.dataset_id}, got error: {e}')
 
-        data_dict = {subset_name: {'test': data_list} for subset_name in data_file_dict.keys()}
+        data_dict = {subset_name: {'test': data_item_dict[subset_name]} for subset_name in data_file_dict.keys()}
 
         return data_dict
 
@@ -112,9 +112,13 @@ class GeneralQAAdapter(DataAdapter):
         """
         res = dict()
         if 'AverageRouge' in self.metric_list:
+            from evalscope.metrics.rouge_metric import compute_rouge_score_one_sample_zh
+
             rouge_dict = compute_rouge_score_one_sample_zh([pred], [gold])
             res.update(rouge_dict)
         if 'AverageBLEU' in self.metric_list:
+            from evalscope.metrics import bleu_ngram_one_sample
+
             bleu_dict = bleu_ngram_one_sample(pred, gold)
             res.update(bleu_dict)
         return res

evalscope/benchmarks/simple_qa/simple_qa_adapter.py

@@ -148,6 +148,7 @@ class SimpleQAAdapter(DataAdapter):
             'is_correct': 1 if res == 'A' else 0,
             'is_incorrect': 1 if res == 'B' else 0,
             'is_not_attempted': 1 if res == 'C' else 0,
+            'judge_response': grading_response,
         }
 
     def compute_metric(self, review_res_list: List[dict], **kwargs) -> List[dict]:

evalscope/benchmarks/tool_bench/tool_bench_adapter.py

@@ -0,0 +1,67 @@
+from typing import Dict, List
+
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import EvalType, OutputType
+from evalscope.metrics import Metric, mean, metric_registry
+
+
+@Benchmark.register(
+    name='tool_bench',
+    pretty_name='ToolBench-Static',
+    dataset_id='AI-ModelScope/ToolBench-Static',
+    subset_list=['in_domain', 'out_of_domain'],
+    metric_list=['Act.EM', 'Plan.EM', 'F1', 'HalluRate', 'Rouge-L'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+)
+class ToolBenchAdapter(DataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        metric_registry.register(Metric(name='Rouge-L', object=mean))
+        metric_registry.register(Metric(name='Act.EM', object=mean))
+        metric_registry.register(Metric(name='Plan.EM', object=mean))
+        metric_registry.register(Metric(name='F1', object=mean))
+        metric_registry.register(Metric(name='HalluRate', object=mean))
+
+    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
+        """
+        Generate model prompt from input data.
+        """
+        messages = input_d['messages']
+        # use prepared messages
+        return self.gen_prompt_data(prompt='', messages=messages)
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        """
+        Parse the raw input labels (gold).
+        """
+        return input_d
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+        """
+        Parse the predicted result and extract proper answer.
+        """
+        return result
+
+    def match(self, gold: dict, pred: str) -> Dict:
+        """
+        Match the gold answer and the predicted answer.
+        """
+        from .utils import calculate_metrics
+
+        data = {
+            'target': gold['target'],
+            'predictions': pred,
+            'tools': gold['tools'],
+        }
+        metrics = calculate_metrics(data)
+        return metrics
+
+    def compute_metric(self, review_res_list: List[dict], **kwargs) -> Dict:
+        # aggregate review results
+        res_dict = super().compute_dict_metric(review_res_list, **kwargs)
+
+        return super().compute_metric(res_dict, **kwargs)

evalscope/benchmarks/tool_bench/utils.py

@@ -0,0 +1,202 @@
+import json
+from rouge import Rouge
+
+
+def evaluate_rougel(cand_list: list, ref_list: list):
+    if len(ref_list) == 0:
+        return 0
+    rouge = Rouge()
+    rouge_score = rouge.get_scores(hyps=cand_list, refs=ref_list, avg=True)
+    rougel = rouge_score['rouge-l']['f']
+    return rougel
+
+
+def evaluate_action_em(cand_list: list, ref_list: list):
+    if len(ref_list) == 0:
+        return 0
+    em = 0
+    for cand, ref in zip(cand_list, ref_list):
+        em += (1 if cand == ref else 0)
+    return em / len(cand_list)
+
+
+def evaluate_action_input_f1(action_pred: list, action_ref: list, cand_list: list, ref_list: list):
+    easy_f1 = []
+    hard_f1 = []
+    f1 = []
+    for i in range(len(action_pred)):
+        ref_action = action_ref[i]
+        pred_action = action_pred[i]
+
+        ref_input = ref_list[i]
+        cand_input = cand_list[i]
+
+        if ref_action != pred_action:
+            easy_f1.append(0)
+            hard_f1.append(0)
+            f1.append(0)
+        else:
+            try:
+                ref_input_json = json.loads(ref_input)
+                try:
+                    cand_input_json = json.loads(cand_input)
+                    half_match = 0
+                    full_match = 0
+                    if ref_input_json == {}:
+                        if cand_input_json == {}:
+                            easy_f1.append(1)
+                            f1.append(1)
+                        else:
+                            easy_f1.append(0)
+                            f1.append(0)
+                    else:
+                        for k, v in ref_input_json.items():
+                            if k in cand_input_json.keys():
+                                if cand_input_json[k] == v:
+                                    full_match += 1
+                                else:
+                                    half_match += 1
+
+                        recall = (0.5 * half_match + full_match) / (len(ref_input_json) + 1e-30)
+                        precision = (0.5 * half_match + full_match) / (len(cand_input_json) + 1e-30)
+                        hard_f1.append((2 * recall * precision) / (recall + precision))
+                        f1.append((2 * recall * precision) / (recall + precision))
+                except Exception:
+                    # cand_input = cand_input.replace("\n","").replace("\"","")
+                    # ref_input = cand_input.replace("\n","").replace("\"","")
+                    # rouge = Rouge()
+                    # rouge_score = rouge.get_scores(hyps=[cand_input], refs=[ref_input], avg=True)
+                    if ref_input_json == {}:
+                        easy_f1.append(0)
+                    else:
+                        hard_f1.append(0)
+                    # hard_f1.append(rouge_score["rouge-l"]["f"])
+                    # f1.append(rouge_score["rouge-l"]["f"])
+                    f1.append(0)
+            except Exception:
+                pass
+
+    # 检查列表是否为空，如果为空则返回0
+    easy_f1_avg = sum(easy_f1) / len(easy_f1) if easy_f1 else 0
+    hard_f1_avg = sum(hard_f1) / len(hard_f1) if hard_f1 else 0
+    f1_avg = sum(f1) / len(f1) if f1 else 0
+
+    return easy_f1_avg, hard_f1_avg, f1_avg
+
+
+def parse_action(text):
+    action = 'None'
+    action_input = '{}'
+    if 'Action Input:' in text:
+        input_idx = text.rindex('Action Input:')
+        action_input = text[input_idx + len('Action Input:'):].strip()
+    else:
+        action_input = '{}'
+
+    if 'Action:' in text:
+        action_idx = text.rindex('Action:')
+        action = text[action_idx + len('Action:'):].strip()
+        if 'Action Input:' in action:
+            input_idx = action.index('Action Input:')
+            action = action[:input_idx].strip()
+    else:
+        action = 'none'
+    return action, action_input
+
+
+def parse_output(text):
+    action, action_input = parse_action(text)
+    if action == 'Finish':
+        try:
+            action_input = json.loads(action_input)
+            # print(action_input)
+            # print(json.dumps(action_input,indent=2))
+            return_type = action_input['return_type']
+            if return_type == 'give_answer':
+                if 'final_answer' in action_input.keys():
+                    answer = str(action_input['final_answer'])
+                    if answer.strip() in ['', '.', ',']:
+                        answer = 'None'
+                else:
+                    answer = 'None'
+                return 'finish', action, action_input, answer
+            else:
+                return 'give up', None, None, None
+        except Exception:
+            return 'give up', None, None, None
+    else:
+        plan = 'call'
+        answer = None
+        return plan, action, action_input, answer
+
+
+def calculate_metrics(data):
+    """
+    Calculate the metrics for the given data.
+    """
+    plan_ref = []
+    plan_pred = []
+    hallu_cases = []
+    answer_ref = []
+    action_ref = []
+    action_input_ref = []
+    answer_pred = []
+    action_pred = []
+    action_input_pred = []
+    hallu_pred = 0
+
+    reference = data['target']
+    prediction = data['predictions']
+    ref_plan, ref_action, ref_input, ref_ans = parse_output(reference)
+    # ref_plan: call
+    # ref_action: spott
+    # ref_input: {"is_id": "city center" }
+    # ref_ans: None
+
+    pred_plan, pred_action, pred_input, pred_ans = parse_output(prediction)
+    if ref_action is not None and ref_action == 'invalid_hallucination_function_name':
+        return {}
+    if pred_action is not None and ref_action != 'none' and ref_action not in [t['name'] for t in data['tools']]:
+        return {}
+
+    if pred_action is not None and pred_action != 'none' and pred_action not in [t['name'] for t in data['tools']]:
+        hallu_pred += 1
+        hallu_cases.append(data)
+
+    plan_ref.append(ref_plan)
+    plan_pred.append(pred_plan)
+    if ref_plan == 'give up':
+        pass
+    elif ref_plan == 'finish':
+        answer_ref.append(ref_ans)
+        if pred_ans is None:
+            answer_pred.append('none')
+        else:
+            answer_pred.append(pred_ans)
+    else:
+        action_ref.append(ref_action)
+        action_input_ref.append(ref_input)
+        if pred_action is None:
+            action_pred.append('none')
+        else:
+            action_pred.append(pred_action)
+
+        if pred_input is None:
+            action_input_pred.append('{}')
+        else:
+            action_input_pred.append(pred_input)
+
+    metric = {}
+    rouge = evaluate_rougel(answer_pred, answer_ref)
+    plan_em = evaluate_action_em(cand_list=plan_pred, ref_list=plan_ref)
+    action_em = evaluate_action_em(cand_list=action_pred, ref_list=action_ref)
+    easy_f1, hard_f1, f1 = evaluate_action_input_f1(action_pred, action_ref, action_input_pred, action_input_ref)
+    hallu_rate = hallu_pred
+    metric['Act.EM'] = action_em
+    metric['F1'] = f1
+    metric['HalluRate'] = hallu_rate
+    metric['plan_em'] = plan_em
+    metric['Easy_F1'] = easy_f1
+    metric['Hard_F1'] = hard_f1
+    metric['Rouge-L'] = rouge
+    return metric

evalscope/benchmarks/utils.py

@@ -13,6 +13,7 @@ class PromptData:
     system_prompt: Optional[str] = None
     multi_choices: Optional[List[str]] = None
     id: Optional[str] = None
+    messages: Optional[List[dict]] = None
 
     def to_dict(self) -> Dict:
         return {k: v for k, v in asdict(self).items() if v is not None}
@@ -21,7 +22,7 @@ class PromptData:
 def preprocess_decorator(func):
 
     @wraps(func)
-    def wrapper(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT):
+    def wrapper(self, result: str, raw_input_d: dict = None, **kwargs):
         if result is None:
             result = ''
         filters = self.config_kwargs.get('filters', None)
@@ -29,6 +30,6 @@ def preprocess_decorator(func):
             # Apply filters to the resultply filters to the result
             for filter_name, filter_value in filters.items():
                 result = Filter.apply(filter_name, result, filter_value)
-        return func(self, result, raw_input_d, eval_type)
+        return func(self, result, raw_input_d, **kwargs)
 
     return wrapper