evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/benchmarks/data_collection/data_collection_adapter.py

@@ -1,28 +1,37 @@
-import math
+import copy
 import os
-import re
-from typing import Any, Optional
+from collections import defaultdict
+from typing import Any, Dict, List
 
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import DEFAULT_DATASET_CACHE_DIR, EvalType, HubType
-from evalscope.utils.io_utils import jsonl_to_list
+from evalscope.api.benchmark import BenchmarkMeta, DataAdapter, DefaultDataAdapter
+from evalscope.api.dataset import DatasetDict, LocalDataLoader, Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.metric.scorer import AggScore, SampleScore
+from evalscope.api.registry import get_benchmark, register_benchmark
+from evalscope.config import TaskConfig
+from evalscope.constants import DataCollection, Tags
+from evalscope.report.generator import ReportGenerator
+from evalscope.report.report import Report
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
 
-@Benchmark.register(
-    name='data_collection',
-    dataset_id='',  # dataset_id need to be set
-    description='Data collection',
-    subset_list=['default'],
-    metric_list=['AverageAccuracy'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
-    prompt_template='',
+@register_benchmark(
+    BenchmarkMeta(
+        name=DataCollection.NAME,
+        dataset_id='',  # dataset_id need to be set
+        description='Custom Data collection, mixing multiple evaluation datasets for '
+        'a unified evaluation, aiming to use less data to achieve a more comprehensive '
+        'assessment of the model\'s capabilities. '
+        '[Usage Reference](https://evalscope.readthedocs.io/zh-cn/latest/advanced_guides/collection/index.html)',
+        tags=[Tags.CUSTOM],
+        metric_list=['acc'],
+        eval_split='test',
+        prompt_template='',
+    )
 )
-class DataCollectionAdapter(DataAdapter):
+class DataCollectionAdapter(DefaultDataAdapter):
 
     def __init__(self, **kwargs):
         """
@@ -30,43 +39,176 @@ class DataCollectionAdapter(DataAdapter):
         """
         super().__init__(**kwargs)
 
-    def load(self,
-             dataset_name_or_path: str = None,
-             subset_list: list = None,
-             work_dir: Optional[str] = DEFAULT_DATASET_CACHE_DIR,
-             datasets_hub: str = HubType.MODELSCOPE,
-             **kwargs) -> dict:
-        """
-        Load the dataset. Remote and local datasets are supported.
-        """
-        dataset_name_or_path = os.path.expanduser(dataset_name_or_path or self.dataset_id)
-        subset_list = subset_list or self.subset_list
-
+    def load(self):
         # Try to load dataset from local disk
+        dataset_name_or_path = self.dataset_id
         if os.path.exists(dataset_name_or_path):
             logger.info(f'Loading dataset from {dataset_name_or_path}')
-            dataset = jsonl_to_list(dataset_name_or_path)
-            if len(dataset) == 0:
-                raise ValueError(f'Local dataset is empty: {dataset_name_or_path}')
+            dataset_path = dataset_name_or_path
         else:
             from modelscope import dataset_snapshot_download
 
             # Load dataset from remote
-            logger.info(f'Loading dataset from {datasets_hub}: > dataset_name: {dataset_name_or_path}')
+            logger.info(f'Loading dataset from modelscope: > dataset_name: {dataset_name_or_path}')
+            # download dataset snapshot
+            dataset_path = dataset_snapshot_download(dataset_name_or_path, allow_file_pattern='*.jsonl')
+
+        dataset = LocalDataLoader(
+            data_id_or_path=dataset_path,
+            split=self.eval_split,
+            sample_fields=self.record_to_sample,
+            subset='test',  # NOTE: using hardcoded test subset
+            limit=self.limit,
+            repeats=self.repeats,
+            shuffle=self.shuffle,
+        ).load()
+
+        test_dataset = DatasetDict({self.default_subset: dataset})
+
+        return test_dataset, None
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a data record to a Sample object. Every record is a DatasetEntry.
+
+        Args:
+            record (Dict[str, Any]): Input data record.
+
+        Returns:
+            Sample: Sample object with input, target, and metadata.
+        """
+        from evalscope.collections import DatasetEntry
+
+        entry = DatasetEntry.model_validate(record)
+        sample = Sample.model_validate(entry.prompt)
+
+        record_without_prompt = copy.deepcopy(record)
+        del record_without_prompt['prompt']
+        sample.metadata[DataCollection.INFO] = record_without_prompt  # keep all metadata
+        return sample
+
+    def _post_process_samples(self):
+        """Post process of each sample"""
+        self._initialize_adapters()
+
+    def _initialize_adapters(self):
+        """Init adapters for each dataset and create dataset id map"""
+        self.dataset_adapters: Dict[str, DataAdapter] = {}
+        self.dataset_name_map = defaultdict(lambda: defaultdict(list))
+
+        # load dataset args
+        dataset_args = copy.deepcopy(self._task_config.dataset_args)
+
+        # Iterate through each sample in the dataset
+        dataset = self.test_dataset[self.default_subset]
+        for sample in dataset:
+            collection_info = sample.metadata.get(DataCollection.INFO, {})
+            dataset_name = collection_info.get('dataset_name', '')
+            subset_name = collection_info.get('subset_name', '')
+            # create id mapping
+            self.dataset_name_map[dataset_name][subset_name].append(sample.id)
+
+            # update dataset args
+            cur_dataset_args = dataset_args.get(dataset_name, {})
+
+            # Initialize dataset adapter
+            if dataset_name not in self.dataset_adapters:
+                config = TaskConfig(dataset_args={dataset_name: cur_dataset_args})
+                self.dataset_adapters[dataset_name] = get_benchmark(dataset_name, config=config)
+
+    def _get_adapter(self, metadata: Dict[str, Any]) -> DataAdapter:
+        collection_info = metadata.get(DataCollection.INFO, {})
+        dataset_name = collection_info.get('dataset_name', '')
+        return self.dataset_adapters.get(dataset_name)
+
+    def run_inference(self, model, sample, output_dir, **kwargs) -> TaskState:
+        data_adapter = self._get_adapter(sample.metadata)
+        if not data_adapter:
+            raise ValueError(f'No data adapter found for sample: {sample}')
+
+        return data_adapter.run_inference(model, sample, output_dir, **kwargs)
+
+    def calculate_metrics(self, task_state) -> SampleScore:
+        data_adapter = self._get_adapter(task_state.metadata)
+        if not data_adapter:
+            raise ValueError(f'No data adapter found for task state: {task_state}')
+
+        return data_adapter.calculate_metrics(task_state)
+
+    def aggregate_scores(self, sample_scores: List[SampleScore]):
+        import pandas as pd
+        from tabulate import tabulate
+
+        data = []
+        for sample_score in sample_scores:
+            collection_info = sample_score.sample_metadata[DataCollection.INFO]
+            main_score = sample_score.score.main_value
+            main_metric = sample_score.score.main_score_name
+
+            # use main score
+            data.append(
+                dict(
+                    task_type=collection_info['task_type'],
+                    categories=tuple(collection_info['categories']),
+                    dataset_name=collection_info['dataset_name'],
+                    subset_name=collection_info['subset_name'],
+                    tags=collection_info['tags'],
+                    sample_id=sample_score.sample_id,
+                    metric=main_metric,
+                    score=main_score
+                )
+            )
+
+        df = pd.DataFrame(data)
+
+        def aggregate_and_sort(df, group_by_cols):
+            # aggregate by group_by_cols, and calculate average_score and count
+            report_df = df.groupby(group_by_cols) \
+                .agg(average_score=('score', 'mean'), count=('score', 'size')) \
+                .reset_index()
+            report_df['average_score'] = report_df['average_score'].round(4)
+            report_df = report_df.sort_values(by='count', ascending=False) \
+                .to_dict(orient='records')
+            return report_df
+
+        # multi-level aggregation
+        subset_report_df = aggregate_and_sort(df, ['task_type', 'metric', 'dataset_name', 'subset_name'])
+        dataset_report_df = aggregate_and_sort(df, ['task_type', 'metric', 'dataset_name'])
+        task_report_df = aggregate_and_sort(df, ['task_type', 'metric'])
+
+        # explode tags to multiple rows
+        df_exploded_tags = df.explode('tags')
+        tag_report_df = aggregate_and_sort(df_exploded_tags, ['tags', 'metric'])
 
-            dataset_path = dataset_snapshot_download(
-                dataset_name_or_path, cache_dir=work_dir, allow_file_pattern='*.jsonl')
-            # find the jsonl file
-            dataset_files = [os.path.join(dataset_path, f) for f in os.listdir(dataset_path) if f.endswith('.jsonl')]
-            dataset = jsonl_to_list(dataset_files[0])
+        # process multi-level categories
+        df_categories = df.copy()
+        # multi-level aggregation for categories
+        max_depth = df_categories['categories'].apply(len).max()
+        for level in range(max_depth):
+            df_categories[f'category{level}'] = df_categories['categories'].apply(
+                lambda x: x[level] if len(x) > level else ''
+            )
+        category_report_df = aggregate_and_sort(
+            df_categories, [f'category{level}' for level in range(max_depth)] + ['metric']
+        )
 
-        return dataset
+        # convert to dict format
+        report_dict = {
+            'subset_level': subset_report_df,
+            'dataset_level': dataset_report_df,
+            'task_level': task_report_df,
+            'tag_level': tag_report_df,
+            'category_level': category_report_df,
+        }
 
-    def get_gold_answer(self, input_d: Any) -> Any:
-        return super().get_gold_answer(input_d)
+        # record report
+        for level, data in report_dict.items():
+            table = tabulate(data, headers='keys', tablefmt='pretty', showindex=False)
+            logger.info(f'{level} Report:\n{table}')
 
-    def match(self, gold: Any, pred: Any) -> Any:
-        return super().match(gold, pred)
+        return df
 
-    def parse_pred_result(self, result: Any, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> Any:
-        return super().parse_pred_result(result, raw_input_d, eval_type)
+    def generate_report(self, scores, model_name, output_dir, **kwargs) -> Report:
+        df = scores[self.default_subset]
+        report = ReportGenerator.gen_collection_report(df, self.name, model_name)
+        return report

evalscope/benchmarks/docmath/docmath_adapter.py

@@ -1,6 +1,14 @@
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
-from evalscope.metrics import LLMJudge
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
 
 TEMPLATE_0SHOT = """Please read the following text and answer the question below.
 
@@ -13,73 +21,123 @@ TEMPLATE_0SHOT = """Please read the following text and answer the question below
 Format your response as follows: "Therefore, the answer is (insert answer here)"."""
 
 
-@Benchmark.register(
-    name='docmath',
-    pretty_name='DocMath',
-    tags=['Reasoning', 'Mathematics', 'Long Context'],
-    description=
-    'DocMath-Eval is a comprehensive benchmark focused on numerical reasoning within specialized domains. It requires the model to comprehend long and specialized documents and perform numerical reasoning to answer the given question.',  # noqa: E501
-    dataset_id='yale-nlp/DocMath-Eval',
-    metric_list=['AverageAccuracy'],
-    subset_list=['complong_testmini', 'compshort_testmini', 'simplong_testmini', 'simpshort_testmini'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
-    prompt_template=TEMPLATE_0SHOT,
+@register_benchmark(
+    BenchmarkMeta(
+        name='docmath',
+        pretty_name='DocMath',
+        tags=[Tags.REASONING, Tags.MATH, Tags.LONG_CONTEXT],
+        description=
+        'DocMath-Eval is a comprehensive benchmark focused on numerical reasoning within specialized domains. It requires the model to comprehend long and specialized documents and perform numerical reasoning to answer the given question.',  # noqa: E501
+        dataset_id='yale-nlp/DocMath-Eval',
+        metric_list=['acc'],
+        subset_list=['complong_testmini', 'compshort_testmini', 'simplong_testmini', 'simpshort_testmini'],
+        eval_split='test',
+        prompt_template=TEMPLATE_0SHOT,
+    )
 )
-class DocMathAdapter(DataAdapter):
+class DocMathAdapter(DefaultDataAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
+        self._use_llm_judge = True  # Enable LLM judge for DocMath
+        self.split_as_subset = True  # Use split as subset for DocMath
 
-    def load(self, **kwargs):
-        # default load mini test
-        kwargs['split_as_subset'] = True
-        data_dict = super().load(**kwargs)
-        return data_dict
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from input data.
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
         """
-        context = context = '\n'.join(input_d['paragraphs'])
-        question = input_d['question']
-        prompt = self.prompt_template.format(context=context, question=question)
-        return self.gen_prompt_data(prompt)
+        Convert a data record to a Sample object.
 
-    def get_gold_answer(self, input_d: dict) -> str:
-        """
-        Parse the raw input labels (gold).
-        """
-        return input_d['ground_truth']
+        Args:
+            record (Dict[str, Any]): Input data record.
 
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+        Returns:
+            Sample: Sample object with input, target, and metadata.
+        """
+        ground_truth = record['ground_truth']
+
+        return Sample(
+            input=record['question'],
+            target=str(ground_truth),
+            metadata={
+                'question_id': record.get('question_id', ''),
+                'paragraphs': record['paragraphs'],
+                'answer_type': type(ground_truth).__name__
+            }
+        )
+
+    def format_prompt_template(self, sample):
+        context = '\n'.join(sample.metadata['paragraphs'])
+        question = sample.input
+        return self.prompt_template.format(context=context, question=question)
+
+    def extract_answer(self, prediction: str, task_state: TaskState):
         """
-        Parse the predicted result and extract proper answer.
+        Extract the answer from the model prediction.
         """
         from .utils import extract_answer
 
-        extracted_answer = extract_answer(result)
+        extracted_answer = extract_answer(prediction)
         return extracted_answer
 
-    def match(self, gold: str, pred: str) -> float:
+    def match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
         """
-        Match the gold answer and the predicted answer.
+        Calculate accuracy score by matching prediction with reference.
         """
         from .utils import get_acc
 
-        return get_acc(prediction=pred, gt=gold)
-
-    def llm_match(self, gold: str, pred: str, judge: LLMJudge, **kwargs) -> float:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        answer_type = task_state.metadata.get('answer_type', 'unknown')
+        accuracy = get_acc(prediction=filtered_prediction, gt=reference, answer_type=answer_type)
+        score.value = {'acc': accuracy}
+        score.main_score_name = 'acc'
+
+        return score
+
+    def llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        """
+        Use LLM judge to evaluate the prediction against the reference.
+        """
         from .utils import GENERAL_ORM_PROMPT, ORM_USER_TEMPLATE
 
-        raw_input = kwargs.get('raw_input', None)
-        question = raw_input['question']
-        # get grading response
-        prompt = ORM_USER_TEMPLATE.format(problem=question, answer_1=gold, answer_2=pred)
-        orm_response = judge(prompt=prompt, system_prompt=GENERAL_ORM_PROMPT)
-        # parse grading response
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        question = task_state.metadata.get('question', '')
+
+        # Get grading response
+        prompt = ORM_USER_TEMPLATE.format(problem=question, answer_1=reference, answer_2=filtered_prediction)
+        orm_response = self.llm_judge.judge(prompt, system_prompt=GENERAL_ORM_PROMPT)
+
+        # Parse grading response
         if 'YES' in orm_response:
-            return 1.0
+            accuracy = 1.0
         else:
-            return 0.0
+            accuracy = 0.0
+
+        score.value = {'acc': accuracy}
+        score.explanation = f'LLM judge: {orm_response}'
+        score.metadata = {
+            'source': 'llm_judge',
+            'judge_strategy': self.judge_strategy,
+            'model': self.llm_judge.model_id
+        }
+        score.main_score_name = 'acc'
+
+        return score

evalscope/benchmarks/docmath/utils.py

@@ -193,23 +193,22 @@ def compare_two_numbers(p, gt):
     return within_eps(pred=p, gt=gt)
 
 
-def get_acc(prediction, gt, cot=True):
+def get_acc(prediction, gt, answer_type, cot=True):
     try:
         if cot:
             prediction = normalize(prediction)
         else:
             prediction = float(prediction)
 
-        answer_type = type(gt).__name__
         assert answer_type in ['int', 'float', 'float64', 'bool'], answer_type
         if isinstance(prediction, (str, int, float, bool)) or isinstance(prediction, list):
             # Comparing prediction against the reference
             if answer_type in ['bool']:
-                acc = int(prediction == gt)
+                acc = int(prediction == bool(gt))
             elif answer_type == 'int':
-                acc = int(compare_two_numbers(prediction, gt))
+                acc = int(compare_two_numbers(prediction, int(gt)))
             elif answer_type == 'float' or answer_type == 'float64':
-                acc = int(compare_two_numbers(prediction, gt))
+                acc = int(compare_two_numbers(prediction, float(gt)))
             else:
                 acc = 0
         else:

evalscope/benchmarks/drop/drop_adapter.py

@@ -1,8 +1,13 @@
+import ast
 import re
-from typing import List
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -28,54 +33,82 @@ Answer:  43
 '''  # noqa: E501
 
 
-@Benchmark.register(
-    name='drop',
-    pretty_name='DROP',
-    tags=['Reasoning'],
-    description=
-    'The DROP (Discrete Reasoning Over Paragraphs) benchmark is designed to evaluate the reading comprehension and reasoning capabilities of AI models. It includes a variety of tasks that require models to read passages and answer questions based on the content.',  # noqa: E501
-    dataset_id='AI-ModelScope/DROP',
-    metric_list=['AverageAccuracy'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='validation',
-    prompt_template=
-    'You will be asked to read a passage and answer a question.{drop_examples}# Your Task\n\n---\n{query}\n\nThink step by step, then write a line of the form "Answer: $ANSWER" at the end of your response.',  # noqa: E501
+@register_benchmark(
+    BenchmarkMeta(
+        name='drop',
+        pretty_name='DROP',
+        tags=[Tags.REASONING],
+        description=
+        'The DROP (Discrete Reasoning Over Paragraphs) benchmark is designed to evaluate the reading comprehension and reasoning capabilities of AI models. It includes a variety of tasks that require models to read passages and answer questions based on the content.',  # noqa: E501
+        dataset_id='AI-ModelScope/DROP',
+        metric_list=['acc'],
+        few_shot_num=3,
+        train_split=None,
+        eval_split='validation',
+        prompt_template=
+        'You will be asked to read a passage and answer a question. {drop_examples}\n# Your Task\n\n---\n{query}\n\nThink step by step, then write a line of the form "Answer: $ANSWER" at the end of your response.',  # noqa: E501
+    )
 )
-class DROPAdapter(DataAdapter):
+class DROPAdapter(DefaultDataAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        few_shot_num = kwargs.get('few_shot_num', 0)
-        if few_shot_num != 0:
+        if self.few_shot_num != 0:
             self.few_shot_num = 3
             logger.info(f'Few shot num is set to {self.few_shot_num} for DROP dataset by system.')
         else:
             self.few_shot_num = 0
 
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
         """
-        Generate model prompt from input data.
+        Convert a data record to a Sample object.
+
+        Args:
+            record (Dict[str, Any]): Input data record.
+
+        Returns:
+            Sample: Sample object with input, target, and metadata.
         """
-        drop_examples = '' if self.few_shot_num == 0 else DROP_EXAMPLES
-        query = f"Passage: {input_d['passage']}\nQuestion: {input_d['question']}"
-        prompt = self.prompt_template.format(
+
+        # Parse gold answers
+        gold_answers = self._get_gold_answers(record)
+
+        return Sample(
+            input=record['question'],
+            target=str(gold_answers),
+            metadata={
+                'passage': record['passage'],
+                'answer': record['answer'],
+                'validated_answers': record['validated_answers']
+            }
+        )
+
+    def format_prompt_template(self, sample: Sample) -> str:
+        drop_examples = ''
+        query = f"Passage: {sample.metadata['passage']}\nQuestion: {sample.input}"
+
+        return self.prompt_template.format(
             drop_examples=drop_examples,
             query=query,
         )
-        return self.gen_prompt_data(prompt)
 
-    def get_gold_answer(self, input_d: dict) -> List[str]:
+    def format_fewshot_template(self, fewshot, sample):
+        drop_examples = DROP_EXAMPLES
+        query = f"Passage: {sample.metadata['passage']}\nQuestion: {sample.input}"
+
+        return self.prompt_template.format(
+            drop_examples=drop_examples,
+            query=query,
+        )
+
+    def _get_gold_answers(self, input_d: dict) -> List[str]:
         """
         Parse the raw input labels (gold).
         """
 
         def _flatten_validated_answers(validated_answers):
-            """Flattens a dict of lists of validated answers.
-            {"number": ['1', '8'], ...}
-            -> [{"number": ['1'], ...}, {"number": ['8'], ...}]
-            """
+            """Flattens a dict of lists of validated answers."""
             valid_answers = []
             for i in range(len(validated_answers['number'])):
                 valid_answers.append({
@@ -96,24 +129,36 @@ class DROPAdapter(DataAdapter):
             answers.append(answer)
         return answers
 
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+    def extract_answer(self, prediction: str, task_state: TaskState):
         """
-        Parse the predicted result and extract proper answer.
+        Extract the answer from the model prediction.
         """
-        match = re.search(r'(?i)Answer\s*:\s*([^\n]+)', result)
-        extracted_answer = match.group(1) if match else result
+        match = re.search(r'(?i)Answer\s*:\s*([^\n]+)', prediction)
+        extracted_answer = match.group(1) if match else prediction
         return extracted_answer
 
-    def match(self, gold: List[str], pred: str) -> float:
+    def match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
         """
-        Match the gold answer and the predicted answer.
+        Calculate accuracy score by matching prediction with reference answers.
         """
         from .utils import _answer_to_bags
 
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
         max_em = 0
-        for gold_answer in gold:
+        reference = ast.literal_eval(reference) if isinstance(reference, str) else reference
+        for gold_answer in reference:
             # Convert the answers to bags of answers
-            predicted_bags = _answer_to_bags(pred)
+            predicted_bags = _answer_to_bags(filtered_prediction)
             gold_bags = _answer_to_bags(gold_answer)
 
             if set(predicted_bags[0]) == set(gold_bags[0]) and len(predicted_bags[0]) == len(gold_bags[0]):
@@ -124,7 +169,10 @@ class DROPAdapter(DataAdapter):
             if gold_answer[0].strip():
                 max_em = max(max_em, exact_match)
 
-        return max_em
+        score.value = {'acc': max_em}
+        score.main_score_name = 'acc'
+
+        return score
 
     @staticmethod
     def parse_answer(answer):