evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/benchmarks/live_code_bench/live_code_bench_adapter.py

@@ -1,88 +1,138 @@
-from tqdm import tqdm
-
-from evalscope.benchmarks import Benchmark, DataAdapter
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages.chat_message import ChatMessageUser
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import convert_numpy_types
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
 
-@Benchmark.register(
-    name='live_code_bench',
-    pretty_name='Live-Code-Bench',
-    tags=['Coding'],
-    description=
-    'Live Code Bench is a benchmark for evaluating code generation models on real-world coding tasks. It includes a variety of programming problems with test cases to assess the model\'s ability to generate correct and efficient code solutions.',  # noqa: E501
-    dataset_id='AI-ModelScope/code_generation_lite',
-    subset_list=['release_latest'],
-    metric_list=['Pass@1'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
-    extra_params={
-        'start_date': None,
-        'end_date': None,
-        'timeout': 6,
-        'debug': False
-    },
-    system_prompt=
-    'You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program.',  # noqa: E501
-    prompt_template=
-    '### Question:\n{question_content}\n\n{format_prompt} ### Answer: (use the provided format with backticks)\n\n',  # noqa: E501
+@register_benchmark(
+    BenchmarkMeta(
+        name='live_code_bench',
+        pretty_name='Live-Code-Bench',
+        tags=[Tags.CODING],
+        description=
+        'Live Code Bench is a benchmark for evaluating code generation models on real-world coding tasks. It includes a variety of programming problems with test cases to assess the model\'s ability to generate correct and efficient code solutions.',  # noqa: E501
+        dataset_id='AI-ModelScope/code_generation_lite',
+        subset_list=['release_latest'],
+        metric_list=['Pass@1'],
+        eval_split='test',
+        prompt_template=
+        '### Question:\n{question_content}\n\n{format_prompt} ### Answer: (use the provided format with backticks)\n\n',
+        extra_params={
+            'start_date': None,
+            'end_date': None,
+            'timeout': 6,
+            'debug': False
+        },
+    )
 )
-class LiveCodeBenchAdapter(DataAdapter):
+class LiveCodeBenchAdapter(DefaultDataAdapter):
+    """
+    Live Code Bench adapter using the new data processing framework.
+    """
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        extra_params = kwargs.get('extra_params', {})
-
-        self.timeout = extra_params.get('timeout', 6)
-        self.debug = extra_params.get('debug', False)
-        self.start_date = extra_params.get('start_date')
-        self.end_date = extra_params.get('end_date')
-
-    def load(self, **kwargs) -> dict:
-        from .load_utils import filter_date, transform
-
-        # Note: need trust_remote_code=True to load the python script
-        dataset_dict = super().load(trust_remote_code=True, **kwargs)
-        new_dataset_dict = {}
-        for subset_key, dataset in dataset_dict.items():
-            datasets = dataset[self.eval_split]
-            filtered_datasets = filter_date(datasets, start_date=self.start_date, end_date=self.end_date)
-
-            transformed_datasets = [transform(item) for item in tqdm(filtered_datasets, desc='Transforming data')]
-            new_dataset_dict[subset_key] = {self.eval_split: transformed_datasets}
-        return new_dataset_dict
-
-    def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate the prompt for the model input.
-        """
-        format_prompt = input_d['format_prompt']
-        question_content = input_d['question_content']
+        self.timeout = self.extra_params.get('timeout', 6)
+        self.debug = self.extra_params.get('debug', False)
+        self.start_date = self.extra_params.get('start_date')
+        self.end_date = self.extra_params.get('end_date')
+
+        self.save_metadata = False  # Don't save metadata, since they are large
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """Convert a data record to a Sample object."""
+        from .load_utils import transform
+
+        record = transform(record)
+
+        question_content = record['question_content']
+        format_prompt = record['format_prompt']
         full_prompt = self.prompt_template.format(question_content=question_content, format_prompt=format_prompt)
 
-        return self.gen_prompt_data(full_prompt)
+        return Sample(
+            input=[ChatMessageUser(content=full_prompt)],
+            target='',
+            metadata={
+                'evaluation_sample': record['evaluation_sample'],
+                'contest_date': record['contest_date']
+            }
+        )
 
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Extract the gold answer from the input dict.
-        return input_d
+    def sample_filter(self, sample):
+        from .load_utils import filter_date
 
-    def match(self, gold: dict, pred: str) -> float:
-        from .evaluate_utils import codegen_metrics
+        return filter_date(sample.metadata['contest_date'], start_date=self.start_date, end_date=self.end_date)
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        """Extract code from the prediction."""
         from .extract_utils import extract_code_generation
+        return extract_code_generation(prediction)
 
-        ext_pred = extract_code_generation(pred)
-
-        references = [{'input_output': gold['evaluation_sample']}]
-        predictions = [[ext_pred]]
-        metrics, eval_results, final_metadata = codegen_metrics(
-            references,
-            predictions,
-            k_list=[1],
-            num_process_evaluate=1,
-            timeout=self.timeout,
-            debug=self.debug,
+    def match_score(
+        self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState
+    ) -> Score:
+        from .evaluate_utils import codegen_metrics
+
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
         )
-        return metrics['pass@1'] / 100  # convert to point scale
+
+        references = [{'input_output': task_state.metadata['evaluation_sample']}]
+        predictions = [[filtered_prediction]]
+
+        try:
+            metrics, eval_results, final_metadata = codegen_metrics(
+                references,
+                predictions,
+                k_list=[1],
+                num_process_evaluate=1,
+                timeout=self.timeout,
+                debug=self.debug,
+            )
+            pass_rate = metrics['pass@1'] / 100  # convert to point scale
+
+            score.value = {'pass': float(pass_rate > 0)}
+            score.explanation = f"Pass@1: {metrics['pass@1']}%"
+
+            # Convert numpy types to native Python types for JSON serialization
+            serializable_eval_results = convert_numpy_types(eval_results)
+            serializable_final_metadata = convert_numpy_types(final_metadata)
+
+            score.metadata = {
+                'pass_rate': float(pass_rate),
+                'timeout': self.timeout,
+                'debug': self.debug,
+                'eval_results': serializable_eval_results,
+                'final_metadata': serializable_final_metadata
+            }
+        except Exception as e:
+            score.value = {'pass': False}
+            score.explanation = f'Evaluation failed: {str(e)}'
+            score.metadata = {'error': str(e)}
+
+        score.main_score_name = 'pass'
+        return score
+
+    def aggregate_scores(self, sample_scores):
+        from evalscope.metrics.metric import PassAtK
+
+        # calculate pass@k here
+        agg_list = []
+        for metric in self.metric_list:
+            if metric.lower().startswith('pass@'):
+                k = int(metric.split('@')[1])
+                # Get the scores for this metric
+                agg = PassAtK(k)
+                agg_list.extend(agg(sample_scores))
+        return agg_list

evalscope/benchmarks/live_code_bench/load_utils.py

@@ -32,8 +32,8 @@ def transform(item):
         private_test_cases = json.loads(item['private_test_cases'])
     except Exception as e:  # noqa: F841
         private_test_cases = json.loads(
-            pickle.loads(zlib.decompress(base64.b64decode(private_test_cases.encode('utf-8'))  # type: ignore
-                                         )))  # type: ignore
+            pickle.loads(zlib.decompress(base64.b64decode(private_test_cases.encode('utf-8'))))
+        )
 
     # load metadata
     metadata = json.loads(item['metadata'])
@@ -47,25 +47,17 @@ def transform(item):
     return item
 
 
-def filter_date(dataset, start_date=None, end_date=None):
-    new_dataset = []
-
-    for item in dataset:
-        contest_date = datetime.fromisoformat(item['contest_date'])
-        if start_date is not None:
-            p_start_date = datetime.strptime(start_date, '%Y-%m-%d')
-            if p_start_date > contest_date:
-                continue
+def filter_date(contest_date, start_date=None, end_date=None) -> bool:
 
-        if end_date is not None:
-            p_end_date = datetime.strptime(end_date, '%Y-%m-%d')
-            if p_end_date < contest_date:
-                continue
+    contest_date = datetime.fromisoformat(contest_date)
+    if start_date is not None:
+        p_start_date = datetime.strptime(start_date, '%Y-%m-%d')
+        if p_start_date > contest_date:
+            return False
 
-        new_dataset.append(item)
+    if end_date is not None:
+        p_end_date = datetime.strptime(end_date, '%Y-%m-%d')
+        if p_end_date < contest_date:
+            return False
 
-    if start_date or end_date:
-        logger.info(
-            f'Filtered dataset with start_date: {start_date}, end_date: {end_date}, remaining items: {len(new_dataset)}'
-        )
-    return new_dataset
+    return True

evalscope/benchmarks/live_code_bench/testing_util.py

@@ -4,18 +4,22 @@ import faulthandler
 import json
 import numpy as np
 import platform
+
 # to run the solution files we're using a timing based approach
 import signal
 import sys
 import time
+
 # used for debugging to time steps
 from datetime import datetime
 from decimal import Decimal
 from enum import Enum
 from functools import partial
 from io import StringIO
+
 # from pyext import RuntimeModule
 from types import ModuleType
+
 # used for testing the code that reads from input
 from unittest.mock import mock_open, patch
 
@@ -342,8 +346,8 @@ def grade_stdio(
             return all_results, WA_send_args
 
         for output_line_idx, (
-                stripped_prediction_line,
-                stripped_gt_out_line,
+            stripped_prediction_line,
+            stripped_gt_out_line,
         ) in enumerate(zip(stripped_prediction_lines, stripped_gt_out_lines)):
             WA_send_args['error_message'] = (
                 f'Wrong answer at {output_line_idx=}: {truncatefn(stripped_prediction_line)} != {truncatefn(stripped_gt_out_line)}'

evalscope/benchmarks/maritime_bench/maritime_bench_adapter.py

@@ -1,82 +1,56 @@
 from typing import Any
 
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType, OutputType
-from evalscope.metrics import exact_match
-from evalscope.metrics.completion_parsers import ResponseParser
-
-SUBSET_LIST = ['default']
-
-
-@Benchmark.register(
-    name='maritime_bench',
-    pretty_name='MaritimeBench',
-    tags=['Maritime', 'MCQ', 'Knowledge'],
-    description=
-    'MaritimeBench is a benchmark for evaluating AI models on maritime-related multiple-choice questions. It consists of questions related to maritime knowledge, where the model must select the correct answer from given options.',  # noqa: E501
-    dataset_id='HiDolphin/MaritimeBench',
-    model_adapter=OutputType.GENERATION,
-    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
-    subset_list=SUBSET_LIST,
-    metric_list=['AverageAccuracy'],
-    eval_split='test',
-    prompt_template=
-    '题目来自于{subset_name}请回答单选题。要求只输出选项，不输出解释，将选项放在<>里，直接输出答案。示例：\n\n题目：在船舶主推进动力装置中，传动轴系在运转中承受以下复杂的应力和负荷，但不包括______。\n选项：\nA. 电磁力\nB. 压拉应力\nC. 弯曲应力\nD. 扭应力\n答：<A> 当前题目\n {query}',  # noqa: E501
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+
+MARITIME_PROMPT_TEMPLATE = '请回答单选题。要求只输出选项，不输出解释，将选项放在[]里，直接输出答案。示例：\n\n题目：在船舶主推进动力装置中，传动轴系在运转中承受以下复杂的应力和负荷，但不包括______。\n选项：\nA. 电磁力\nB. 压拉应力\nC. 弯曲应力\nD. 扭应力\n答：[A]\n 当前题目\n {question}\n选项：\n{choices}'  # noqa: E501
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='maritime_bench',
+        pretty_name='MaritimeBench',
+        tags=[Tags.CHINESE, Tags.MULTIPLE_CHOICE, Tags.KNOWLEDGE],
+        description=
+        'MaritimeBench is a benchmark for evaluating AI models on maritime-related multiple-choice questions. It consists of questions related to maritime knowledge, where the model must select the correct answer from given options.',  # noqa: E501
+        dataset_id='HiDolphin/MaritimeBench',
+        metric_list=['acc'],
+        few_shot_num=0,
+        eval_split='test',
+        prompt_template=MARITIME_PROMPT_TEMPLATE,
+    )
 )
-class MaritimeBenchAdapter(DataAdapter):
+class MaritimeBenchAdapter(MultiChoiceAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        self.choices = ['A', 'B', 'C', 'D']
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> Any:
-
-        prefix = ''
-        query = prefix + input_d['question'] + '\n'
-        available_choices = []
-        for option in self.choices:
-            if option in input_d and input_d[option]:
-                query += option + ':' + input_d[option] + '\n'
-                available_choices.append(option)
-
-        full_prompt = self.prompt_template.format(subset_name=subset_name, query=query)
-        return self.gen_prompt_data(full_prompt, choices=available_choices)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        """
-        Parse the raw input labels (gold).
-
-        Args:
-            input_d: input raw data. Depending on the dataset.
-
-        Returns:
-            The parsed input. e.g. gold answer ... Depending on the dataset.
-        """
-        return input_d['answer']
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
-        """
-        Parse the raw model prediction (pred).
-
-        Args:
-            pred: model prediction. Depending on the model.
-
-        Returns:
-            The parsed prediction. e.g. model answer... Depending on the model.
-        """
-
-        return ResponseParser.parse_bracketed_answer(result, options=self.choices)
-
-    def match(self, gold: Any, pred: Any) -> Any:
-        """
-        Match the gold answer with the predicted answer.
-
-        Args:
-            gold: The gold answer.
-            pred: The predicted answer.
-
-        Returns:
-            The result of the match.
-        """
-        return exact_match(gold=gold, pred=pred)
+        self.reformat_subset = True
+
+    def record_to_sample(self, record) -> Sample:
+        # Extract available choices from the record
+        choices = []
+        choice_letters = ['A', 'B', 'C', 'D']
+        for letter in choice_letters:
+            if letter in record and record[letter]:
+                choices.append(record[letter])
+
+        return Sample(
+            input=record['question'],
+            choices=choices,
+            target=record['answer'],
+        )
+
+    def format_prompt_template(self, sample):
+        choices = '\n'.join([f'{chr(65 + i)}. {choice}' for i, choice in enumerate(sample.choices)])
+        return MARITIME_PROMPT_TEMPLATE.format(question=sample.input, choices=choices)
+
+    def extract_answer(self, prediction, task_state):
+        # use regex to extract the answer from the prediction
+        import re
+        match = re.search(r'\[([A-D])\]', prediction)
+        if match:
+            return match.group(1)
+        return ''

evalscope/benchmarks/math_500/math_500_adapter.py

@@ -1,58 +1,51 @@
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import extract_answer, math_equal, strip_answer_string
-from evalscope.utils.logger import get_logger
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from typing import Any, Dict
 
-# flake8: noqa
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
 
-@Benchmark.register(
-    name='math_500',
-    pretty_name='MATH-500',
-    tags=['Mathematics'],
-    description=
-    "MATH-500 is a benchmark for evaluating mathematical reasoning capabilities of AI models. It consists of 500 diverse math problems across five levels of difficulty, designed to test a model's ability to solve complex mathematical problems by generating step-by-step solutions and providing the correct final answer.",  # noqa: E501
-    dataset_id='AI-ModelScope/MATH-500',
-    subset_list=['Level 1', 'Level 2', 'Level 3', 'Level 4', 'Level 5'],
-    metric_list=['AveragePass@1'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
-    prompt_template='{query}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
+@register_benchmark(
+    BenchmarkMeta(
+        name='math_500',
+        pretty_name='MATH-500',
+        tags=[Tags.MATH, Tags.REASONING],
+        description=
+        "MATH-500 is a benchmark for evaluating mathematical reasoning capabilities of AI models. It consists of 500 diverse math problems across five levels of difficulty, designed to test a model's ability to solve complex mathematical problems by generating step-by-step solutions and providing the correct final answer.",  # noqa: E501
+        dataset_id='AI-ModelScope/MATH-500',
+        subset_list=['Level 1', 'Level 2', 'Level 3', 'Level 4', 'Level 5'],
+        metric_list=[{
+            'acc': {
+                'numeric': True
+            }
+        }],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        prompt_template='{question}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
+    )
 )
-class Math500Adapter(DataAdapter):
+class Math500Adapter(DefaultDataAdapter):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-    def load(self, **kwargs):
-        # default load all levels
-        kwargs['subset_list'] = ['default']
-        data_dict = super().load(**kwargs)
-        return self.reformat_subset(data_dict, subset_key='level', format='Level {}')
-
-    def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate the prompt for the model input.
-        """
-        problem = input_d['problem']
-        full_prompt = self.prompt_template.format(query=problem)
-
-        return self.gen_prompt_data(full_prompt)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Extract the gold answer from the input dict.
-        return strip_answer_string(input_d['answer'])
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
-        """
-        Parse the model output to get the answer. Could be the best choice index.
-        """
-        # Note: Use same extraction method for both of checkpoint/service/custom
-        result = strip_answer_string(extract_answer(result))
-        return result
-
-    def match(self, gold: str, pred: str) -> float:
-        res = math_equal(pred, gold)
-        return 1.0 if res else 0.0
+        self.reformat_subset = True
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        return Sample(
+            input=record['problem'],
+            target=record['answer'],
+            subset_key=f"Level {record['level']}",
+            metadata={
+                'question_id': record['unique_id'],
+                'solution': record['solution'],
+            },
+        )

evalscope/benchmarks/math_vista/math_vista_adapter.py

@@ -0,0 +1,129 @@
+# flake8: noqa: E501
+import re
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import MultipleChoiceTemplate, parse_answers, prompt
+
+logger = get_logger()
+
+SUBSET_LIST = ['default']
+
+OPEN_PROMPT = """
+Solve the following problem step by step. The last line of your response should be of the form "ANSWER: $ANSWER" (without quotes) where $ANSWER is the answer to the problem.
+
+{question}
+
+Remember to put your answer on its own line at the end in the form "ANSWER: $ANSWER" (without quotes) where $ANSWER is the answer to the problem, and you do not need to use a \\boxed command.
+"""
+
+MULT_CHOICE_PROMPT = MultipleChoiceTemplate.SINGLE_ANSWER_COT
+
+MULTI_CHOICE_TYPE = 'multi_choice'
+OPEN_TYPE = 'free_form'
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='math_vista',
+        pretty_name='MathVista',
+        dataset_id='evalscope/MathVista',
+        tags=[Tags.MATH, Tags.REASONING, Tags.MULTIPLE_CHOICE, Tags.MULTI_MODAL],
+        description=
+        'MathVista is a consolidated Mathematical reasoning benchmark within Visual contexts. It consists of three newly created datasets, IQTest, FunctionQA, and PaperQA, which address the missing visual domains and are tailored to evaluate logical reasoning on puzzle test figures, algebraic reasoning over functional plots, and scientific reasoning with academic paper figures, respectively. It also incorporates 9 MathQA datasets and 19 VQA datasets from the literature, which significantly enrich the diversity and complexity of visual perception and mathematical reasoning challenges within our benchmark. In total, MathVista includes 6,141 examples collected from 31 different datasets.',
+        subset_list=SUBSET_LIST,
+        metric_list=['acc'],
+        eval_split='testmini',
+        prompt_template=OPEN_PROMPT,
+    )
+)
+class MathVistaAdapter(VisionLanguageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        content_list, answers_list = MathVistaAdapter.create_content_and_answers_list(record)
+
+        if record['question_type'] == 'multi_choice':
+            label_answer = self.get_option_label(answers_list, record['answer'])
+            return Sample(
+                input=[ChatMessageUser(content=content_list)],
+                choices=answers_list,
+                target=label_answer,
+                metadata={
+                    'question_type': record['question_type'],
+                    'answer_type': record['answer_type'],
+                    **record['metadata'],
+                }
+            )
+        elif record['question_type'] == 'free_form':
+            return Sample(
+                input=[ChatMessageUser(content=content_list)],
+                target=record['answer'],
+                metadata={
+                    'precision': record['precision'],
+                    'question_type': record['question_type'],
+                    'answer_type': record['answer_type'],
+                    **record['metadata'],
+                }
+            )
+        else:
+            raise ValueError(f"Unexpected question_type: {record['question_type']}")
+
+    def get_option_label(self, options, value):
+        try:
+            index = options.index(value)
+            return chr(ord('A') + index)
+        except ValueError:
+            logger.warning(f"Answer '{value}' not found in options: {options}. This may cause evaluation issues.")
+            return value
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        question_type = task_state.metadata['question_type']
+        if question_type == MULTI_CHOICE_TYPE:
+            answers = parse_answers(task_state)
+            return ''.join(sorted(list(answers)))
+        elif question_type == OPEN_TYPE:
+            pattern = r'ANSWER:\s*(.*)'
+            match = re.search(pattern, prediction)
+            if match:
+                return match.group(1).strip()
+            return ''
+        else:
+            raise ValueError(f'Unsupported question type: {question_type}')
+
+    @staticmethod
+    def create_content_and_answers_list(record: dict[str, Any], ) -> tuple[list[Content], list[str]]:
+        """
+            Create a list of content elements and a list of answers from a record.
+
+            Args:
+                record (dict): The record containing question, images, and options.
+
+
+            Returns:
+                tuple: A tuple containing:
+                    - content_list (list): A list of content elements (text and images).
+                    - answers_list (list): A list of possible answers (for multiple-choice questions).
+        """
+        question_type = record['question_type']
+        if question_type == MULTI_CHOICE_TYPE:
+            answers_list = record['choices']
+            input_text = prompt(question=record['question'], choices=answers_list, template=MULT_CHOICE_PROMPT)
+            content_list: list[Content] = [ContentText(text=input_text)]
+        else:
+            answers_list: list[str] = []
+            content_list: list[Content] = [ContentText(text=OPEN_PROMPT.format(question=record['question']))]
+        image = record['decoded_image']
+        if image:
+            image_base64 = bytes_to_base64(image['bytes'], format='jpg', add_header=True)
+            content_list.append(ContentImage(image=image_base64))
+        return content_list, answers_list