opencompass 0.3.2__tar.gz → 0.3.2.post1__tar.gz

{opencompass-0.3.2 → opencompass-0.3.2.post1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: opencompass
-Version: 0.3.2
+Version: 0.3.2.post1
 Summary: A comprehensive toolkit for large model evaluation
 Home-page: https://github.com/open-compass/opencompass
 Author: OpenCompass Contributors

opencompass-0.3.2.post1/opencompass/__init__.py

@@ -0,0 +1 @@
+__version__ = '0.3.2.post1'

{opencompass-0.3.2 → opencompass-0.3.2.post1}/opencompass/utils/__init__.py

@@ -10,5 +10,6 @@ from .lark import *  # noqa
 from .logging import *  # noqa
 from .menu import *  # noqa
 from .model_postprocessors import *  # noqa
+from .postprocessors import *  # noqa
 from .prompt import *  # noqa
 from .text_postprocessors import *  # noqa

opencompass-0.3.2.post1/opencompass/utils/postprocessors/xfinder/extractor.py

@@ -0,0 +1,175 @@
+import json
+import time
+from logging import getLogger
+
+import requests
+from openai import OpenAI
+
+from .xfinder_utils import PROMPT_TEMPLATE
+
+Instruction = """I will provide you with a question, output sentences along with an answer range. The output sentences are the response of the question provided. The answer range could either describe the type of answer expected or list all possible valid answers. Using the information provided, you must accurately and precisely determine and extract the intended key answer from the output sentences. Please don't have your subjective thoughts about the question.
+First, you need to determine whether the content of the output sentences is relevant to the given question. If the entire output sentences are unrelated to the question (meaning the output sentences are not addressing the question), then output [No valid answer].
+Otherwise, ignore the parts of the output sentences that have no relevance to the question and then extract the key answer that matches the answer range.
+Below are some special cases you need to be aware of:
+    (1) If the output sentences present multiple different answers, carefully determine if the later provided answer is a correction or modification of a previous one. If so, extract this corrected or modified answer as the final response. Conversely, if the output sentences fluctuate between multiple answers without a clear final answer, you should output [No valid answer].
+    (2) If the answer range is a list and the key answer in the output sentences is not explicitly listed among the candidate options in the answer range, also output [No valid answer].
+
+""" # noqa
+
+
+class Extractor:
+
+    def __init__(
+        self,
+        model_name,
+        model_path=None,
+        url=None,
+        temperature=0,
+        max_tokens=3000,
+        api_key='EMPTY',
+        SYSTEM='You are a help assistant tasked with extracting the precise key answer from given output sentences. You must only provide the extracted key answer without including any additional text.'  # noqa
+    ):
+        self.model_name = model_name
+        self.PROMPT_TEMPLATE = PROMPT_TEMPLATE[model_name]
+        self.SYSTEM = SYSTEM
+        self.model_path = model_path
+        self.url = url
+        self.api_key = api_key
+        self.temperature = temperature
+        self.max_tokens = max_tokens
+        self.mode = 'API' if self.url is not None else 'Local'
+        self.logger = getLogger(__name__)
+
+        if self.mode == 'Local':
+            from vllm import LLM, SamplingParams
+            self.sampling_params = SamplingParams(temperature=self.temperature,
+                                                  max_tokens=self.max_tokens,
+                                                  stop=[
+                                                      '<|endoftext|>',
+                                                      '<|im_end|>', '<eoa>',
+                                                      '<||>', '<end_of_turn>',
+                                                      '<|eot_id|>'
+                                                  ])
+            self.llm = LLM(model=self.model_path, gpu_memory_utilization=0.5)
+
+    @staticmethod
+    def prepare_input(item):
+        user_input = Instruction + \
+            "Question: \"\"\"" + item['question'] + "\"\"\"\n\n" + \
+            "Output sentences: \"\"\"" + item['llm_output'] + "\"\"\"\n\n" + \
+            'Answer range: ' + item['standard_answer_range'] + '\n\n' + \
+            'Key extracted answer: '
+
+        return user_input
+
+    def gen_output(self, query):
+        if self.mode == 'API':
+            # return self.send_request(query)
+            return self.openai_infer(query)
+        else:
+            return self.offline_infer(query)
+
+    def send_request(self, query: str) -> str:
+        """Send a request to the model's API and return the response.
+
+        Args:
+            query (str): The input query.
+
+        Returns:
+            str: The extracted answer (xFinder's output).
+        """
+        prompt = self.PROMPT_TEMPLATE.format(system=self.SYSTEM, input=query)
+        payload = json.dumps({
+            'prompt':
+            prompt,
+            'temperature':
+            self.temperature,
+            'max_tokens':
+            self.max_tokens,
+            'stop': [
+                '<|endoftext|>', '<|im_end|>', '<eoa>', '<||>',
+                '<end_of_turn>', '<|eot_id|>'
+            ],
+        })
+        headers = {'Content-Type': 'application/json'}
+        res = requests.request('POST', self.url, headers=headers, data=payload)
+        res = res.json()['text'][0]
+        res = res.replace(prompt, '')
+        # res = requests.post(self.url, json=payload)
+        # res = res.json()['text']
+        res = res.strip()
+        return res
+
+    def openai_infer(self, query: str, retry=9) -> str:
+        """Perform inference on the OpenAI model.
+
+        Args:
+            query (str): The input query.
+
+        Returns:
+            str: The extracted answer (xFinder's output).
+        """
+        if isinstance(self.url, list):
+            # Randomly api for better load balancing
+            import random
+            self.url = random.choice(self.url)
+        self.client = OpenAI(
+            api_key=self.api_key,
+            base_url=self.url,
+        )
+        self.retry = retry
+
+        t = time.time()
+        retry = self.retry
+        response = ''
+        while retry > 0:
+            try:
+                chat_response = self.client.chat.completions.create(
+                    model=self.client.models.list().data[0].id
+                    if self.model_name == '' else self.model_name,
+                    messages=[
+                        {
+                            'role': 'system',
+                            'content': self.SYSTEM
+                        },
+                        {
+                            'role': 'user',
+                            'content': query
+                        },
+                    ],
+                    stop=[
+                        '<|endoftext|>', '<|im_end|>', '<eoa>', '<||>',
+                        '<end_of_turn>', '<|eot_id|>'
+                    ],
+                    temperature=self.temperature,
+                    max_tokens=self.max_tokens,
+                )
+                js_response = json.loads(chat_response.model_dump_json())
+                response = js_response['choices'][0]['message']['content']
+                break
+            except Exception as e:
+                self.logger.info(f'Error: {e}')
+                self.logger.info(f'{self.url} is down. Retrying...')
+                self.logger.info(f'Time elapsed: {time.time() - t} seconds')
+                time.sleep(6)
+                retry -= 1
+        if retry == 0:
+            response = 'Error: Failed to get response.'
+            self.logger.info(f'{response} after {self.retry} tries.')
+            raise ValueError('The api is down')
+        return response.strip()
+
+    def offline_infer(self, query: str) -> str:
+        """Perform inference on the local xFinder model.
+
+        Args:
+            query (str): The input query.
+
+        Returns:
+            str: The extracted answer (xFinder's output).
+        """
+        prompt = self.PROMPT_TEMPLATE.format(system=self.SYSTEM, input=query)
+        res = self.llm.generate(prompt, self.sampling_params)
+        res = res[0]
+        res = res.outputs[0].text.strip()
+        return res

opencompass-0.3.2.post1/opencompass/utils/postprocessors/xfinder/xfinder_utils/PROMPT_TEMPLATE.py

@@ -0,0 +1,14 @@
+PROMPT_TEMPLATE = {
+    'xFinder-qwen1505':
+    """<|System|>:{system}
+<|User|>:{input}
+<|Bot|>:""",
+    'xFinder-llama38it':
+    """<|start_header_id|>system<|end_header_id|>
+
+{system}<|eot_id|><|start_header_id|>user<|end_header_id|>
+
+{input}<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+""",
+}

opencompass-0.3.2.post1/opencompass/utils/postprocessors/xfinder/xfinder_utils/__init__.py

@@ -0,0 +1,3 @@
+from .convert_data import *  # noqa
+from .data_process import *  # noqa
+from .PROMPT_TEMPLATE import *  # noqa

opencompass-0.3.2.post1/opencompass/utils/postprocessors/xfinder/xfinder_utils/convert_data.py

@@ -0,0 +1,123 @@
+# Convert OpenCompass prediction data to XFinder format
+import copy
+import json
+import re
+
+xfinder_template = {
+    'math': {
+        'model_name':
+        '',
+        'dataset':
+        '',
+        'key_answer_type':
+        'math',
+        'question':
+        '',
+        'llm_output':
+        '',
+        'correct_answer':
+        '',
+        'standard_answer_range':
+        'a(n) number / set / vector / matrix / interval / expression / function / equation / inequality'  # noqa
+    },
+    'alphabet_option': {
+        'model_name': '',
+        'dataset': '',
+        'key_answer_type': 'alphabet_option',
+        'question': '',
+        'llm_output': '.',
+        'correct_answer': '',
+        'standard_answer_range': []
+    },
+    'categorical_label': {
+        'model_name': '',
+        'dataset': '',
+        'key_answer_type': '',
+        'question': '',
+        'llm_output': '',
+        'correct_answer': '',
+        'standard_answer_range': []
+    },
+    'short_text': {
+        'model_name': '',
+        'dataset': '',
+        'key_answer_type': 'short_text',
+        'question': '',
+        'llm_output': '',
+        'correct_answer': '',
+        'standard_answer_range': []
+    }
+}
+
+
+def parse_options(text: str):
+    lines = text.split('\n')
+    parsed_options = []
+    option_pattern = r'^[A-Z]\)|[A-Z]\.|[A-Z]\)|[A-Z]:|\([A-Z]\)'
+    for line in lines:
+        line = line.strip()
+        match = re.match(option_pattern, line)
+        if match:
+            option = ''
+            # 等于第一个属于选项的字符
+            for c in line:
+                if c.isalpha():
+                    option = c
+                    break
+            content_start = match.end() + 1
+            content = line[content_start:].strip()
+            parsed_options.append([option, content])
+
+    return parsed_options
+
+
+def convert_to_xfinder_format(typ, data, model_name='', dataset_name=''):
+    assert typ in xfinder_template.keys(), f'Invalid type {typ}'
+    format_data = []
+    for item in data:
+        template = copy.deepcopy(xfinder_template[typ])
+        question = item['origin_prompt'][-1]['prompt']
+        llm_output = item['prediction']
+        correct_answer = item['reference'] if item['reference'] else item[
+            'gold']
+        template['correct_answer'] = correct_answer
+        template['model_name'] = model_name
+        template['dataset'] = dataset_name
+        template['question'] = question
+        template['llm_output'] = llm_output
+        try:
+            assert typ in list(xfinder_template.keys())
+            if typ == 'alphabet_option':
+                options = parse_options(question)
+                template['standard_answer_range'] = options
+            elif typ == 'short_text':
+                template['standard_answer_range'] = item['gold']
+            elif typ == 'categorical_label':
+                pass
+        except Exception as e:
+            print(f'Error when parsing question options: {e}, skipping...')
+            continue
+
+        format_data.append(template)
+    return format_data
+
+
+if __name__ == '__main__':
+    # Test
+    example_data = {
+        'origin_prompt': [{
+            'role':
+            'HUMAN',
+            'prompt':
+            'Alice, Bob, Claire, Dave, and Eve are dancers at a square dance. At the start of a song, they each have a partner: Alice is dancing with Ophelia, Bob is dancing with Jamie, Claire is dancing with Melissa, Dave is dancing with Rodrigo, and Eve is dancing with Patrick.\nThroughout the song, the dancers often trade partners. First, Claire and Bob switch partners. Then, Claire and Eve switch partners. Then, Claire and Bob switch partners. Then, Eve and Dave switch partners. Finally, Claire and Alice switch partners. At the end of the dance, Alice is dancing with\nOptions:\n(A) Ophelia\n(B) Jamie\n(C) Melissa\n(D) Rodrigo\n(E) Patrick'  # noqa
+        }],
+        'origin_prediction':
+        '\n 答案: B) 前者小于后者',
+        'prediction':
+        'B',
+        'reference':
+        'A'
+    }
+    example_data = convert_to_xfinder_format('alphabet_option', [example_data],
+                                             'GPT-3', 'OpenAI')
+    print(json.dumps(example_data, indent=4, ensure_ascii=False))

opencompass-0.3.2.post1/opencompass/utils/postprocessors/xfinder/xfinder_utils/data_process.py

@@ -0,0 +1,24 @@
+import ast
+
+
+class DataProcessor:
+
+    def __init__(self):
+        pass
+
+    def read_data(self, data):
+        for item in data:
+            if isinstance(item['standard_answer_range'],
+                          str) and item['key_answer_type'] != 'math':
+                try:
+                    item['standard_answer_range'] = ast.literal_eval(
+                        item['standard_answer_range'])
+                except Exception as e:
+                    print(f'Error: {e}')
+                    print('Please check the form of standard_answer_range')
+                    exit(0)
+
+            item['standard_answer_range'] = str(item['standard_answer_range'])
+            item['key_answer_type'] = str(item['key_answer_type'])
+
+        return data

{opencompass-0.3.2 → opencompass-0.3.2.post1}/opencompass.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: opencompass
-Version: 0.3.2
+Version: 0.3.2.post1
 Summary: A comprehensive toolkit for large model evaluation
 Home-page: https://github.com/open-compass/opencompass
 Author: OpenCompass Contributors

{opencompass-0.3.2 → opencompass-0.3.2.post1}/opencompass.egg-info/SOURCES.txt

@@ -2019,4 +2019,11 @@ opencompass/utils/model_postprocessors.py
 opencompass/utils/prompt.py
 opencompass/utils/run.py
 opencompass/utils/text_postprocessors.py
-opencompass/utils/types.py
+opencompass/utils/types.py
+opencompass/utils/postprocessors/__init__.py
+opencompass/utils/postprocessors/xfinder/__init__.py
+opencompass/utils/postprocessors/xfinder/extractor.py
+opencompass/utils/postprocessors/xfinder/xfinder_utils/PROMPT_TEMPLATE.py
+opencompass/utils/postprocessors/xfinder/xfinder_utils/__init__.py
+opencompass/utils/postprocessors/xfinder/xfinder_utils/convert_data.py
+opencompass/utils/postprocessors/xfinder/xfinder_utils/data_process.py

opencompass-0.3.2/opencompass/__init__.py

@@ -1 +0,0 @@
-__version__ = '0.3.2'