evalscope 0.5.2__py3-none-any.whl → 0.5.4__py3-none-any.whl

evalscope/third_party/longbench_write/eval.py

@@ -0,0 +1,284 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+# Copyright (c) ZhipuAI, Inc. and its affiliates.
+import multiprocessing
+import os
+import json
+import random
+import re
+from concurrent.futures import ThreadPoolExecutor
+
+import matplotlib.pyplot as plt
+import numpy as np
+import requests
+from tqdm import tqdm
+
+from evalscope.utils import jsonl_to_list
+from evalscope.utils import get_logger
+
+logger = get_logger()
+
+"""
+This script is used to evaluate results of predictions for the LongWriter model.
+Refer to https://github.com/THUDM/LongWriter for more details.
+
+EvalLength:
+    Evaluate the length of the generated responses.
+    Metrics:
+        score_l: The average score of the length evaluation.
+
+EvalQuality:
+    Evaluate the quality of the generated responses by using Judge Model.
+    Metrics:
+        score_q: The average score of the quality evaluation.
+"""
+
+
+class EvalLength:
+
+    EVAL_L = 'eval_length'
+
+    def __init__(self, model: str, pred_path: str, output_dir: str):
+        self.model = model
+        self.pred_path = pred_path
+        self.output_dir = output_dir
+
+        self.model_id_path = self.model.strip(os.sep).replace(os.sep, '__')
+
+    @staticmethod
+    def score(x, y):
+        if y > x:
+            return 100 * max(0, 1. - (y / x - 1) / 3)
+        else:
+            return 100 * max(0, 1. - (x / y - 1) / 2)
+
+    def eval(self, dump_res: bool = True):
+        # example = {"prompt": "Write an outline for a short 100-word blog post about xxx", "type": "Community Forum", "length": 100, "response_length": 103, "response": "I. Introduction A. xxx"}
+        predictions = [json.loads(line) for line in open(self.pred_path, encoding='utf-8')]
+        x, y, scores = [], [], []
+
+        for pred in tqdm(predictions, total=len(predictions), desc=f'Process of eval_l: '):
+            x.append(pred["length"])
+            y.append(pred["response_length"])
+            scores.append(self.score(pred["length"], pred["response_length"]))
+
+        avg_score_l = np.mean(scores)
+        logger.info(f'Average score of length evaluation: {avg_score_l:.2f}')
+
+        # Dump to output file
+        if dump_res:
+            output_res_path = f'{self.output_dir}/{self.model_id_path}/{self.EVAL_L}.jsonl'
+            with open(output_res_path, 'w') as f:
+                f.write(json.dumps({'score_l': avg_score_l, 'scores': scores}, ensure_ascii=False) + '\n')
+                logger.info(f'Successfully dumped evaluation results to {output_res_path}')
+
+        return x, y, scores
+
+    def plot(self, x: list, y: list):
+        plt = self.plot_img(x, y)
+        output_pic_path = f'{self.output_dir}/{self.model_id_path}/eval_length_scatter.png'
+        plt.savefig(output_pic_path)
+        logger.info(f'Successfully saved scatter plot to {output_pic_path}')
+
+    @staticmethod
+    def plot_img(x: list, y: list):
+        # set plt size 6x6
+        plt.figure(figsize=(6, 6))
+        lmt = 25000
+        # plot x, y
+        plt.scatter(x, y, s=100, c='r', alpha=0.3)
+        # plot x=y
+        plt.plot([0, lmt], [0, lmt], 'k--')
+        plt.xscale('log')
+        plt.yscale('log')
+        plt.xlim(50, lmt)
+        plt.ylim(50, lmt)
+        plt.xlabel('Required Length', fontsize=20, fontweight='bold')
+        plt.ylabel('Output Length', fontsize=20, fontweight='bold')
+        plt.xticks(fontsize=24)
+        plt.yticks(fontsize=24)
+        plt.tight_layout()
+
+        return plt
+
+
+class EvalQuality:
+
+        EVAL_Q = 'eval_quality'
+        OPENAI_BASE_URL = 'https://api.openai.com/v1/chat/completions'
+        DIMS = ["Relevance", "Accuracy", "Coherence", "Clarity", "Breadth and Depth", "Reading Experience"]
+
+        def __init__(self,
+                     model: str,
+                     pred_path: str,
+                     output_dir: str,
+                     prompt_template_path: str,
+                     openai_api_key: str = None,
+                     openai_api_base: str = OPENAI_BASE_URL,
+                     openai_gpt_model: str = 'gpt-4o-2024-05-13',
+                     generation_kwargs: dict = None,
+                     proc_num: int = 8):
+
+            self.model = model
+            self.openai_api_base = openai_api_base
+            self.pred_path = pred_path
+            self.output_dir = output_dir
+            self.proc_num = proc_num
+            self.eval_scores = []
+
+            assert os.path.exists(self.pred_path), f'Prediction file not found: {self.pred_path}'
+
+            # Default: temperature=0.5, max_new_tokens=1024, stop=None
+            if generation_kwargs is None:
+                self.generation_kwargs = dict({
+                    'max_new_tokens': 1024,
+                    'temperature': 0.5,
+                    'stop': None,
+                })
+            else:
+                self.generation_kwargs = generation_kwargs
+
+            self.prompt_template: str = open(prompt_template_path, 'r', encoding='utf-8').read()
+
+            self.model_id_path = self.model.strip(os.sep).replace(os.sep, '__')
+            self.output_res_path = f'{self.output_dir}/{self.model_id_path}/{self.EVAL_Q}.jsonl'
+
+            self.openai_api_key: str = openai_api_key
+            self.openai_gpt_model = openai_gpt_model
+            assert self.openai_api_key, 'Please set `OPENAI_API_KEY` in environment variables.'
+
+        def get_response_gpt4(self, prompt, temperature=0.5, max_new_tokens=1024, stop=None):
+            tries = 0
+            while tries < 1:
+                tries += 1
+                try:
+                    headers = {
+                        'Authorization': "Bearer {}".format(self.openai_api_key),
+                    }
+                    messages = [
+                        {'role': 'user', 'content': prompt},
+                    ]
+                    resp = requests.post(self.openai_api_base, json={
+                        "model": self.openai_gpt_model,
+                        "messages": messages,
+                        "temperature": temperature,
+                        "max_tokens": max_new_tokens,
+                        "stop": stop,
+                    }, headers=headers, timeout=600)
+                    if resp.status_code != 200:
+                        raise Exception(resp.text)
+                    resp = resp.json()
+                    logger.info(f'>>gpt resp: {resp}')
+                    break
+                except KeyboardInterrupt as e:
+                    raise e
+                except Exception as e:
+                    if "maximum context length" in str(e):
+                        raise e
+                    elif "triggering" in str(e):
+                        return 'Trigger OpenAI\'s content management policy'
+                    logger.error("Error Occurs: \"%s\"        Retry ..." % (str(e)))
+            else:
+                logger.error("Max tries. Failed.")
+                return "Max tries. Failed."
+            try:
+                return resp["choices"][0]["message"]["content"]
+            except:
+                return ''
+
+        @staticmethod
+        def extract_info(pattern, text):
+            match = re.search(pattern, text, re.DOTALL)
+            if match:
+                return match.group(1)
+            else:
+                return None
+
+        def process_data(self, item):
+            # for item in tqdm(items, total=len(items), desc=f'Process of eval_q: '):
+            prompt = self.prompt_template.replace('$INST$', item['prompt']).replace('$RESPONSE$', item["response"])
+            scores = None
+            output = self.get_response_gpt4(prompt, **self.generation_kwargs)
+            try:
+                if '```json' in output:
+                    output = self.extract_info(r'```json\n(.*?)\n```', output)
+                output = output.replace('\n', '')
+                scores = json.loads(output)
+                for dim in self.DIMS:
+                    if dim not in scores:
+                        logger.warning(f'Cannot find score for dimension: {dim} in scores {scores}.')
+                        scores = None
+            except Exception as e:
+                logger.error(f'Error occurs during process data: {str(e)}')
+
+            if scores is None:
+                logger.error(f'Failed to extract scores for item: {item}')
+            else:
+                logger.info(f'>>scores: {scores}')
+                item['scores'] = scores
+
+            return item
+
+        def eval(self):
+
+            data_all = jsonl_to_list(self.pred_path)
+            total = len(data_all)
+            assert total > 0, f'No data found in prediction file: {self.pred_path}'
+
+            random.shuffle(data_all)
+
+            with ThreadPoolExecutor() as executor:
+                self.eval_scores = list(executor.map(self.process_data, data_all))
+
+            # self.process_data(items=data)
+            logger.info(f'>>self.eval_scores: {self.eval_scores}')
+
+            total_score = dict()
+            for dim in self.DIMS:
+                # scores = [float(score[dim]) if dim in score else 3 for score in self.eval_scores]
+                scores = [float(item['scores'][dim]) if 'scores' in item and dim in item['scores'] else 3 for item in self.eval_scores]
+                total_score[dim] = ((sum(scores) / len(scores)) - 1) * 25
+            total_score['total'] = sum(total_score.values()) / len(total_score)
+            logger.info(f'Total score of quality evaluation: {total_score["total"]:.2f}')
+
+            output_res_path: str = f'{self.output_dir}/{self.model_id_path}/{self.EVAL_Q}.jsonl'
+            with open(output_res_path, 'w', encoding='utf-8') as fout:
+                fout.write(json.dumps(total_score, ensure_ascii=False) + '\n')
+
+
+def run_eval(model: str,
+             pred_path: str,
+             output_dir: str,
+             prompt_template_path: str,
+             openai_api_key: str,
+             openai_api_base: str,
+             openai_gpt_model: str,
+             generation_kwargs: dict,
+             proc_num: int,
+             stage: list,
+             ):
+    logger.info(f'Got eval stages: {stage}')
+
+    if 'eval_l' in stage:
+        logger.info(f'Processing evaluation of length for model: {model}')
+        eval_length = EvalLength(model=model,
+                                 pred_path=pred_path,
+                                 output_dir=output_dir)
+        x, y, _ = eval_length.eval()
+        eval_length.plot(x, y)
+    else:
+        logger.warning(f'*** Skip `eval_l` stage ***')
+
+    if 'eval_q' in stage:
+        logger.info(f'Processing evaluation of quality for model: {model}')
+        eval_quality = EvalQuality(model=model,
+                                   pred_path=pred_path,
+                                   output_dir=output_dir,
+                                   prompt_template_path=prompt_template_path,
+                                   openai_api_key=openai_api_key,
+                                   openai_api_base=openai_api_base,
+                                   openai_gpt_model=openai_gpt_model,
+                                   generation_kwargs=generation_kwargs,
+                                   proc_num=proc_num)
+        eval_quality.eval()
+    else:
+        logger.warning('*** Skip `eval_q` stage ***')

evalscope/third_party/longbench_write/infer.py

@@ -0,0 +1,217 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+# Copyright (c) ZhipuAI, Inc. and its affiliates.
+
+import os
+import json
+from typing import List
+
+import torch
+import numpy as np
+import random
+from modelscope import AutoTokenizer, AutoModelForCausalLM
+from tqdm import tqdm
+
+from evalscope.third_party.longbench_write.utils import count_words
+from evalscope.models.api import OpenaiApi
+from evalscope.utils import get_logger
+
+logger = get_logger()
+
+DEFAULT_PROC_NUM = 8
+
+"""
+This script is used to generate predictions for the LongWriter model.
+Refer to https://github.com/THUDM/LongWriter for more details.
+"""
+
+
+def get_pred(rank, world_size, data, path, max_new_tokens, temperature, tokenizer, fout):
+    device = torch.device(f'cuda:{rank}')
+    model = AutoModelForCausalLM.from_pretrained(path, trust_remote_code=True, torch_dtype=torch.bfloat16).to(device)
+    model = model.eval()
+
+    for dt in tqdm(data, total=len(data), desc=f'Infer on rank-{rank}: '):
+        prompt = dt['prompt']
+        if "llama" in path.lower():
+            prompt = f"[INST]{prompt}[/INST]"
+            input = tokenizer(prompt, truncation=False, return_tensors="pt").to(device)
+            context_length = input.input_ids.shape[-1]
+            output = model.generate(
+                **input,
+                max_new_tokens=max_new_tokens,
+                num_beams=1,
+                do_sample=True,
+                temperature=temperature,
+            )[0]
+            response = tokenizer.decode(output[context_length:], skip_special_tokens=True)
+        else:
+            response, history = model.chat(tokenizer, prompt, history=[], max_new_tokens=max_new_tokens,
+                                           temperature=temperature)
+        dt["response_length"], _ = count_words(response)
+        dt["response"] = response
+
+        logger.info(dt)
+
+        fout.write(json.dumps(dt, ensure_ascii=False) + '\n')
+        fout.flush()
+
+    logger.info(f'Successfully generated predictions for {len(data)} samples.')
+
+
+def seed_everything(seed):
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+    np.random.seed(seed)
+    random.seed(seed)
+    torch.backends.cudnn.benchmark = False
+    torch.backends.cudnn.deterministic = True
+    torch.cuda.manual_seed_all(seed)
+
+
+# def run_infer(model: str,
+#               data_path: str,
+#               output_dir: str,
+#               generation_kwargs: dict = None,
+#               enable: bool = True, ):
+#     """
+#     Process inference for LongWriter model.
+#
+#     Args:
+#         model: The model id of the LongWriter model on ModelScope, or local model path.
+#         data_path: The path to the data file.
+#         output_dir: The output directory for the predictions.
+#         generation_kwargs: The generation arguments for the model.
+#             Attributes: `max_new_tokens`: The maximum number of tokens to generate. `temperature`: The temperature
+#         enable: Whether to run infer process.
+#     """
+#     model_id_path: str = os.path.join(output_dir, model.strip(os.sep).replace(os.sep, '__'))
+#
+#     if not enable:
+#         logger.warning('*** Skip `infer` stage ***')
+#         return f'{model_id_path}/pred.jsonl'
+#
+#     seed_everything(42)
+#
+#     os.makedirs(model_id_path, exist_ok=True)
+#     fout = open(f'{model_id_path}/pred.jsonl', 'w', encoding='utf-8')
+#
+#     if generation_kwargs is None:
+#         generation_kwargs = dict({
+#             'max_new_tokens': 32768,
+#             'temperature': 0.5
+#         })
+#
+#     tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
+#     world_size = torch.cuda.device_count()
+#
+#     logger.info(f'>>Input data path: {data_path}')
+#     with open(data_path, encoding='utf-8') as f:
+#         data = [json.loads(line) for line in f]
+#
+#     data_subsets = [data[i::world_size] for i in range(world_size)]
+#     processes = []
+#     for rank in range(world_size):
+#         p = mp.Process(target=get_pred,
+#                        args=(rank, world_size, data_subsets[rank], model, generation_kwargs.get('max_new_tokens'), generation_kwargs.get('temperature'), tokenizer, fout))
+#         p.start()
+#         processes.append(p)
+#
+#     for p in processes:
+#         p.join()
+#
+#     logger.info(f'Finish generating predictions for {model}.')
+#     logger.info(f'Predictions are saved in {model_id_path}/pred.jsonl.')
+#
+#     return f'{model_id_path}/pred.jsonl'
+
+
+def run_infer(model: str,
+              data_path: str,
+              output_dir: str,
+              api_config: dict,
+              generation_kwargs: dict = None,
+              enable: bool = True, ):
+    """
+    Process inference for LongWriter model.
+
+    Args:
+        model: The model id of the LongWriter model on ModelScope, or local model path.
+        data_path: The path to the data file.
+        output_dir: The output directory for the predictions.
+        api_config: The configuration for the OpenAI API inference.
+            Attributes:
+                `openai_api_key`: The OpenAI API key. Default is None for custom model serving.
+                `openai_api_base`: The OpenAI API base URL.
+                `is_chat`: Whether to chat. Default is True.
+                `verbose`: Whether to print verbose information. Default is False.
+        generation_kwargs: The generation arguments for the model.
+            Attributes: `max_new_tokens`: The maximum number of tokens to generate. `temperature`: The temperature
+        enable: Whether to run infer process.
+    """
+    model_id_path: str = os.path.join(output_dir, model.strip(os.sep).replace(os.sep, '__'))
+
+    if not enable:
+        logger.warning('*** Skip `infer` stage ***')
+        return f'{model_id_path}/pred.jsonl'
+
+    seed_everything(42)
+
+    if generation_kwargs is None:
+        generation_kwargs = dict({
+            'max_new_tokens': 32768,
+            'temperature': 0.5,
+            'repetition_penalty': 1.0,
+        })
+
+    # Prepare inputs
+    logger.info(f'>>Input data path: {data_path}')
+    # TODO: add load data from ms
+    with open(data_path, encoding='utf-8') as f:
+        data_list = [json.loads(line) for line in f]
+
+    logger.info(f'Input example: {data_list[0]}')
+
+    api_client = OpenaiApi(model=model,
+                           openai_api_key=None,
+                           openai_api_base=api_config.get('openai_api_base', 'http://127.0.0.1:8000/v1/chat/completions'),
+                           max_new_tokens=generation_kwargs.get('max_new_tokens', 4096),
+                           temperature=generation_kwargs.get('temperature', 0.0),
+                           repetition_penalty=generation_kwargs.get('repetition_penalty', 1.0),
+                           is_chat=api_config.get('is_chat', True),
+                           verbose=api_config.get('verbose', False),
+                           )
+
+    # TODO: ONLY FOR TEST  generate_simple
+    results: List[str] = api_client.generate_simple(inputs=[example['prompt'] for example in data_list])
+    assert len(results) == len(data_list), f'Error: The number of predictions {len(results)} is not equal to the number of inputs {len(data_list)}.'
+    logger.info(f'Finish generating predictions with {len(data_list)} samples for {model}')
+
+    # Outputs
+    os.makedirs(model_id_path, exist_ok=True)
+    output_pred_file: str = f'{model_id_path}/pred.jsonl'
+    with open(output_pred_file, 'w', encoding='utf-8') as f:
+        for dt, res in zip(data_list, results):
+            dt["response_length"], _ = count_words(res)
+            dt["response"] = res
+            f.write(json.dumps(dt, ensure_ascii=False) + '\n')
+
+    logger.info(f'Predictions are saved in {output_pred_file}')
+
+    return output_pred_file
+
+
+if __name__ == '__main__':
+    # ZhipuAI/LongWriter-glm4-9b, ZhipuAI/LongWriter-llama3.1-8b
+    api_config = dict(openai_api_key=None,
+                      openai_api_base='http://127.0.0.1:8000/v1/chat/completions',
+                      is_chat=True,
+                      verbose=True,)
+
+    run_infer(model='ZhipuAI/LongWriter-glm4-9b',
+              data_path='resources/longbench_write.jsonl',
+              output_dir='outputs',
+              api_config=api_config,
+              generation_kwargs=dict({
+                      'max_new_tokens': 32768,
+                      'temperature': 0.5})
+              )

evalscope/third_party/longbench_write/longbench_write.py

@@ -0,0 +1,88 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+from typing import Union
+
+from evalscope.third_party.longbench_write.infer import run_infer
+from evalscope.third_party.longbench_write.eval import run_eval
+from evalscope.utils import yaml_to_dict, json_to_dict
+from evalscope.utils import get_logger
+
+logger = get_logger()
+
+"""
+Entry file for LongWriter evaluation.
+"""
+
+
+def run_task(task_cfg: Union[str, dict]):
+
+    if isinstance(task_cfg, str):
+        if task_cfg.endswith('.yaml'):
+            task_cfg: dict = yaml_to_dict(task_cfg)
+        elif task_cfg.endswith('.json'):
+            task_cfg: dict = json_to_dict(task_cfg)
+        else:
+            raise ValueError(f'Unsupported file format: {task_cfg}, should be yaml or json file.')
+
+    # Parse task configuration
+    stage: list = task_cfg.get('stage', ['infer', 'eval_l', 'eval_q'])
+    model: str = task_cfg.get('model')
+    input_data_path: str = task_cfg.get('input_data_path')
+    output_dir: str = task_cfg.get('output_dir')
+
+    infer_config: dict = task_cfg.get('infer_config')
+    eval_config: dict = task_cfg.get('eval_config')
+    assert infer_config is not None and eval_config is not None, 'Please provide infer_config and eval_config.'
+
+    # Run inference process
+    pred_res_path = run_infer(model=model,
+                              data_path=input_data_path or os.path.join(os.path.dirname(__file__), 'resources/longbench_write.jsonl'),
+                              output_dir=output_dir,
+                              api_config=dict(
+                                  openai_api_key=infer_config.get('openai_api_key'),
+                                  openai_api_base=infer_config.get('openai_api_base'),
+                                  is_chat=infer_config.get('is_chat', True),
+                                  verbose=infer_config.get('verbose', False),
+                              ),
+                              generation_kwargs=infer_config.get('generation_kwargs'),
+                              enable='infer' in stage)
+
+    # Run eval process
+    run_eval(model=model,
+             pred_path=pred_res_path,
+             output_dir=output_dir,
+             prompt_template_path=os.path.join(os.path.dirname(__file__), 'resources/judge.txt'),
+             openai_api_key=eval_config.get('openai_api_key'),
+             openai_api_base=eval_config.get('openai_api_base'),
+             openai_gpt_model=eval_config.get('openai_gpt_model'),
+             generation_kwargs=eval_config.get('generation_kwargs'),
+             proc_num=eval_config.get('proc_num', 16),
+             stage=stage)
+
+
+if __name__ == '__main__':
+    # Note: evaluation task configuration can also be loaded from yaml or json file.
+    # task_cfg = os.path.join(os.path.dirname(__file__), 'default_task.yaml')
+    # task_cfg = os.path.join(os.path.dirname(__file__), 'default_task.json')
+    task_cfg = dict(stage=['infer', 'eval_l', 'eval_q'],
+                    model='ZhipuAI/LongWriter-glm4-9b',  # or /path/to/your_model_dir
+                    input_data_path=None,
+                    output_dir='./outputs',
+
+                    infer_config={
+                        'openai_api_base': 'http://127.0.0.1:8000/v1/chat/completions',
+                        'is_chat': True,
+                        'verbose': False,
+                        'generation_kwargs': {'max_new_tokens': 32768, 'temperature': 0.5, 'repetition_penalty': 1.0},
+                    },
+
+                    eval_config={
+                        'openai_api_key': 'YOUR_OPENAI_API_KEY',
+                        'openai_api_base': 'https://api.openai.com/v1/chat/completions',
+                        'openai_gpt_model': 'gpt-4o-2024-05-13',
+                        'generation_kwargs': {'max_new_tokens': 1024, 'temperature': 0.5, 'stop': None},
+                        'proc_num': 16,
+                    },
+                    )
+
+    run_task(task_cfg=task_cfg)

evalscope/third_party/longbench_write/resources/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.

evalscope/third_party/longbench_write/resources/judge.txt

@@ -0,0 +1,31 @@
+You are an expert in evaluating text quality. Please evaluate the quality of an AI assistant's response to a user's writing request. Be as strict as possible.
+
+You need to evaluate across the following six dimensions, with scores ranging from 1 to 5. The scoring criteria from 5 to 1 for each dimension are as follows:
+
+1. Relevance: From content highly relevant and fully applicable to the user's request to completely irrelevant or inapplicable.
+
+2. Accuracy: From content completely accurate with no factual errors or misleading information to content with numerous errors and highly misleading.
+
+3. Coherence: From clear structure with smooth logical connections to disorganized structure with no coherence.
+
+4. Clarity: From clear language, rich in detail, and easy to understand to confusing expression with minimal details.
+
+5. Breadth and Depth: From both broad and deep content with a lot of information to seriously lacking breadth and depth with minimal information.
+
+6. Reading Experience: From excellent reading experience, engaging and easy to understand content to very poor reading experience, boring and hard to understand content.
+
+Please evaluate the quality of the following response to a user's request according to the above requirements.
+
+<User Request>
+
+$INST$
+
+</User Request>
+
+<Response>
+
+$RESPONSE$
+
+</Response>
+
+Please evaluate the quality of the response. You must first provide a brief analysis of its quality, then give a comprehensive analysis with scores for each dimension. The output must strictly follow the JSON format: {"Analysis": ..., "Relevance": ..., "Accuracy": ..., "Coherence": ..., "Clarity": ..., "Breadth and Depth": ..., "Reading Experience": ...}. You do not need to consider whether the response meets the user's length requirements in your evaluation. Ensure that only one integer between 1 and 5 is output for each dimension score.