evalscope 0.15.1__py3-none-any.whl → 0.16.1__py3-none-any.whl

evalscope/config.py

@@ -13,29 +13,20 @@ from evalscope.models import CustomModel, DummyCustomModel
 from evalscope.utils import gen_hash
 from evalscope.utils.io_utils import dict_to_yaml, json_to_dict, yaml_to_dict
 from evalscope.utils.logger import get_logger
+from evalscope.utils.utils import parse_int_or_float
 
 logger = get_logger()
 
 cur_path = os.path.dirname(os.path.abspath(__file__))
 
-DEFAULT_MODEL_ARGS = {'revision': 'master', 'precision': 'torch.float16'}
-DEFAULT_GENERATION_CONFIG = {
-    'max_length': 2048,
-    'max_new_tokens': 512,
-    'do_sample': False,
-    'top_k': 50,
-    'top_p': 1.0,
-    'temperature': 1.0,
-}
-
 
 @dataclass
 class TaskConfig:
     # Model-related arguments
     model: Union[str, 'CustomModel', None] = None
     model_id: Optional[str] = None
-    model_args: Optional[Dict] = field(default_factory=lambda: DEFAULT_MODEL_ARGS | {})
-    model_task: Optional[str] = ModelTask.TEXT_GENERATION
+    model_args: Dict = field(default_factory=dict)
+    model_task: str = ModelTask.TEXT_GENERATION
 
     # Template-related arguments
     template_type: Optional[str] = None  # Deprecated, will be removed in v1.0.0.
@@ -48,14 +39,14 @@ class TaskConfig:
     dataset_hub: str = HubType.MODELSCOPE
 
     # Generation configuration arguments
-    generation_config: Optional[Dict] = field(default_factory=lambda: DEFAULT_GENERATION_CONFIG | {})
+    generation_config: Dict = field(default_factory=dict)
 
     # Evaluation-related arguments
     eval_type: str = EvalType.CHECKPOINT
     eval_backend: str = EvalBackend.NATIVE
     eval_config: Union[str, Dict, None] = None
     stage: str = EvalStage.ALL
-    limit: Optional[int] = None
+    limit: Optional[Union[int, float]] = None
     eval_batch_size: Optional[int] = None
 
     # Cache and working directory arguments
@@ -65,6 +56,7 @@ class TaskConfig:
     outputs: Optional[str] = None  # Deprecated, will be removed in v1.0.0.
 
     # Debug and runtime mode arguments
+    ignore_errors: bool = False
     debug: bool = False
     dry_run: bool = False
     seed: Optional[int] = 42
@@ -76,7 +68,8 @@ class TaskConfig:
     # LLMJudge arguments
     judge_strategy: str = JudgeStrategy.AUTO
     judge_worker_num: int = 1
-    judge_model_args: Optional[Dict] = field(default_factory=lambda: {})
+    judge_model_args: Optional[Dict] = field(default_factory=dict)
+    analysis_report: bool = False
 
     def __post_init__(self):
         if self.model is None:
@@ -95,6 +88,50 @@ class TaskConfig:
         if self.eval_batch_size is None:
             self.eval_batch_size = 8 if self.eval_type == EvalType.SERVICE else 1
 
+        # Post process limit
+        if self.limit is not None:
+            self.limit = parse_int_or_float(self.limit)
+
+        # Set default generation_config and model_args
+        self.__init_default_generation_config()
+        self.__init_default_model_args()
+
+    def __init_default_generation_config(self):
+        if self.generation_config:
+            return
+        if self.model_task == ModelTask.IMAGE_GENERATION:
+            self.generation_config = {
+                'height': 1024,
+                'width': 1024,
+                'num_inference_steps': 50,
+                'guidance_scale': 9.0,
+            }
+        elif self.model_task == ModelTask.TEXT_GENERATION:
+            if self.eval_type == EvalType.CHECKPOINT:
+                self.generation_config = {
+                    'max_length': 2048,
+                    'max_new_tokens': 512,
+                    'do_sample': False,
+                    'top_k': 50,
+                    'top_p': 1.0,
+                    'temperature': 1.0,
+                }
+            elif self.eval_type == EvalType.SERVICE:
+                self.generation_config = {
+                    'max_tokens': 2048,
+                    'temperature': 0.0,
+                }
+
+    def __init_default_model_args(self):
+        if self.model_args:
+            return
+        if self.model_task == ModelTask.TEXT_GENERATION:
+            if self.eval_type == EvalType.CHECKPOINT:
+                self.model_args = {
+                    'revision': 'master',
+                    'precision': 'torch.float16',
+                }
+
     def to_dict(self):
         result = self.__dict__.copy()
         if isinstance(self.model, CustomModel):
@@ -218,9 +255,9 @@ def parse_task_config(task_cfg) -> TaskConfig:
     elif isinstance(task_cfg, str):
         extension = os.path.splitext(task_cfg)[-1]
         logger.info(f'Args: Task config is provided with {extension} file type.')
-        if extension in ['yaml', 'yml']:
+        if extension in ['.yaml', '.yml']:
             task_cfg = TaskConfig.from_yaml(task_cfg)
-        elif extension == 'json':
+        elif extension == '.json':
             task_cfg = TaskConfig.from_json(task_cfg)
         else:
             raise ValueError('Args: Unsupported file extension.')

evalscope/evaluator/evaluator.py

@@ -13,7 +13,7 @@ from evalscope.benchmarks import DataAdapter
 from evalscope.config import TaskConfig
 from evalscope.constants import AnswerKeys, DumpMode, EvalStage, EvalType, JudgeStrategy, ReviewKeys
 from evalscope.models import BaseModelAdapter
-from evalscope.report import Report, gen_table
+from evalscope.report import Report, gen_report_table
 from evalscope.utils import dict_torch_dtype_to_str, gen_hash
 from evalscope.utils.io_utils import OutputsStructure, dump_jsonl_data, jsonl_to_list
 from evalscope.utils.logger import get_logger
@@ -46,7 +46,6 @@ class Evaluator(object):
         self.dataset_name = data_adapter.name
         self.dataset_name_or_path = os.path.expanduser(data_adapter.dataset_id)
         self.model_name = task_cfg.model_id
-        self.custom_task_name = f'{self.model_name}_{self.dataset_name}'
 
         self.data_adapter = data_adapter
         self.model_adapter = model_adapter
@@ -79,8 +78,16 @@ class Evaluator(object):
         # Limit and index prompts
         limited_prompts = defaultdict(list)
         for subset_name, prompts_list in prompts.items():
-            limit = self.task_cfg.limit or len(prompts_list)
-            for index, prompt in enumerate(prompts_list[:limit]):
+            # If limit is None, use all prompts
+            if self.task_cfg.limit is None:
+                limit = len(prompts_list)
+            else:
+                if isinstance(self.task_cfg.limit, int):
+                    limit = self.task_cfg.limit
+                elif isinstance(self.task_cfg.limit, float):
+                    limit = int(len(prompts_list) * self.task_cfg.limit)
+            # Limit the number of prompts
+            for index, prompt in enumerate(prompts_list[:min(limit, len(prompts_list))]):
                 prompt[AnswerKeys.INDEX] = index
                 limited_prompts[subset_name].append(prompt)
 
@@ -97,13 +104,23 @@ class Evaluator(object):
         answer_d[AnswerKeys.ANSWER_ID] = answer_id
         answer_d[AnswerKeys.SUBSET_NAME] = subset_name
         answer_d[AnswerKeys.RAW_INPUT] = input_d[AnswerKeys.RAW_INPUT]
-        # answer_d[AnswerKeys.ORIGIN_PROMPT] = input_d
         answer_d[AnswerKeys.INDEX] = input_d[AnswerKeys.INDEX]
         return answer_d
 
     def _get_answer(self, input_prompts, subset_name, infer_cfg) -> List[dict]:
         answers_list = []
-        answer_ds: List[dict] = self.model_adapter.predict(inputs=input_prompts, infer_cfg=infer_cfg)
+        try:
+            # get answer from model
+            answer_ds: List[dict] = self.model_adapter.predict(inputs=input_prompts, infer_cfg=infer_cfg)
+        except Exception as e:
+            logger.error(f'Failed to get answer for {input_prompts}, due to {e}')
+            # if ignore_errors is True, continue to next input
+            if self.task_cfg.ignore_errors:
+                logger.warning('`ignore_errors` is set to True. Dropping this prompt and continuing with evaluation.')
+                return answers_list
+            else:
+                raise e
+        # process answer
         for answer_d, input_prompt in zip(answer_ds, input_prompts):
             answer_id = self._generate_answer_id(self.model_adapter.model_cfg, input_prompt, infer_cfg)
             processed_answer = self._process_answer(answer_d, input_prompt, subset_name, answer_id)
@@ -197,16 +214,17 @@ class Evaluator(object):
             reviewer_spec = {}
 
         review_res = deepcopy(answer_d)
-        choices = review_res[AnswerKeys.CHOICES]
-        if len(choices) == 0:
-            review_res[ReviewKeys.REVIEWED] = False
+        if AnswerKeys.CHOICES not in review_res:
+            review_res[AnswerKeys.CHOICES] = []
+            review_res[ReviewKeys.REVIEWED] = True
             review_res[ReviewKeys.REVIEW_ID] = None
             review_res[ReviewKeys.REVIEWER_SPEC] = reviewer_spec
             review_res[ReviewKeys.REVIEW_TIME] = time.time()
+            logger.warning(f'No choices found for answer dict: {review_res}')
             return review_res
 
         rev_choices = []
-        for choice in choices:
+        for choice in review_res[AnswerKeys.CHOICES]:
             raw_input_d: dict = review_res[AnswerKeys.RAW_INPUT]
             answer_content = choice[ReviewKeys.MESSAGE][ReviewKeys.CONTENT]
             gold_content = self.data_adapter.get_gold_answer(raw_input_d)
@@ -280,11 +298,20 @@ class Evaluator(object):
         review_file_path = os.path.join(self.outputs_structure.reviews_dir, self.model_name, review_file_name)
         os.makedirs(os.path.dirname(review_file_path), exist_ok=True)
 
+        # Load existing reviews if using cache
+        existing_reviews = {}
         if self.use_cache and os.path.exists(review_file_path):
-            logger.info(f'Updating the review file: {review_file_path} ...')
-            os.remove(review_file_path)
+            with open(review_file_path, 'r') as f:
+                for line in f:
+                    review = json.loads(line.strip())
+                    existing_reviews[review['index']] = review
+            logger.info(f'Reusing review result from {review_file_path}, got {len(existing_reviews)} reviews.')
 
         def process_single_review(answer_d):
+            # Check if review already exists in cache
+            if self.use_cache and answer_d['index'] in existing_reviews:
+                return existing_reviews[answer_d['index']]
+
             review_id, reviewer_spec = self._generate_review_id(answer_d)
             # Get review
             review_d = self._get_review(answer_d=answer_d, review_id=review_id, reviewer_spec=reviewer_spec)
@@ -299,8 +326,9 @@ class Evaluator(object):
             for future in tqdm(as_completed(futures), total=len(futures), desc=f'Reviewing({subset_name}): '):
                 review_d = future.result()
                 reviews_list.append(review_d)
-                # Dump reviews
-                dump_jsonl_data(review_d, review_file_path, dump_mode=DumpMode.APPEND)
+                # Dump new reviews only if not using cache or review is new
+                if not self.use_cache or review_d['index'] not in existing_reviews:
+                    dump_jsonl_data(review_d, review_file_path, dump_mode=DumpMode.APPEND)
 
         return reviews_list
 
@@ -315,17 +343,24 @@ class Evaluator(object):
         Returns:
             The metric result. Depends on the metric function in data_adapter.
         """
+        # Get max choices
+        choices_lengths = [
+            len(review_d[AnswerKeys.CHOICES]) for review_d in reviews_list if review_d.get(ReviewKeys.REVIEWED)
+        ]
+        if choices_lengths:
+            max_choices = max(choices_lengths)
+        else:
+            max_choices = 0
 
+        # Get review result
         review_res_list = []
-        max_choices = max(
-            len(review_d[AnswerKeys.CHOICES]) for review_d in reviews_list if review_d[ReviewKeys.REVIEWED])
         for review_d in reviews_list:
             if not review_d[ReviewKeys.REVIEWED]:
-                logger.warning(f'Review not finished for answer_id: {review_d[AnswerKeys.ANSWER_ID]}')
+                logger.warning(f'Review not finished for answer_id: {review_d[AnswerKeys.ANSWER_ID]}, skipping ...')
                 continue
 
             if len(review_d[AnswerKeys.CHOICES]) == 0:
-                logger.warning(f'No choices found for answer_id: {review_d[AnswerKeys.ANSWER_ID]}')
+                logger.warning(f'No choices found for answer_id: {review_d[AnswerKeys.ANSWER_ID]}, skipping ...')
                 continue
             elif len(review_d[AnswerKeys.CHOICES]) == 1 and max_choices == 1:
                 review_res = review_d[AnswerKeys.CHOICES][0][ReviewKeys.REVIEW][ReviewKeys.RESULT]
@@ -343,41 +378,45 @@ class Evaluator(object):
 
         return metric_score
 
-    def dump_report(self, reviews_score_all: List[dict], use_table: bool = True):
+    def dump_report(self, reviews_score_all: List[dict]):
         """
         Get report for total reviews of specific dataset.
         It is required to rewrite this method to support your own evaluator.
 
         Args:
             reviews_score_all: reviews score list. Generated by func self.data_adapter.compute_metric().
-            use_table: whether to generate table for reports. Default to True.
 
         Returns: None
         """
+        report_path = os.path.join(self.outputs_structure.reports_dir, self.model_name)
+        os.makedirs(report_path, exist_ok=True)
         # Get report map
         report_map: Report = self.data_adapter.gen_report(
-            subset_score_map=reviews_score_all,
-            report_name=self.custom_task_name,
-            model_name=self.model_name,
-            dataset_name=self.dataset_name)
-
-        # Dump report
-        report_path: str = os.path.join(self.outputs_structure.reports_dir, self.model_name,
-                                        self.dataset_name + '.json')
-        os.makedirs(os.path.dirname(report_path), exist_ok=True)
+            subset_score_map=reviews_score_all, model_name=self.model_name)
 
-        # Write report
-        with open(report_path, 'w') as f:
-            f.write(json.dumps(report_map.to_dict(), ensure_ascii=False, indent=4))
-        logger.info(f'Dump report: {report_path} \n')
+        # Post process report
+        self.data_adapter.post_process_report(report_map, report_path=report_path)
 
         # Make table
-        if use_table:
-            try:
-                report_table: str = gen_table([self.outputs_structure.reports_dir])
-                logger.info(f'Report table: \n{report_table} \n')
-            except Exception:
-                logger.error('Failed to generate report table.')
+        try:
+            report_table = gen_report_table(report_map)
+            logger.info(f'{self.dataset_name_or_path} report table: \n{report_table} \n')
+        except Exception:
+            logger.error('Failed to generate report table.')
+
+        # Make report analysis
+        if self.task_cfg.analysis_report:
+            logger.info('Generating report analysis, please wait ...')
+            analysis = report_map.generate_analysis(self.task_cfg.judge_model_args)
+            logger.info('Report analysis:\n%s', analysis)
+        else:
+            logger.info('Skipping report analysis (`analysis_report=False`).')
+
+        # Dump report
+        report_file = os.path.join(report_path, f'{self.dataset_name}.json')
+        report_map.to_json(report_file)
+        logger.info(f'Dump report to: {report_file} \n')
+
         return report_map
 
     def eval(self, **kwargs) -> dict:
@@ -403,7 +442,7 @@ class Evaluator(object):
             stage == 'review': return the reviews_map
         """
 
-        logger.info(f'**** Start evaluating on dataset {self.dataset_name_or_path} ****')
+        logger.info(f'Start evaluating on dataset {self.dataset_name_or_path}')
 
         reviews_score_all = {}  # {subset_name: (score, num)}
         stage_answers_dict = {}
@@ -433,6 +472,6 @@ class Evaluator(object):
         # Generate report
         report_map = self.dump_report(reviews_score_all)
 
-        logger.info(f'**** Evaluation finished on {self.dataset_name_or_path} ****\n')
+        logger.info(f'Evaluation finished on {self.dataset_name_or_path}')
 
         return report_map

evalscope/metrics/__init__.py

@@ -9,7 +9,7 @@ if TYPE_CHECKING:
     from .metrics import (bleu_ngram_one_sample, exact_match, macro_mean, mean, micro_mean, simple_f1_score,
                           weighted_mean)
     from .named_metrics import Metric, metric_registry
-    from .rouge_metric import compute_rouge_score_one_sample_zh
+    from .rouge_metric import compute_rouge_score, compute_rouge_score_one_sample, compute_rouge_score_one_sample_zh
 
 else:
     _import_structure = {
@@ -28,6 +28,8 @@ else:
         ],
         'rouge_metric': [
             'compute_rouge_score_one_sample_zh',
+            'compute_rouge_score',
+            'compute_rouge_score_one_sample',
         ],
         'llm_judge': [
             'LLMJudge',

evalscope/metrics/bundled_rouge_score/rouge_scorer.py

@@ -44,20 +44,25 @@ from evalscope.utils import get_logger
 
 logger = get_logger()
 
-# Deal with nltk punkt_tab.zip tokenizer file to avoid downloading issue
-try:
-    nltk_dir = os.path.join(os.path.expanduser('~'), 'nltk_data/tokenizers')
-    os.makedirs(nltk_dir, exist_ok=True)
-    punkt_path = os.path.join(nltk_dir, 'punkt_tab.zip')
-    punkt_tab_url = 'https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/open_data/nltk_data/punkt_tab.zip'
-
-    if not os.path.exists(punkt_path):
-        os.system(f'wget --timeout=10 --tries=3 -P {nltk_dir} {punkt_tab_url}')
-        os.system(f'unzip {punkt_path} -d {nltk_dir}')
-    else:
-        logger.debug(f'{punkt_path} already exists, skipping download')
-except Exception as e:
-    logger.error(f'Try to download punkt_tab.zip for nltk failed: {e}')
+
+def check_nltk_data():
+    """
+    Check if nltk data is available in the system.
+    If not, download the necessary data files.
+    """
+    try:
+        nltk_dir = os.path.join(os.path.expanduser('~'), 'nltk_data/tokenizers')
+        os.makedirs(nltk_dir, exist_ok=True)
+        punkt_path = os.path.join(nltk_dir, 'punkt_tab.zip')
+        punkt_tab_url = 'https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/open_data/nltk_data/punkt_tab.zip'
+
+        if not os.path.exists(punkt_path):
+            os.system(f'wget --timeout=10 --tries=3 -P {nltk_dir} {punkt_tab_url}')
+            os.system(f'unzip {punkt_path} -d {nltk_dir}')
+        else:
+            logger.debug(f'{punkt_path} already exists, skipping download')
+    except Exception as e:
+        logger.error(f'Try to download punkt_tab.zip for nltk failed: {e}')
 
 
 class RougeScorer(scoring.BaseScorer):
@@ -83,11 +88,11 @@ class RougeScorer(scoring.BaseScorer):
     """
 
     def __init__(self, rouge_types, use_stemmer=False, split_summaries=False, tokenizer=None):
-
         self.rouge_types = rouge_types
         if tokenizer:
             self._tokenizer = tokenizer
         else:
+            check_nltk_data()
             self._tokenizer = tokenizers.DefaultTokenizer(use_stemmer)
             logging.info('Using default tokenizer.')
 

evalscope/metrics/llm_judge.py

@@ -22,6 +22,9 @@ B: INCORRECT
 Just return the letters "A" or "B", with no text around it.
 """  # noqa: E501
 
+DEFAULT_JUDGE_MODEL = 'Qwen/Qwen3-235B-A22B'
+DEFAULT_API_URL = 'https://api-inference.modelscope.cn/v1/'
+
 
 class LLMJudge:
     """
@@ -47,25 +50,25 @@ class LLMJudge:
             prompt_template (str, optional): Prompt template for the judge
             generation_config (dict, optional): Generation configuration for the judge
         """
-        self.api_key = api_key or os.environ.get('OPENAI_API_KEY', 'EMPTY')
-        self.api_url = api_url or os.environ.get('OPENAI_API_BASE', 'https://api.openai.com/v1')
-        self.model_id = model_id or os.environ.get('LOCAL_LLM', 'gpt-4')
+        self.api_key = api_key or os.environ.get('MODELSCOPE_SDK_TOKEN', 'EMPTY')
+        self.api_url = api_url or os.environ.get('MODELSCOPE_API_BASE', DEFAULT_API_URL)
+        self.model_id = model_id or os.environ.get('MODELSCOPE_JUDGE_LLM', DEFAULT_JUDGE_MODEL)
         self.system_prompt = system_prompt or os.environ.get('JUDGE_SYSTEM_PROMPT', None)
         self.prompt_template = prompt_template or os.environ.get('JUDGE_PROMPT_TEMPLATE', DEFAULT_PROMPT_TEMPLATE)
-        self.generation_config = generation_config
+        self.generation_config = generation_config or {}
 
         from evalscope.models import ServerModelAdapter
 
         # Initialize ServerModelAdapter
         self.server_adapter = ServerModelAdapter(api_url=self.api_url, model_id=self.model_id, api_key=self.api_key)
 
-    def __call__(self, prompt: str, system_prompt: Optional[str] = None) -> float:
+    def __call__(self, prompt: str, system_prompt: Optional[str] = None) -> str:
         """
         Args:
             prompt (str): The prompt to evaluate
             system_prompt (str, optional): The system prompt to use for the evaluation
         Returns:
-            float: The score of the evaluation
+            str: The response from the LLM
         """
         input_data = {'data': [prompt], 'system_prompt': system_prompt or self.system_prompt}
 
@@ -74,6 +77,10 @@ class LLMJudge:
         if self.generation_config:
             infer_cfg.update(self.generation_config)
 
+        if self.model_id == DEFAULT_JUDGE_MODEL:
+            # Disable thinking for the default judge model
+            infer_cfg['enable_thinking'] = self.generation_config.get('enable_thinking', False)
+
         try:
             # Send request using ServerModelAdapter
             response = self.server_adapter.process_single_input(input_data, infer_cfg)
@@ -82,8 +89,8 @@ class LLMJudge:
             llm_response = response.get('choices', [{}])[0].get('message', {}).get('content', '')
             return llm_response
         except Exception as e:
-            logger.error(f'Error during LLM evaluation: {e}')
-            return None
+            logger.error(f'Error occurred during {self.model_id}@{self.api_url} LLM judge evaluation: {e}')
+            return ''
 
     def build_prompt(self, pred: str, gold: str, question: Optional[str] = None):
         if question is None:

evalscope/metrics/math_parser.py

@@ -4,7 +4,7 @@ The logic in this file largely borrows from Qwen2.5-Math codebase at https://git
 # flake8: noqa
 import re
 import regex
-from latex2sympy2 import latex2sympy
+from latex2sympy2_extended import latex2sympy
 from math import isclose
 from sympy import N, simplify
 from sympy.parsing.latex import parse_latex

evalscope/metrics/rouge_metric.py

@@ -19,10 +19,6 @@ class DummyTokenizer:
         return text.split()
 
 
-scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], tokenizer=DummyTokenizer())
-zh_scorer = Rouge()
-
-
 def is_contains_chinese(strs):
     for _char in strs:
         if '\u4e00' <= _char <= '\u9fa5':
@@ -51,6 +47,7 @@ def compute_rouge_score(predict_l, reference_l):
 
 def compute_rouge_score_one_sample_zh(predict, reference):
     result = dict()
+    zh_scorer = Rouge()
     for p, r in zip(predict, reference):
         p = ' '.join(jieba.cut(p)) if is_contains_chinese(p) else p
         r = ' '.join(jieba.cut(r)) if is_contains_chinese(r) else r
@@ -60,21 +57,22 @@ def compute_rouge_score_one_sample_zh(predict, reference):
         except Exception as e:
             logger.warning(f'rouge score error: {p} {r} {e}')
             continue
-        result['rouge-1-r'] = score['rouge-1']['r']
-        result['rouge-1-p'] = score['rouge-1']['p']
-        result['rouge-1-f'] = score['rouge-1']['f']
-        result['rouge-2-r'] = score['rouge-2']['r']
-        result['rouge-2-p'] = score['rouge-2']['p']
-        result['rouge-2-f'] = score['rouge-2']['f']
-        result['rouge-l-r'] = score['rouge-l']['r']
-        result['rouge-l-p'] = score['rouge-l']['p']
-        result['rouge-l-f'] = score['rouge-l']['f']
+        result['Rouge-1-R'] = score['rouge-1']['r']
+        result['Rouge-1-P'] = score['rouge-1']['p']
+        result['Rouge-1-F'] = score['rouge-1']['f']
+        result['Rouge-2-R'] = score['rouge-2']['r']
+        result['Rouge-2-P'] = score['rouge-2']['p']
+        result['Rouge-2-F'] = score['rouge-2']['f']
+        result['Rouge-L-R'] = score['rouge-l']['r']
+        result['Rouge-L-P'] = score['rouge-l']['p']
+        result['Rouge-L-F'] = score['rouge-l']['f']
 
     return result
 
 
 def compute_rouge_score_one_sample(predict, reference):
     result = dict()
+    scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], tokenizer=DummyTokenizer())
     for p, r in zip(predict, reference):
         try:
             score = scorer.score(p, r)