evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/metrics/bundled_rouge_score/rouge_scorer.py

@@ -44,20 +44,25 @@ from evalscope.utils import get_logger
 
 logger = get_logger()
 
-# Deal with nltk punkt_tab.zip tokenizer file to avoid downloading issue
-try:
-    nltk_dir = os.path.join(os.path.expanduser('~'), 'nltk_data/tokenizers')
-    os.makedirs(nltk_dir, exist_ok=True)
-    punkt_path = os.path.join(nltk_dir, 'punkt_tab.zip')
-    punkt_tab_url = 'https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/open_data/nltk_data/punkt_tab.zip'
-
-    if not os.path.exists(punkt_path):
-        os.system(f'wget --timeout=10 --tries=3 -P {nltk_dir} {punkt_tab_url}')
-        os.system(f'unzip {punkt_path} -d {nltk_dir}')
-    else:
-        logger.debug(f'{punkt_path} already exists, skipping download')
-except Exception as e:
-    logger.error(f'Try to download punkt_tab.zip for nltk failed: {e}')
+
+def check_nltk_data():
+    """
+    Check if nltk data is available in the system.
+    If not, download the necessary data files.
+    """
+    try:
+        nltk_dir = os.path.join(os.path.expanduser('~'), 'nltk_data/tokenizers')
+        os.makedirs(nltk_dir, exist_ok=True)
+        punkt_path = os.path.join(nltk_dir, 'punkt_tab.zip')
+        punkt_tab_url = 'https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/open_data/nltk_data/punkt_tab.zip'
+
+        if not os.path.exists(punkt_path):
+            os.system(f'wget --timeout=10 --tries=3 -P {nltk_dir} {punkt_tab_url}')
+            os.system(f'unzip {punkt_path} -d {nltk_dir}')
+        else:
+            logger.debug(f'{punkt_path} already exists, skipping download')
+    except Exception as e:
+        logger.error(f'Try to download punkt_tab.zip for nltk failed: {e}')
 
 
 class RougeScorer(scoring.BaseScorer):
@@ -83,11 +88,11 @@ class RougeScorer(scoring.BaseScorer):
     """
 
     def __init__(self, rouge_types, use_stemmer=False, split_summaries=False, tokenizer=None):
-
         self.rouge_types = rouge_types
         if tokenizer:
             self._tokenizer = tokenizer
         else:
+            check_nltk_data()
             self._tokenizer = tokenizers.DefaultTokenizer(use_stemmer)
             logging.info('Using default tokenizer.')
 

evalscope/metrics/llm_judge.py

@@ -0,0 +1,211 @@
+import os
+import re
+from typing import Any, Dict, List, Optional
+
+from evalscope.api.messages import ChatMessage, ChatMessageSystem, ChatMessageUser
+from evalscope.constants import JudgeScoreType
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+DEFAULT_PROMPT_TEMPLATE = """Your job is to look at a question, a gold target, and a predicted answer, and return a letter "A" or "B" to indicate whether the predicted answer is correct or incorrect.
+
+[Question]
+{question}
+
+[Reference Answer]
+{gold}
+
+[Predicted Answer]
+{pred}
+
+Evaluate the model's answer based on correctness compared to the reference answer.
+Grade the predicted answer of this new question as one of:
+A: CORRECT
+B: INCORRECT
+
+Just return the letters "A" or "B", with no text around it.
+"""  # noqa: E501
+
+
+DEFAULT_NUMERIC_SCORE_TEMPLATE = """Please act as an impartial judge and evaluate the quality of the response provided by an AI assistant to the user question displayed below. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of the response.
+Begin your evaluation by providing a short explanation. Be as objective as possible.
+After providing your explanation, you must rate the response on a scale of 0 (worst) to 1 (best) by strictly following this format: \"[[rating]]\", for example: \"Rating: [[0.5]]\"
+
+[Question]
+{question}
+
+[Response]
+{pred}
+"""  # noqa: E501
+
+DEFAULT_JUDGE_MODEL = 'Qwen/Qwen3-235B-A22B'
+DEFAULT_API_URL = 'https://api-inference.modelscope.cn/v1/'
+
+
+class LLMJudge:
+    """
+    A metric that uses LLM to judge the quality of model predictions by comparing them with reference answers.
+    """
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        api_url: Optional[str] = None,
+        model_id: Optional[str] = None,
+        system_prompt: Optional[str] = None,
+        prompt_template: Optional[str] = None,
+        generation_config: Optional[Dict[str, Any]] = None,
+        score_pattern: Optional[str] = None,
+        score_mapping: Optional[Dict[str, float]] = None,
+        score_type: str = JudgeScoreType.PATTERN,  # 'pattern', 'numeric'
+        **kwargs
+    ):
+        """
+        Initialize LLMJudge metric.
+
+        Args:
+            api_key (str, optional): API key for OpenAI or compatible service
+            api_base (str, optional): API base URL
+            model_id (str, optional): Model ID for LLM
+            system_prompt (str, optional): System prompt for the judge
+            prompt_template (str, optional): Prompt template for the judge
+            generation_config (dict, optional): Generation configuration for the judge
+            score_pattern (str, optional): Regex pattern to extract score from LLM response
+            score_mapping (dict, optional): Mapping from extracted score to float value
+            score_type (str, optional): Type of score extraction strategy ('pattern', 'numeric') defaults to 'pattern'.
+                - 'pattern': Use score_pattern and score_mapping to extract categorical scores
+                - 'numeric': Treat the extracted value as a direct numerical score
+        """
+        self.api_key = api_key or os.environ.get('MODELSCOPE_SDK_TOKEN', 'EMPTY')
+        self.api_url = api_url or os.environ.get('MODELSCOPE_API_BASE', DEFAULT_API_URL)
+        self.model_id = model_id or os.environ.get('MODELSCOPE_JUDGE_LLM', DEFAULT_JUDGE_MODEL)
+        self.system_prompt = system_prompt or os.environ.get('JUDGE_SYSTEM_PROMPT', None)
+        self.generation_config = generation_config or {'temperature': 0.0, 'max_tokens': 1024}
+
+        # Default score mapping for A/B pattern
+        self.score_type = score_type
+        if self.score_type == JudgeScoreType.NUMERIC:
+            self.score_pattern = score_pattern or r'\[\[(\d+(?:\.\d+)?)\]\]'
+            self.prompt_template = prompt_template or os.environ.get(
+                'JUDGE_PROMPT_TEMPLATE', DEFAULT_NUMERIC_SCORE_TEMPLATE
+            )
+        elif self.score_type == JudgeScoreType.PATTERN:
+            self.score_pattern = score_pattern or r'(A|B)'
+            self.prompt_template = prompt_template or os.environ.get('JUDGE_PROMPT_TEMPLATE', DEFAULT_PROMPT_TEMPLATE)
+        else:
+            raise ValueError(f"Invalid score_type: {self.score_type}. Must be 'pattern' or 'numeric'.")
+        self.score_mapping = score_mapping or {'A': 1.0, 'B': 0.0}
+
+        self._init_server_adapter()
+
+    def _init_server_adapter(self):
+        from evalscope.api.model import GenerateConfig, get_model
+
+        self.model = get_model(
+            model=self.model_id,
+            eval_type='openai_api',
+            base_url=self.api_url,
+            api_key=self.api_key,
+            config=GenerateConfig(**self.generation_config),
+        )
+
+    def judge(
+        self,
+        prompt: str = '',
+        system_prompt: Optional[str] = None,
+        messages: Optional[List[ChatMessage]] = None
+    ) -> str:
+        """
+        Generate a response from the LLM based on the provided prompt and context.
+        If messages is provided, it will be used as the input context.
+
+        Args:
+            prompt (str): The prompt to evaluate
+            system_prompt (str, optional): The system prompt to use for the evaluation
+            messages (List[ChatMessage], optional): A list of chat messages to include in the evaluation
+        Returns:
+            str: The response from the LLM
+        """
+        # parse messages
+        if messages is not None:
+            input_messages = messages
+        else:
+            system_content = system_prompt or self.system_prompt
+            input_messages = [ChatMessageUser(content=prompt)]
+            if system_content:
+                input_messages.insert(0, ChatMessageSystem(content=system_content))
+        try:
+            # Send request using ServerModelAdapter
+            response = self.model.generate(input_messages)
+
+            # Extract content from response
+            llm_response = response.completion
+            return llm_response
+        except Exception as e:
+            logger.error(f'Error occurred during {self.model_id}@{self.api_url} LLM judge evaluation: {e}')
+            return ''
+
+    def build_prompt(self, pred: str, gold: str, question: Optional[str] = None):
+        if question is None:
+            question = 'Not provided'
+
+        # check variables in prompt_template
+        prompt = self.prompt_template
+        if '{question}' in self.prompt_template:
+            prompt = prompt.replace('{question}', question)
+        if '{pred}' in self.prompt_template:
+            prompt = prompt.replace('{pred}', pred)
+        if '{gold}' in self.prompt_template:
+            prompt = prompt.replace('{gold}', gold)
+        return prompt
+
+    def get_score(self, response: str) -> float:
+        """
+        Extract score from LLM response using the configured pattern and mapping.
+
+        Args:
+            response (str): The response from the LLM
+
+        Returns:
+            float: The numeric score extracted from the response
+        """
+        if response is None:
+            return 0.0
+
+        # choose extraction method based on score_type
+        if self.score_type == JudgeScoreType.NUMERIC:
+            return self._extract_numeric_score(response)
+        elif self.score_type == JudgeScoreType.PATTERN:
+            return self._extract_pattern_score(response)
+
+    def _extract_numeric_score(self, response: str) -> Optional[float]:
+        """extract numeric score from the response using the score_pattern"""
+        match = re.search(self.score_pattern, response)
+
+        if match:
+            # try to convert each captured group to float
+            for group in match.groups():
+                if group is not None:
+                    try:
+                        return float(group)
+                    except (ValueError, TypeError):
+                        continue
+
+            # if not found in groups, try the whole match
+            try:
+                return float(match.group(0))
+            except (ValueError, TypeError):
+                logger.warning(f'Failed to convert any extracted value to float from: {match.group(0)}')
+
+        return None
+
+    def _extract_pattern_score(self, response: str) -> float:
+        """use the score_pattern to extract categorical scores"""
+        match = re.search(self.score_pattern, response)
+        if match:
+            answer = match.group(0)
+            return self.score_mapping.get(answer, 0.0)
+        else:
+            logger.warning(f"No match found for pattern '{self.score_pattern}' in response: {response}")
+            return 0.0