evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/filters/__init__.py

@@ -0,0 +1,2 @@
+from .extraction import *
+from .selection import *

evalscope/filters/extraction.py

@@ -0,0 +1,126 @@
+import re
+from typing import List
+
+from evalscope.api.filter import Filter
+from evalscope.api.registry import register_filter
+
+
+@register_filter('regex')
+class RegexFilter(Filter):
+    """A filter that extracts values from text using regex pattern matching.
+
+    This filter applies a regex pattern to each model response and extracts matched values.
+    If no match is found, returns a fallback value. Useful for extracting structured data
+    (like numbers) from unstructured model outputs.
+    """
+
+    def __init__(
+        self,
+        regex_pattern: str = r'#### (\-?[0-9\.\,]+)',
+        group_select: int = 0,
+        fallback: str = '[invalid]',
+    ) -> None:
+        """
+        pass a string `regex` to run `re.compile(r"regex")` on.
+        `fallback` defines the output returned if no matches for the regex are located.
+        """
+        self.regex_pattern = regex_pattern
+        self.regex = re.compile(regex_pattern)
+        self.group_select = group_select
+        self.fallback = fallback
+
+    def apply(self, instance: List[str]) -> List[str]:
+        """Apply regex pattern to each string in the instance list."""
+        filtered = []
+        for resp in instance:
+            match = self.regex.findall(resp)
+            if match:
+                match = match[self.group_select]
+                if isinstance(match, tuple):
+                    match = [m for m in match if m]
+                    if match:
+                        match = match[0]
+                    else:
+                        match = self.fallback
+                match = match.strip()
+            else:
+                match = self.fallback
+            filtered.append(match)
+        return filtered
+
+
+@register_filter('regex_pos')
+class POSFilter(Filter):
+    """ """
+
+    def __init__(
+        self,
+        regex_pattern: str = r"\['(.*?)'\]",
+        group_select=0,
+        fallback=None,
+    ) -> None:
+        """
+        pass a string `regex` to run `re.compile(r"regex")` on.
+        `fallback` defines the output returned if no matches for the regex are located.
+        """
+        if fallback is None:
+            fallback = ['invalid']
+        self.regex_pattern = regex_pattern
+        self.regex = re.compile(regex_pattern)
+        self.group_select = group_select
+        self.fallback = fallback
+
+    def apply(self, instance: List[str]) -> List[str]:
+        """Extract POS tags from each string in the instance list."""
+
+        def extract_tagged_tokens(text):
+            # Extract tagged tokens list from text input using regex
+            tokens = re.findall(r"\('([^']*)', '([^']*)'\)", text)
+            return [(token, pos) for token, pos in tokens]
+
+        def extract_pos_tags(result):
+            pos_tags = []
+            if isinstance(result, str):
+                result = extract_tagged_tokens(result)
+            pos_tags.extend(pos for _, pos in result)
+            return pos_tags if pos_tags else self.fallback
+
+        filtered = []
+        for resp in instance:
+            match = extract_pos_tags(resp)
+            filtered.append(str(match))
+        return filtered
+
+
+@register_filter('remove_whitespace')
+class WhitespaceFilter(Filter):
+    """Filters out leading whitespace from responses."""
+
+    def apply(self, instance: List[str]) -> List[str]:
+        """Remove leading whitespace from each string in the instance list."""
+        filtered_resp = []
+        for resp in instance:
+            resp = resp.lstrip()
+            filtered_resp.append(resp)
+        return filtered_resp
+
+
+@register_filter('remove_until')
+class RemoveUntilFilter(Filter):
+    """Filters out all text until a specified delimiter is found."""
+
+    def __init__(self, delimiter: str) -> None:
+        self.delimiter = delimiter
+
+    def apply(self, instance: List[str]) -> List[str]:
+        """Remove all text until the delimiter from each string in the instance list."""
+        filtered_resp = []
+        for resp in instance:
+            resp = resp.split(self.delimiter, 1)[-1]
+            filtered_resp.append(resp)
+        return filtered_resp
+
+
+@register_filter('extract')
+class ExtractFilter(RegexFilter):
+    ...

evalscope/filters/selection.py

@@ -0,0 +1,57 @@
+from collections import Counter
+from typing import List
+
+from evalscope.api.filter import Filter
+from evalscope.api.registry import register_filter
+
+
+@register_filter('take_first')
+class TakeFirstFilter(Filter):
+
+    def __init__(self) -> None:
+        """
+        Can define custom behavior here, if an individual instantiation of a Filter class should have state.
+        """
+
+    def apply(self, instance: List[str]) -> List[str]:
+        """
+        Take only the first response from the instance list.
+        """
+        return [instance[0]] if instance else []
+
+
+@register_filter('take_first_k')
+class TakeKFilter(Filter):
+
+    def __init__(self, **kwargs) -> None:
+        self.k = kwargs.pop('k')
+        super().__init__(**kwargs)
+
+    def apply(self, instance: List[str]) -> List[str]:
+        """
+        Take the first k responses from the instance list.
+        """
+        assert len(instance) >= self.k, (
+            f'Need at least {self.k} responses to take first {self.k}, but got {len(instance)} only!'
+        )
+        return instance[:self.k]
+
+
+@register_filter('majority_vote')
+class MajorityVoteFilter(Filter):
+
+    def __init__(self) -> None:
+        """
+        Can define custom behavior here, if an individual instantiation of a Filter class should have state.
+        """
+
+    def apply(self, instance: List[str]) -> List[str]:
+        """
+        Select the response that occurs most frequently in the instance list.
+        """
+        if not instance:
+            return []
+
+        counts = Counter(instance)
+        vote = counts.most_common(1)[0][0]
+        return [vote]

evalscope/metrics/__init__.py

@@ -4,12 +4,18 @@ from typing import TYPE_CHECKING
 from evalscope.utils.import_utils import _LazyModule
 
 if TYPE_CHECKING:
-    from .completion_parsers import ResponseParser, lmsys_parser, ranking_parser
     from .llm_judge import DEFAULT_NUMERIC_SCORE_TEMPLATE, DEFAULT_PROMPT_TEMPLATE, LLMJudge
     from .math_parser import extract_answer, math_equal, strip_answer_string
-    from .metrics import (bleu_ngram_one_sample, exact_match, macro_mean, mean, micro_mean, simple_f1_score,
-                          weighted_mean)
-    from .named_metrics import Metric, metric_registry
+    from .metric import PassAtK
+    from .metrics import (
+        bleu_ngram_one_sample,
+        exact_match,
+        macro_mean,
+        mean,
+        micro_mean,
+        simple_f1_score,
+        weighted_mean,
+    )
     from .rouge_metric import compute_rouge_score, compute_rouge_score_one_sample, compute_rouge_score_one_sample_zh
 
 else:
@@ -23,9 +29,8 @@ else:
             'simple_f1_score',
             'weighted_mean',
         ],
-        'named_metrics': [
-            'Metric',
-            'metric_registry',
+        'metric': [
+            'PassAtK',
         ],
         'rouge_metric': [
             'compute_rouge_score_one_sample_zh',
@@ -41,12 +46,7 @@ else:
             'extract_answer',
             'math_equal',
             'strip_answer_string',
-        ],
-        'completion_parsers': [
-            'ResponseParser',
-            'lmsys_parser',
-            'ranking_parser',
-        ],
+        ]
     }
 
     import sys

evalscope/metrics/llm_judge.py

@@ -2,6 +2,7 @@ import os
 import re
 from typing import Any, Dict, List, Optional
 
+from evalscope.api.messages import ChatMessage, ChatMessageSystem, ChatMessageUser
 from evalscope.constants import JudgeScoreType
 from evalscope.utils.logger import get_logger
 
@@ -48,17 +49,18 @@ class LLMJudge:
     """
 
     def __init__(
-            self,
-            api_key: Optional[str] = None,
-            api_url: Optional[str] = None,
-            model_id: Optional[str] = None,
-            system_prompt: Optional[str] = None,
-            prompt_template: Optional[str] = None,
-            generation_config: Optional[Dict[str, Any]] = None,
-            score_pattern: Optional[str] = None,
-            score_mapping: Optional[Dict[str, float]] = None,
-            score_type: str = JudgeScoreType.PATTERN,  # 'pattern', 'numeric'
-            **kwargs):
+        self,
+        api_key: Optional[str] = None,
+        api_url: Optional[str] = None,
+        model_id: Optional[str] = None,
+        system_prompt: Optional[str] = None,
+        prompt_template: Optional[str] = None,
+        generation_config: Optional[Dict[str, Any]] = None,
+        score_pattern: Optional[str] = None,
+        score_mapping: Optional[Dict[str, float]] = None,
+        score_type: str = JudgeScoreType.PATTERN,  # 'pattern', 'numeric'
+        **kwargs
+    ):
         """
         Initialize LLMJudge metric.
 
@@ -79,14 +81,15 @@ class LLMJudge:
         self.api_url = api_url or os.environ.get('MODELSCOPE_API_BASE', DEFAULT_API_URL)
         self.model_id = model_id or os.environ.get('MODELSCOPE_JUDGE_LLM', DEFAULT_JUDGE_MODEL)
         self.system_prompt = system_prompt or os.environ.get('JUDGE_SYSTEM_PROMPT', None)
-        self.generation_config = generation_config or {}
+        self.generation_config = generation_config or {'temperature': 0.0, 'max_tokens': 1024}
 
         # Default score mapping for A/B pattern
         self.score_type = score_type
         if self.score_type == JudgeScoreType.NUMERIC:
             self.score_pattern = score_pattern or r'\[\[(\d+(?:\.\d+)?)\]\]'
-            self.prompt_template = prompt_template or os.environ.get('JUDGE_PROMPT_TEMPLATE',
-                                                                     DEFAULT_NUMERIC_SCORE_TEMPLATE)
+            self.prompt_template = prompt_template or os.environ.get(
+                'JUDGE_PROMPT_TEMPLATE', DEFAULT_NUMERIC_SCORE_TEMPLATE
+            )
         elif self.score_type == JudgeScoreType.PATTERN:
             self.score_pattern = score_pattern or r'(A|B)'
             self.prompt_template = prompt_template or os.environ.get('JUDGE_PROMPT_TEMPLATE', DEFAULT_PROMPT_TEMPLATE)
@@ -97,36 +100,47 @@ class LLMJudge:
         self._init_server_adapter()
 
     def _init_server_adapter(self):
-        from evalscope.models import ServerModelAdapter
-
-        # Initialize ServerModelAdapter
-        self.server_adapter = ServerModelAdapter(api_url=self.api_url, model_id=self.model_id, api_key=self.api_key)
-
-    def __call__(self, prompt: str, system_prompt: Optional[str] = None) -> str:
+        from evalscope.api.model import GenerateConfig, get_model
+
+        self.model = get_model(
+            model=self.model_id,
+            eval_type='openai_api',
+            base_url=self.api_url,
+            api_key=self.api_key,
+            config=GenerateConfig(**self.generation_config),
+        )
+
+    def judge(
+        self,
+        prompt: str = '',
+        system_prompt: Optional[str] = None,
+        messages: Optional[List[ChatMessage]] = None
+    ) -> str:
         """
+        Generate a response from the LLM based on the provided prompt and context.
+        If messages is provided, it will be used as the input context.
+
         Args:
             prompt (str): The prompt to evaluate
             system_prompt (str, optional): The system prompt to use for the evaluation
+            messages (List[ChatMessage], optional): A list of chat messages to include in the evaluation
         Returns:
             str: The response from the LLM
         """
-        input_data = {'data': [prompt], 'system_prompt': system_prompt or self.system_prompt}
-
-        # Inference configuration
-        infer_cfg = {'temperature': 0.0, 'max_tokens': 1024}
-        if self.generation_config:
-            infer_cfg.update(self.generation_config)
-
-        if self.model_id == DEFAULT_JUDGE_MODEL:
-            # Disable thinking for the default judge model
-            infer_cfg['enable_thinking'] = self.generation_config.get('enable_thinking', False)
-
+        # parse messages
+        if messages is not None:
+            input_messages = messages
+        else:
+            system_content = system_prompt or self.system_prompt
+            input_messages = [ChatMessageUser(content=prompt)]
+            if system_content:
+                input_messages.insert(0, ChatMessageSystem(content=system_content))
         try:
             # Send request using ServerModelAdapter
-            response = self.server_adapter.process_single_input(input_data, infer_cfg)
+            response = self.model.generate(input_messages)
 
             # Extract content from response
-            llm_response = response.get('choices', [{}])[0].get('message', {}).get('content', '')
+            llm_response = response.completion
             return llm_response
         except Exception as e:
             logger.error(f'Error occurred during {self.model_id}@{self.api_url} LLM judge evaluation: {e}')

evalscope/metrics/math_parser.py

@@ -153,9 +153,11 @@ def strip_answer_string(string):
 
     # cdot
     # string = string.replace("\\cdot", "")
-    if (string.startswith('{') and string.endswith('}') and string.isalnum()
-            or string.startswith('(') and string.endswith(')') and string.isalnum()
-            or string.startswith('[') and string.endswith(']') and string.isalnum()):
+    if (
+        string.startswith('{') and string.endswith('}') and string.isalnum()
+        or string.startswith('(') and string.endswith(')') and string.isalnum()
+        or string.startswith('[') and string.endswith(']') and string.isalnum()
+    ):
         string = string[1:-1]
 
     # inf
@@ -387,9 +389,8 @@ def math_equal(
 
     ## deal with [], (), {}
     pred_str, ref_str = prediction, reference
-    if (prediction.startswith('[') and prediction.endswith(']')
-            and not reference.startswith('(')) or (prediction.startswith('(') and prediction.endswith(')')
-                                                   and not reference.startswith('[')):
+    if (prediction.startswith('[') and prediction.endswith(']') and not reference.startswith('(')
+        ) or (prediction.startswith('(') and prediction.endswith(')') and not reference.startswith('[')):
         pred_str = pred_str.strip('[]()')
         ref_str = ref_str.strip('[]()')
     for s in ['{', '}', '(', ')']:
@@ -399,25 +400,29 @@ def math_equal(
         return True
 
     ## [a, b] vs. [c, d], return a==c and b==d
-    if (regex.match(r'(\(|\[).+(\)|\])', prediction) is not None
-            and regex.match(r'(\(|\[).+(\)|\])', reference) is not None):
+    if (
+        regex.match(r'(\(|\[).+(\)|\])', prediction) is not None
+        and regex.match(r'(\(|\[).+(\)|\])', reference) is not None
+    ):
         pred_parts = prediction[1:-1].split(',')
         ref_parts = reference[1:-1].split(',')
         if len(pred_parts) == len(ref_parts):
-            if all(
-                [math_equal(pred_parts[i], ref_parts[i], include_percentage, is_close)
-                 for i in range(len(pred_parts))]):
+            if all([
+                math_equal(pred_parts[i], ref_parts[i], include_percentage, is_close) for i in range(len(pred_parts))
+            ]):
                 return True
     if ((prediction.startswith('\\begin{pmatrix}') or prediction.startswith('\\begin{bmatrix}'))
-            and (prediction.endswith('\\end{pmatrix}') or prediction.endswith('\\end{bmatrix}'))
-            and (reference.startswith('\\begin{pmatrix}') or reference.startswith('\\begin{bmatrix}'))
-            and (reference.endswith('\\end{pmatrix}') or reference.endswith('\\end{bmatrix}'))):
+        and (prediction.endswith('\\end{pmatrix}') or prediction.endswith('\\end{bmatrix}'))
+        and (reference.startswith('\\begin{pmatrix}') or reference.startswith('\\begin{bmatrix}'))
+        and (reference.endswith('\\end{pmatrix}') or reference.endswith('\\end{bmatrix}'))):
         pred_lines = [
-            line.strip() for line in prediction[len('\\begin{pmatrix}'):-len('\\end{pmatrix}')].split('\\\\')
+            line.strip()
+            for line in prediction[len('\\begin{pmatrix}'):-len('\\end{pmatrix}')].split('\\\\')
             if line.strip()
         ]
         ref_lines = [
-            line.strip() for line in reference[len('\\begin{pmatrix}'):-len('\\end{pmatrix}')].split('\\\\')
+            line.strip()
+            for line in reference[len('\\begin{pmatrix}'):-len('\\end{pmatrix}')].split('\\\\')
             if line.strip()
         ]
         matched = True
@@ -427,12 +432,12 @@ def math_equal(
                 ref_parts = ref_line.split('&')
                 if len(pred_parts) == len(ref_parts):
                     if not all([
-                            math_equal(
-                                pred_parts[i],
-                                ref_parts[i],
-                                include_percentage,
-                                is_close,
-                            ) for i in range(len(pred_parts))
+                        math_equal(
+                            pred_parts[i],
+                            ref_parts[i],
+                            include_percentage,
+                            is_close,
+                        ) for i in range(len(pred_parts))
                     ]):
                         matched = False
                         break