evalscope 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/utils/logger.py

@@ -28,22 +28,41 @@ logging.getLogger('datasets').setLevel(logging.WARNING)
 logging.getLogger('httpx').setLevel(logging.WARNING)
 logging.getLogger('modelscope').setLevel(logging.ERROR)
 
+info_set = set()
+warning_set = set()
+
+
+def info_once(self, msg, *args, **kwargs):
+    hash_id = kwargs.get('hash_id') or msg
+    if hash_id in info_set:
+        return
+    info_set.add(hash_id)
+    self.info(msg)
+
+
+def warning_once(self, msg, *args, **kwargs):
+    hash_id = kwargs.get('hash_id') or msg
+    if hash_id in warning_set:
+        return
+    warning_set.add(hash_id)
+    self.warning(msg)
+
 
 def get_logger(
     log_file: Optional[str] = None,
     name: Optional[str] = None,
     log_level: int = DEFAULT_LEVEL,
     file_mode: str = 'w',
-    force=False
+    force: bool = False,
 ):
     """Get logging logger
 
     Args:
-        log_file: Log filename, if specified, file handler will be added to
-            logger
-        log_level: Logging level.
-        file_mode: Specifies the mode to open the file, if filename is
-            specified (if filemode is unspecified, it defaults to 'w').
+        log_file: Log filename. If specified, a file handler will be added to the logger.
+        name: Logical component name. Used to derive the logger name.
+        log_level: Logging level to set.
+        file_mode: Mode to open the file when log_file is provided (default 'w').
+        force: If True, reconfigure the existing logger (levels, formatters, handlers).
     """
 
     if name:
@@ -58,7 +77,7 @@ def get_logger(
             logger.setLevel(log_level)
             for handler in logger.handlers:
                 handler.setLevel(log_level)
-                # 区分不同类型的 handler，使用相应的格式化器
+                # Select formatter by handler type
                 if isinstance(handler, logging.FileHandler):
                     handler.setFormatter(
                         plain_detailed_formatter if log_level == logging.DEBUG else plain_simple_formatter
@@ -67,6 +86,7 @@ def get_logger(
                     handler.setFormatter(
                         color_detailed_formatter if log_level == logging.DEBUG else color_simple_formatter
                     )
+            # Ensure file handler points to current log_file (replace if needed)
             add_file_handler_if_needed(logger, log_file, file_mode, log_level)
         return logger
 
@@ -88,7 +108,7 @@ def get_logger(
     handlers = [stream_handler]
 
     if is_worker0 and log_file is not None:
-        file_handler = logging.FileHandler(log_file, file_mode)
+        file_handler = logging.FileHandler(log_file, file_mode, encoding='utf-8')
         handlers.append(file_handler)
 
     for handler in handlers:
@@ -118,23 +138,54 @@ def configure_logging(debug: bool, log_file: Optional[str] = None):
         get_logger(log_level=logging.DEBUG, force=True)
 
 
-def add_file_handler_if_needed(logger, log_file, file_mode, log_level):
-    for handler in logger.handlers:
-        if isinstance(handler, logging.FileHandler):
-            return
+def add_file_handler_if_needed(
+    logger: logging.Logger,
+    log_file: Optional[str],
+    file_mode: str,
+    log_level: int,
+) -> None:
+    """Ensure logger has a FileHandler targeting log_file.
+    - If no FileHandler exists, add one.
+    - If a FileHandler exists but points to a different file, replace it.
+    """
+    if log_file is None:
+        return
 
+    # Only worker-0 writes files
     if iutil.find_spec('torch') is not None:
         from modelscope.utils.torch_utils import is_master
-
         is_worker0 = is_master()
     else:
         is_worker0 = True
 
-    if is_worker0 and log_file is not None:
-        file_handler = logging.FileHandler(log_file, file_mode)
-        file_handler.setFormatter(plain_detailed_formatter if log_level == logging.DEBUG else plain_simple_formatter)
-        file_handler.setLevel(log_level)
-        logger.addHandler(file_handler)
+    if not is_worker0:
+        return
+
+    target_path = os.path.abspath(log_file)
+    existing_file_handlers = [h for h in logger.handlers if isinstance(h, logging.FileHandler)]
+
+    # If there is a FileHandler already pointing to the target file, nothing to do.
+    for fh in existing_file_handlers:
+        try:
+            if os.path.abspath(getattr(fh, 'baseFilename', '')) == target_path:
+                return
+        except Exception:
+            # If any issue retrieving baseFilename, fall through to replacement
+            pass
+
+    # Replace all existing FileHandlers with the new one
+    for fh in existing_file_handlers:
+        try:
+            logger.removeHandler(fh)
+            fh.flush()
+            fh.close()
+        except Exception:
+            pass
+
+    file_handler = logging.FileHandler(target_path, file_mode, encoding='utf-8')
+    file_handler.setFormatter(plain_detailed_formatter if log_level == logging.DEBUG else plain_simple_formatter)
+    file_handler.setLevel(log_level)
+    logger.addHandler(file_handler)
 
 
 def warn_once(logger: Logger, message: str) -> None:

evalscope/utils/model_utils.py

@@ -3,6 +3,8 @@ import random
 from enum import Enum
 from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union
 
+from evalscope.utils.import_utils import check_import
+
 if TYPE_CHECKING:
     from transformers import GenerationConfig
 
@@ -67,7 +69,8 @@ def seed_everything(seed: int):
     """
     random.seed(seed)
     np.random.seed(seed)
-    try:
+
+    if check_import('torch', raise_warning=False):
         import torch
 
         torch.manual_seed(seed)
@@ -75,5 +78,3 @@ def seed_everything(seed: int):
             torch.cuda.manual_seed_all(seed)
             torch.backends.cudnn.deterministic = True
             torch.backends.cudnn.benchmark = False
-    except ImportError:
-        pass

evalscope/utils/multi_choices.py

@@ -1,11 +1,8 @@
 # flake8: noqa: E501
-from __future__ import annotations
-
 import re
-from typing import TYPE_CHECKING, List, Optional
+from typing import List, Optional, Union
 
-if TYPE_CHECKING:
-    from evalscope.api.evaluator import Choices, Target, TaskState
+from evalscope.api.evaluator import Choices, Target, TaskState
 
 FEW_SHOT_TEMPLATE = r"""Here are some examples of how to answer similar questions:
 
@@ -84,10 +81,27 @@ def answer_options(choices: Choices) -> str:
     return '\n'.join([f'{answer_character(i)}) {choices[j].value}' for i, j in enumerate(indexes)])
 
 
-def prompt(question: str, choices: Choices, template: str, fewshot: Optional[str] = None) -> str:
+def format_letter_choices(choices: Union[Choices, List[str]]) -> str:
+    """
+    Returns the `choices` formatted as a letter list, e.g.:
+
+    ["choice 1", "choice 2", "choice 3"] ->
+        "A,B,C"
+    """
+    if isinstance(choices, list):
+        choices = Choices(choices)
+
+    indexes = list(range(len(choices)))
+
+    return ','.join([f'{answer_character(i)}' for i in indexes])
+
+
+def prompt(question: str, choices: Union[Choices, List[str]], template: str, fewshot: Optional[str] = None) -> str:
+    if isinstance(choices, list):
+        choices = Choices(choices)
 
     choices_text = answer_options(choices)
-    letters = ','.join(answer_character(i) for i in range(len(choices)))
+    letters = format_letter_choices(choices)
     if not fewshot:
         return template.format(
             choices=choices_text,
@@ -122,6 +136,14 @@ def format_example(
     return f'{question}\n{choices_text}\nANSWER: {answer.text}'
 
 
+def _fallback_parse_answer(completion: str) -> Optional[set[str]]:
+    # Fallback to find the last upper case letter
+    for letter in reversed(completion):
+        if letter.isupper():
+            return {letter}
+    return None
+
+
 def parse_answers(state: TaskState, multiple_correct: bool = False) -> set[str]:
     """
     Convenience function for extracting answers from the state output.
@@ -150,6 +172,11 @@ def parse_answers(state: TaskState, multiple_correct: bool = False) -> set[str]:
             state.output.completion,
         )
 
+    if match is None:
+        fallback_answer = _fallback_parse_answer(state.output.completion)
+        if fallback_answer:
+            return fallback_answer
+
     if match is None:
         return set()
 
@@ -200,6 +227,11 @@ def parse_answers_zh(state: TaskState, multiple_correct: bool = False) -> set[st
     pattern = r'答案\s*[:：]\s*([A-Za-z0-9,，]+)'
     match = re.search(pattern, state.output.completion, flags=re.MULTILINE)
 
+    if match is None:
+        fallback_answer = _fallback_parse_answer(state.output.completion)
+        if fallback_answer:
+            return fallback_answer
+
     if match is None:
         return set()
 

evalscope/utils/ner.py

@@ -0,0 +1,377 @@
+import re
+from typing import Any, Dict, List, Set, Tuple
+
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+PROMPT_TEMPLATE = """
+You are a named entity recognition system that identifies the following entity types:
+{entities}
+
+Process the provided text and mark all named entities with XML-style tags.
+
+For example:
+<person>John Smith</person> works at <organization>Google</organization> in <location>Mountain View</location>.
+
+Available entity tags: {entity_list}
+
+INSTRUCTIONS:
+1. Wrap your entire response in <response>...</response> tags.
+2. Inside these tags, include the original text with entity tags inserted.
+3. Do not change the original text in any way (preserve spacing, punctuation, case, etc.).
+4. Tag ALL entities you can identify using the exact tag names provided.
+5. Do not include explanations, just the tagged text.
+6. If entity spans overlap, choose the most specific entity type.
+7. Ensure every opening tag has a matching closing tag.
+
+Text to process:
+{text}
+""".lstrip()
+
+FEWSHOT_TEMPLATE = """
+Here are some examples of named entity recognition:
+
+{fewshot}
+
+You are a named entity recognition system that identifies the following entity types:
+{entities}
+
+Process the provided text and mark all named entities with XML-style tags.
+
+For example:
+<person>John Smith</person> works at <organization>Google</organization> in <location>Mountain View</location>.
+
+Available entity tags: {entity_list}
+
+INSTRUCTIONS:
+1. Wrap your entire response in <response>...</response> tags.
+2. Inside these tags, include the original text with entity tags inserted.
+3. Do not change the original text in any way (preserve spacing, punctuation, case, etc.).
+4. Tag ALL entities you can identify using the exact tag names provided.
+5. Do not include explanations, just the tagged text.
+6. If entity spans overlap, choose the most specific entity type.
+7. Ensure every opening tag has a matching closing tag.
+
+Text to process:
+{text}
+""".lstrip()
+
+# Common error patterns to handle in XML predictions
+DEFAULT_TAG_FIX_PATTERNS = [
+    # Fix mismatched tags
+    (r'<(\w+)>(.*?)</\w+>', r'<\1>\2</\1>'),
+]
+
+
+def create_target_text(tokens: List[str], ner_tags: List[str], entity_type_map: Dict[str, str]) -> str:
+    """
+    Create annotated text from tokens and NER tags.
+    Handles BIO tagging scheme conversion to inline XML-style tags.
+
+    Args:
+        tokens: List of text tokens
+        ner_tags: List of BIO tags corresponding to tokens
+        entity_type_map: Mapping from BIO entity types to user-friendly tag names
+
+    Returns:
+        String with XML-style entity markup wrapped in <response> tags
+    """
+    result = []
+    current_entity = None
+    entity_tokens = []
+
+    for i, (token, tag) in enumerate(zip(tokens, ner_tags)):
+        if tag.startswith('B-'):  # Beginning of entity
+            # Close previous entity if exists
+            if current_entity:
+                entity_type = entity_type_map.get(current_entity, '')
+                if entity_type:
+                    result.append(f'<{entity_type.lower()}>{" ".join(entity_tokens)}</{entity_type.lower()}>')
+                else:
+                    result.append(' '.join(entity_tokens))
+                entity_tokens = []
+
+            current_entity = tag[2:]  # Remove B- prefix
+            entity_tokens.append(token)
+        elif tag.startswith('I-') and current_entity and tag[2:] == current_entity:  # Inside entity
+            entity_tokens.append(token)
+        else:  # Outside any entity (O tag)
+            if current_entity:  # Close previous entity
+                entity_type = entity_type_map.get(current_entity, '')
+                if entity_type:
+                    result.append(f'<{entity_type.lower()}>{" ".join(entity_tokens)}</{entity_type.lower()}>')
+                else:
+                    result.append(' '.join(entity_tokens))
+                current_entity = None
+                entity_tokens = []
+
+            result.append(token)
+
+    # Handle any remaining entity at end of sequence
+    if current_entity:
+        entity_type = entity_type_map.get(current_entity, '')
+        if entity_type:
+            result.append(f'<{entity_type.lower()}>{" ".join(entity_tokens)}</{entity_type.lower()}>')
+        else:
+            result.append(' '.join(entity_tokens))
+
+    # Wrap the entire response in <response> tags as required by the pipeline
+    return f'<response>{" ".join(result)}</response>'
+
+
+def clean_prediction(text: str, tag_fix_patterns: List[Tuple[str, str]] = None) -> str:
+    """
+    Clean and fix common XML errors in model predictions.
+
+    Args:
+        text: The prediction text to clean
+        tag_fix_patterns: List of regex patterns and replacements to fix common XML errors
+
+    Returns:
+        Cleaned text with fixed XML tags
+    """
+    if tag_fix_patterns is None:
+        tag_fix_patterns = DEFAULT_TAG_FIX_PATTERNS
+
+    cleaned = text
+
+    # Extract content from response tags if present
+    response_match = re.search(r'<response>(.*?)</response>', cleaned, re.DOTALL)
+    if response_match:
+        cleaned = response_match.group(1)
+
+    # Apply fix patterns for common XML errors
+    for pattern, replacement in tag_fix_patterns:
+        cleaned = re.sub(pattern, replacement, cleaned)
+
+    return cleaned
+
+
+def extract_entities_from_text(text: str, reverse_entity_map: Dict[str, str]) -> List[Tuple]:
+    """
+    Extract entities from tagged text with robust error handling.
+
+    Args:
+        text: Text with XML entity tags
+        reverse_entity_map: Mapping from user-friendly tag names to BIO entity types
+
+    Returns:
+        List of (entity_type, entity_text, start_idx, end_idx) tuples
+    """
+    entities = []
+
+    # Define regex pattern to find XML-style entity tags - handle potential errors
+    pattern = r'<(\w+)>(.*?)</\1>'
+
+    try:
+        for match in re.finditer(pattern, text):
+            entity_type = match.group(1).lower()  # Normalize type to lowercase
+            entity_text = match.group(2)
+            start_idx = match.start()
+            end_idx = match.end()
+
+            # Map back to entity types if possible
+            mapped_type = reverse_entity_map.get(entity_type)
+
+            if mapped_type:
+                entities.append((mapped_type, entity_text, start_idx, end_idx))
+            else:
+                # Unknown entity type but still count it for evaluation
+                entities.append((entity_type, entity_text, start_idx, end_idx))
+
+    except Exception as e:
+        logger.warning(f'Error parsing entities in text: {str(e)}')
+
+    # Handle malformed XML by trying to find additional tag patterns
+    # This is a fallback for when the model produces incorrect tags
+    unclosed_pattern = r'<(\w+)>(.*?)(?=<|$)'
+    try:
+        # Find potential unclosed tags
+        for match in re.finditer(unclosed_pattern, text):
+            # Skip if already part of a well-formed tag
+            if any(start_idx <= match.start() < end_idx for _, _, start_idx, end_idx in entities):
+                continue
+
+            entity_type = match.group(1).lower()
+            entity_text = match.group(2)
+            start_idx = match.start()
+            end_idx = match.end()
+
+            # Map back to entity types
+            mapped_type = reverse_entity_map.get(entity_type)
+            if mapped_type:
+                entities.append((mapped_type, entity_text, start_idx, end_idx))
+
+    except Exception as e:
+        logger.warning(f'Error handling malformed tags: {str(e)}')
+
+    return entities
+
+
+def xml_to_bio_tags(xml_text: str, original_tokens: List[str], reverse_entity_map: Dict[str, str]) -> List[str]:
+    """
+    Convert XML-annotated text back to BIO tags aligned with the original tokens.
+
+    Args:
+        xml_text: Text with XML entity annotations
+        original_tokens: Original tokens to align with
+        reverse_entity_map: Mapping from user-friendly tag names to BIO entity types
+
+    Returns:
+        List of BIO tags corresponding to the original tokens
+    """
+    # Extract entities with their character positions
+    entities = extract_entities_from_text(xml_text, reverse_entity_map)
+
+    # Initialize all tags as 'O'
+    bio_tags = ['O'] * len(original_tokens)
+
+    # Reconstruct the original text to find character positions for each token
+    original_text = ' '.join(original_tokens)
+
+    # Track token start positions in the original text
+    token_positions = []
+    pos = 0
+    for token in original_tokens:
+        token_pos = original_text.find(token, pos)
+        if token_pos == -1:
+            # Fallback: just use the current position if we can't find the exact match
+            token_positions.append(pos)
+        else:
+            token_positions.append(token_pos)
+            pos = token_pos + len(token)
+
+    # Add token end positions
+    token_ends = [pos + len(token) for pos, token in zip(token_positions, original_tokens)]
+
+    # Map entities to tokens based on character positions
+    for entity_type, entity_text, start_pos, end_pos in entities:
+        # Extract the context from the XML text to help locate the correct entity occurrence
+        # Get some context before and after the entity in the XML text
+        context_start = max(0, start_pos - 20)
+        context_end = min(len(xml_text), end_pos + 20)
+
+        # Extract context without XML tags
+        context_before = re.sub(r'<[^>]+>', '', xml_text[context_start:start_pos])
+        context_after = re.sub(r'<[^>]+>', '', xml_text[end_pos:context_end])
+
+        # Use context to find the correct entity position in original text
+        search_pos = 0
+        entity_start = -1
+
+        while search_pos < len(original_text):
+            # Find the next occurrence of the entity
+            potential_start = original_text.find(entity_text, search_pos)
+            if potential_start == -1:
+                break
+
+            # Check if the context matches
+            potential_context_start = max(0, potential_start - len(context_before))
+            potential_context_end = min(len(original_text), potential_start + len(entity_text) + len(context_after))
+
+            before_match = context_before.strip() in original_text[potential_context_start:potential_start].strip()
+            after_match = context_after.strip() in original_text[potential_start
+                                                                 + len(entity_text):potential_context_end].strip()
+
+            # If context matches or we can't find a better match, use this position
+            if before_match or after_match or search_pos > len(original_text) // 2:
+                entity_start = potential_start
+                break
+
+            # Move search position forward
+            search_pos = potential_start + 1
+
+        # If we couldn't find the entity with context, fall back to the first occurrence
+        if entity_start == -1:
+            entity_start = original_text.find(entity_text)
+            if entity_start == -1:
+                continue
+
+        entity_end = entity_start + len(entity_text)
+
+        # Find tokens that overlap with this entity
+        for i, (token_start, token_end) in enumerate(zip(token_positions, token_ends)):
+            if token_start <= entity_end and token_end >= entity_start:
+                # This token overlaps with the entity
+                if bio_tags[i] == 'O':
+                    # Start of entity
+                    if i == 0 or bio_tags[i - 1] == 'O' or not bio_tags[i - 1].endswith(entity_type):
+                        bio_tags[i] = f'B-{entity_type}'
+                    else:
+                        # Continuation of entity
+                        bio_tags[i] = f'I-{entity_type}'
+
+    return bio_tags
+
+
+def calculate_bio_metrics(pred_tags: List[str], gold_tags: List[str], tokens: List[str]) -> Tuple[int, int, int]:
+    """
+    Calculate metrics by comparing BIO tag sequences.
+
+    Args:
+        pred_tags: Predicted BIO tags
+        gold_tags: Gold standard BIO tags
+        tokens: Original tokens
+
+    Returns:
+        Tuple of (true_positives, false_positives, false_negatives)
+    """
+    # Extract entity spans from BIO tags
+    pred_spans = extract_spans_from_bio(pred_tags, tokens)
+    gold_spans = extract_spans_from_bio(gold_tags, tokens)
+
+    # Calculate metrics
+    true_positives = len(pred_spans.intersection(gold_spans))
+    false_positives = len(pred_spans - gold_spans)
+    false_negatives = len(gold_spans - pred_spans)
+
+    return true_positives, false_positives, false_negatives
+
+
+def extract_spans_from_bio(tags: List[str], tokens: List[str]) -> Set[Tuple]:
+    """
+    Extract entity spans from BIO tags.
+
+    Args:
+        tags: List of BIO tags
+        tokens: List of tokens corresponding to the tags
+
+    Returns:
+        Set of (entity_type, start_idx, end_idx, text) tuples
+    """
+    spans = set()
+    current_entity = None
+    start_idx = None
+    entity_tokens = []
+
+    for i, (token, tag) in enumerate(zip(tokens, tags)):
+        if tag.startswith('B-'):  # Beginning of entity
+            # Close previous entity if exists
+            if current_entity:
+                entity_type = current_entity
+                entity_text = ' '.join(entity_tokens)
+                spans.add((entity_type, start_idx, i - 1, entity_text))
+                entity_tokens = []
+
+            current_entity = tag[2:]  # Remove B- prefix
+            start_idx = i
+            entity_tokens.append(token)
+        elif tag.startswith('I-') and current_entity:  # Inside entity
+            entity_tokens.append(token)
+        elif tag == 'O':  # Outside any entity
+            if current_entity:  # Close previous entity
+                entity_type = current_entity
+                entity_text = ' '.join(entity_tokens)
+                spans.add((entity_type, start_idx, i - 1, entity_text))
+                current_entity = None
+                start_idx = None
+                entity_tokens = []
+
+    # Handle any remaining entity at end of sequence
+    if current_entity:
+        entity_type = current_entity
+        entity_text = ' '.join(entity_tokens)
+        spans.add((entity_type, start_idx, len(tokens) - 1, entity_text))
+
+    return spans

evalscope/version.py

@@ -1,4 +1,4 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-__version__ = '1.0.0'
-__release_datetime__ = '2025-08-25 12:00:00'
+__version__ = '1.2.0'
+__release_datetime__ = '2025-11-11 12:00:00'