evalscope 1.0.2__py3-none-any.whl → 1.1.1__py3-none-any.whl

evalscope/api/benchmark/__init__.py

@@ -1,3 +1,10 @@
-from .adapters import DefaultDataAdapter, ImageEditAdapter, MultiChoiceAdapter, Text2ImageAdapter, VisionLanguageAdapter
+from .adapters import (
+    DefaultDataAdapter,
+    ImageEditAdapter,
+    MultiChoiceAdapter,
+    NERAdapter,
+    Text2ImageAdapter,
+    VisionLanguageAdapter,
+)
 from .benchmark import DataAdapter
 from .meta import BenchmarkMeta

evalscope/api/benchmark/adapters/__init__.py

@@ -1,5 +1,6 @@
 from .default_data_adapter import DefaultDataAdapter
 from .image_edit_adapter import ImageEditAdapter
 from .multi_choice_adapter import MultiChoiceAdapter
+from .ner_adapter import NERAdapter
 from .text2image_adapter import Text2ImageAdapter
 from .vision_language_adapter import VisionLanguageAdapter

evalscope/api/benchmark/adapters/default_data_adapter.py

@@ -128,6 +128,9 @@ class DefaultDataAdapter(DataAdapter):
             for sample in self.test_dataset[subset]:
                 if isinstance(sample.input, str):
                     sample.input = self.process_sample_str_input(sample, subset)
+                elif isinstance(sample.input, list):
+                    # Handle list[ChatMessage] and add system prompt if needed
+                    sample.input = self.process_sample_messages_input(sample, subset)
 
     def process_sample_str_input(self, sample: Sample, subset: str) -> List[ChatMessage]:
         """
@@ -142,6 +145,15 @@ class DefaultDataAdapter(DataAdapter):
             input_messages.insert(0, ChatMessageSystem(content=self.system_prompt))
         return input_messages
 
+    def process_sample_messages_input(self, sample: Sample, subset: str) -> List[ChatMessage]:
+        """
+        Normalize a sample's existing List[ChatMessage] input and ensure system prompt is set once.
+        """
+        messages = list(sample.input)  # shallow copy to avoid in-place mutations
+        if self.system_prompt and not any(isinstance(m, ChatMessageSystem) for m in messages):
+            messages = [ChatMessageSystem(content=self.system_prompt)] + messages
+        return messages
+
     def process_sample_input(self, sample: Sample, subset: str) -> str:
         """
         Process a single sample's input by applying prompt templates and few-shot formatting.

evalscope/api/benchmark/adapters/ner_adapter.py

@@ -0,0 +1,212 @@
+from typing import Any, Dict, List, Set, Tuple
+
+from evalscope.api.dataset import Sample
+from evalscope.api.metric.scorer import AggScore, SampleScore, Score
+from evalscope.utils.import_utils import check_import
+from evalscope.utils.logger import get_logger
+from evalscope.utils.ner import (
+    DEFAULT_TAG_FIX_PATTERNS,
+    calculate_bio_metrics,
+    clean_prediction,
+    create_target_text,
+    extract_entities_from_text,
+    extract_spans_from_bio,
+    xml_to_bio_tags,
+)
+from .default_data_adapter import DefaultDataAdapter
+
+logger = get_logger()
+
+
+class NERAdapter(DefaultDataAdapter):
+    """
+    Base adapter class for Named Entity Recognition (NER) tasks.
+
+    This adapter handles converting between BIO tagging schemes and XML-style entity markup,
+    and provides evaluation metrics using seqeval.
+
+    Subclasses should define their entity types and register the benchmark.
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        # Define mapping from BIO tags to user-friendly tag names
+        self.entity_type_map = {}
+        # Add descriptions for each entity type
+        self.entity_descriptions = {}
+
+        # These will be initialized in setup_entity_mappings
+        self.reverse_entity_map = {}
+        self.entity_list = []
+        self.entities_description = ''
+
+        # Define common error patterns to handle
+        self.tag_fix_patterns = DEFAULT_TAG_FIX_PATTERNS
+
+        check_import('seqeval', 'seqeval', raise_error=True, feature_name='NER metrics')
+        # Note: setup_entity_mappings() should be called by subclasses
+        # after they define their entity_type_map and entity_descriptions
+
+    def setup_entity_mappings(self):
+        """
+        Setup entity mappings and descriptions for prompt formatting.
+        This should be called after entity_type_map and entity_descriptions are defined.
+        """
+        # Reverse mapping for converting back from prediction to evaluation
+        self.reverse_entity_map = {v.lower(): k for k, v in self.entity_type_map.items()}
+
+        # Create list of tags for prompt formatting
+        self.entity_list = [f'<{ent.lower()}>' for ent in self.entity_type_map.values()]
+
+        # Create description of entities for prompt
+        self.entities_description = ', '.join([
+            f'{self.entity_type_map[tag]} ({self.entity_descriptions[tag]})' for tag in self.entity_type_map
+        ])
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a record with tokens and NER tags into a Sample.
+        Creates both the raw text input and annotated text target.
+        """
+        tokens: List[str] = record['tokens']
+        ner_tags: List[str] = record['ner_tags']
+
+        # Create the input text by joining tokens
+        input_text = ' '.join(tokens)
+
+        # Process tokens and tags to create annotated target text
+        target_text = create_target_text(tokens, ner_tags, self.entity_type_map)
+
+        # Store tokens and tags in metadata for evaluation
+        metadata = {'tokens': tokens, 'ner_tags': ner_tags}
+
+        return Sample(input=input_text, target=target_text, metadata=metadata)
+
+    def format_prompt_template(self, sample):
+        """
+        Format the prompt with entity types, available tags, and text to annotate.
+        """
+        return self.prompt_template.format(
+            entities=self.entities_description, entity_list=', '.join(self.entity_list), text=sample.input
+        )
+
+    def format_fewshot_template(self, fewshot, sample):
+        """
+        Format the few-shot prompt with all required parameters.
+        """
+        return self.few_shot_prompt_template.format(
+            fewshot=fewshot,
+            entities=self.entities_description,
+            entity_list=', '.join(self.entity_list),
+            text=sample.input
+        )
+
+    def sample_to_fewshot(self, sample: Sample) -> str:
+        """
+        Format a sample as a few-shot example showing original and annotated text.
+        """
+        if not sample.metadata:
+            return ''
+
+        # Format few-shot examples to match the expected response format
+        return f'Input:\n{sample.input}\n\nOutput:\n{sample.target}'
+
+    def match_score(self, original_prediction, filtered_prediction, reference, task_state) -> Score:
+        """
+        Evaluate named entity recognition performance using seqeval.
+        """
+        from seqeval.metrics import accuracy_score, f1_score, precision_score, recall_score
+
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        try:
+            # Get the original tokens and tags from the reference metadata
+            original_tokens = task_state.metadata['tokens']
+            original_tags = task_state.metadata['ner_tags']
+
+            if not original_tokens or len(original_tokens) == 0:
+                if hasattr(reference, 'metadata') and reference.metadata:
+                    original_tokens = reference.metadata['tokens']
+                    original_tags = reference.metadata['ner_tags']
+
+            # Clean and normalize the prediction
+            cleaned_prediction = clean_prediction(filtered_prediction, self.tag_fix_patterns)
+
+            # Convert XML-style prediction back to BIO tags aligned with original tokens
+            pred_bio_tags = xml_to_bio_tags(cleaned_prediction, original_tokens, self.reverse_entity_map)
+
+            # Use seqeval to calculate metrics
+            # Note: seqeval expects lists of lists (one per sequence)
+            y_true = [original_tags]
+            y_pred = [pred_bio_tags]
+
+            precision = precision_score(y_true, y_pred)
+            recall = recall_score(y_true, y_pred)
+            f1 = f1_score(y_true, y_pred)
+            accuracy = accuracy_score(y_true, y_pred)
+
+            score.value = {'precision': precision, 'recall': recall, 'f1_score': f1, 'accuracy': accuracy}
+
+            # Store tags for aggregation (proper micro-averaging in aggregate_scores)
+            # This way aggregate_scores can compute metrics across all samples at once,
+            # which gives you true micro-averaged scores rather than averaged macro scores.
+            score.metadata = {'y_true': original_tags, 'y_pred': pred_bio_tags}
+        except Exception as e:
+            logger.warning(f'Error evaluating NER prediction: {str(e)}')
+            score.value = {'precision': 0.0, 'recall': 0.0, 'f1_score': 0.0, 'accuracy': 0.0}
+
+        return score
+
+    def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]:
+        """
+        Aggregate metrics across all samples using seqeval.
+        """
+        from seqeval.metrics import accuracy_score, f1_score, precision_score, recall_score
+
+        # Collect all predictions and references
+        y_true_all = []
+        y_pred_all = []
+
+        for ss in sample_scores:
+            # Extract the BIO tags from metadata if available
+            # You may need to store these during match_score
+            if hasattr(ss.score, 'metadata') and 'y_true' in ss.score.metadata and 'y_pred' in ss.score.metadata:
+                y_true_all.append(ss.score.metadata['y_true'])
+                y_pred_all.append(ss.score.metadata['y_pred'])
+
+        if not y_true_all:
+            # Fallback: calculate averages from individual scores
+            num_samples = len(sample_scores)
+            avg_precision = sum(ss.score.value.get('precision', 0.0) for ss in sample_scores) / num_samples
+            avg_recall = sum(ss.score.value.get('recall', 0.0) for ss in sample_scores) / num_samples
+            avg_f1 = sum(ss.score.value.get('f1_score', 0.0) for ss in sample_scores) / num_samples
+            avg_accuracy = sum(ss.score.value.get('accuracy', 0.0) for ss in sample_scores) / num_samples
+        else:
+            # Use seqeval for micro-averaged metrics across all samples
+            avg_precision = precision_score(y_true_all, y_pred_all)
+            avg_recall = recall_score(y_true_all, y_pred_all)
+            avg_f1 = f1_score(y_true_all, y_pred_all)
+            avg_accuracy = accuracy_score(y_true_all, y_pred_all)
+
+        num_samples = len(sample_scores)
+
+        agg_scores = [
+            AggScore(
+                metric_name='precision',
+                score=avg_precision,
+                num=num_samples,
+                metadata={'type': 'seqeval-micro-average'}
+            ),
+            AggScore(
+                metric_name='recall', score=avg_recall, num=num_samples, metadata={'type': 'seqeval-micro-average'}
+            ),
+            AggScore(metric_name='f1_score', score=avg_f1, num=num_samples, metadata={'type': 'seqeval-micro-average'}),
+            AggScore(
+                metric_name='accuracy', score=avg_accuracy, num=num_samples, metadata={'type': 'seqeval-accuracy'}
+            )
+        ]
+
+        return agg_scores

evalscope/api/benchmark/benchmark.py

@@ -216,6 +216,13 @@ class DataAdapter(LLMJudgeMixin, SandboxMixin, ABC):
         """
         return self._benchmark_meta.train_split
 
+    @train_split.setter
+    def train_split(self, value: str):
+        """
+        Set the train split of the benchmark.
+        """
+        self._benchmark_meta.train_split = value
+
     @property
     def eval_split(self) -> Optional[str]:
         """
@@ -223,6 +230,13 @@ class DataAdapter(LLMJudgeMixin, SandboxMixin, ABC):
         """
         return self._benchmark_meta.eval_split
 
+    @eval_split.setter
+    def eval_split(self, value: str):
+        """
+        Set the eval split of the benchmark.
+        """
+        self._benchmark_meta.eval_split = value
+
     @property
     def prompt_template(self) -> Optional[str]:
         """

evalscope/api/dataset/dataset.py

@@ -347,3 +347,24 @@ class DatasetDict:
             cur_dataset.reindex(group_size=repeats)
             dataset_dict[key] = cur_dataset
         return cls(dataset_dict)
+
+    @classmethod
+    def from_dataset_dicts(cls, dataset_dicts: List['DatasetDict']) -> 'DatasetDict':
+        """
+        Create a DatasetDict by merging multiple DatasetDicts.
+
+        Args:
+            dataset_dicts (List[DatasetDict]): List of DatasetDicts to merge.
+
+        Returns:
+            DatasetDict: A new DatasetDict containing the merged datasets.
+        """
+        merged_dict = defaultdict(list)
+        for dataset_dict in dataset_dicts:
+            for key, dataset in dataset_dict.items():
+                merged_dict[key].extend(dataset.samples)
+        # Create a MemoryDataset for each subset key
+        final_dict = {}
+        for key, samples in merged_dict.items():
+            final_dict[key] = MemoryDataset(samples, name=key)
+        return cls(final_dict)

evalscope/api/dataset/loader.py

@@ -8,7 +8,7 @@ from typing import Callable, Dict, List, Optional, Union
 from evalscope.api.dataset.utils import record_to_sample_fn
 from evalscope.constants import DEFAULT_EVALSCOPE_CACHE_DIR, HubType
 from evalscope.utils import get_logger
-from evalscope.utils.io_utils import csv_to_list, gen_hash, jsonl_to_list, safe_filename
+from evalscope.utils.io_utils import csv_to_list, gen_hash, jsonl_to_list, safe_filename, tsv_to_list
 from .dataset import Dataset, FieldSpec, MemoryDataset, Sample
 from .utils import data_to_samples, shuffle_choices_if_requested
 
@@ -168,7 +168,11 @@ class LocalDataLoader(DataLoader):
         dataset = []
 
         # Check for JSONL or CSV files in the specified path
-        for ext, loader in [('.jsonl', jsonl_to_list), ('.csv', csv_to_list)]:
+        for ext, loader in [
+            ('.jsonl', jsonl_to_list),
+            ('.csv', csv_to_list),
+            ('.tsv', tsv_to_list),
+        ]:
             # Check if the file exists with the given extension
             if os.path.isfile(path) and path.endswith(ext):
                 file_paths = [path]

evalscope/api/mixin/sandbox_mixin.py

@@ -1,7 +1,6 @@
-import asyncio
-import threading
-from typing import TYPE_CHECKING, Any, Dict, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, Optional
 
+from evalscope.utils.function_utils import AsyncioLoopRunner, thread_safe
 from evalscope.utils.logger import get_logger
 
 if TYPE_CHECKING:
@@ -24,25 +23,10 @@ class SandboxMixin:
         self._sandbox_id: Optional[str] = None
         """Sandbox ID."""
 
-        self._loop: Optional[asyncio.AbstractEventLoop] = None
-        """Event loop for async operations."""
-
-        # Initialize sandbox synchronously by running async methods
-        if self.use_sandbox:
-            self._loop = asyncio.new_event_loop()
-
-            # Start the loop in a separate thread
-            def run_loop():
-                asyncio.set_event_loop(self._loop)
-                self._loop.run_forever()
-
-            self._loop_thread = threading.Thread(target=run_loop, daemon=True)
-            self._loop_thread.start()
-
-            # Wait for initialization
-            future = asyncio.run_coroutine_threadsafe(self._async_init(), self._loop)
-            future.result()
+        # Lazy init state
+        self._initialized: bool = False
 
+        # NOTE: Initialization is deferred.
         super().__init__()
 
     async def _async_init(self):
@@ -70,6 +54,25 @@ class SandboxMixin:
         """Get the sandbox ID."""
         return self._sandbox_id
 
+    @thread_safe
+    def ensure_sandbox_ready(self) -> bool:
+        """
+        Ensure the sandbox loop, manager, and sandbox instance are initialized.
+        This method is thread-safe and idempotent.
+        """
+        if not self.use_sandbox:
+            return False
+
+        if self._initialized and self._manager and self._sandbox_id:
+            return True
+
+        # Initialize manager and sandbox using the class-level runner
+        AsyncioLoopRunner.run(self.init_sandbox_manager_async())
+        AsyncioLoopRunner.run(self.init_sandbox_async())
+
+        self._initialized = True
+        return True
+
     async def init_sandbox_manager_async(self) -> Optional['SandboxManager']:
         """Initialize the sandbox manager asynchronously."""
         if self._manager is not None:
@@ -100,13 +103,7 @@ class SandboxMixin:
         if not self.use_sandbox:
             return None
 
-        # Use the dedicated loop if available
-        if self._loop and not self._loop.is_closed():
-            future = asyncio.run_coroutine_threadsafe(self.init_sandbox_manager_async(), self._loop)
-            return future.result()
-        else:
-            # Fallback for cases where no loop is available
-            return asyncio.run(self.init_sandbox_manager_async())
+        return AsyncioLoopRunner.run(self.init_sandbox_manager_async())
 
     async def init_sandbox_async(self) -> Optional[str]:
         """Initialize the sandbox instance asynchronously."""
@@ -141,17 +138,12 @@ class SandboxMixin:
         if not self.use_sandbox:
             return None
 
-        # Use the dedicated loop if available
-        if self._loop and not self._loop.is_closed():
-            future = asyncio.run_coroutine_threadsafe(self.init_sandbox_async(), self._loop)
-            return future.result()
-        else:
-            # Fallback for cases where no loop is available
-            return asyncio.run(self.init_sandbox_async())
+        return AsyncioLoopRunner.run(self.init_sandbox_async())
 
     def execute_code_in_sandbox(self, code: str, timeout: int = 60, language: str = 'python') -> Dict[str, Any]:
         """Execute code in the sandbox."""
-        if not self._sandbox_id or not self._manager:
+        # Lazy, thread-safe initialization
+        if not self.ensure_sandbox_ready():
             logger.warning('Sandbox is not initialized.')
             return {'error': 'Sandbox is not initialized.'}
 
@@ -175,30 +167,16 @@ class SandboxMixin:
                 )
             return result
 
-        # Use the dedicated loop if available
-        if self._loop and not self._loop.is_closed():
-            future = asyncio.run_coroutine_threadsafe(_execute_async(), self._loop)
-            result = future.result(timeout + 10)  # Add some buffer to the timeout
-        else:
-            # Fallback for cases where no loop is available
-            result = asyncio.run(_execute_async())
-
+        # Execute in background loop via class-level runner
+        result = AsyncioLoopRunner.run(_execute_async(), timeout=timeout + 10)
         return result.model_dump(exclude_none=True)
 
     def sandbox_finalize(self, *args, **kwargs):
         """Finalize the sandbox manager."""
         if self._manager:
             try:
-                if self._loop and not self._loop.is_closed():
-                    # Stop the manager using the dedicated loop
-                    future = asyncio.run_coroutine_threadsafe(self._manager.stop(), self._loop)
-                    future.result(timeout=30)
-
-                    # Stop the event loop
-                    self._loop.call_soon_threadsafe(self._loop.stop)
-                    if hasattr(self, '_loop_thread'):
-                        self._loop_thread.join(timeout=5)
-
+                # Stop the manager but keep the shared loop alive
+                AsyncioLoopRunner.run(self._manager.stop(), timeout=30)
                 logger.info('Sandbox manager finalized.')
             except Exception as e:
                 logger.warning(f'Error finalizing sandbox manager: {e}')

evalscope/api/model/generate_config.py

@@ -108,6 +108,12 @@ class GenerateConfig(BaseModel):
     extra_body: Optional[Dict[str, Any]] = Field(default=None)
     """Extra body to be sent with requests to OpenAI compatible servers. OpenAI, vLLM, and SGLang only."""
 
+    extra_query: Optional[Dict[str, Any]] = Field(default=None)
+    """Extra query parameters to be sent with requests to OpenAI compatible servers. OpenAI, vLLM, and SGLang only."""
+
+    extra_headers: Optional[Dict[str, str]] = Field(default=None)
+    """Extra headers to be sent with requests to OpenAI compatible servers. OpenAI, vLLM, and SGLang only."""
+
     height: Optional[int] = Field(default=None)
     """Image height for image generation model only"""
 

evalscope/app/ui/multi_model.py

@@ -204,7 +204,12 @@ def create_multi_model_tab(sidebar: 'SidebarComponents', lang: str):
         data_score_df_b, _ = get_single_dataset_df(report_df_b, dataset_name)
 
         # Get subset choices - should be same for both models
-        subsets = data_score_df_a[ReportKey.subset_name].unique().tolist()
+        # Only select the subsets that Cat.0 is not '-'
+        df_for_subsets = data_score_df_a.copy()
+        subsets = sorted(
+            df_for_subsets.loc[df_for_subsets[f'{ReportKey.category_prefix}0'].ne('-'),
+                               ReportKey.subset_name].dropna().unique().tolist()
+        )
 
         return gr.update(choices=subsets, value=None), None
 

evalscope/app/ui/single_model.py

@@ -134,11 +134,17 @@ def create_single_model_tab(sidebar: 'SidebarComponents', lang: str):
     )
     def update_single_report_dataset(dataset_name, report_list):
         logger.debug(f'Updating single report dataset: {dataset_name}')
-        report_df = get_data_frame(report_list=report_list)
+        report_df = get_data_frame(report_list=report_list, flatten_metrics=True, flatten_categories=True)
         analysis = get_report_analysis(report_list, dataset_name)
         data_score_df, styler = get_single_dataset_df(report_df, dataset_name)
         data_score_plot = plot_single_dataset_scores(data_score_df)
-        subsets = data_score_df[ReportKey.subset_name].unique().tolist()
+        # Only select the subsets that Cat.0 is not '-'
+        df_for_subsets = data_score_df.copy()
+        subsets = sorted(
+            df_for_subsets.loc[df_for_subsets[f'{ReportKey.category_prefix}0'].ne('-'),
+                               ReportKey.subset_name].dropna().unique().tolist()
+        )
+
         logger.debug(f'subsets: {subsets}')
         return data_score_plot, styler, gr.update(choices=subsets, value=None), None, analysis
 

evalscope/app/utils/data_utils.py

@@ -168,9 +168,10 @@ def get_model_prediction(work_dir: str, model_name: str, dataset_name: str, subs
             'Index': str(review_result.index),
             'Input': review_result.input.replace('\n', '\n\n'),  # for markdown
             'Metadata': metadata,
-            'Generated': prediction,
+            'Generated': prediction or '',  # Ensure no None value
             'Gold': target,
-            'Pred': extracted_prediction if extracted_prediction != prediction else '*Same as Generated*',
+            'Pred': (extracted_prediction if extracted_prediction != prediction else '*Same as Generated*')
+            or '',  # Ensure no None value
             'Score': score.model_dump(exclude_none=True),
             'NScore': normalize_score(score.main_value)
         }

evalscope/app/utils/visualization.py

@@ -18,7 +18,7 @@ logger = get_logger()
 def plot_single_report_scores(df: pd.DataFrame):
     if df is None:
         return None
-    logger.debug(f'df: {df}')
+    logger.debug(f'df: \n{df}')
     plot = px.bar(df, x=df[ReportKey.dataset_name], y=df[ReportKey.score], text=df[ReportKey.score])
 
     width = DEFAULT_BAR_WIDTH if len(df[ReportKey.dataset_name]) <= 5 else None
@@ -36,7 +36,7 @@ def plot_single_report_sunburst(report_list: List[Report]):
         df = get_data_frame(report_list=report_list, flatten_metrics=False)
         categories = sorted([i for i in df.columns if i.startswith(ReportKey.category_prefix)])
         path = [ReportKey.dataset_name] + categories + [ReportKey.subset_name]
-    logger.debug(f'df: {df}')
+    logger.debug(f'df: \n{df}')
     df[categories] = df[categories].fillna('default')  # NOTE: fillna for empty categories
 
     plot = px.sunburst(