evalscope 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/api/mixin/sandbox_mixin.py

@@ -0,0 +1,182 @@
+from typing import TYPE_CHECKING, Any, Dict, Optional
+
+from evalscope.utils.function_utils import AsyncioLoopRunner, thread_safe
+from evalscope.utils.logger import get_logger
+
+if TYPE_CHECKING:
+    from ms_enclave.sandbox.manager import SandboxManager
+
+    from evalscope.config import TaskConfig
+
+logger = get_logger()
+
+
+class SandboxMixin:
+    """Sandbox mixin for sandboxed code execution."""
+
+    def __init__(self, task_config: 'TaskConfig'):
+        self._task_config = task_config
+
+        self._manager: Optional['SandboxManager'] = None
+        """Sandbox manager instance."""
+
+        self._sandbox_id: Optional[str] = None
+        """Sandbox ID."""
+
+        # Lazy init state
+        self._initialized: bool = False
+
+        # NOTE: Initialization is deferred.
+        super().__init__()
+
+    async def _async_init(self):
+        """Async initialization helper."""
+        await self.init_sandbox_manager_async()
+        await self.init_sandbox_async()
+
+    @property
+    def use_sandbox(self) -> bool:
+        """
+        Return whether to use sandbox for the benchmark.
+        """
+        if not self._task_config:
+            return False
+        else:
+            return self._task_config.use_sandbox
+
+    @property
+    def sandbox_manager(self) -> Optional['SandboxManager']:
+        """Get the sandbox manager instance."""
+        return self._manager
+
+    @property
+    def sandbox_id(self) -> Optional[str]:
+        """Get the sandbox ID."""
+        return self._sandbox_id
+
+    @thread_safe
+    def ensure_sandbox_ready(self) -> bool:
+        """
+        Ensure the sandbox loop, manager, and sandbox instance are initialized.
+        This method is thread-safe and idempotent.
+        """
+        if not self.use_sandbox:
+            return False
+
+        if self._initialized and self._manager and self._sandbox_id:
+            return True
+
+        # Initialize manager and sandbox using the class-level runner
+        AsyncioLoopRunner.run(self.init_sandbox_manager_async())
+        AsyncioLoopRunner.run(self.init_sandbox_async())
+
+        self._initialized = True
+        return True
+
+    async def init_sandbox_manager_async(self) -> Optional['SandboxManager']:
+        """Initialize the sandbox manager asynchronously."""
+        if self._manager is not None:
+            return self._manager
+
+        if not self.use_sandbox:
+            return None
+
+        from ms_enclave.sandbox.manager import HttpSandboxManager, LocalSandboxManager
+
+        manager_config = self._task_config.sandbox_manager_config or {}
+        if manager_config.get('base_url'):
+            # Remote manager
+            self._manager = HttpSandboxManager(**manager_config)
+        else:
+            # Local manager
+            self._manager = LocalSandboxManager(**manager_config)
+
+        await self._manager.start()
+        logger.info('Sandbox manager initialized.')
+        return self._manager
+
+    def init_sandbox_manager(self) -> Optional['SandboxManager']:
+        """Initialize the sandbox manager."""
+        if self._manager is not None:
+            return self._manager
+
+        if not self.use_sandbox:
+            return None
+
+        return AsyncioLoopRunner.run(self.init_sandbox_manager_async())
+
+    async def init_sandbox_async(self) -> Optional[str]:
+        """Initialize the sandbox instance asynchronously."""
+        if self._sandbox_id is not None:
+            return self._sandbox_id
+
+        if not self.use_sandbox:
+            return None
+
+        from ms_enclave.sandbox.model import DockerSandboxConfig, SandboxType
+
+        sandbox_config = self._task_config.sandbox_config or DockerSandboxConfig(
+            image='python:3.11-slim', tools_config={
+                'shell_executor': {},
+                'python_executor': {}
+            }
+        )
+        sandbox_type = self._task_config.sandbox_type or SandboxType.DOCKER
+
+        self._sandbox_id = await self._manager.create_sandbox(sandbox_type=sandbox_type, config=sandbox_config)
+
+        sandbox_info = await self._manager.get_sandbox_info(self._sandbox_id)
+
+        logger.info(f'Sandbox of type {sandbox_type} initialized. Info: {sandbox_info.model_dump(exclude_none=True)}')
+        return self._sandbox_id
+
+    def init_sandbox(self) -> Optional[str]:
+        """Initialize the sandbox instance."""
+        if self._sandbox_id is not None:
+            return self._sandbox_id
+
+        if not self.use_sandbox:
+            return None
+
+        return AsyncioLoopRunner.run(self.init_sandbox_async())
+
+    def execute_code_in_sandbox(self, code: str, timeout: int = 60, language: str = 'python') -> Dict[str, Any]:
+        """Execute code in the sandbox."""
+        # Lazy, thread-safe initialization
+        if not self.ensure_sandbox_ready():
+            logger.warning('Sandbox is not initialized.')
+            return {'error': 'Sandbox is not initialized.'}
+
+        from ms_enclave.sandbox.model import ExecutionStatus, ToolResult
+
+        async def _execute_async():
+            if language.lower() == 'python':
+                tool_name = 'python_executor'
+                parameters = {'code': code, 'timeout': timeout}
+                result = await self._manager.execute_tool(self._sandbox_id, tool_name, parameters)
+            elif language.lower() == 'shell':
+                tool_name = 'shell_executor'
+                parameters = {'command': code, 'timeout': timeout}
+                result = await self._manager.execute_tool(self._sandbox_id, tool_name, parameters)
+            else:
+                logger.warning(f"Unsupported language: {language}. Supported languages are 'python' and 'shell'.")
+                result = ToolResult(
+                    status=ExecutionStatus.ERROR,
+                    tool_name='code_executor',
+                    output=f"Unsupported language: {language}. Supported languages are 'python' and 'shell'."
+                )
+            return result
+
+        # Execute in background loop via class-level runner
+        result = AsyncioLoopRunner.run(_execute_async(), timeout=timeout + 10)
+        return result.model_dump(exclude_none=True)
+
+    def sandbox_finalize(self, *args, **kwargs):
+        """Finalize the sandbox manager."""
+        if self._manager:
+            try:
+                # Stop the manager but keep the shared loop alive
+                AsyncioLoopRunner.run(self._manager.stop(), timeout=30)
+                logger.info('Sandbox manager finalized.')
+            except Exception as e:
+                logger.warning(f'Error finalizing sandbox manager: {e}')

evalscope/api/model/generate_config.py

@@ -25,9 +25,7 @@ class ResponseSchema(BaseModel):
 
 class GenerateConfig(BaseModel):
     """Model generation options."""
-
-    max_retries: Optional[int] = Field(default=None)
-    """Maximum number of times to retry request (defaults to unlimited)."""
+    model_config = {'extra': 'allow'}
 
     timeout: Optional[int] = Field(default=None)
     """Request timeout (in seconds)."""
@@ -38,9 +36,6 @@ class GenerateConfig(BaseModel):
     stream: Optional[bool] = Field(default=None)
     """Whether to stream the response (default is model specific)."""
 
-    system_message: Optional[str] = Field(default=None)
-    """Override the default system message."""
-
     max_tokens: Optional[int] = Field(default=None)
     """The maximum number of tokens that can be generated in the completion (default is model specific)."""
 
@@ -62,6 +57,9 @@ class GenerateConfig(BaseModel):
     presence_penalty: Optional[float] = Field(default=None)
     """Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics. OpenAI, Google, Grok, Groq, vLLM, and SGLang only."""
 
+    repetition_penalty: Optional[float] = Field(default=None)
+    """Exponential penalty applied to existing tokens in the generated text. 1.0 means no penalty. OpenAI, HuggingFace, and vLLM only."""
+
     logit_bias: Optional[Dict[int, float]] = Field(default=None)
     """Map token Ids to an associated bias value from -100 to 100 (e.g. "42=10,43=-10"). OpenAI, Grok, Grok, and vLLM only."""
 
@@ -113,6 +111,12 @@ class GenerateConfig(BaseModel):
     extra_body: Optional[Dict[str, Any]] = Field(default=None)
     """Extra body to be sent with requests to OpenAI compatible servers. OpenAI, vLLM, and SGLang only."""
 
+    extra_query: Optional[Dict[str, Any]] = Field(default=None)
+    """Extra query parameters to be sent with requests to OpenAI compatible servers. OpenAI, vLLM, and SGLang only."""
+
+    extra_headers: Optional[Dict[str, str]] = Field(default=None)
+    """Extra headers to be sent with requests to OpenAI compatible servers. OpenAI, vLLM, and SGLang only."""
+
     height: Optional[int] = Field(default=None)
     """Image height for image generation model only"""
 

evalscope/api/model/model.py

@@ -318,7 +318,7 @@ def get_model_with_task_config(task_config: 'TaskConfig') -> Model:
 
 @thread_safe
 def get_model(
-    model: str,
+    model: Union[str, Model, ModelAPI],
     eval_type: str,
     base_url: Optional[str] = None,
     api_key: Optional[str] = None,
@@ -346,6 +346,9 @@ def get_model(
     if isinstance(model, Model):
         return model
 
+    if isinstance(model, ModelAPI):
+        return Model(model, config, model_args)
+
     # see if we can return a memoized model instance
     # (exclude mockllm since custom_outputs is an infinite generator)
     model_cache_key: str = ''
@@ -362,7 +365,7 @@ def get_model(
 
     logger.info(
         f'Creating model {model} with eval_type={eval_type} '
-        f'base_url={base_url}, api_key={api_key}, config={config}, model_args={model_args}'
+        f'base_url={base_url}, config={config.model_dump(exclude_none=True)}, model_args={model_args}'
     )
 
     # find a matching model type

evalscope/api/tool/tool_info.py

@@ -1,7 +1,7 @@
 import inspect
 from dataclasses import dataclass
 from docstring_parser import Docstring, parse
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_validator
 from typing import Any, Callable, Dict, List, Literal, Optional, TypeAlias, Union, get_args, get_type_hints
 
 from evalscope.utils.json_schema import JSONSchema, JSONType, json_schema, python_type_to_json_type

evalscope/app/app.py

@@ -6,6 +6,7 @@ import argparse
 from evalscope.utils.logger import configure_logging
 from .arguments import add_argument
 from .ui import create_app_ui
+from .utils.env_utils import setup_env
 
 
 def create_app(args: argparse.Namespace):
@@ -17,6 +18,8 @@ def create_app(args: argparse.Namespace):
     """
     configure_logging(debug=args.debug)
 
+    setup_env(args)
+
     demo = create_app_ui(args)
 
     demo.launch(

evalscope/app/ui/multi_model.py

@@ -204,7 +204,12 @@ def create_multi_model_tab(sidebar: 'SidebarComponents', lang: str):
         data_score_df_b, _ = get_single_dataset_df(report_df_b, dataset_name)
 
         # Get subset choices - should be same for both models
-        subsets = data_score_df_a[ReportKey.subset_name].unique().tolist()
+        # Only select the subsets that Cat.0 is not '-'
+        df_for_subsets = data_score_df_a.copy()
+        subsets = sorted(
+            df_for_subsets.loc[df_for_subsets[f'{ReportKey.category_prefix}0'].ne('-'),
+                               ReportKey.subset_name].dropna().unique().tolist()
+        )
 
         return gr.update(choices=subsets, value=None), None
 

evalscope/app/ui/single_model.py

@@ -134,11 +134,17 @@ def create_single_model_tab(sidebar: 'SidebarComponents', lang: str):
     )
     def update_single_report_dataset(dataset_name, report_list):
         logger.debug(f'Updating single report dataset: {dataset_name}')
-        report_df = get_data_frame(report_list=report_list)
+        report_df = get_data_frame(report_list=report_list, flatten_metrics=True, flatten_categories=True)
         analysis = get_report_analysis(report_list, dataset_name)
         data_score_df, styler = get_single_dataset_df(report_df, dataset_name)
         data_score_plot = plot_single_dataset_scores(data_score_df)
-        subsets = data_score_df[ReportKey.subset_name].unique().tolist()
+        # Only select the subsets that Cat.0 is not '-'
+        df_for_subsets = data_score_df.copy()
+        subsets = sorted(
+            df_for_subsets.loc[df_for_subsets[f'{ReportKey.category_prefix}0'].ne('-'),
+                               ReportKey.subset_name].dropna().unique().tolist()
+        )
+
         logger.debug(f'subsets: {subsets}')
         return data_score_plot, styler, gr.update(choices=subsets, value=None), None, analysis
 
@@ -198,9 +204,9 @@ def create_single_model_tab(sidebar: 'SidebarComponents', lang: str):
 
         # Process the data for display
         input_md = row['Input'] + '\n\n' + process_model_prediction(row['Metadata'])
-        generated_md = process_model_prediction(row['Generated'])
-        gold_md = process_model_prediction(row['Gold'])
-        pred_md = convert_markdown_image(process_model_prediction(row['Pred']))
+        generated_md = convert_markdown_image(row['Generated'])
+        gold_md = convert_markdown_image(row['Gold'])
+        pred_md = process_model_prediction(row['Pred'])
         score_md = process_json_content(row['Score'])
         nscore_val = float(row['NScore']) if not pd.isna(row['NScore']) else 0.0
 

evalscope/app/utils/data_utils.py

@@ -2,7 +2,6 @@
 Data loading and processing utilities for the Evalscope dashboard.
 """
 import glob
-import numpy as np
 import os
 import pandas as pd
 from typing import Any, Dict, List, Union
@@ -160,17 +159,19 @@ def get_model_prediction(work_dir: str, model_name: str, dataset_name: str, subs
             if f'{sample_dataset_name}/{sample_subset_name}' != subset_name:
                 continue
 
-        prediction = sample_score.score.prediction
-        target = review_result.target
-        extracted_prediction = sample_score.score.extracted_prediction
         score = sample_score.score
+        metadata = sample_score.sample_metadata
+        prediction = score.prediction
+        target = review_result.target
+        extracted_prediction = score.extracted_prediction
         raw_d = {
             'Index': str(review_result.index),
             'Input': review_result.input.replace('\n', '\n\n'),  # for markdown
-            'Metadata': sample_score.sample_metadata,
-            'Generated': prediction if prediction != extracted_prediction else '*Same as Pred*',
+            'Metadata': metadata,
+            'Generated': prediction or '',  # Ensure no None value
             'Gold': target,
-            'Pred': extracted_prediction,
+            'Pred': (extracted_prediction if extracted_prediction != prediction else '*Same as Generated*')
+            or '',  # Ensure no None value
             'Score': score.model_dump(exclude_none=True),
             'NScore': normalize_score(score.main_value)
         }

evalscope/app/utils/env_utils.py

@@ -0,0 +1,12 @@
+# flake8: noqa
+import os
+
+
+def setup_env(args):
+    compat_dsw_gradio(args)
+
+
+def compat_dsw_gradio(args) -> None:
+    if ('JUPYTER_NAME' in os.environ) and ('dsw-'
+                                           in os.environ['JUPYTER_NAME']) and ('GRADIO_ROOT_PATH' not in os.environ):
+        os.environ['GRADIO_ROOT_PATH'] = f"/{os.environ['JUPYTER_NAME']}/proxy/{args.server_port}"

evalscope/app/utils/text_utils.py

@@ -2,11 +2,9 @@
 Text processing utilities for the Evalscope dashboard.
 """
 import json
-import numpy as np
 import os
-import pandas as pd
 import re
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 
 from evalscope.utils.logger import get_logger
 from ..constants import LATEX_DELIMITERS
@@ -14,15 +12,19 @@ from ..constants import LATEX_DELIMITERS
 logger = get_logger()
 
 
-def convert_markdown_image(text):
-    if not os.path.isfile(text):
-        return text
-    # Convert the image path to a markdown image tag
-    if text.endswith('.png') or text.endswith('.jpg') or text.endswith('.jpeg'):
-        text = os.path.abspath(text)
-        image_tag = f'![image](gradio_api/file={text})'
-        logger.debug(f'Converting image path to markdown: {text} -> {image_tag}')
+def convert_markdown_image(text: str):
+    if text.startswith('data:image'):
+        # Convert base64 image data to a markdown image tag
+        image_tag = f'![image]({text})'
+        logger.debug(f'Converting base64 image data to markdown: {text[:30]}... -> {image_tag[:40]}...')
         return image_tag
+    elif os.path.isfile(text):
+        # Convert the image path to a markdown image tag
+        if text.endswith('.png') or text.endswith('.jpg') or text.endswith('.jpeg'):
+            text = os.path.abspath(text)
+            image_tag = f'![image](gradio_api/file={text})'
+            logger.debug(f'Converting image path to markdown: {text} -> {image_tag}')
+            return image_tag
     return text
 
 
@@ -85,7 +87,7 @@ def process_model_prediction_old(item: Any, max_length: int = 2048) -> str:
     return result
 
 
-def process_model_prediction(item: Any, max_length: int = 32000) -> str:
+def process_model_prediction(item: Any, max_length: Optional[int] = None) -> str:
     if isinstance(item, (dict, list)):
         result = json.dumps(item, ensure_ascii=False, indent=2)
         result = f'```json\n{result}\n```'

evalscope/app/utils/visualization.py

@@ -18,7 +18,7 @@ logger = get_logger()
 def plot_single_report_scores(df: pd.DataFrame):
     if df is None:
         return None
-    logger.debug(f'df: {df}')
+    logger.debug(f'df: \n{df}')
     plot = px.bar(df, x=df[ReportKey.dataset_name], y=df[ReportKey.score], text=df[ReportKey.score])
 
     width = DEFAULT_BAR_WIDTH if len(df[ReportKey.dataset_name]) <= 5 else None
@@ -36,7 +36,7 @@ def plot_single_report_sunburst(report_list: List[Report]):
         df = get_data_frame(report_list=report_list, flatten_metrics=False)
         categories = sorted([i for i in df.columns if i.startswith(ReportKey.category_prefix)])
         path = [ReportKey.dataset_name] + categories + [ReportKey.subset_name]
-    logger.debug(f'df: {df}')
+    logger.debug(f'df: \n{df}')
     df[categories] = df[categories].fillna('default')  # NOTE: fillna for empty categories
 
     plot = px.sunburst(

evalscope/arguments.py

@@ -2,7 +2,7 @@
 import argparse
 import json
 
-from evalscope.constants import EvalBackend, EvalType, JudgeStrategy, ModelTask, OutputType
+from evalscope.constants import EvalBackend, EvalType, JudgeStrategy, ModelTask
 
 
 class ParseStrArgsAction(argparse.Action):
@@ -60,8 +60,7 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--generation-config', type=str, action=ParseStrArgsAction, help='The generation config, should be a string.')  # noqa: E501
 
     # Evaluation-related arguments
-    parser.add_argument('--eval-type', type=str, help='The type for evaluating.',
-                        choices=[EvalType.CHECKPOINT, EvalType.CUSTOM, EvalType.SERVICE])
+    parser.add_argument('--eval-type', type=str, help='The type for evaluating.')
     parser.add_argument('--eval-backend', type=str, help='The evaluation backend to use.',
                         choices=[EvalBackend.NATIVE, EvalBackend.OPEN_COMPASS, EvalBackend.VLM_EVAL_KIT, EvalBackend.RAG_EVAL])  # noqa: E501
     parser.add_argument('--eval-config', type=str, required=False, help='The eval task config file path for evaluation backend.')  # noqa: E501
@@ -77,7 +76,6 @@ def add_argument(parser: argparse.ArgumentParser):
     # Debug and runtime mode arguments
     parser.add_argument('--ignore-errors', action='store_true', default=False, help='Ignore errors during evaluation.')
     parser.add_argument('--debug', action='store_true', default=False, help='Debug mode, will print information for debugging.')  # noqa: E501
-    parser.add_argument('--dry-run', action='store_true', default=False, help='Dry run in single processing mode.')
     parser.add_argument('--seed', type=int, default=42, help='Random seed for reproducibility.')
     parser.add_argument('--api-key', type=str, default='EMPTY', help='The API key for the remote API model.')
     parser.add_argument('--api-url', type=str, default=None, help='The API url for the remote API model.')
@@ -89,6 +87,12 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--judge-model-args', type=json.loads, default='{}', help='The judge model args, should be a json string.')  # noqa: E501
     parser.add_argument('--judge-worker-num', type=int, default=1, help='The number of workers for the judge model.')
     parser.add_argument('--analysis-report', action='store_true', default=False, help='Generate analysis report for the evaluation results using judge model.')  # noqa: E501
+
+    # Sandbox-related arguments
+    parser.add_argument('--use-sandbox', action='store_true', default=False, help='Whether to use sandbox for model evaluation.')  # noqa: E501
+    parser.add_argument('--sandbox-type', type=str, default='docker', help='The sandbox type to use.')  # noqa: E501
+    parser.add_argument('--sandbox-config', type=json.loads, default='{}', help='The sandbox config, should be a json string.')  # noqa: E501
+    parser.add_argument('--sandbox-manager-config', type=json.loads, default='{}', help='The sandbox manager config, should be a json string.')  # noqa: E501
     # yapf: enable
 
 

evalscope/backend/opencompass/backend_manager.py

@@ -47,7 +47,6 @@ class OpenCompassBackendManager(BackendManager):
                     datasets: list, the datasets.
                     models: list, the models.
                     work_dir (Optional): str, the working directory. Default to None, which means the current directory.
-                    dry_run (Optional): bool, the dry-run flag. Default to False.
                     debug (Optional): bool, the debug flag. Default to False.
                     reuse (Optional): str, reuse previous outputs & results. Default to None.
                     generation_kwargs (Optional): dict, the generation config. Default to {}.
@@ -140,7 +139,6 @@ class OpenCompassBackendManager(BackendManager):
             cmd_str = f'python -m run_oc ' \
                       f'--models {" ".join(self.args.models)} ' \
                       f'--datasets {" ".join(self.args.datasets)} ' \
-                      f'{self.get_restore_arg("dry-run", self.args.dry_run)} ' \
                       f'{self.get_arg_with_default("work-dir", self.args.work_dir)}'
 
         elif cmd_mode == CmdMode.SCRIPT:

evalscope/backend/rag_eval/utils/embedding.py

@@ -164,6 +164,13 @@ class CrossEncoderModel(BaseModel):
             max_length=self.max_seq_length,
             automodel_args=self.model_kwargs,
         )
+        self.tokenizer = self.model.tokenizer
+        # set pad token
+        if self.tokenizer.pad_token is None:
+            self.tokenizer.pad_token = self.tokenizer.eos_token
+        if ('pad_token_id' not in self.model.config) or (self.model.config.pad_token_id is None):
+            self.model.config.update({'pad_token_id': self.tokenizer.eos_token_id})
+
         self.supported_encode_params = get_supported_params(self.model.predict)
 
     def predict(self, sentences: List[List[str]], **kwargs) -> Tensor:
@@ -189,6 +196,7 @@ class APIEmbeddingModel(BaseModel):
         self.openai_api_base = kwargs.get('api_base')
         self.openai_api_key = kwargs.get('api_key')
         self.dimensions = kwargs.get('dimensions')
+        self.check_embedding_ctx_length = kwargs.get('check_embedding_ctx_length', False)
         self.framework = ['API']
 
         self.model = OpenAIEmbeddings(
@@ -196,7 +204,7 @@ class APIEmbeddingModel(BaseModel):
             openai_api_base=self.openai_api_base,
             openai_api_key=self.openai_api_key,
             dimensions=self.dimensions,
-            check_embedding_ctx_length=False
+            check_embedding_ctx_length=self.check_embedding_ctx_length,
         )
 
         super().__init__(model_name_or_path=self.model_name, **kwargs)