evalscope 0.13.1__py3-none-any.whl → 0.14.0__py3-none-any.whl

evalscope/perf/benchmark.py

@@ -9,7 +9,7 @@ import threading
 import time
 from http import HTTPStatus
 from tqdm import tqdm
-from typing import List
+from typing import AsyncGenerator, List
 
 from evalscope.perf.arguments import Arguments
 from evalscope.perf.http_client import AioHttpClient, test_connection
@@ -18,95 +18,73 @@ from evalscope.perf.utils.benchmark_util import BenchmarkData, BenchmarkMetrics
 from evalscope.perf.utils.db_util import create_result_table, get_result_db_path, insert_benchmark_data, summary_result
 from evalscope.perf.utils.handler import add_signal_handlers, exception_handler
 from evalscope.perf.utils.local_server import start_app
+from evalscope.perf.utils.log_utils import init_swanlab, init_wandb
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
-query_send_completed_event = asyncio.Event()
+
 data_process_completed_event = asyncio.Event()
 
 
 @exception_handler
-async def dispatch_requests_worker(request_queue: asyncio.Queue, args: Arguments):
+async def get_requests(args: Arguments) -> AsyncGenerator[dict, None]:
     query_generator_class = ApiRegistry(args.api)
     query_generator = query_generator_class(args.tokenizer_path)
 
     def load_prompt(prompt_path_or_text):
-        """Load the prompt from a file or directly from the input text."""
         if prompt_path_or_text.startswith('@'):
             with open(prompt_path_or_text[1:], 'r', encoding='utf-8') as file:
                 return file.read()
         return prompt_path_or_text
 
-    async def dispatch_request(request):
-        """Dispatch a single request with optional rate limiting."""
-        await request_queue.put(request)
-        if args.rate != -1:
-            interval = np.random.exponential(1.0 / args.rate)
-            await asyncio.sleep(interval)
-
-    async def dispatch_requests_from_prompt(messages):
-        """Generate and dispatch requests based on the given prompt."""
+    async def generate_requests_from_prompt(messages):
         request = query_generator.build_request(messages, args)
-        if args.number is None:
-            await dispatch_request(request)
-            return 1
         for _ in range(args.number):
-            await dispatch_request(request)
-        return args.number
+            yield request
 
-    async def dispatch_requests_from_dataset():
-        """Generate and dispatch requests based on the dataset."""
-        total_query_count = 0
+    async def generate_requests_from_dataset():
         message_generator_class = DatasetRegistry(args.dataset)
         message_generator = message_generator_class(args)
 
+        count = 0
         for messages in message_generator:
             request = query_generator.build_request(messages, args)
-            if request is None:
-                continue
-            await dispatch_request(request)
-            total_query_count += 1
-            if args.number and total_query_count >= args.number:
-                break
-
-        return total_query_count
+            if request is not None:
+                yield request
+                count += 1
+                if args.number and count >= args.number:
+                    break
 
-    # Load prompt or dataset and dispatch requests accordingly
     if args.prompt:
         prompt = load_prompt(args.prompt)
-        messages = [{'role': 'user', 'content': prompt}]
-        total_queries = await dispatch_requests_from_prompt(messages)
+        messages = [{'role': 'user', 'content': prompt}] if args.apply_chat_template else prompt
+        generator = generate_requests_from_prompt(messages)
     elif args.dataset:
-        total_queries = await dispatch_requests_from_dataset()
+        generator = generate_requests_from_dataset()
     else:
         raise Exception('Either prompt or dataset is required!')
 
-    return total_queries
+    async for request in generator:
+        yield request
+        if args.rate != -1:
+            interval = np.random.exponential(1.0 / args.rate)
+            await asyncio.sleep(interval)
 
 
 @exception_handler
-async def send_requests_worker(
-    task_id,
-    request_queue: asyncio.Queue,
+async def send_request(
+    semaphore: asyncio.Semaphore,
+    request: dict,
     benchmark_data_queue: asyncio.Queue,
     args: Arguments,
 ):
-    client = AioHttpClient(args)
-    async with client:
-        while not (query_send_completed_event.is_set() and request_queue.empty()):
-            try:
-                # Attempt to get a request from the queue with a timeout
-                request = await asyncio.wait_for(request_queue.get(), timeout=0.0001)
-                request_queue.task_done()
-            except asyncio.TimeoutError:
-                # If timeout, continue to the next iteration
-                continue
-
-            # Initialize benchmark data for the current request
+    async with semaphore:
+        client = AioHttpClient(args)
+        async with client:
             benchmark_data = BenchmarkData(request=request)
+            benchmark_data.start_time = time.perf_counter()
             collected_messages = []
             try:
-                # Send the request and process the response
                 async for is_error, state_code, response_data in client.post(request):
                     if is_error or state_code != HTTPStatus.OK:
                         logger.error(f'Request: {request} failed, state_code: {state_code}, data: {response_data}')
@@ -124,35 +102,28 @@ async def send_requests_worker(
                 logger.exception(e)
                 logger.error(f'Request query: {request} exception')
             finally:
-                # Record completion time and collected messages
                 benchmark_data.completed_time = time.perf_counter()
                 benchmark_data.response_messages = collected_messages
                 await benchmark_data_queue.put(benchmark_data)
 
 
 @exception_handler
-async def statistic_benchmark_metric_worker(benchmark_data_queue: asyncio.Queue, args: Arguments):
+async def statistic_benchmark_metric(benchmark_data_queue: asyncio.Queue, args: Arguments):
     metrics = BenchmarkMetrics(concurrency=args.parallel)
 
     api_plugin_class = ApiRegistry(args.api)
     api_plugin = api_plugin_class(args.tokenizer_path)
 
     result_db_path = get_result_db_path(args)
-    # Initialize wandb
-    if args.wandb_api_key:
-        import datetime
-        import wandb
-        os.environ['WANDB_SILENT'] = 'true'
-        os.environ['WANDB_DIR'] = args.outputs_dir
 
-        wandb.login(key=args.wandb_api_key)
-        current_time = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
-        name = args.name if args.name else f'{args.model_id}_{current_time}'
-        wandb.init(project='perf_benchmark', name=name, config=args.to_dict())
+    if args.wandb_api_key:
+        init_wandb(args)
+    if args.swanlab_api_key:
+        init_swanlab(args)
 
     collected_benchmark_data = []
 
-    with tqdm(desc='Processing') as pbar:
+    with tqdm(desc='Processing', total=args.number) as pbar:
         while not (data_process_completed_event.is_set() and benchmark_data_queue.empty()):
             try:
                 # Attempt to get benchmark data from the queue with a timeout
@@ -171,9 +142,13 @@ async def statistic_benchmark_metric_worker(benchmark_data_queue: asyncio.Queue,
             # Create a message with the updated metrics
             message = metrics.create_message()
 
-            # Log the message to wandb if the api key is provided
+            # Log the message to wandb\swanlab if the api key is provided
             if args.wandb_api_key:
+                import wandb
                 wandb.log(message)
+            if args.swanlab_api_key:
+                import swanlab
+                swanlab.log(message)
 
             # Log the message to the logger every n queries
             if int(metrics.n_total_queries) % args.log_every_n_query == 0:
@@ -194,17 +169,12 @@ async def statistic_benchmark_metric_worker(benchmark_data_queue: asyncio.Queue,
 
 
 @exception_handler
-async def start_server(args: Arguments) -> bool:
+async def connect_test(args: Arguments) -> bool:
     if args.api.startswith('local'):
         #  start local server
         server = threading.Thread(target=start_app, args=(copy.deepcopy(args), ), daemon=True)
         server.start()
 
-        if args.dataset.startswith('speed_benchmark'):
-            args.url = f'http://127.0.0.1:{args.port}/v1/completions'
-        else:
-            args.url = f'http://127.0.0.1:{args.port}/v1/chat/completions'
-
     if (not args.no_test_connection) and (not await test_connection(args)):
         raise TimeoutError('Test connection failed')
 
@@ -216,39 +186,23 @@ async def benchmark(args: Arguments) -> None:
         add_signal_handlers(loop)
 
     # init queue
-    request_queue = asyncio.Queue()
     benchmark_data_queue = asyncio.Queue()
-
     # reset event
-    query_send_completed_event.clear()
     data_process_completed_event.clear()
-
-    async def create_send_request_tasks():
-        tasks: List[asyncio.Task] = []
-        for idx in range(args.parallel):
-            task = asyncio.create_task(send_requests_worker(idx, request_queue, benchmark_data_queue, args))
-            tasks.append(task)
-        return tasks
-
-    async def run_tasks():
-        await start_server(args)
-
-        dispatch_task = asyncio.create_task(dispatch_requests_worker(request_queue, args))
-        statistic_benchmark_metric_task = asyncio.create_task(
-            statistic_benchmark_metric_worker(benchmark_data_queue, args))
-        send_request_tasks = await create_send_request_tasks()
-
-        expected_number_of_queries = await dispatch_task
-        await request_queue.join()
-        query_send_completed_event.set()
-
-        await asyncio.gather(*send_request_tasks, return_exceptions=True)
-        await benchmark_data_queue.join()
-        data_process_completed_event.set()
-
-        metrics, result_db_path = await statistic_benchmark_metric_task
-        summary_result(args, metrics, expected_number_of_queries, result_db_path)
-
-        await asyncio.sleep(0.250)
-
-    await run_tasks()
+    # test connection
+    await connect_test(args)
+    # start statistic benchmark metric
+    statistic_benchmark_metric_task = asyncio.create_task(statistic_benchmark_metric(benchmark_data_queue, args))
+    # start send request
+    semaphore = asyncio.Semaphore(args.parallel)
+    send_request_tasks: List[asyncio.Task] = []
+    async for request in get_requests(args):
+        task = asyncio.create_task(send_request(semaphore, request, benchmark_data_queue, args))
+        send_request_tasks.append(task)
+
+    await asyncio.gather(*send_request_tasks, return_exceptions=True)
+    await benchmark_data_queue.join()
+    data_process_completed_event.set()
+
+    metrics, result_db_path = await statistic_benchmark_metric_task
+    summary_result(args, metrics, result_db_path)

evalscope/perf/http_client.py

@@ -24,7 +24,6 @@ class AioHttpClient:
         self.connect_timeout = args.connect_timeout
         self.client = aiohttp.ClientSession(
             timeout=aiohttp.ClientTimeout(connect=self.connect_timeout, sock_read=self.read_timeout),
-            connector=aiohttp.TCPConnector(limit=1),
             trace_configs=[self._create_trace_config()] if args.debug else [])
 
     def _create_trace_config(self):
@@ -144,7 +143,7 @@ async def test_connection(args: Arguments) -> bool:
     async def attempt_connection():
         client = AioHttpClient(args)
         async with client:
-            if 'chat/completions' in args.url:
+            if args.apply_chat_template:
                 request = {
                     'messages': [{
                         'role': 'user',
@@ -164,7 +163,7 @@ async def test_connection(args: Arguments) -> bool:
             is_error, state_code, response_data = await asyncio.wait_for(
                 attempt_connection(), timeout=args.connect_timeout)
             if not is_error:
-                logger.info('Connection successful.')
+                logger.info('Test connection successful.')
                 return True
             logger.warning(f'Retrying...  <{state_code}> {response_data}')
         except Exception as e:

evalscope/perf/plugin/api/custom_api.py

@@ -24,7 +24,7 @@ class CustomPlugin(ApiPluginBase):
         """
         super().__init__(model_path=mode_path)
         if mode_path is not None:
-            from transformers import AutoTokenizer
+            from modelscope import AutoTokenizer
             self.tokenizer = AutoTokenizer.from_pretrained(mode_path)
         else:
             self.tokenizer = None

evalscope/perf/plugin/api/openai_api.py

@@ -24,7 +24,7 @@ class OpenaiPlugin(ApiPluginBase):
         """
         super().__init__(model_path=mode_path)
         if mode_path is not None:
-            from transformers import AutoTokenizer
+            from modelscope import AutoTokenizer
             self.tokenizer = AutoTokenizer.from_pretrained(mode_path)
         else:
             self.tokenizer = None
@@ -94,9 +94,11 @@ class OpenaiPlugin(ApiPluginBase):
             payload['top_p'] = param.top_p
         if param.top_k is not None:
             payload['top_k'] = param.top_k
+        if param.extra_args is not None:
+            payload.update(param.extra_args)
         return payload
 
-    def parse_responses(self, responses, request: Any = None, **kwargs) -> Dict:
+    def parse_responses(self, responses, request: Any = None, **kwargs) -> tuple[int, int]:
         """Parser responses and return number of request and response tokens.
         Only one response for non-stream, multiple responses for stream.
         """

evalscope/perf/plugin/datasets/custom.py

@@ -18,4 +18,7 @@ class CustomDatasetPlugin(DatasetPluginBase):
             prompt = item.strip()
             if len(prompt) > self.query_parameters.min_prompt_length and len(
                     prompt) < self.query_parameters.max_prompt_length:
-                yield [{'role': 'user', 'content': prompt}]
+                if self.query_parameters.apply_chat_template:
+                    yield [{'role': 'user', 'content': prompt}]
+                else:
+                    yield prompt

evalscope/perf/plugin/datasets/line_by_line.py

@@ -19,4 +19,7 @@ class LineByLineDatasetPlugin(DatasetPluginBase):
             prompt = item.strip()
             if len(prompt) > self.query_parameters.min_prompt_length and len(
                     prompt) < self.query_parameters.max_prompt_length:
-                yield [{'role': 'user', 'content': prompt}]
+                if self.query_parameters.apply_chat_template:
+                    yield [{'role': 'user', 'content': prompt}]
+                else:
+                    yield prompt

evalscope/perf/plugin/datasets/longalpaca.py

@@ -24,4 +24,7 @@ class LongAlpacaDatasetPlugin(DatasetPluginBase):
             prompt = item['instruction'].strip()
             if len(prompt) > self.query_parameters.min_prompt_length and len(
                     prompt) < self.query_parameters.max_prompt_length:
-                yield [{'role': 'user', 'content': prompt}]
+                if self.query_parameters.apply_chat_template:
+                    yield [{'role': 'user', 'content': prompt}]
+                else:
+                    yield prompt

evalscope/perf/plugin/datasets/openqa.py

@@ -29,4 +29,7 @@ class OpenqaDatasetPlugin(DatasetPluginBase):
             prompt = item['question'].strip()
             if (len(prompt) > self.query_parameters.min_prompt_length
                     and len(prompt) < self.query_parameters.max_prompt_length):
-                yield [{'role': 'user', 'content': prompt}]
+                if self.query_parameters.apply_chat_template:
+                    yield [{'role': 'user', 'content': prompt}]
+                else:
+                    yield prompt

evalscope/perf/plugin/datasets/random_dataset.py

@@ -23,8 +23,12 @@ class RandomDatasetPlugin(DatasetPluginBase):
         self.number = self.query_parameters.number or 1
 
     def build_messages(self) -> Iterator[List[Dict]]:
-        min_prompt_length = self.query_parameters.min_prompt_length - self.template_len
-        max_prompt_length = self.query_parameters.max_prompt_length - self.template_len + 1
+        if self.query_parameters.apply_chat_template:
+            min_prompt_length = self.query_parameters.min_prompt_length - self.template_len
+            max_prompt_length = self.query_parameters.max_prompt_length - self.template_len + 1
+        else:
+            min_prompt_length = self.query_parameters.min_prompt_length
+            max_prompt_length = self.query_parameters.max_prompt_length + 1
 
         assert min_prompt_length >= 0, f'min_prompt_length should be greater than or equal to the template length {self.template_len}.'  # noqa: E501
         assert max_prompt_length >= min_prompt_length, 'max_prompt_length should be greater than or equal to min_prompt_length.'  # noqa: E501
@@ -34,10 +38,13 @@ class RandomDatasetPlugin(DatasetPluginBase):
         offsets = np.random.randint(0, self.tokenizer.vocab_size, size=self.number)
 
         for i in range(self.number):
-            prompt_ids = (offsets[i] + i + np.arange(input_lens[i])) % self.tokenizer.vocab_size
-            prompt = self.tokenizer.decode(
-                self.prefix_ids + prompt_ids.tolist(), skip_special_tokens=False, clean_up_tokenization_spaces=False)
-            yield [{'role': 'user', 'content': prompt}]
+            prompt_ids = ((offsets[i] + i + np.arange(input_lens[i])) % self.tokenizer.vocab_size).tolist()
+            prompt = self.tokenizer.decode(self.prefix_ids + prompt_ids)
+
+            if self.query_parameters.apply_chat_template:
+                yield [{'role': 'user', 'content': prompt}]
+            else:
+                yield prompt
 
     def get_random_inputs(self, length: int) -> List[int]:
         if length <= 0:

evalscope/perf/plugin/datasets/speed_benchmark.py

@@ -3,6 +3,9 @@ from typing import Dict, Iterator, List, Tuple
 from evalscope.perf.arguments import Arguments
 from evalscope.perf.plugin.datasets.base import DatasetPluginBase
 from evalscope.perf.plugin.registry import register_dataset
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
 
 
 @register_dataset('speed_benchmark')
@@ -18,6 +21,14 @@ class SpeedBenchmarkDatasetPlugin(DatasetPluginBase):
     def __init__(self, query_parameters: Arguments):
         super().__init__(query_parameters)
 
+        url = self.query_parameters.url
+        if url.endswith('v1/chat/completions'):
+            logger.warning(
+                'The API URL is not set correctly for `speed_benchmark`. Using `v1/completions` instead of `v1/chat/completions` by system.'  # noqa
+            )
+            url = url.replace('v1/chat/completions', 'v1/completions')
+            self.query_parameters.url = url
+
     def build_messages(self) -> Iterator[List[Dict]]:
         for input_len in self.INPUT_LENGTH:
             for _ in range(self.REPEAT):

evalscope/perf/utils/benchmark_util.py

@@ -11,7 +11,7 @@ logger = get_logger()
 @dataclass
 class BenchmarkData:
     request: Any = None
-    start_time: float = field(default_factory=time.perf_counter)
+    start_time: float = 0.0
     completed_time: float = 0.0
     chunk_times: List[float] = field(default_factory=list)
     success: bool = False
@@ -73,7 +73,9 @@ class BenchmarkMetrics:
     avg_chunk_time: float = -1
     avg_prompt_tokens: float = -1
     avg_completion_tokens: float = -1
-    avg_token_per_seconds: float = -1
+    avg_input_token_per_seconds: float = -1
+    avg_output_token_per_seconds: float = -1
+    avg_total_token_per_seconds: float = -1
     avg_time_per_token: float = -1
     qps: float = -1
 
@@ -111,22 +113,26 @@ class BenchmarkMetrics:
             self.avg_chunk_time = self.total_chunks_time / self.n_total_chunks
             self.avg_prompt_tokens = self.n_total_prompt_tokens / self.n_succeed_queries
             self.avg_completion_tokens = self.n_total_completion_tokens / self.n_succeed_queries
-            self.avg_token_per_seconds = self.n_total_completion_tokens / self.total_time
+            self.avg_input_token_per_seconds = self.n_total_prompt_tokens / self.total_first_chunk_latency
+            self.avg_output_token_per_seconds = self.n_total_completion_tokens / self.total_time
+            self.avg_total_token_per_seconds = (self.n_total_prompt_tokens
+                                                + self.n_total_completion_tokens) / self.total_time
             self.avg_time_per_token = self.n_time_per_output_token / self.n_succeed_queries
             self.qps = self.n_succeed_queries / self.total_time
         except ZeroDivisionError as e:
             logger.exception(e)
             return
 
-    def create_message(self, default_ndigits=3):
+    def create_message(self, default_ndigits=4):
         message = {
             'Time taken for tests (s)': round(self.total_time, default_ndigits),
             'Number of concurrency': self.concurrency,
             'Total requests': int(self.n_total_queries),
             'Succeed requests': self.n_succeed_queries,
             'Failed requests': self.n_failed_queries,
-            'Throughput(average tokens/s)': round(self.avg_token_per_seconds, default_ndigits),
-            'Average QPS': round(self.qps, default_ndigits),
+            'Output token throughput (tok/s)': round(self.avg_output_token_per_seconds, default_ndigits),
+            'Total token throughput (tok/s)': round(self.avg_total_token_per_seconds, default_ndigits),
+            'Request throughput (req/s)': round(self.qps, default_ndigits),
             'Average latency (s)': round(self.avg_latency, default_ndigits),
             'Average time to first token (s)': round(self.avg_first_chunk_latency, default_ndigits),
             'Average time per output token (s)': round(self.avg_time_per_token, default_ndigits),

evalscope/perf/utils/db_util.py

@@ -175,7 +175,7 @@ def get_percentile_results(result_db_path: str) -> Dict[str, List[float]]:
 
     metrics = {
         'TTFT (s)': [row[FIRST_CHUNK_LATENCY_INDEX] for row in rows],
-        'TPOT (s)':
+        'ITL (s)':
         inter_token_latencies_all,
         'Latency (s)': [row[LATENCY_INDEX] for row in rows],
         'Input tokens': [row[PROMPT_TOKENS_INDEX] for row in rows],
@@ -194,12 +194,12 @@ def get_percentile_results(result_db_path: str) -> Dict[str, List[float]]:
     return results
 
 
-def summary_result(args: Arguments, metrics: BenchmarkMetrics, expected_number_of_queries: int, result_db_path: str):
+def summary_result(args: Arguments, metrics: BenchmarkMetrics, result_db_path: str):
     result_path = os.path.dirname(result_db_path)
     write_json_file(args.to_dict(), os.path.join(result_path, 'benchmark_args.json'))
 
     data = metrics.create_message()
-    data.update({'Expected number of requests': expected_number_of_queries, 'Result DB path': result_db_path})
+    data.update({'Expected number of requests': args.number, 'Result DB path': result_db_path})
     write_json_file(data, os.path.join(result_path, 'benchmark_summary.json'))
 
     # Print summary in a table

evalscope/perf/utils/log_utils.py

@@ -0,0 +1,41 @@
+import os
+
+from evalscope.perf.arguments import Arguments
+
+
+def init_wandb(args: Arguments) -> None:
+    """
+    Initialize WandB for logging.
+    """
+    # Initialize wandb if the api key is provided
+    import datetime
+    try:
+        import wandb
+    except ImportError:
+        raise RuntimeError('Cannot import wandb. Please install it with command: \n pip install wandb')
+    os.environ['WANDB_SILENT'] = 'true'
+    os.environ['WANDB_DIR'] = args.outputs_dir
+
+    wandb.login(key=args.wandb_api_key)
+    current_time = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
+    name = args.name if args.name else f'{args.model_id}_{current_time}'
+    wandb.init(project='perf_benchmark', name=name, config=args.to_dict())
+
+
+def init_swanlab(args: Arguments) -> None:
+    import datetime
+    try:
+        import swanlab
+    except ImportError:
+        raise RuntimeError('Cannot import swanlab. Please install it with command: \n pip install swanlab')
+    os.environ['SWANLAB_LOG_DIR'] = args.outputs_dir
+    if not args.swanlab_api_key == 'local':
+        swanlab.login(api_key=args.swanlab_api_key)
+    current_time = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
+    name = args.name if args.name else f'{args.model_id}_{current_time}'
+    swanlab.config.update({'framework': '📏evalscope'})
+    swanlab.init(
+        project='perf_benchmark',
+        name=name,
+        config=args.to_dict(),
+        mode='local' if args.swanlab_api_key == 'local' else None)

evalscope/report/app.py

@@ -44,7 +44,7 @@ def scan_for_report_folders(root_path):
                 continue
             datasets = []
             for dataset_item in glob.glob(os.path.join(model_item, '*.json')):
-                datasets.append(os.path.basename(dataset_item).split('.')[0])
+                datasets.append(os.path.splitext(os.path.basename(dataset_item))[0])
             datasets = DATASET_TOKEN.join(datasets)
             reports.append(
                 f'{os.path.basename(folder)}{REPORT_TOKEN}{os.path.basename(model_item)}{MODEL_TOKEN}{datasets}')
@@ -253,17 +253,17 @@ def process_model_prediction(item: Any):
 
 
 def normalize_score(score):
-    if isinstance(score, bool):
-        return 1.0 if score else 0.0
-    elif isinstance(score, dict):
-        for key in score:
-            return float(score[key])
-        return 0.0
-    else:
-        try:
-            return float(score)
-        except (ValueError, TypeError):
+    try:
+        if isinstance(score, bool):
+            return 1.0 if score else 0.0
+        elif isinstance(score, dict):
+            for key in score:
+                return float(score[key])
             return 0.0
+        else:
+            return float(score)
+    except (ValueError, TypeError):
+        return 0.0
 
 
 def get_model_prediction(work_dir: str, model_name: str, dataset_name: str, subset_name: str):

evalscope/run.py

@@ -58,10 +58,17 @@ def setup_work_directory(task_cfg: TaskConfig, run_time: str):
 
     outputs = OutputsStructure(outputs_dir=task_cfg.work_dir)
 
+    # Unify the output directory structure
     if task_cfg.eval_backend == EvalBackend.OPEN_COMPASS:
         task_cfg.eval_config['time_str'] = run_time
     elif task_cfg.eval_backend == EvalBackend.VLM_EVAL_KIT:
         task_cfg.eval_config['work_dir'] = task_cfg.work_dir
+    elif task_cfg.eval_backend == EvalBackend.RAG_EVAL:
+        from evalscope.backend.rag_eval import Tools
+        if task_cfg.eval_config['tool'].lower() == Tools.MTEB:
+            task_cfg.eval_config['eval']['output_folder'] = task_cfg.work_dir
+        elif task_cfg.eval_config['tool'].lower() == Tools.CLIP_BENCHMARK:
+            task_cfg.eval_config['eval']['output_dir'] = task_cfg.work_dir
     return outputs
 
 

evalscope/summarizer.py

@@ -105,7 +105,8 @@ class Summarizer:
                             summary_res: dict = csv_to_list(summary_file_path)[0]
                         elif summary_file_path.endswith('json'):
                             summary_res: dict = json_to_dict(summary_file_path)
-                        file_name = os.path.basename(summary_file_path).split('.')[0]
+                        base_name = os.path.basename(summary_file_path)
+                        file_name = os.path.splitext(base_name)[0]
                         final_res_list.append({file_name: summary_res})
 
             elif eval_backend == EvalBackend.THIRD_PARTY: