evalscope 0.7.1__py3-none-any.whl → 0.8.0__py3-none-any.whl

evalscope/perf/plugin/api/dashscope_api.py

@@ -1,8 +1,7 @@
+import json
 import os
 from typing import Any, Dict, Iterator, List
 
-import json
-
 from evalscope.perf.arguments import Arguments
 from evalscope.perf.plugin.api.base import ApiPluginBase
 from evalscope.perf.plugin.registry import register_api

evalscope/perf/plugin/api/openai_api.py

@@ -1,8 +1,7 @@
-import os
-from typing import Any, Dict, Iterator, List
-
 import json
+import os
 from transformers import AutoTokenizer
+from typing import Any, Dict, Iterator, List
 
 from evalscope.perf.arguments import Arguments
 from evalscope.perf.plugin.api.base import ApiPluginBase
@@ -151,6 +150,6 @@ class OpenaiPlugin(ApiPluginBase):
         elif input_tokens is None and output_tokens is None:  # no usage info get.
             input_tokens = 0
             output_tokens = 0
-            logger.warning('No usage info get.')
+            logger.warning('No usage information found. Please specify `--tokenizer-path` to generate usage details.')
 
         return input_tokens, output_tokens

evalscope/perf/plugin/datasets/base.py

@@ -1,9 +1,8 @@
+import json
 import sys
 from abc import abstractmethod
 from typing import Any, Dict, Iterator, List, Tuple
 
-import json
-
 from evalscope.perf.arguments import Arguments
 
 

evalscope/perf/plugin/datasets/flickr8k.py

@@ -1,9 +1,8 @@
 import base64
 from io import BytesIO
-from typing import Any, Dict, Iterator, List
-
 from modelscope.msdatasets import MsDataset
 from PIL import Image
+from typing import Any, Dict, Iterator, List
 
 from evalscope.perf.arguments import Arguments
 from evalscope.perf.plugin.datasets.base import DatasetPluginBase

evalscope/perf/plugin/datasets/longalpaca.py

@@ -1,6 +1,5 @@
-from typing import Any, Dict, Iterator, List
-
 from modelscope import MsDataset
+from typing import Any, Dict, Iterator, List
 
 from evalscope.perf.arguments import Arguments
 from evalscope.perf.plugin.datasets.base import DatasetPluginBase

evalscope/perf/plugin/datasets/openqa.py

@@ -1,8 +1,7 @@
+import json
 import subprocess
 from typing import Any, Dict, Iterator, List
 
-import json
-
 from evalscope.perf.arguments import Arguments
 from evalscope.perf.plugin.datasets.base import DatasetPluginBase
 from evalscope.perf.plugin.registry import register_dataset

evalscope/perf/utils/analysis_result.py

@@ -1,9 +1,8 @@
 import base64
+import json
 import pickle
 import sqlite3
 
-import json
-
 result_db_path = '/mnt/data/data/user/maoyunlin.myl/eval-scope/outputs/qwen2.5_benchmark_20241111_160543.db'
 con = sqlite3.connect(result_db_path)
 query_sql = "SELECT request, response_messages, prompt_tokens, completion_tokens \

evalscope/perf/utils/benchmark_util.py

@@ -1,9 +1,8 @@
 import time
+import torch
 from dataclasses import dataclass, field
 from typing import Any, List, Optional, Tuple
 
-import torch
-
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()

evalscope/perf/utils/db_util.py

@@ -1,11 +1,10 @@
 import base64
+import json
 import os
 import pickle
 import sqlite3
 import sys
 from datetime import datetime
-
-import json
 from tabulate import tabulate
 
 from evalscope.perf.arguments import Arguments
@@ -88,15 +87,19 @@ def insert_benchmark_data(cursor: sqlite3.Cursor, benchmark_data: BenchmarkData)
         cursor.execute(query, common_columns)
 
 
-def get_result_db_path(name, model):
+def get_output_path(args: Arguments) -> str:
     current_time = datetime.now().strftime('%Y%m%d_%H%M%S')
-    output_dir = './outputs'
-    result_db_path = os.path.join(output_dir, f'{name or model}_perf', current_time, 'benchmark_data.db')
+    output_path = os.path.join(args.outputs_dir, current_time, f'{args.name or args.model_id}')
+    if not os.path.exists(output_path):
+        os.makedirs(output_path, exist_ok=True)
+    logger.info(f'Save the result to: {output_path}')
+    return output_path
+
 
-    if not os.path.exists(os.path.dirname(result_db_path)):
-        os.makedirs(os.path.dirname(result_db_path), exist_ok=True)
+def get_result_db_path(args: Arguments):
+    result_db_path = os.path.join(args.outputs_dir, 'benchmark_data.db')
 
-    logger.info(f'Save the result to: {result_db_path}')
+    logger.info(f'Save the data base to: {result_db_path}')
     if os.path.exists(result_db_path):
         logger.warning('The db file exists, delete it and start again!.')
         sys.exit(1)

evalscope/perf/utils/local_server.py

@@ -1,16 +1,15 @@
 import os
 import subprocess
-from contextlib import asynccontextmanager
-from dataclasses import dataclass
-
 import torch
 import uvicorn
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from sse_starlette.sse import EventSourceResponse
 
 from evalscope.perf.arguments import Arguments
-from evalscope.perf.utils.chat_service import ChatCompletionRequest, ChatService, ModelList, TextCompletionRequest
+from evalscope.utils.chat_service import ChatCompletionRequest, ChatService, ModelList, TextCompletionRequest
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -66,9 +65,9 @@ async def lifespan(app: FastAPI):
         torch.cuda.empty_cache()
 
 
-def create_app(args) -> FastAPI:
+def create_app(model, attn_implementation=None) -> FastAPI:
     app = FastAPI(lifespan=lifespan)
-    chat_service = ChatService(model_path=args.model, attn_implementation=args.attn_implementation)
+    chat_service = ChatService(model_path=model, attn_implementation=attn_implementation)
 
     app.add_middleware(
         CORSMiddleware,
@@ -98,18 +97,25 @@ def create_app(args) -> FastAPI:
 
 def start_app(args: Arguments):
     if args.api == 'local':
-        app = create_app(args)
-        uvicorn.run(app, host='0.0.0.0', port=8877, workers=1)
+        app = create_app(args.model, args.attn_implementation)
+        uvicorn.run(app, host='0.0.0.0', port=args.port, workers=1)
 
     elif args.api == 'local_vllm':
         os.environ['VLLM_USE_MODELSCOPE'] = 'True'
-
+        # yapf: disable
         proc = subprocess.Popen([
-            'python', '-m', 'vllm.entrypoints.openai.api_server', '--model', args.model, '--served-model-name',
-            os.path.basename(args.model), '--tensor-parallel-size',
-            str(torch.cuda.device_count()), '--max-model-len', '32768', '--gpu-memory-utilization', '0.9', '--host',
-            '0.0.0.0', '--port', '8877', '--disable-log-requests', '--disable-log-stats'
+            'python', '-m', 'vllm.entrypoints.openai.api_server',
+            '--model', args.model,
+            '--served-model-name', args.model,
+            '--tensor-parallel-size', str(torch.cuda.device_count()),
+            '--max-model-len', '32768',
+            '--gpu-memory-utilization', '0.9',
+            '--host', '0.0.0.0',
+            '--port', args.port,
+            '--disable-log-requests',
+            '--disable-log-stats',
         ])
+        # yapf: enable
         import atexit
 
         def on_exit():

evalscope/registry/config/cfg_arena_zhihu.yaml

@@ -21,7 +21,7 @@ answers_gen:
         model_id_or_path: /mnt/data/data/user/maoyunlin.myl/output/qwen2-7b-instruct/v25-20240809-113533/checkpoint-309-merged
         revision: NULL       # revision of model, default is NULL
         precision: torch.float16
-        enable: true           # enable or disable this model     
+        enable: true           # enable or disable this model
         template_type: default-generation
         generation_config:
             do_sample: true

evalscope/registry/tasks/arc.yaml

@@ -1,5 +1,5 @@
 model_args:    # model args should be followed by benchmark requirements
-  revision: default
+  revision: master
   precision: torch.float16
   device_map: auto
 #  model_name_or_path: qwen/qwen-7b-chat
@@ -22,8 +22,7 @@ model: null   # Note: to be implemented as CustomModel
 eval_type: custom
 datasets:
   - arc
-outputs: null    # structure: configs, logs, predictions, reviews, reports # TODO: need to parse
 use_cache: false
 stage: all
-dataset_hub: ModelScope    # `Local` or `ModelScope`
+dataset_hub: modelscope    # `Local` or `ModelScope`
 limit: null

evalscope/registry/tasks/bbh.yaml

@@ -1,5 +1,5 @@
 model_args:    # model args should be followed by benchmark requirements
-  revision: default
+  revision: master
   precision: torch.float16
   device_map: auto
 #  model_name_or_path: qwen/qwen-7b-chat
@@ -20,8 +20,7 @@ model: null   # Note: to be implemented as CustomModel
 eval_type: custom
 datasets:
   - bbh
-outputs: null    # structure: configs, logs, predictions, reviews, reports # TODO: need to parse
 use_cache: false
 stage: all
-dataset_hub: ModelScope    # `Local` or `ModelScope`
-limit: null
+dataset_hub: modelscope    # `Local` or `ModelScope`
+limit: null

evalscope/registry/tasks/bbh_mini.yaml

@@ -1,5 +1,5 @@
 model_args:    # model args should be followed by benchmark requirements
-  revision: default
+  revision: master
   precision: torch.float16
   device_map: auto
 #  model_name_or_path: qwen/qwen-7b-chat
@@ -20,8 +20,7 @@ model: null   # Note: to be implemented as CustomModel
 eval_type: custom
 datasets:
   - bbh
-outputs: null    # structure: configs, logs, predictions, reviews, reports # TODO: need to parse
 use_cache: false
 stage: all
-dataset_hub: ModelScope    # `Local` or `ModelScope`
-limit: null
+dataset_hub: modelscope    # `Local` or `ModelScope`
+limit: null

evalscope/registry/tasks/ceval.yaml

@@ -1,5 +1,5 @@
 model_args:    # model args should be followed by benchmark requirements
-  revision: default
+  revision: master
   precision: torch.float16
   device_map: auto
 #  model_name_or_path: qwen/qwen-7b-chat
@@ -23,5 +23,5 @@ datasets:
 outputs: null    # structure: configs, logs, predictions, reviews, reports # TODO: need to parse
 use_cache: false
 stage: all
-dataset_hub: ModelScope    # `Local` or `ModelScope`
-limit: null
+dataset_hub: modelscope    # `Local` or `ModelScope`
+limit: null

evalscope/registry/tasks/ceval_mini.yaml

@@ -1,5 +1,5 @@
 model_args:    # model args should be followed by benchmark requirements
-  revision: default
+  revision: master
   precision: torch.float16
   device_map: auto
 #  model_name_or_path: qwen/qwen-7b-chat
@@ -20,8 +20,7 @@ model: null   # Note: to be implemented as CustomModel
 eval_type: custom
 datasets:
   - ceval
-outputs: null    # structure: configs, logs, predictions, reviews, reports # TODO: need to parse
 use_cache: false
 stage: all
-dataset_hub: ModelScope    # `Local` or `ModelScope`
-limit: null
+dataset_hub: modelscope    # `Local` or `ModelScope`
+limit: null

evalscope/registry/tasks/cmmlu.yaml

@@ -1,5 +1,5 @@
 model_args:    # model args should be followed by benchmark requirements
-  revision: default
+  revision: master
   precision: torch.float16
   device_map: auto
 #  model_name_or_path: qwen/qwen-7b-chat
@@ -23,5 +23,5 @@ datasets:
 outputs: null    # structure: configs, logs, predictions, reviews, reports # TODO: need to parse
 use_cache: false
 stage: all
-dataset_hub: ModelScope    # `Local` or `ModelScope`
-limit: null
+dataset_hub: modelscope    # `Local` or `ModelScope`
+limit: null

evalscope/registry/tasks/eval_qwen-7b-chat_v100.yaml

@@ -24,5 +24,5 @@ datasets:
 outputs: ./outputs/eval_qwen-7b-chat_v100    # Directory to save the outputs, structure: logs, predictions, reviews, reports
 use_cache: false
 stage: all
-dataset_hub: ModelScope    # `Local` or `ModelScope`
+dataset_hub: modelscope    # `Local` or `ModelScope`
 limit: 10

evalscope/registry/tasks/general_qa.yaml

@@ -1,5 +1,5 @@
 model_args:    # model args should be followed by benchmark requirements
-  revision: default
+  revision: master
   precision: torch.float16
   device_map: auto
 #  model_name_or_path: qwen/qwen-7b-chat

evalscope/registry/tasks/gsm8k.yaml

@@ -1,5 +1,5 @@
 model_args:    # model args should be followed by benchmark requirements
-  revision: default
+  revision: master
   precision: torch.float16
   device_map: auto
 #  model_name_or_path: qwen/qwen-7b-chat
@@ -25,5 +25,5 @@ datasets:
 outputs: null    # structure: configs, logs, predictions, reviews, reports # TODO: need to parse
 use_cache: false
 stage: all
-dataset_hub: ModelScope    # `Local` or `ModelScope`
+dataset_hub: modelscope    # `Local` or `ModelScope`
 limit: null

evalscope/registry/tasks/mmlu.yaml

@@ -1,5 +1,5 @@
 model_args:    # model args should be followed by benchmark requirements
-  revision: default
+  revision: master
   precision: torch.float16
   device_map: auto
 #  model_name_or_path: qwen/qwen-7b-chat
@@ -25,5 +25,5 @@ datasets:
 outputs: null    # structure: configs, logs, predictions, reviews, reports # TODO: need to parse
 use_cache: true
 stage: all
-dataset_hub: ModelScope    # `Local` or `ModelScope`
-limit: null
+dataset_hub: modelscope    # `Local` or `ModelScope`
+limit: null

evalscope/registry/tasks/mmlu_mini.yaml

@@ -1,5 +1,5 @@
 model_args:    # model args should be followed by benchmark requirements
-  revision: default
+  revision: master
   precision: torch.float16
   device_map: auto
 #  model_name_or_path: qwen/qwen-7b-chat
@@ -23,5 +23,5 @@ datasets:
 outputs: null    # structure: configs, logs, predictions, reviews, reports # TODO: need to parse
 use_cache: false
 stage: all
-dataset_hub: ModelScope    # `Local` or `ModelScope`
-limit: null
+dataset_hub: modelscope    # `Local` or `ModelScope`
+limit: null