evalscope 0.16.0__py3-none-any.whl → 0.16.1__py3-none-any.whl

tests/cli/test_run.py

@@ -13,7 +13,7 @@ from evalscope.run import run_task
 from evalscope.utils import is_module_installed, test_level_list
 from evalscope.utils.logger import get_logger
 
-os.environ['LOG_LEVEL'] = 'DEBUG'
+os.environ['EVALSCOPE_LOG_LEVEL'] = 'DEBUG'
 
 logger = get_logger()
 
@@ -311,7 +311,8 @@ class TestRun(unittest.TestCase):
                 # 'maritime_bench',
                 # 'drop',
                 # 'winogrande',
-                'tool_bench',
+                # 'tool_bench',
+                'frames',
             ],
             dataset_args={
                 'mmlu': {
@@ -384,9 +385,10 @@ class TestRun(unittest.TestCase):
                 'temperature': 0,
                 'n': 1,
                 'max_tokens': 4096,
+                # 'extra_headers':{'key': 'value'},
             },
             # ignore_errors=True,
-            use_cache='outputs/20250519_142106'
+            # use_cache='outputs/20250519_142106'
         )
 
         run_task(task_cfg=task_cfg)
@@ -427,24 +429,27 @@ class TestRun(unittest.TestCase):
         from evalscope.config import TaskConfig
 
         task_cfg = TaskConfig(
-            model='qwen2.5-0.5b-instruct',
+            model='qwen-plus',
             api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
             api_key= env.get('DASHSCOPE_API_KEY'),
             eval_type=EvalType.SERVICE,
             datasets=[
                 # 'math_500',
-                # 'aime24',
+                'aime24',
                 # 'competition_math',
                 # 'arc',
-                # 'gsm8k'
+                # 'gsm8k',
                 # 'truthful_qa',
                 # 'simple_qa',
-                'chinese_simpleqa',
+                # 'chinese_simpleqa',
                 # 'live_code_bench',
                 # 'humaneval',
                 # 'general_qa',
                 # 'alpaca_eval',
-                # 'arena_hard'
+                # 'arena_hard',
+                # 'frames',
+                # 'docmath',
+                # 'needle_haystack',
             ],
             dataset_args={
                 'competition_math': {
@@ -469,13 +474,16 @@ class TestRun(unittest.TestCase):
                         '中华文化'
                     ]
                 },
+                'frames': {
+                    'local_path': '/root/.cache/modelscope/hub/datasets/iic/frames'
+                }
             },
             eval_batch_size=10,
-            limit=10,
+            limit=1,
             judge_strategy=JudgeStrategy.AUTO,
             judge_worker_num=5,
             judge_model_args={
-                'model_id': 'qwen2.5-7b-instruct',
+                'model_id': 'qwen2.5-72b-instruct',
                 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
                 'api_key': env.get('DASHSCOPE_API_KEY'),
                 'generation_config': {
@@ -491,7 +499,9 @@ class TestRun(unittest.TestCase):
             },
             timeout=60000,
             stream=True,
-            use_cache='outputs/20250519_142551'
+            analysis_report=True,
+            # debug=True,
+            # use_cache='outputs/20250602_135859'
         )
 
         run_task(task_cfg=task_cfg)

tests/rag/test_mteb.py

@@ -46,11 +46,11 @@ class TestMTEB(unittest.TestCase):
                 ],
                 'eval': {
                     'tasks': [
-                        # 'TNews',
-                        # 'CLSClusteringS2S',
+                        'TNews',
+                        'CLSClusteringS2S',
                         'T2Reranking',
-                        # 'T2Retrieval',
-                        # 'ATEC',
+                        'T2Retrieval',
+                        'ATEC',
                     ],
                     'verbosity': 2,
                     'overwrite_results': True,
@@ -85,7 +85,7 @@ class TestMTEB(unittest.TestCase):
                     ],
                     'verbosity': 2,
                     'overwrite_results': True,
-                    'limits': 30,
+                    'limits': 10,
                 },
             },
         )