evalscope 0.13.2__py3-none-any.whl → 0.15.0__py3-none-any.whl

tests/cli/test_all.py

@@ -4,13 +4,12 @@ from dotenv import dotenv_values
 env = dotenv_values('.env')
 
 import os
-import subprocess
 import unittest
 
 from evalscope.config import TaskConfig
 from evalscope.constants import EvalType, JudgeStrategy, OutputType
 from evalscope.run import run_task
-from evalscope.utils import is_module_installed, test_level_list
+from evalscope.utils import test_level_list
 from evalscope.utils.logger import get_logger
 
 os.environ['LOG_LEVEL'] = 'DEBUG'
@@ -46,6 +45,7 @@ datasets=[
         'chinese_simpleqa',
         'alpaca_eval',
         'arena_hard',
+        'maritime_bench',
 ]
 
 dataset_args={
@@ -134,8 +134,8 @@ class TestRun(unittest.TestCase):
             eval_type=EvalType.SERVICE,
             datasets=datasets,
             dataset_args=dataset_args,
-            eval_batch_size=2,
-            limit=2,
+            eval_batch_size=1,
+            limit=1,
             stream=True,
             generation_config={
                 'temperature': 0,

tests/cli/test_collection.py

@@ -80,4 +80,5 @@ class TestCollection(unittest.TestCase):
                 'api_key': os.getenv('DASHSCOPE_API_KEY'),
             }
         )
-        run_task(task_cfg=task_cfg)
+        res = run_task(task_cfg=task_cfg)
+        print(res)

tests/cli/test_run.py

@@ -137,7 +137,7 @@ class TestRun(unittest.TestCase):
                     'subset_list': ['gsm8k'],
                 },
                 'musr': {
-                    'subset_list': ['murder_mysteries']
+                    'subset_list': ['murder_mysteries'],
                 },
                 'general_mcq': {
                     'local_path': 'custom_eval/text/mcq',  # 自定义数据集路径
@@ -209,17 +209,23 @@ class TestRun(unittest.TestCase):
         task_cfg = TaskConfig(
             model='Qwen/Qwen2.5-0.5B-Instruct',
             datasets=[
-                'iquiz',
+                # 'iquiz',
                 # 'math_500',
                 # 'aime24',
-                # 'competition_math'
+                # 'competition_math',
+                'mmlu',
             ],
             dataset_args={
                 'competition_math': {
                     'subset_list': ['Level 4', 'Level 5']
-                }
+                },
+                'mmlu': {
+                    'subset_list': ['elementary_mathematics', 'high_school_european_history', 'nutrition'],
+                    'few_shot_num': 0
+                },
             },
-            limit=5
+            limit=10,
+            eval_batch_size=10,
         )
 
         run_task(task_cfg=task_cfg)
@@ -263,7 +269,7 @@ class TestRun(unittest.TestCase):
             datasets=[
                 # 'iquiz',
                 # 'ifeval',
-                # 'mmlu',
+                'mmlu',
                 # 'mmlu_pro',
                 # 'musr',
                 # 'process_bench',
@@ -281,9 +287,10 @@ class TestRun(unittest.TestCase):
                 # 'ceval',
                 # 'hellaswag',
                 # 'general_mcq',
-                'general_qa'
+                # 'general_qa'
                 # 'super_gpqa',
-                # 'mmlu_redux'
+                # 'mmlu_redux',
+                # 'maritime_bench'
             ],
             dataset_args={
                 'mmlu': {
@@ -322,7 +329,8 @@ class TestRun(unittest.TestCase):
                     'subset_list': ['gsm8k'],
                 },
                 'musr': {
-                    'subset_list': ['murder_mysteries']
+                    'subset_list': ['murder_mysteries'],
+                    'local_path': '/root/.cache/modelscope/hub/datasets/AI-ModelScope/MuSR'
                 },
                 'general_mcq': {
                     'local_path': 'custom_eval/text/mcq',  # 自定义数据集路径
@@ -353,10 +361,9 @@ class TestRun(unittest.TestCase):
             stream=False,
             generation_config={
                 'temperature': 0,
-                'n': 2,
+                'n': 1,
                 'max_tokens': 4096,
-            },
-            use_cache='outputs/20250326_202848',
+            }
         )
 
         run_task(task_cfg=task_cfg)

tests/perf/test_perf.py

@@ -104,7 +104,7 @@ class TestPerf(unittest.TestCase):
         task_cfg = Arguments(
             parallel=20,
             model='Qwen2.5-0.5B-Instruct',
-            url='http://127.0.0.1:8801/v1/chat/completions',
+            url='http://127.0.0.1:8801/v1/completions',
             api='openai',
             dataset='random',
             min_tokens=1024,
@@ -112,10 +112,10 @@ class TestPerf(unittest.TestCase):
             prefix_length=0,
             min_prompt_length=1024,
             max_prompt_length=1024,
-            number=40,
+            number=20,
             tokenizer_path='Qwen/Qwen2.5-0.5B-Instruct',
             seed=None,
-            debug= True,
+            extra_args={'ignore_eos': True}
         )
         run_perf_benchmark(task_cfg)
 

tests/rag/test_clip_benchmark.py

@@ -45,7 +45,6 @@ class TestCLIPBenchmark(unittest.TestCase):
                     'num_workers': 1,
                     'verbose': True,
                     'skip_existing': False,
-                    'output_dir': 'outputs',
                     'cache_dir': 'cache',
                     'limit': 1000,
                 },

tests/rag/test_mteb.py

@@ -1,8 +1,9 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-import subprocess
 import unittest
+from dotenv import dotenv_values
 
+env = dotenv_values('.env')
 from evalscope.run import run_task
 from evalscope.utils import is_module_installed, test_level_list
 from evalscope.utils.logger import get_logger
@@ -45,14 +46,13 @@ class TestMTEB(unittest.TestCase):
                 ],
                 'eval': {
                     'tasks': [
-                        'TNews',
-                        'CLSClusteringS2S',
+                        # 'TNews',
+                        # 'CLSClusteringS2S',
                         'T2Reranking',
-                        'T2Retrieval',
-                        'ATEC',
+                        # 'T2Retrieval',
+                        # 'ATEC',
                     ],
                     'verbosity': 2,
-                    'output_folder': 'outputs',
                     'overwrite_results': True,
                     'limits': 500,
                 },
@@ -61,6 +61,37 @@ class TestMTEB(unittest.TestCase):
 
         run_task(task_cfg)
 
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_one_stage_api(self):
+        from evalscope import TaskConfig
+        task_cfg = TaskConfig(
+            eval_backend='RAGEval',
+            eval_config={
+                'tool': 'MTEB',
+                'model': [
+                    {
+                        'model_name': 'text-embedding-v3',
+                        'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+                        'api_key': env.get('DASHSCOPE_API_KEY', 'EMPTY'),
+                        'dimensions': 1024,
+                        'encode_kwargs': {
+                            'batch_size': 10,
+                        },
+                    }
+                ],
+                'eval': {
+                    'tasks': [
+                        'T2Retrieval',
+                    ],
+                    'verbosity': 2,
+                    'overwrite_results': True,
+                    'limits': 30,
+                },
+            },
+        )
+
+        run_task(task_cfg)
+
     @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
     def test_run_two_stage_mteb(self):
         task_cfg = {
@@ -92,7 +123,6 @@ class TestMTEB(unittest.TestCase):
                 'eval': {
                     'tasks': ['MedicalRetrieval', 'T2Retrieval'],
                     'verbosity': 2,
-                    'output_folder': 'outputs',
                     'overwrite_results': True,
                     # 'limits': 10,
                     'top_k': 10,
@@ -124,7 +154,6 @@ class TestMTEB(unittest.TestCase):
                     'tasks': ['CustomRetrieval'],
                     'dataset_path': 'custom_eval/text/retrieval',
                     'verbosity': 2,
-                    'output_folder': 'outputs',
                     'overwrite_results': True,
                     'limits': 500,
                 },

tests/rag/test_ragas.py

@@ -5,7 +5,7 @@ from dotenv import dotenv_values
 env = dotenv_values('.env')
 import unittest
 
-from evalscope.run import run_task
+from evalscope import TaskConfig, run_task
 from evalscope.utils import is_module_installed, test_level_list
 from evalscope.utils.logger import get_logger
 
@@ -37,13 +37,10 @@ class TestRAGAS(unittest.TestCase):
                     'docs': ['README_zh.md'],
                     'test_size': 5,
                     'output_file': 'outputs/testset.json',
-                    'distribution': {
-                        'simple': 0.5,
-                        'multi_context': 0.4,
-                        'reasoning': 0.1,
-                    },
                     'generator_llm': {
-                        'model_name_or_path': 'qwen/Qwen2-7B-Instruct',
+                        'model_name': 'qwen-plus',  # 自定义聊天模型名称
+                        'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',  # 自定义基础URL
+                        'api_key': env.get('DASHSCOPE_API_KEY', 'EMPTY'),  # 自定义API密钥
                     },
                     'embeddings': {
                         'model_name_or_path': 'AI-ModelScope/m3e-base',
@@ -87,32 +84,38 @@ class TestRAGAS(unittest.TestCase):
 
     @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
     def test_run_rag_eval_api(self):
-        task_cfg = {
-            'eval_backend': 'RAGEval',
-            'eval_config': {
-                'tool': 'RAGAS',
-                'eval': {
-                    'testset_file':
-                    'outputs/testset.json',
-                    'critic_llm': {
-                        'model_name': 'gpt-4o-mini',  # 自定义聊天模型名称
-                        'api_base': 'http://127.0.0.1:8088/v1',  # 自定义基础URL
-                        'api_key': 'xxxx',  # 你的API密钥
+        from evalscope.backend.rag_eval.ragas.arguments import EvaluationArguments
+        task_cfg = TaskConfig(
+            eval_backend='RAGEval',
+            eval_config=dict(
+                tool='RAGAS',
+                eval=EvaluationArguments(
+                    testset_file='outputs/testset_chinese_with_answer_small.json',
+                    critic_llm={
+                        'model_name': 'qwen-plus',  # 自定义聊天模型名称
+                        'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',  # 自定义基础URL
+                        'api_key': env.get('DASHSCOPE_API_KEY', 'EMPTY'),  # 自定义API密钥
                     },
-                    'embeddings': {
-                        'model_name_or_path': 'AI-ModelScope/m3e-base',
+                    embeddings={
+                        'model_name': 'text-embedding-v1',
+                        'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+                        'api_key': env.get('DASHSCOPE_API_KEY', 'EMPTY'),
+                        'dimensions': 1024,
+                        'encode_kwargs': {
+                            'batch_size': 10,
+                        },
                     },
-                    'metrics': [
+                    metrics=[
                         'Faithfulness',
                         'AnswerRelevancy',
                         'ContextPrecision',
                         'AnswerCorrectness',
-                        'MultiModalFaithfulness',
-                        'MultiModalRelevance',
+                        # 'MultiModalFaithfulness',
+                        # 'MultiModalRelevance',
                     ],
-                },
-            },
-        }
+                ),
+            ),
+        )
 
         logger.info(f'>> Start to run task: {task_cfg}')
 

tests/vlm/test_vlmeval.py

@@ -1,6 +1,7 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
+from dotenv import dotenv_values
 
-import subprocess
+env = dotenv_values('.env')
 import unittest
 
 from evalscope.run import run_task
@@ -56,5 +57,40 @@ class TestVLMEval(unittest.TestCase):
         assert len(report_list) > 0, f'Failed to get report list: {report_list}'
 
 
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_vlm_api(self):
+        task_cfg = {
+            'eval_backend': 'VLMEvalKit',
+            'eval_config': {
+                'data': ['SEEDBench_IMG', 'ChartQA_TEST'],
+                'limit': 5,
+                'mode': 'all',
+                'model': [
+                    {'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions',
+                    'key': env.get('DASHSCOPE_API_KEY'),
+                    'name': 'CustomAPIModel',
+                    'temperature': 0.0,
+                    'type': 'qwen2.5-vl-7b-instruct',
+                    'img_size': -1,
+                    'video_llm': False,
+                    'max_tokens': 512,}
+                    ],
+                'nproc': 5,
+                'reuse': False,
+            },
+            'work_dir': 'outputs',
+            # 'use_cache': 'outputs/20241216_142838'
+        }
+
+        logger.info(f'>> Start to run task: {task_cfg}')
+
+        run_task(task_cfg)
+
+        logger.info('>> Start to get the report with summarizer ...')
+        report_list = Summarizer.get_report_from_cfg(task_cfg)
+        logger.info(f'\n>>The report list: {report_list}')
+
+        assert len(report_list) > 0, f'Failed to get report list: {report_list}'
+
 if __name__ == '__main__':
     unittest.main(buffer=False)

evalscope/backend/vlm_eval_kit/custom_dataset.py

@@ -1,46 +0,0 @@
-import numpy as np
-import os
-from vlmeval.dataset.image_base import ImageBaseDataset
-from vlmeval.dataset.image_vqa import CustomVQADataset
-from vlmeval.smp import d2df, dump, load
-
-
-class CustomDataset:
-
-    def load_data(self, dataset):
-        # customize the loading of the dataset
-        data_path = os.path.join(os.path.expanduser('~/LMUData'), f'{dataset}.tsv')
-        return load(data_path)
-
-    def build_prompt(self, line):
-        msgs = ImageBaseDataset.build_prompt(self, line)
-        # add a hint or custom instruction here
-        msgs[-1]['value'] += '\nAnswer the question using a single word or phrase.'
-        return msgs
-
-    def evaluate(self, eval_file, **judge_kwargs):
-        data = load(eval_file)
-        assert 'answer' in data and 'prediction' in data
-        data['prediction'] = [str(x) for x in data['prediction']]
-        data['answer'] = [str(x).lower() for x in data['answer']]
-
-        print(data)
-
-        # ========compute the evaluation metrics as you need =========
-        # exact match
-        result = np.mean(data['answer'] == data['prediction'])
-        ret = {'Overall': result}
-        ret = d2df(ret).round(2)
-
-        # save the result
-        suffix = eval_file.split('.')[-1]
-        result_file = eval_file.replace(f'.{suffix}', '_acc.csv')
-        dump(ret, result_file)
-        return ret
-        # ============================================================
-
-
-# override the default dataset class
-CustomVQADataset.load_data = CustomDataset.load_data
-CustomVQADataset.build_prompt = CustomDataset.build_prompt
-CustomVQADataset.evaluate = CustomDataset.evaluate

evalscope/benchmarks/live_code_bench/execute_utils.py

@@ -1,267 +0,0 @@
-# Copyright 2020 The HuggingFace Datasets Authors and the
-# current dataset script contributor.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This code is adapted from OpenAI's release
-# https://github.com/openai/human-eval/blob/master/human_eval/execution.py
-
-import contextlib
-import faulthandler
-import io
-import multiprocessing
-import os
-import platform
-import signal
-import tempfile
-
-BASE_IMPORTS = """from itertools import accumulate, chain, combinations, count, permutations, product, groupby, islice, repeat
-from copy import deepcopy
-from string import ascii_lowercase
-from math import floor, log2, log10, sqrt, comb, gcd, ceil, inf, isqrt
-from collections import defaultdict, deque, Counter
-from bisect import bisect, bisect_left, bisect_right, insort
-from heapq import heappush, heappop, heapify, merge
-from functools import reduce, cache, lru_cache
-from random import randrange, shuffle
-from operator import itemgetter, sub
-from re import search as re_search  # Assuming 're' refers to a regex search
-from os.path import commonprefix
-from typing import List, Tuple, Dict, Set, Optional, Union, Any, Callable, Iterable, Iterator, Generator
-import copy
-import string
-import math
-import collections
-import bisect
-import heapq
-import functools
-import random
-import itertools
-import operator
-import re
-import numpy as np
-import pandas as pd
-from math import log, prod  # 'log' and 'prod' are functions in the math module
-from collections import deque, defaultdict, Counter, OrderedDict
-from itertools import accumulate, permutations, combinations, product, groupby, islice, chain, repeat, zip_longest, cycle
-from functools import lru_cache, reduce, partial
-# from sortedcontainers import SortedList, SortedDict, SortedSet
-# import sortedcontainers
-from operator import iand
-import sys
-"""  # noqa: E501
-
-
-def codeexecute_check_correctness(check_program, timeout=3):
-    """Evaluates the functional correctness of a completion by running the test
-    suite provided in the problem.
-
-    :param completion_id: an optional completion ID so we can match
-        the results later even if execution finishes asynchronously.
-    """
-    manager = multiprocessing.Manager()
-    result = manager.list()
-
-    p = multiprocessing.Process(target=unsafe_execute, args=(check_program, result, timeout))
-    p.start()
-    p.join(timeout=timeout + 1)
-    if p.is_alive():
-        p.kill()
-
-    if not result:
-        result.append('timed out')
-
-    return result[0] == 'passed'
-
-
-def unsafe_execute(check_program, result, timeout):
-
-    with create_tempdir():
-
-        # These system calls are needed when cleaning up tempdir.
-        import os
-        import shutil
-
-        rmtree = shutil.rmtree
-        rmdir = os.rmdir
-        chdir = os.chdir
-
-        # Disable functionalities that can make destructive changes
-        # to the test.
-        reliability_guard()
-
-        # Run program.
-        try:
-            exec_globals = {}
-            with swallow_io():
-                with time_limit(timeout):
-                    exec(check_program, exec_globals)
-            result.append('passed')
-        except TimeoutException:
-            result.append('timed out')
-        except BaseException as e:
-            result.append(f'failed: {e}')
-
-        # Needed for cleaning up.
-        shutil.rmtree = rmtree
-        os.rmdir = rmdir
-        os.chdir = chdir
-
-
-@contextlib.contextmanager
-def time_limit(seconds):
-
-    def signal_handler(signum, frame):
-        raise TimeoutException('Timed out!')
-
-    signal.setitimer(signal.ITIMER_REAL, seconds)
-    signal.signal(signal.SIGALRM, signal_handler)
-    try:
-        yield
-    finally:
-        signal.setitimer(signal.ITIMER_REAL, 0)
-
-
-@contextlib.contextmanager
-def swallow_io():
-    stream = WriteOnlyStringIO()
-    with contextlib.redirect_stdout(stream):
-        with contextlib.redirect_stderr(stream):
-            with redirect_stdin(stream):
-                yield
-
-
-@contextlib.contextmanager
-def create_tempdir():
-    with tempfile.TemporaryDirectory() as dirname:
-        with chdir(dirname):
-            yield dirname
-
-
-class TimeoutException(Exception):
-    pass
-
-
-class WriteOnlyStringIO(io.StringIO):
-    """StringIO that throws an exception when it's read from."""
-
-    def read(self, *args, **kwargs):
-        raise OSError
-
-    def readline(self, *args, **kwargs):
-        raise OSError
-
-    def readlines(self, *args, **kwargs):
-        raise OSError
-
-    def readable(self, *args, **kwargs):
-        """Returns True if the IO object can be read."""
-        return False
-
-
-class redirect_stdin(contextlib._RedirectStream):  # type: ignore
-    _stream = 'stdin'
-
-
-@contextlib.contextmanager
-def chdir(root):
-    if root == '.':
-        yield
-        return
-    cwd = os.getcwd()
-    os.chdir(root)
-    try:
-        yield
-    except BaseException as exc:
-        raise exc
-    finally:
-        os.chdir(cwd)
-
-
-def reliability_guard(maximum_memory_bytes=None):
-    """This disables various destructive functions and prevents the generated
-    code from interfering with the test (e.g. fork bomb, killing other
-    processes, removing filesystem files, etc.)
-
-    WARNING This function is NOT a security sandbox. Untrusted code, including,
-    model- generated code, should not be blindly executed outside of one. See
-    the Codex paper for more information about OpenAI's code sandbox, and
-    proceed with caution.
-    """
-
-    if maximum_memory_bytes is not None:
-        import resource
-
-        resource.setrlimit(resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes))
-        resource.setrlimit(resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes))
-        if not platform.uname().system == 'Darwin':
-            resource.setrlimit(resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes))
-
-    faulthandler.disable()
-
-    import builtins
-
-    builtins.exit = None
-    builtins.quit = None
-
-    import os
-
-    os.environ['OMP_NUM_THREADS'] = '1'
-
-    os.kill = None
-    os.system = None
-    os.putenv = None
-    os.remove = None
-    os.removedirs = None
-    os.rmdir = None
-    os.fchdir = None
-    os.setuid = None
-    os.fork = None
-    os.forkpty = None
-    os.killpg = None
-    os.rename = None
-    os.renames = None
-    os.truncate = None
-    os.replace = None
-    os.unlink = None
-    os.fchmod = None
-    os.fchown = None
-    os.chmod = None
-    os.chown = None
-    os.chroot = None
-    os.fchdir = None
-    os.lchflags = None
-    os.lchmod = None
-    os.lchown = None
-    os.getcwd = None
-    os.chdir = None
-
-    import shutil
-
-    shutil.rmtree = None
-    shutil.move = None
-    shutil.chown = None
-
-    import subprocess
-
-    subprocess.Popen = None  # type: ignore
-
-    __builtins__['help'] = None
-
-    import sys
-
-    sys.modules['ipdb'] = None
-    sys.modules['joblib'] = None
-    sys.modules['resource'] = None
-    sys.modules['psutil'] = None
-    sys.modules['tkinter'] = None