evalscope 1.0.1__py3-none-any.whl → 1.1.0__py3-none-any.whl

tests/rag/test_ragas.py

@@ -1,128 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-import os
-from dotenv import dotenv_values
-
-from tests.utils import test_level_list
-
-env = dotenv_values('.env')
-import unittest
-
-from evalscope import TaskConfig, run_task
-from evalscope.utils.import_utils import is_module_installed
-from evalscope.utils.logger import get_logger
-
-logger = get_logger()
-
-
-class TestRAGAS(unittest.TestCase):
-
-    def setUp(self) -> None:
-        self._check_env('ragas')
-
-    def tearDown(self) -> None:
-        pass
-
-    @staticmethod
-    def _check_env(module_name: str):
-        if is_module_installed(module_name):
-            logger.info(f'{module_name} is installed.')
-        else:
-            raise ModuleNotFoundError(f'run: pip install {module_name}')
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_generate_dataset(self):
-        task_cfg = {
-            'eval_backend': 'RAGEval',
-            'eval_config': {
-                'tool': 'RAGAS',
-                'testset_generation': {
-                    'docs': ['README_zh.md'],
-                    'test_size': 5,
-                    'output_file': 'outputs/testset.json',
-                    'generator_llm': {
-                        'model_name': 'qwen-plus',  # 自定义聊天模型名称
-                        'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',  # 自定义基础URL
-                        'api_key': env.get('DASHSCOPE_API_KEY', 'EMPTY'),  # 自定义API密钥
-                    },
-                    'embeddings': {
-                        'model_name_or_path': 'AI-ModelScope/m3e-base',
-                    },
-                    'language': 'chinese',
-                },
-            },
-        }
-
-        logger.info(f'>> Start to run task: {task_cfg}')
-
-        run_task(task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_rag_eval(self):
-        task_cfg = {
-            'eval_backend': 'RAGEval',
-            'eval_config': {
-                'tool': 'RAGAS',
-                'eval': {
-                    'testset_file': 'outputs/testset_chinese_with_answer.json',
-                    'critic_llm': {
-                        'model_name_or_path': 'Qwen/Qwen2.5-7B-Instruct',
-                    },
-                    'embeddings': {
-                        'model_name_or_path': 'AI-ModelScope/m3e-base',
-                    },
-                    'metrics': [
-                        'Faithfulness',
-                        'AnswerRelevancy',
-                        'ContextPrecision',
-                        'AnswerCorrectness',
-                    ],
-                },
-            },
-        }
-
-        logger.info(f'>> Start to run task: {task_cfg}')
-
-        run_task(task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_rag_eval_api(self):
-        from evalscope.backend.rag_eval.ragas.arguments import EvaluationArguments
-        task_cfg = TaskConfig(
-            eval_backend='RAGEval',
-            eval_config=dict(
-                tool='RAGAS',
-                eval=EvaluationArguments(
-                    testset_file='outputs/testset_chinese_with_answer_small.json',
-                    critic_llm={
-                        'model_name': 'qwen-plus',  # 自定义聊天模型名称
-                        'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',  # 自定义基础URL
-                        'api_key': env.get('DASHSCOPE_API_KEY', 'EMPTY'),  # 自定义API密钥
-                    },
-                    embeddings={
-                        'model_name': 'text-embedding-v1',
-                        'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-                        'api_key': env.get('DASHSCOPE_API_KEY', 'EMPTY'),
-                        'dimensions': 1024,
-                        'encode_kwargs': {
-                            'batch_size': 10,
-                        },
-                    },
-                    metrics=[
-                        'Faithfulness',
-                        'AnswerRelevancy',
-                        'ContextPrecision',
-                        'AnswerCorrectness',
-                        # 'MultiModalFaithfulness',
-                        # 'MultiModalRelevance',
-                    ],
-                ),
-            ),
-        )
-
-        logger.info(f'>> Start to run task: {task_cfg}')
-
-        run_task(task_cfg)
-
-
-if __name__ == '__main__':
-    unittest.main(buffer=False)

tests/swift/__init__.py

@@ -1 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.

tests/swift/test_run_swift_eval.py

@@ -1,146 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-
-import json
-import os
-import requests
-import subprocess
-import time
-import unittest
-
-from evalscope.backend.opencompass import OpenCompassBackendManager
-from evalscope.run import run_task
-from evalscope.summarizer import Summarizer
-from evalscope.utils.import_utils import is_module_installed
-from evalscope.utils.logger import get_logger
-from tests.utils import test_level_list
-
-logger = get_logger(__name__)
-
-DEFAULT_CHAT_MODEL_URL = 'http://127.0.0.1:8000/v1/chat/completions'
-DEFAULT_BASE_MODEL_URL = 'http://127.0.0.1:8001/v1/completions'
-
-
-class TestRunSwiftEval(unittest.TestCase):
-
-    def setUp(self) -> None:
-        logger.info('Init env for swift-eval UTs ...\n')
-
-        self.model_name = 'qwen2_5-0_5b-instruct'
-        assert is_module_installed('evalscope'), 'Please install `evalscope` from pypi or source code.'
-
-        if not is_module_installed('opencompass'):
-            logger.warning('Note: installing ms-opencompass ...')
-            subprocess.run('pip3 install ms-opencompass -U', shell=True, check=True)
-
-        if not is_module_installed('swift'):
-            logger.warning('Note: installing ms-swift ...')
-            subprocess.run('pip3 install ms-swift[llm]', shell=True, check=True)
-
-        logger.warning('vllm not installed, use native swift deploy service instead.')
-
-        logger.info('\nStaring run swift deploy ...')
-        self.process_swift_deploy = subprocess.Popen(
-            f'swift deploy --model_type {self.model_name} --infer_backend pt', text=True, shell=True)
-        if self.process_swift_deploy.stderr:
-            logger.info(f'swift deploy log info: {self.process_swift_deploy.stderr}')
-
-        self.all_datasets = OpenCompassBackendManager.list_datasets()
-        assert len(self.all_datasets) > 0, f'Failed to list datasets from OpenCompass backend: {self.all_datasets}'
-
-    def tearDown(self) -> None:
-        # Stop the swift deploy model service
-        logger.warning('\nStopping swift deploy ...')
-        self.process_swift_deploy.terminate()
-        self.process_swift_deploy.wait()
-        logger.info('Process swift-deploy terminated successfully.')
-
-    @staticmethod
-    def find_and_kill_pid(pids: list):
-        if len(pids) > 0:
-            for pid in pids:
-                subprocess.run(['kill', str(pid)])
-                logger.warning(f'Killed process {pid}.')
-        else:
-            logger.info('No pids found.')
-
-    @staticmethod
-    def find_and_kill_service(service_name):
-        try:
-            # find pid
-            result = subprocess.run(['ps', '-ef'], stdout=subprocess.PIPE, text=True)
-
-            lines = result.stdout.splitlines()
-            pids = []
-            for line in lines:
-                if service_name in line and 'grep' not in line:
-                    parts = line.split()
-                    pid = parts[1]
-                    pids.append(pid)
-
-            if not pids:
-                logger.info(f'No process found for {service_name}.')
-            else:
-                for pid in pids:
-                    subprocess.run(['kill', pid])
-                    logger.warning(f'Killed process {pid} for service {service_name}.')
-        except Exception as e:
-            logger.error(f'An error occurred: {e}')
-
-    @staticmethod
-    def check_service_status(url: str, data: dict, retries: int = 30, delay: int = 10):
-        for i in range(retries):
-            try:
-                logger.info(f'Attempt {i + 1}: Checking service at {url} ...')
-                response = requests.post(
-                    url, data=json.dumps(data), headers={'Content-Type': 'application/json'}, timeout=30)
-                if response.status_code == 200:
-                    logger.info(f'Service at {url} is available !\n\n')
-                    return True
-                else:
-                    logger.info(f'Service at {url} returned status code {response.status_code}.')
-            except requests.exceptions.RequestException as e:
-                logger.info(f'Attempt {i + 1}: An error occurred: {e}')
-
-            time.sleep(delay)
-
-        logger.info(f'Service at {url} is not available after {retries} retries.')
-        return False
-
-    @unittest.skipUnless(1 in test_level_list(), 'skip test in current test level')
-    def test_run_task(self):
-        # Prepare the config
-        task_cfg = dict(
-            eval_backend='OpenCompass',
-            eval_config={
-                'datasets': ['cmb', 'bbh', 'ceval', 'ARC_e', 'gsm8k'],
-                'models': [
-                    {
-                        'path': self.model_name,
-                        'openai_api_base': DEFAULT_CHAT_MODEL_URL,
-                        'batch_size': 8
-                    },
-                ],
-                'work_dir': 'outputs/llama3_eval_result',
-                'reuse': None,  # string, `latest` or timestamp, e.g. `20230516_144254`, default to None
-                'limit': 5,  # string or int or float, e.g. `[2:5]`, 5, 5.0, default to None, it means run all examples
-            },
-        )
-
-        # Check the service status
-        data = {'model': self.model_name, 'messages': [{'role': 'user', 'content': 'who are you?'}]}
-        assert self.check_service_status(
-            DEFAULT_CHAT_MODEL_URL, data=data), f'Failed to check service status: {DEFAULT_CHAT_MODEL_URL}'
-
-        # Submit the task
-        logger.info(f'Start to run UT with cfg: {task_cfg}')
-        run_task(task_cfg=task_cfg)
-
-        # Get the final report with summarizer
-        report_list = Summarizer.get_report_from_cfg(task_cfg)
-        logger.info(f'>>The report list:\n{report_list}')
-
-        assert len(report_list) > 0, f'Failed to get report list: {report_list}'
-
-
-if __name__ == '__main__':
-    unittest.main()

tests/swift/test_run_swift_vlm_eval.py

@@ -1,128 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-import json
-import os
-import requests
-import shutil
-import subprocess
-import time
-import unittest
-
-from evalscope.backend.vlm_eval_kit import VLMEvalKitBackendManager
-from evalscope.run import run_task
-from evalscope.summarizer import Summarizer
-from evalscope.utils.import_utils import is_module_installed
-from evalscope.utils.logger import get_logger
-from tests.utils import test_level_list
-
-logger = get_logger(__name__)
-
-DEFAULT_CHAT_MODEL_URL = 'http://127.0.0.1:8000/v1/chat/completions'
-DEFAULT_API_KEY = 'EMPTY'
-DEFAULT_MODEL_NAME = 'CustomAPIModel'
-DEFAULT_WORK_DIR = 'outputs/qwen-vl-chat'
-
-
-class TestRunSwiftVLMEval(unittest.TestCase):
-
-    def setUp(self) -> None:
-        logger.info('Init env for swift-eval UTs ...\n')
-        assert is_module_installed('evalscope'), 'Please install `llmuses` from pypi or source code.'
-
-        if not is_module_installed('vlmeval'):
-            logger.warning('Note: installing ms-vlmeval ...')
-            subprocess.run('pip3 install ms-vlmeval -U', shell=True, check=True)
-
-        if not is_module_installed('swift'):
-            logger.warning('Note: installing ms-swift ...')
-            subprocess.run('pip3 install ms-swift -U', shell=True, check=True)
-
-        if os.path.exists(DEFAULT_WORK_DIR):
-            shutil.rmtree(DEFAULT_WORK_DIR)
-            logger.info(f'Removed work dir: {os.path.abspath(DEFAULT_WORK_DIR)} \n')
-
-        logger.info('\nStaring run swift deploy ...')
-        self.model_name = 'qwen-vl-chat'
-        self.process_swift_deploy = subprocess.Popen(
-            f'swift deploy --model_type {self.model_name} --infer_backend pt', text=True, shell=True)
-
-        self.all_datasets = VLMEvalKitBackendManager.list_supported_datasets()
-        assert len(self.all_datasets) > 0, f'Failed to list datasets from VLMEvalKit backend: {self.all_datasets}'
-
-    def tearDown(self) -> None:
-        # Stop the swift deploy model service
-        logger.warning('Stopping swift deploy ...')
-        self.process_swift_deploy.terminate()
-        self.process_swift_deploy.wait()
-        logger.info('Process swift-deploy terminated successfully.')
-
-    @staticmethod
-    def _check_env(module_name: str):
-        if is_module_installed(module_name):
-            logger.info(f'{module_name} is installed.')
-        else:
-            raise ModuleNotFoundError(f'run: pip install {module_name}')
-
-    @staticmethod
-    def check_service_status(url: str, data: dict, retries: int = 20, delay: int = 10):
-        for i in range(retries):
-            try:
-                logger.info(f'Attempt {i + 1}: Checking service at {url} ...')
-                response = requests.post(
-                    url, data=json.dumps(data), headers={'Content-Type': 'application/json'}, timeout=30)
-                if response.status_code == 200:
-                    logger.info(f'Service at {url} is available !\n\n')
-                    return True
-                else:
-                    logger.info(f'Service at {url} returned status code {response.status_code}.')
-            except requests.exceptions.RequestException as e:
-                logger.info(f'Attempt {i + 1}: An error occurred: {e}')
-
-            time.sleep(delay)
-
-        logger.info(f'Service at {url} is not available after {retries} retries.')
-        return False
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_api(self):
-        api_base = DEFAULT_CHAT_MODEL_URL
-        task_cfg = {
-            'eval_backend': 'VLMEvalKit',
-            'eval_config': {
-                'data': ['SEEDBench_IMG', 'ChartQA_TEST'],
-                'limit':
-                30,
-                'mode':
-                'all',
-                'model': [{
-                    'api_base': api_base,  # swfit deploy model api
-                    'key': DEFAULT_API_KEY,
-                    'name': DEFAULT_MODEL_NAME,  # must be CustomAPIModel for swift
-                    'temperature': 0.0,
-                    'type': self.model_name
-                }],  # swift model type
-                'nproc':
-                1,
-                'reuse':
-                True,
-                'work_dir':
-                DEFAULT_WORK_DIR
-            }
-        }
-
-        # Check the service status
-        data = {'model': self.model_name, 'messages': [{'role': 'user', 'content': 'who are you?'}]}
-        assert self.check_service_status(api_base, data=data), f'Failed to check service status: {api_base}'
-
-        logger.info(f'>> Start to run task: {task_cfg}')
-
-        run_task(task_cfg)
-
-        logger.info('>> Start to get the report with summarizer ...')
-        report_list = Summarizer.get_report_from_cfg(task_cfg)
-        logger.info(f'\n>> The report list: {report_list}')
-
-        assert len(report_list) > 0, f'Failed to get report list: {report_list}'
-
-
-if __name__ == '__main__':
-    unittest.main(buffer=False)

tests/swift/test_run_swift_vlm_jugde_eval.py

@@ -1,157 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-import json
-import os
-import requests
-import shutil
-import subprocess
-import time
-import unittest
-
-from evalscope.backend.vlm_eval_kit import VLMEvalKitBackendManager
-from evalscope.run import run_task
-from evalscope.summarizer import Summarizer
-from evalscope.utils.import_utils import is_module_installed
-from evalscope.utils.logger import get_logger
-from tests.utils import test_level_list
-
-logger = get_logger(__name__)
-
-DEFAULT_CHAT_MODEL_URL = 'http://127.0.0.1:8000/v1/chat/completions'
-DEFAULT_JUDGE_MODEL_URL = 'http://127.0.0.1:8866/v1/chat/completions'
-
-DEFAULT_API_KEY = 'EMPTY'
-DEFAULT_MODEL_NAME = 'CustomAPIModel'
-DEFAULT_WORK_DIR = 'outputs'
-
-
-class TestRunSwiftVLMEval(unittest.TestCase):
-
-    def setUp(self) -> None:
-        logger.info('Init env for swift-eval UTs ...\n')
-        assert is_module_installed('llmuses'), 'Please install `llmuses` from pypi or source code.'
-
-        logger.warning('Note: installing ms-vlmeval ...')
-        subprocess.run('pip3 install ms-vlmeval -U', shell=True, check=True)
-
-        logger.warning('Note: installing ms-swift ...')
-        subprocess.run('pip3 install ms-swift -U', shell=True, check=True)
-
-        if os.path.exists(DEFAULT_WORK_DIR):
-            shutil.rmtree(DEFAULT_WORK_DIR)
-            logger.info(f'Removed work dir: {os.path.abspath(DEFAULT_WORK_DIR)} \n')
-
-        logger.info('\nStaring run swift deploy ...')
-        self.model_name = 'qwen-vl-chat'
-        self.process_swift_deploy = subprocess.Popen(
-            f'swift deploy --model_type {self.model_name}',
-            text=True,
-            shell=True,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE)
-
-        logger.info('\nStaring run swift deploy judge ...')
-        self.judge_model_name = 'qwen2-7b-instruct'
-        self.process_swift_deploy_judge = subprocess.Popen(
-            f'swift deploy --model_type {self.judge_model_name} \
-                                                    --port 8866',
-            text=True,
-            shell=True,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE)
-
-        self.all_datasets = VLMEvalKitBackendManager.list_supported_datasets()
-        assert len(self.all_datasets) > 0, f'Failed to list datasets from VLMEvalKit backend: {self.all_datasets}'
-
-    def tearDown(self) -> None:
-        # Stop the swift deploy model service
-        logger.warning('\nStopping swift deploy ...')
-        self.process_swift_deploy.terminate()
-        self.process_swift_deploy.wait()
-
-        self.process_swift_deploy_judge.terminate()
-        self.process_swift_deploy_judge.wait()
-
-        logger.info('Process swift-deploy terminated successfully.')
-
-    @staticmethod
-    def _check_env(module_name: str):
-        if is_module_installed(module_name):
-            logger.info(f'{module_name} is installed.')
-        else:
-            raise ModuleNotFoundError(f'run: pip install {module_name}')
-
-    @staticmethod
-    def check_service_status(url: str, data: dict, retries: int = 20, delay: int = 10):
-        for i in range(retries):
-            try:
-                logger.info(f'Attempt {i + 1}: Checking service at {url} ...')
-                response = requests.post(
-                    url, data=json.dumps(data), headers={'Content-Type': 'application/json'}, timeout=30)
-                if response.status_code == 200:
-                    logger.info(f'Service at {url} is available !\n\n')
-                    return True
-                else:
-                    logger.info(f'Service at {url} returned status code {response.status_code}.')
-            except requests.exceptions.RequestException as e:
-                logger.info(f'Attempt {i + 1}: An error occurred: {e}')
-
-            time.sleep(delay)
-
-        logger.info(f'Service at {url} is not available after {retries} retries.')
-        return False
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_api_with_judge(self):
-        # Check the judge service status
-        data = {'model': self.judge_model_name, 'messages': [{'role': 'user', 'content': 'who are you?'}]}
-        assert self.check_service_status(DEFAULT_JUDGE_MODEL_URL, data=data), \
-            f'Failed to check judge service status: {DEFAULT_JUDGE_MODEL_URL}'
-
-        api_base = DEFAULT_CHAT_MODEL_URL
-        task_cfg = {
-            'eval_backend': 'VLMEvalKit',
-            'eval_config': {
-                'LOCAL_LLM':
-                self.judge_model_name,  # judge model id
-                'OPENAI_API_BASE':
-                DEFAULT_JUDGE_MODEL_URL,  # judge model api
-                'OPENAI_API_KEY':
-                DEFAULT_API_KEY,
-                'data': ['SEEDBench_IMG', 'ChartQA_TEST'],
-                'limit':
-                20,
-                'mode':
-                'all',
-                'model': [{
-                    'api_base': api_base,  # swfit deploy model api
-                    'key': DEFAULT_API_KEY,
-                    'name': DEFAULT_MODEL_NAME,  # must be CustomAPIModel for swift
-                    'temperature': 0.0,
-                    'type': self.model_name
-                }],  # swift model type
-                'nproc':
-                1,
-                'reuse':
-                True,
-                'work_dir':
-                DEFAULT_WORK_DIR
-            }
-        }
-
-        # Check the service status
-        data = {'model': self.model_name, 'messages': [{'role': 'user', 'content': 'who are you?'}]}
-        assert self.check_service_status(api_base, data=data), f'Failed to check service status: {api_base}'
-
-        logger.info(f'>> Start to run task: {task_cfg}')
-
-        run_task(task_cfg)
-
-        logger.info('>> Start to get the report with summarizer ...')
-        report_list = Summarizer.get_report_from_cfg(task_cfg)
-        logger.info(f'\n>> The report list: {report_list}')
-
-        assert len(report_list) > 0, f'Failed to get report list: {report_list}'
-
-
-if __name__ == '__main__':
-    unittest.main(buffer=False)

tests/test_run_all.py

@@ -1,12 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-
-import subprocess
-
-if __name__ == '__main__':
-    cmd = f'TEST_LEVEL_LIST=0,1 python3 -m unittest discover tests'
-    run_res = subprocess.run(cmd, text=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-
-    if run_res.returncode == 0:
-        print(f'>>test_run_all stdout: {run_res.stdout}')
-    else:
-        print(f'>>test_run_all stderr: {run_res.stderr}')

tests/utils.py

@@ -1,13 +0,0 @@
-import os
-
-TEST_LEVEL_LIST = [0, 1]
-# Example: export TEST_LEVEL_LIST=0,1
-TEST_LEVEL_LIST_STR = 'TEST_LEVEL_LIST'
-
-
-def test_level_list():
-    global TEST_LEVEL_LIST
-    if TEST_LEVEL_LIST_STR in os.environ:
-        TEST_LEVEL_LIST = [int(x) for x in os.environ[TEST_LEVEL_LIST_STR].split(',')]
-
-    return TEST_LEVEL_LIST

tests/vlm/__init__.py

@@ -1 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.