evalscope 1.0.2__py3-none-any.whl → 1.1.1__py3-none-any.whl

tests/rag/test_clip_benchmark.py

@@ -1,87 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-import os
-import subprocess
-import unittest
-
-from evalscope.run import run_task
-from evalscope.utils.import_utils import is_module_installed
-from evalscope.utils.logger import get_logger
-from tests.utils import test_level_list
-
-logger = get_logger()
-
-
-class TestCLIPBenchmark(unittest.TestCase):
-
-    def setUp(self) -> None:
-        self._check_env('webdataset')
-
-    def tearDown(self) -> None:
-        pass
-
-    @staticmethod
-    def _check_env(module_name: str):
-        if is_module_installed(module_name):
-            logger.info(f'{module_name} is installed.')
-        else:
-            raise ModuleNotFoundError(f'run: pip install {module_name}')
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_task(self):
-        task_cfg = {
-            'eval_backend': 'RAGEval',
-            'eval_config': {
-                'tool': 'clip_benchmark',
-                'eval': {
-                    'models': [
-                        {
-                            'model_name': 'AI-ModelScope/chinese-clip-vit-large-patch14-336px',
-                        }
-                    ],
-                    'dataset_name': [
-                        'muge',
-                        'mnist',
-                        'flickr8k'
-                    ],
-                    'split': 'test',
-                    'batch_size': 128,
-                    'num_workers': 1,
-                    'verbose': True,
-                    'skip_existing': False,
-                    'cache_dir': 'cache',
-                    'limit': 1000,
-                },
-            },
-        }
-
-        run_task(task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_custom(self):
-        task_cfg = {
-            'eval_backend': 'RAGEval',
-            'eval_config': {
-                'tool': 'clip_benchmark',
-                'eval': {
-                    'models': [
-                        {
-                            'model_name': 'AI-ModelScope/chinese-clip-vit-large-patch14-336px',
-                        }
-                    ],
-                    'dataset_name': ['custom'],
-                    'data_dir': 'custom_eval/multimodal/text-image-retrieval',
-                    'split': 'test',
-                    'batch_size': 128,
-                    'num_workers': 1,
-                    'verbose': True,
-                    'skip_existing': False,
-                    'limit': 1000,
-                },
-            },
-        }
-
-        run_task(task_cfg)
-
-
-if __name__ == '__main__':
-    unittest.main(buffer=False)

tests/rag/test_mteb.py

@@ -1,213 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-
-import unittest
-from dotenv import dotenv_values
-
-from tests.utils import test_level_list
-
-env = dotenv_values('.env')
-from evalscope.run import run_task
-from evalscope.utils.import_utils import is_module_installed
-from evalscope.utils.logger import get_logger
-
-logger = get_logger()
-
-
-class TestMTEB(unittest.TestCase):
-
-    def setUp(self) -> None:
-        self._check_env('mteb')
-
-    def tearDown(self) -> None:
-        pass
-
-    @staticmethod
-    def _check_env(module_name: str):
-        if is_module_installed(module_name):
-            logger.info(f'{module_name} is installed.')
-        else:
-            raise ModuleNotFoundError(f'run: pip install {module_name}')
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_one_stage_mteb(self):
-        task_cfg = {
-            'eval_backend': 'RAGEval',
-            'eval_config': {
-                'tool': 'MTEB',
-                'model': [
-                    {
-                        'model_name_or_path': 'AI-ModelScope/m3e-base',
-                        'pooling_mode': None,  # load from model config
-                        'max_seq_length': 512,
-                        'prompt': '',
-                        'model_kwargs': {'torch_dtype': 'auto'},
-                        'encode_kwargs': {
-                            'batch_size': 128,
-                        },
-                    }
-                ],
-                'eval': {
-                    'tasks': [
-                        'TNews',
-                        'CLSClusteringS2S',
-                        'T2Reranking',
-                        'T2Retrieval',
-                        'ATEC',
-                    ],
-                    'verbosity': 2,
-                    'overwrite_results': True,
-                    'limits': 500,
-                },
-            },
-        }
-
-        run_task(task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_one_stage_api(self):
-        from evalscope import TaskConfig
-        task_cfg = TaskConfig(
-            eval_backend='RAGEval',
-            eval_config={
-                'tool': 'MTEB',
-                'model': [
-                    {
-                        'model_name': 'text-embedding-v3',
-                        'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-                        'api_key': env.get('DASHSCOPE_API_KEY', 'EMPTY'),
-                        'dimensions': 1024,
-                        'encode_kwargs': {
-                            'batch_size': 10,
-                        },
-                    }
-                ],
-                'eval': {
-                    'tasks': [
-                        'T2Retrieval',
-                    ],
-                    'verbosity': 2,
-                    'overwrite_results': True,
-                    'limits': 10,
-                },
-            },
-        )
-
-        run_task(task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_two_stage_mteb(self):
-        task_cfg = {
-            'eval_backend': 'RAGEval',
-            'eval_config': {
-                'tool': 'MTEB',
-                'model': [
-                    {
-                        'model_name_or_path': 'AI-ModelScope/m3e-base',
-                        'is_cross_encoder': False,
-                        'max_seq_length': 512,
-                        'prompt': '',
-                        'model_kwargs': {'torch_dtype': 'auto'},
-                        'encode_kwargs': {
-                            'batch_size': 64,
-                        },
-                    },
-                    {
-                        'model_name_or_path': 'BAAI/bge-reranker-v2-m3',
-                        'is_cross_encoder': True,
-                        'max_seq_length': 512,
-                        'prompt': '为这个问题生成一个检索用的表示',
-                        'model_kwargs': {'torch_dtype': 'auto'},
-                        'encode_kwargs': {
-                            'batch_size': 32,
-                        },
-                    },
-                ],
-                'eval': {
-                    'tasks': [
-                        'MedicalRetrieval',
-                        'T2Retrieval'
-                    ],
-                    'verbosity': 2,
-                    'overwrite_results': True,
-                    'limits': 10,
-                    'top_k': 10,
-                },
-            },
-        }
-
-        run_task(task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_two_stage_api(self):
-        task_cfg = {
-            'eval_backend': 'RAGEval',
-            'eval_config': {
-                'tool': 'MTEB',
-                'model': [
-                    {
-                        'model_name': 'text-embedding-v3',
-                        'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-                        'api_key': env.get('DASHSCOPE_API_KEY', 'EMPTY'),
-                        'dimensions': 1024,
-                        'encode_kwargs': {
-                            'batch_size': 10,
-                        },
-                    },
-                    {
-                        'model_name': 'text-embedding-v3',
-                        'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-                        'api_key': env.get('DASHSCOPE_API_KEY', 'EMPTY'),
-                        'dimensions': 1024,
-                        'encode_kwargs': {
-                            'batch_size': 10,
-                        },
-                    },
-                ],
-                'eval': {
-                    'tasks': [
-                        'MedicalRetrieval',
-                        # 'T2Retrieval'
-                    ],
-                    'verbosity': 2,
-                    'overwrite_results': True,
-                    'limits': 10,
-                    'top_k': 10,
-                },
-            },
-        }
-
-        run_task(task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_custom(self):
-        task_cfg = {
-            'eval_backend': 'RAGEval',
-            'eval_config': {
-                'tool': 'MTEB',
-                'model': [
-                    {
-                        'model_name_or_path': 'AI-ModelScope/m3e-base',
-                        'pooling_mode': None,  # load from model config
-                        'max_seq_length': 512,
-                        'prompt': '',
-                        'model_kwargs': {'torch_dtype': 'auto'},
-                        'encode_kwargs': {
-                            'batch_size': 128,
-                        },
-                    }
-                ],
-                'eval': {
-                    'tasks': ['CustomRetrieval'],
-                    'dataset_path': 'custom_eval/text/retrieval',
-                    'verbosity': 2,
-                    'overwrite_results': True,
-                    'limits': 500,
-                },
-            },
-        }
-
-        run_task(task_cfg)
-
-
-if __name__ == '__main__':
-    unittest.main(buffer=False)

tests/rag/test_ragas.py

@@ -1,128 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-import os
-from dotenv import dotenv_values
-
-from tests.utils import test_level_list
-
-env = dotenv_values('.env')
-import unittest
-
-from evalscope import TaskConfig, run_task
-from evalscope.utils.import_utils import is_module_installed
-from evalscope.utils.logger import get_logger
-
-logger = get_logger()
-
-
-class TestRAGAS(unittest.TestCase):
-
-    def setUp(self) -> None:
-        self._check_env('ragas')
-
-    def tearDown(self) -> None:
-        pass
-
-    @staticmethod
-    def _check_env(module_name: str):
-        if is_module_installed(module_name):
-            logger.info(f'{module_name} is installed.')
-        else:
-            raise ModuleNotFoundError(f'run: pip install {module_name}')
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_generate_dataset(self):
-        task_cfg = {
-            'eval_backend': 'RAGEval',
-            'eval_config': {
-                'tool': 'RAGAS',
-                'testset_generation': {
-                    'docs': ['README_zh.md'],
-                    'test_size': 5,
-                    'output_file': 'outputs/testset.json',
-                    'generator_llm': {
-                        'model_name': 'qwen-plus',  # 自定义聊天模型名称
-                        'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',  # 自定义基础URL
-                        'api_key': env.get('DASHSCOPE_API_KEY', 'EMPTY'),  # 自定义API密钥
-                    },
-                    'embeddings': {
-                        'model_name_or_path': 'AI-ModelScope/m3e-base',
-                    },
-                    'language': 'chinese',
-                },
-            },
-        }
-
-        logger.info(f'>> Start to run task: {task_cfg}')
-
-        run_task(task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_rag_eval(self):
-        task_cfg = {
-            'eval_backend': 'RAGEval',
-            'eval_config': {
-                'tool': 'RAGAS',
-                'eval': {
-                    'testset_file': 'outputs/testset_chinese_with_answer.json',
-                    'critic_llm': {
-                        'model_name_or_path': 'Qwen/Qwen2.5-7B-Instruct',
-                    },
-                    'embeddings': {
-                        'model_name_or_path': 'AI-ModelScope/m3e-base',
-                    },
-                    'metrics': [
-                        'Faithfulness',
-                        'AnswerRelevancy',
-                        'ContextPrecision',
-                        'AnswerCorrectness',
-                    ],
-                },
-            },
-        }
-
-        logger.info(f'>> Start to run task: {task_cfg}')
-
-        run_task(task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_rag_eval_api(self):
-        from evalscope.backend.rag_eval.ragas.arguments import EvaluationArguments
-        task_cfg = TaskConfig(
-            eval_backend='RAGEval',
-            eval_config=dict(
-                tool='RAGAS',
-                eval=EvaluationArguments(
-                    testset_file='outputs/testset_chinese_with_answer_small.json',
-                    critic_llm={
-                        'model_name': 'qwen-plus',  # 自定义聊天模型名称
-                        'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',  # 自定义基础URL
-                        'api_key': env.get('DASHSCOPE_API_KEY', 'EMPTY'),  # 自定义API密钥
-                    },
-                    embeddings={
-                        'model_name': 'text-embedding-v1',
-                        'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-                        'api_key': env.get('DASHSCOPE_API_KEY', 'EMPTY'),
-                        'dimensions': 1024,
-                        'encode_kwargs': {
-                            'batch_size': 10,
-                        },
-                    },
-                    metrics=[
-                        'Faithfulness',
-                        'AnswerRelevancy',
-                        'ContextPrecision',
-                        'AnswerCorrectness',
-                        # 'MultiModalFaithfulness',
-                        # 'MultiModalRelevance',
-                    ],
-                ),
-            ),
-        )
-
-        logger.info(f'>> Start to run task: {task_cfg}')
-
-        run_task(task_cfg)
-
-
-if __name__ == '__main__':
-    unittest.main(buffer=False)

tests/swift/__init__.py

@@ -1 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.

tests/swift/test_run_swift_eval.py

@@ -1,146 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-
-import json
-import os
-import requests
-import subprocess
-import time
-import unittest
-
-from evalscope.backend.opencompass import OpenCompassBackendManager
-from evalscope.run import run_task
-from evalscope.summarizer import Summarizer
-from evalscope.utils.import_utils import is_module_installed
-from evalscope.utils.logger import get_logger
-from tests.utils import test_level_list
-
-logger = get_logger(__name__)
-
-DEFAULT_CHAT_MODEL_URL = 'http://127.0.0.1:8000/v1/chat/completions'
-DEFAULT_BASE_MODEL_URL = 'http://127.0.0.1:8001/v1/completions'
-
-
-class TestRunSwiftEval(unittest.TestCase):
-
-    def setUp(self) -> None:
-        logger.info('Init env for swift-eval UTs ...\n')
-
-        self.model_name = 'qwen2_5-0_5b-instruct'
-        assert is_module_installed('evalscope'), 'Please install `evalscope` from pypi or source code.'
-
-        if not is_module_installed('opencompass'):
-            logger.warning('Note: installing ms-opencompass ...')
-            subprocess.run('pip3 install ms-opencompass -U', shell=True, check=True)
-
-        if not is_module_installed('swift'):
-            logger.warning('Note: installing ms-swift ...')
-            subprocess.run('pip3 install ms-swift[llm]', shell=True, check=True)
-
-        logger.warning('vllm not installed, use native swift deploy service instead.')
-
-        logger.info('\nStaring run swift deploy ...')
-        self.process_swift_deploy = subprocess.Popen(
-            f'swift deploy --model_type {self.model_name} --infer_backend pt', text=True, shell=True)
-        if self.process_swift_deploy.stderr:
-            logger.info(f'swift deploy log info: {self.process_swift_deploy.stderr}')
-
-        self.all_datasets = OpenCompassBackendManager.list_datasets()
-        assert len(self.all_datasets) > 0, f'Failed to list datasets from OpenCompass backend: {self.all_datasets}'
-
-    def tearDown(self) -> None:
-        # Stop the swift deploy model service
-        logger.warning('\nStopping swift deploy ...')
-        self.process_swift_deploy.terminate()
-        self.process_swift_deploy.wait()
-        logger.info('Process swift-deploy terminated successfully.')
-
-    @staticmethod
-    def find_and_kill_pid(pids: list):
-        if len(pids) > 0:
-            for pid in pids:
-                subprocess.run(['kill', str(pid)])
-                logger.warning(f'Killed process {pid}.')
-        else:
-            logger.info('No pids found.')
-
-    @staticmethod
-    def find_and_kill_service(service_name):
-        try:
-            # find pid
-            result = subprocess.run(['ps', '-ef'], stdout=subprocess.PIPE, text=True)
-
-            lines = result.stdout.splitlines()
-            pids = []
-            for line in lines:
-                if service_name in line and 'grep' not in line:
-                    parts = line.split()
-                    pid = parts[1]
-                    pids.append(pid)
-
-            if not pids:
-                logger.info(f'No process found for {service_name}.')
-            else:
-                for pid in pids:
-                    subprocess.run(['kill', pid])
-                    logger.warning(f'Killed process {pid} for service {service_name}.')
-        except Exception as e:
-            logger.error(f'An error occurred: {e}')
-
-    @staticmethod
-    def check_service_status(url: str, data: dict, retries: int = 30, delay: int = 10):
-        for i in range(retries):
-            try:
-                logger.info(f'Attempt {i + 1}: Checking service at {url} ...')
-                response = requests.post(
-                    url, data=json.dumps(data), headers={'Content-Type': 'application/json'}, timeout=30)
-                if response.status_code == 200:
-                    logger.info(f'Service at {url} is available !\n\n')
-                    return True
-                else:
-                    logger.info(f'Service at {url} returned status code {response.status_code}.')
-            except requests.exceptions.RequestException as e:
-                logger.info(f'Attempt {i + 1}: An error occurred: {e}')
-
-            time.sleep(delay)
-
-        logger.info(f'Service at {url} is not available after {retries} retries.')
-        return False
-
-    @unittest.skipUnless(1 in test_level_list(), 'skip test in current test level')
-    def test_run_task(self):
-        # Prepare the config
-        task_cfg = dict(
-            eval_backend='OpenCompass',
-            eval_config={
-                'datasets': ['cmb', 'bbh', 'ceval', 'ARC_e', 'gsm8k'],
-                'models': [
-                    {
-                        'path': self.model_name,
-                        'openai_api_base': DEFAULT_CHAT_MODEL_URL,
-                        'batch_size': 8
-                    },
-                ],
-                'work_dir': 'outputs/llama3_eval_result',
-                'reuse': None,  # string, `latest` or timestamp, e.g. `20230516_144254`, default to None
-                'limit': 5,  # string or int or float, e.g. `[2:5]`, 5, 5.0, default to None, it means run all examples
-            },
-        )
-
-        # Check the service status
-        data = {'model': self.model_name, 'messages': [{'role': 'user', 'content': 'who are you?'}]}
-        assert self.check_service_status(
-            DEFAULT_CHAT_MODEL_URL, data=data), f'Failed to check service status: {DEFAULT_CHAT_MODEL_URL}'
-
-        # Submit the task
-        logger.info(f'Start to run UT with cfg: {task_cfg}')
-        run_task(task_cfg=task_cfg)
-
-        # Get the final report with summarizer
-        report_list = Summarizer.get_report_from_cfg(task_cfg)
-        logger.info(f'>>The report list:\n{report_list}')
-
-        assert len(report_list) > 0, f'Failed to get report list: {report_list}'
-
-
-if __name__ == '__main__':
-    unittest.main()