evalscope 0.6.1__py3-none-any.whl → 0.7.1__py3-none-any.whl

tests/rag/test_mteb.py

@@ -0,0 +1,136 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import subprocess
+import unittest
+from evalscope.utils import test_level_list, is_module_installed
+from evalscope.utils.logger import get_logger
+from evalscope.run import run_task
+
+logger = get_logger()
+
+
+class TestMTEB(unittest.TestCase):
+
+    def setUp(self) -> None:
+        self._check_env('mteb')
+
+    def tearDown(self) -> None:
+        pass
+
+    @staticmethod
+    def _check_env(module_name: str):
+        if is_module_installed(module_name):
+            logger.info(f'{module_name} is installed.')
+        else:
+            raise ModuleNotFoundError(f'run: pip install {module_name}')
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_one_stage_mteb(self):
+        task_cfg = {
+            'eval_backend': 'RAGEval',
+            'eval_config': {
+                'tool': 'MTEB',
+                'model': [
+                    {
+                        'model_name_or_path': 'AI-ModelScope/m3e-base',
+                        'pooling_mode': None,  # load from model config
+                        'max_seq_length': 512,
+                        'prompt': '',
+                        'model_kwargs': {'torch_dtype': 'auto'},
+                        'encode_kwargs': {
+                            'batch_size': 128,
+                        },
+                    }
+                ],
+                'eval': {
+                    'tasks': [
+                        'TNews',
+                        'CLSClusteringS2S',
+                        'T2Reranking',
+                        'T2Retrieval',
+                        'ATEC',
+                    ],
+                    'verbosity': 2,
+                    'output_folder': 'outputs',
+                    'overwrite_results': True,
+                    'limits': 500,
+                },
+            },
+        }
+
+        run_task(task_cfg)
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_two_stage_mteb(self):
+        task_cfg = {
+            'eval_backend': 'RAGEval',
+            'eval_config': {
+                'tool': 'MTEB',
+                'model': [
+                    {
+                        'model_name_or_path': 'AI-ModelScope/m3e-base',
+                        'is_cross_encoder': False,
+                        'max_seq_length': 512,
+                        'prompt': '',
+                        'model_kwargs': {'torch_dtype': 'auto'},
+                        'encode_kwargs': {
+                            'batch_size': 64,
+                        },
+                    },
+                    {
+                        'model_name_or_path': 'OpenBMB/MiniCPM-Reranker',
+                        'is_cross_encoder': True,
+                        'max_seq_length': 512,
+                        'prompt': '为这个问题生成一个检索用的表示',
+                        'model_kwargs': {'torch_dtype': 'auto'},
+                        'encode_kwargs': {
+                            'batch_size': 32,
+                        },
+                    },
+                ],
+                'eval': {
+                    'tasks': ['MedicalRetrieval', 'T2Retrieval'],
+                    'verbosity': 2,
+                    'output_folder': 'outputs',
+                    'overwrite_results': True,
+                    'limits': 10,
+                },
+            },
+        }
+
+        run_task(task_cfg)
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_custom(self):
+        task_cfg = {
+            'eval_backend': 'RAGEval',
+            'eval_config': {
+                'tool': 'MTEB',
+                'model': [
+                    {
+                        'model_name_or_path': 'AI-ModelScope/m3e-base',
+                        'pooling_mode': None,  # load from model config
+                        'max_seq_length': 512,
+                        'prompt': '',
+                        'model_kwargs': {'torch_dtype': 'auto'},
+                        'encode_kwargs': {
+                            'batch_size': 128,
+                        },
+                    }
+                ],
+                'eval': {
+                    'tasks': ['CustomRetrieval'],
+                    'dataset_path': 'custom_eval/text/retrieval',
+                    'verbosity': 2,
+                    'output_folder': 'outputs',
+                    'overwrite_results': True,
+                    'limits': 500,
+                },
+            },
+        }
+
+        run_task(task_cfg)
+
+
+if __name__ == '__main__':
+    unittest.main(buffer=False)

tests/rag/test_ragas.py

@@ -0,0 +1,120 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+import unittest
+from evalscope.utils import test_level_list, is_module_installed
+from evalscope.utils.logger import get_logger
+from evalscope.run import run_task
+
+logger = get_logger()
+
+
+class TestRAGAS(unittest.TestCase):
+
+    def setUp(self) -> None:
+        self._check_env('ragas')
+
+    def tearDown(self) -> None:
+        pass
+
+    @staticmethod
+    def _check_env(module_name: str):
+        if is_module_installed(module_name):
+            logger.info(f'{module_name} is installed.')
+        else:
+            raise ModuleNotFoundError(f'run: pip install {module_name}')
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_generate_dataset(self):
+        task_cfg = {
+            'eval_backend': 'RAGEval',
+            'eval_config': {
+                'tool': 'RAGAS',
+                'testset_generation': {
+                    'docs': ['README_zh.md'],
+                    'test_size': 5,
+                    'output_file': 'outputs/testset.json',
+                    'distribution': {
+                        'simple': 0.5,
+                        'multi_context': 0.4,
+                        'reasoning': 0.1,
+                    },
+                    'generator_llm': {
+                        'model_name_or_path': 'qwen/Qwen2-7B-Instruct',
+                        'template_type': 'qwen',
+                    },
+                    'embeddings': {
+                        'model_name_or_path': 'AI-ModelScope/m3e-base',
+                    },
+                    'language': 'chinese',
+                },
+            },
+        }
+
+        logger.info(f'>> Start to run task: {task_cfg}')
+
+        run_task(task_cfg)
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_rag_eval(self):
+        task_cfg = {
+            'eval_backend': 'RAGEval',
+            'eval_config': {
+                'tool': 'RAGAS',
+                'eval': {
+                    'testset_file': 'outputs/testset_chinese_with_answer.json',
+                    'critic_llm': {
+                        'model_name_or_path': 'qwen/Qwen2-7B-Instruct',
+                        'template_type': 'qwen',
+                    },
+                    'embeddings': {
+                        'model_name_or_path': 'AI-ModelScope/m3e-base',
+                    },
+                    'metrics': [
+                        'Faithfulness',
+                        'AnswerRelevancy',
+                        'ContextPrecision',
+                        'AnswerCorrectness',
+                    ],
+                },
+            },
+        }
+
+        logger.info(f'>> Start to run task: {task_cfg}')
+
+        run_task(task_cfg)
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_rag_eval_api(self):
+        task_cfg = {
+            'eval_backend': 'RAGEval',
+            'eval_config': {
+                'tool': 'RAGAS',
+                'eval': {
+                    'testset_file': 'outputs/testset.json',
+                    'critic_llm': {
+                        'model_name': 'gpt-4o-mini',  # 自定义聊天模型名称
+                        'api_base': 'http://127.0.0.1:8088/v1',  # 自定义基础URL
+                        'api_key': 'xxxx',  # 你的API密钥
+                    },
+                    'embeddings': {
+                        'model_name_or_path': 'AI-ModelScope/m3e-base',
+                    },
+                    'metrics': [
+                        'Faithfulness',
+                        'AnswerRelevancy',
+                        'ContextPrecision',
+                        'AnswerCorrectness',
+                        'MultiModalFaithfulness',
+                        'MultiModalRelevance',
+                    ],
+                },
+            },
+        }
+
+        logger.info(f'>> Start to run task: {task_cfg}')
+
+        run_task(task_cfg)
+
+
+if __name__ == '__main__':
+    unittest.main(buffer=False)

tests/swift/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.

tests/swift/test_run_swift_eval.py

@@ -0,0 +1,146 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import os
+import subprocess
+import time
+import unittest
+
+import json
+import requests
+
+from evalscope.backend.opencompass import OpenCompassBackendManager
+from evalscope.run import run_task
+from evalscope.summarizer import Summarizer
+from evalscope.utils import is_module_installed, test_level_list
+from evalscope.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+DEFAULT_CHAT_MODEL_URL = 'http://127.0.0.1:8000/v1/chat/completions'
+DEFAULT_BASE_MODEL_URL = 'http://127.0.0.1:8001/v1/completions'
+
+
+class TestRunSwiftEval(unittest.TestCase):
+
+    def setUp(self) -> None:
+        logger.info('Init env for swift-eval UTs ...\n')
+
+        self.model_name = 'qwen2_5-0_5b-instruct'
+        assert is_module_installed('evalscope'), 'Please install `evalscope` from pypi or source code.'
+
+        if not is_module_installed('opencompass'):
+            logger.warning('Note: installing ms-opencompass ...')
+            subprocess.run('pip3 install ms-opencompass -U', shell=True, check=True)
+
+        if not is_module_installed('swift'):
+            logger.warning('Note: installing ms-swift ...')
+            subprocess.run('pip3 install ms-swift[llm]', shell=True, check=True)
+
+        logger.warning('vllm not installed, use native swift deploy service instead.')
+
+        logger.info('\nStaring run swift deploy ...')
+        self.process_swift_deploy = subprocess.Popen(
+            f'swift deploy --model_type {self.model_name} --infer_backend pt', text=True, shell=True)
+        if self.process_swift_deploy.stderr:
+            logger.info(f'swift deploy log info: {self.process_swift_deploy.stderr}')
+
+        self.all_datasets = OpenCompassBackendManager.list_datasets()
+        assert len(self.all_datasets) > 0, f'Failed to list datasets from OpenCompass backend: {self.all_datasets}'
+
+    def tearDown(self) -> None:
+        # Stop the swift deploy model service
+        logger.warning('\nStopping swift deploy ...')
+        self.process_swift_deploy.terminate()
+        self.process_swift_deploy.wait()
+        logger.info('Process swift-deploy terminated successfully.')
+
+    @staticmethod
+    def find_and_kill_pid(pids: list):
+        if len(pids) > 0:
+            for pid in pids:
+                subprocess.run(['kill', str(pid)])
+                logger.warning(f'Killed process {pid}.')
+        else:
+            logger.info('No pids found.')
+
+    @staticmethod
+    def find_and_kill_service(service_name):
+        try:
+            # find pid
+            result = subprocess.run(['ps', '-ef'], stdout=subprocess.PIPE, text=True)
+
+            lines = result.stdout.splitlines()
+            pids = []
+            for line in lines:
+                if service_name in line and 'grep' not in line:
+                    parts = line.split()
+                    pid = parts[1]
+                    pids.append(pid)
+
+            if not pids:
+                logger.info(f'No process found for {service_name}.')
+            else:
+                for pid in pids:
+                    subprocess.run(['kill', pid])
+                    logger.warning(f'Killed process {pid} for service {service_name}.')
+        except Exception as e:
+            logger.error(f'An error occurred: {e}')
+
+    @staticmethod
+    def check_service_status(url: str, data: dict, retries: int = 30, delay: int = 10):
+        for i in range(retries):
+            try:
+                logger.info(f'Attempt {i + 1}: Checking service at {url} ...')
+                response = requests.post(
+                    url, data=json.dumps(data), headers={'Content-Type': 'application/json'}, timeout=30)
+                if response.status_code == 200:
+                    logger.info(f'Service at {url} is available !\n\n')
+                    return True
+                else:
+                    logger.info(f'Service at {url} returned status code {response.status_code}.')
+            except requests.exceptions.RequestException as e:
+                logger.info(f'Attempt {i + 1}: An error occurred: {e}')
+
+            time.sleep(delay)
+
+        logger.info(f'Service at {url} is not available after {retries} retries.')
+        return False
+
+    @unittest.skipUnless(1 in test_level_list(), 'skip test in current test level')
+    def test_run_task(self):
+        # Prepare the config
+        task_cfg = dict(
+            eval_backend='OpenCompass',
+            eval_config={
+                'datasets': ['cmb', 'bbh', 'ceval', 'ARC_e', 'gsm8k'],
+                'models': [
+                    {
+                        'path': self.model_name,
+                        'openai_api_base': DEFAULT_CHAT_MODEL_URL,
+                        'batch_size': 8
+                    },
+                ],
+                'work_dir': 'outputs/llama3_eval_result',
+                'reuse': None,  # string, `latest` or timestamp, e.g. `20230516_144254`, default to None
+                'limit': 5,  # string or int or float, e.g. `[2:5]`, 5, 5.0, default to None, it means run all examples
+            },
+        )
+
+        # Check the service status
+        data = {'model': self.model_name, 'messages': [{'role': 'user', 'content': 'who are you?'}]}
+        assert self.check_service_status(
+            DEFAULT_CHAT_MODEL_URL, data=data), f'Failed to check service status: {DEFAULT_CHAT_MODEL_URL}'
+
+        # Submit the task
+        logger.info(f'Start to run UT with cfg: {task_cfg}')
+        run_task(task_cfg=task_cfg)
+
+        # Get the final report with summarizer
+        report_list = Summarizer.get_report_from_cfg(task_cfg)
+        logger.info(f'>>The report list:\n{report_list}')
+
+        assert len(report_list) > 0, f'Failed to get report list: {report_list}'
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/swift/test_run_swift_vlm_eval.py

@@ -0,0 +1,128 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+import shutil
+import subprocess
+import time
+import unittest
+
+import json
+import requests
+
+from evalscope.backend.vlm_eval_kit import VLMEvalKitBackendManager
+from evalscope.run import run_task
+from evalscope.summarizer import Summarizer
+from evalscope.utils import is_module_installed, test_level_list
+from evalscope.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+DEFAULT_CHAT_MODEL_URL = 'http://127.0.0.1:8000/v1/chat/completions'
+DEFAULT_API_KEY = 'EMPTY'
+DEFAULT_MODEL_NAME = 'CustomAPIModel'
+DEFAULT_WORK_DIR = 'outputs/qwen-vl-chat'
+
+
+class TestRunSwiftVLMEval(unittest.TestCase):
+
+    def setUp(self) -> None:
+        logger.info('Init env for swift-eval UTs ...\n')
+        assert is_module_installed('evalscope'), 'Please install `llmuses` from pypi or source code.'
+
+        if not is_module_installed('vlmeval'):
+            logger.warning('Note: installing ms-vlmeval ...')
+            subprocess.run('pip3 install ms-vlmeval -U', shell=True, check=True)
+
+        if not is_module_installed('swift'):
+            logger.warning('Note: installing ms-swift ...')
+            subprocess.run('pip3 install ms-swift -U', shell=True, check=True)
+
+        if os.path.exists(DEFAULT_WORK_DIR):
+            shutil.rmtree(DEFAULT_WORK_DIR)
+            logger.info(f'Removed work dir: {os.path.abspath(DEFAULT_WORK_DIR)} \n')
+
+        logger.info('\nStaring run swift deploy ...')
+        self.model_name = 'qwen-vl-chat'
+        self.process_swift_deploy = subprocess.Popen(
+            f'swift deploy --model_type {self.model_name} --infer_backend pt', text=True, shell=True)
+
+        self.all_datasets = VLMEvalKitBackendManager.list_supported_datasets()
+        assert len(self.all_datasets) > 0, f'Failed to list datasets from VLMEvalKit backend: {self.all_datasets}'
+
+    def tearDown(self) -> None:
+        # Stop the swift deploy model service
+        logger.warning('Stopping swift deploy ...')
+        self.process_swift_deploy.terminate()
+        self.process_swift_deploy.wait()
+        logger.info('Process swift-deploy terminated successfully.')
+
+    @staticmethod
+    def _check_env(module_name: str):
+        if is_module_installed(module_name):
+            logger.info(f'{module_name} is installed.')
+        else:
+            raise ModuleNotFoundError(f'run: pip install {module_name}')
+
+    @staticmethod
+    def check_service_status(url: str, data: dict, retries: int = 20, delay: int = 10):
+        for i in range(retries):
+            try:
+                logger.info(f'Attempt {i + 1}: Checking service at {url} ...')
+                response = requests.post(
+                    url, data=json.dumps(data), headers={'Content-Type': 'application/json'}, timeout=30)
+                if response.status_code == 200:
+                    logger.info(f'Service at {url} is available !\n\n')
+                    return True
+                else:
+                    logger.info(f'Service at {url} returned status code {response.status_code}.')
+            except requests.exceptions.RequestException as e:
+                logger.info(f'Attempt {i + 1}: An error occurred: {e}')
+
+            time.sleep(delay)
+
+        logger.info(f'Service at {url} is not available after {retries} retries.')
+        return False
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_api(self):
+        api_base = DEFAULT_CHAT_MODEL_URL
+        task_cfg = {
+            'eval_backend': 'VLMEvalKit',
+            'eval_config': {
+                'data': ['SEEDBench_IMG', 'ChartQA_TEST'],
+                'limit':
+                30,
+                'mode':
+                'all',
+                'model': [{
+                    'api_base': api_base,  # swfit deploy model api
+                    'key': DEFAULT_API_KEY,
+                    'name': DEFAULT_MODEL_NAME,  # must be CustomAPIModel for swift
+                    'temperature': 0.0,
+                    'type': self.model_name
+                }],  # swift model type
+                'nproc':
+                1,
+                'reuse':
+                True,
+                'work_dir':
+                DEFAULT_WORK_DIR
+            }
+        }
+
+        # Check the service status
+        data = {'model': self.model_name, 'messages': [{'role': 'user', 'content': 'who are you?'}]}
+        assert self.check_service_status(api_base, data=data), f'Failed to check service status: {api_base}'
+
+        logger.info(f'>> Start to run task: {task_cfg}')
+
+        run_task(task_cfg)
+
+        logger.info('>> Start to get the report with summarizer ...')
+        report_list = Summarizer.get_report_from_cfg(task_cfg)
+        logger.info(f'\n>> The report list: {report_list}')
+
+        assert len(report_list) > 0, f'Failed to get report list: {report_list}'
+
+
+if __name__ == '__main__':
+    unittest.main(buffer=False)

tests/swift/test_run_swift_vlm_jugde_eval.py

@@ -0,0 +1,157 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+import shutil
+import subprocess
+import time
+import unittest
+
+import json
+import requests
+
+from evalscope.backend.vlm_eval_kit import VLMEvalKitBackendManager
+from evalscope.run import run_task
+from evalscope.summarizer import Summarizer
+from evalscope.utils import is_module_installed, test_level_list
+from evalscope.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+DEFAULT_CHAT_MODEL_URL = 'http://127.0.0.1:8000/v1/chat/completions'
+DEFAULT_JUDGE_MODEL_URL = 'http://127.0.0.1:8866/v1/chat/completions'
+
+DEFAULT_API_KEY = 'EMPTY'
+DEFAULT_MODEL_NAME = 'CustomAPIModel'
+DEFAULT_WORK_DIR = 'outputs'
+
+
+class TestRunSwiftVLMEval(unittest.TestCase):
+
+    def setUp(self) -> None:
+        logger.info('Init env for swift-eval UTs ...\n')
+        assert is_module_installed('llmuses'), 'Please install `llmuses` from pypi or source code.'
+
+        logger.warning('Note: installing ms-vlmeval ...')
+        subprocess.run('pip3 install ms-vlmeval -U', shell=True, check=True)
+
+        logger.warning('Note: installing ms-swift ...')
+        subprocess.run('pip3 install ms-swift -U', shell=True, check=True)
+
+        if os.path.exists(DEFAULT_WORK_DIR):
+            shutil.rmtree(DEFAULT_WORK_DIR)
+            logger.info(f'Removed work dir: {os.path.abspath(DEFAULT_WORK_DIR)} \n')
+
+        logger.info('\nStaring run swift deploy ...')
+        self.model_name = 'qwen-vl-chat'
+        self.process_swift_deploy = subprocess.Popen(
+            f'swift deploy --model_type {self.model_name}',
+            text=True,
+            shell=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE)
+
+        logger.info('\nStaring run swift deploy judge ...')
+        self.judge_model_name = 'qwen2-7b-instruct'
+        self.process_swift_deploy_judge = subprocess.Popen(
+            f'swift deploy --model_type {self.judge_model_name} \
+                                                    --port 8866',
+            text=True,
+            shell=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE)
+
+        self.all_datasets = VLMEvalKitBackendManager.list_supported_datasets()
+        assert len(self.all_datasets) > 0, f'Failed to list datasets from VLMEvalKit backend: {self.all_datasets}'
+
+    def tearDown(self) -> None:
+        # Stop the swift deploy model service
+        logger.warning('\nStopping swift deploy ...')
+        self.process_swift_deploy.terminate()
+        self.process_swift_deploy.wait()
+
+        self.process_swift_deploy_judge.terminate()
+        self.process_swift_deploy_judge.wait()
+
+        logger.info('Process swift-deploy terminated successfully.')
+
+    @staticmethod
+    def _check_env(module_name: str):
+        if is_module_installed(module_name):
+            logger.info(f'{module_name} is installed.')
+        else:
+            raise ModuleNotFoundError(f'run: pip install {module_name}')
+
+    @staticmethod
+    def check_service_status(url: str, data: dict, retries: int = 20, delay: int = 10):
+        for i in range(retries):
+            try:
+                logger.info(f'Attempt {i + 1}: Checking service at {url} ...')
+                response = requests.post(
+                    url, data=json.dumps(data), headers={'Content-Type': 'application/json'}, timeout=30)
+                if response.status_code == 200:
+                    logger.info(f'Service at {url} is available !\n\n')
+                    return True
+                else:
+                    logger.info(f'Service at {url} returned status code {response.status_code}.')
+            except requests.exceptions.RequestException as e:
+                logger.info(f'Attempt {i + 1}: An error occurred: {e}')
+
+            time.sleep(delay)
+
+        logger.info(f'Service at {url} is not available after {retries} retries.')
+        return False
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_api_with_judge(self):
+        # Check the judge service status
+        data = {'model': self.judge_model_name, 'messages': [{'role': 'user', 'content': 'who are you?'}]}
+        assert self.check_service_status(DEFAULT_JUDGE_MODEL_URL, data=data), \
+            f'Failed to check judge service status: {DEFAULT_JUDGE_MODEL_URL}'
+
+        api_base = DEFAULT_CHAT_MODEL_URL
+        task_cfg = {
+            'eval_backend': 'VLMEvalKit',
+            'eval_config': {
+                'LOCAL_LLM':
+                self.judge_model_name,  # judge model id
+                'OPENAI_API_BASE':
+                DEFAULT_JUDGE_MODEL_URL,  # judge model api
+                'OPENAI_API_KEY':
+                DEFAULT_API_KEY,
+                'data': ['SEEDBench_IMG', 'ChartQA_TEST'],
+                'limit':
+                20,
+                'mode':
+                'all',
+                'model': [{
+                    'api_base': api_base,  # swfit deploy model api
+                    'key': DEFAULT_API_KEY,
+                    'name': DEFAULT_MODEL_NAME,  # must be CustomAPIModel for swift
+                    'temperature': 0.0,
+                    'type': self.model_name
+                }],  # swift model type
+                'nproc':
+                1,
+                'reuse':
+                True,
+                'work_dir':
+                DEFAULT_WORK_DIR
+            }
+        }
+
+        # Check the service status
+        data = {'model': self.model_name, 'messages': [{'role': 'user', 'content': 'who are you?'}]}
+        assert self.check_service_status(api_base, data=data), f'Failed to check service status: {api_base}'
+
+        logger.info(f'>> Start to run task: {task_cfg}')
+
+        run_task(task_cfg)
+
+        logger.info('>> Start to get the report with summarizer ...')
+        report_list = Summarizer.get_report_from_cfg(task_cfg)
+        logger.info(f'\n>> The report list: {report_list}')
+
+        assert len(report_list) > 0, f'Failed to get report list: {report_list}'
+
+
+if __name__ == '__main__':
+    unittest.main(buffer=False)