evalscope 0.6.1__py3-none-any.whl → 0.7.0__py3-none-any.whl

evalscope/registry/data/qa_browser/category_mapping.yaml

@@ -0,0 +1,10 @@
+文本生成: ['写作', '续写生成', '通用写作', '应用文写作', '头脑风暴', '开放对话', '角色扮演']
+文本理解:
+  ['文本点评', '文本摘要', '润色/纠错', '文本分类', '情感分析', '信息抽取', '文本聚类', '序列处理']
+知识问答: ['百科知识问答']
+数学解题: ['数学解题']
+逻辑推理: ['逻辑推理', '阅读理解']
+CODING: ['CODING', '表格处理']
+翻译: ['翻译']
+安全风险: ['有伤害性']
+其他: ['*']

evalscope/registry/data/question.jsonl

@@ -0,0 +1,80 @@
+{"question_id": 1, "text": "How can I improve my time management skills?", "category": "generic"}
+{"question_id": 2, "text": "What are the most effective ways to deal with stress?", "category": "generic"}
+{"question_id": 3, "text": "What are the main differences between Python and JavaScript programming languages?", "category": "generic"}
+{"question_id": 4, "text": "How can I increase my productivity while working from home?", "category": "generic"}
+{"question_id": 5, "text": "Can you explain the basics of quantum computing?", "category": "generic"}
+{"question_id": 6, "text": "What are the differences between plant-based and animal-based protein sources?", "category": "generic"}
+{"question_id": 7, "text": "How can I develop my critical thinking skills?", "category": "generic"}
+{"question_id": 8, "text": "What are the major challenges faced by the education sector today?", "category": "generic"}
+{"question_id": 9, "text": "What are the primary factors that influence consumer behavior?", "category": "generic"}
+{"question_id": 10, "text": "What are the most effective strategies for conflict resolution in the workplace?", "category": "generic"}
+{"question_id": 11, "text": "What are some potential implications of using a single-use plastic bottle versus a reusable bottle on both the environment and human health?", "category": "knowledge"}
+{"question_id": 12, "text": "What factors would you consider when designing an inclusive and accessible public transportation system?", "category": "knowledge"}
+{"question_id": 13, "text": "How can governments utilize fiscal and monetary policies to combat economic recessions?", "category": "knowledge"}
+{"question_id": 14, "text": "How do language and cultural barriers affect the way people communicate and form relationships in multicultural societies?", "category": "knowledge"}
+{"question_id": 15, "text": "Describe a scenario where artificial intelligence could be used to improve the quality and efficiency of healthcare delivery.", "category": "knowledge"}
+{"question_id": 16, "text": "Explain the process of gene editing using CRISPR-Cas9 technology, and discuss its potential applications and ethical implications.", "category": "knowledge"}
+{"question_id": 17, "text": "How do vaccinations work to protect individuals and communities from infectious diseases, and what is herd immunity?", "category": "knowledge"}
+{"question_id": 18, "text": "How do social media platforms influence the way people consume and share news, and what are the potential implications for the spread of misinformation?", "category": "knowledge"}
+{"question_id": 19, "text": "How do cultural, social, and economic factors influence people's food choices, and how can this knowledge be used to promote healthier diets?", "category": "knowledge"}
+{"question_id": 20, "text": "Explain the process of natural selection and how it contributes to the evolution and adaptation of species.", "category": "knowledge"}
+{"question_id": 21, "text": "How would you introduce yourself as a medieval knight at a royal banquet?", "category": "roleplay"}
+{"question_id": 22, "text": "As a pirate captain, what would you say to your crew to motivate them to search for hidden treasure?", "category": "roleplay"}
+{"question_id": 23, "text": "If you were a Shakespearean character, how would you declare your love for someone in a soliloquy?", "category": "roleplay"}
+{"question_id": 24, "text": "As a superhero, how would you explain your origin story to a curious child?", "category": "roleplay"}
+{"question_id": 25, "text": "Imagine you are a time traveler from the year 3000. What technological advancements would you tell people about?", "category": "roleplay"}
+{"question_id": 26, "text": "As a sports commentator, describe the winning play in the final seconds of a championship game.", "category": "roleplay"}
+{"question_id": 27, "text": "Pretend to be a world-famous chef. How would you describe your signature dish to a panel of judges?", "category": "roleplay"}
+{"question_id": 28, "text": "You are a mountain climber reaching the summit of Mount Everest. Describe your emotions and the view from the top.", "category": "roleplay"}
+{"question_id": 29, "text": "As a space colonist on Mars, describe your daily life and the challenges you face living on another planet.", "category": "roleplay"}
+{"question_id": 30, "text": "Pretend to be a character in a post-apocalyptic world. Describe how you survive and the allies you encounter.", "category": "roleplay"}
+{"question_id": 31, "text": "How can you determine if a restaurant is popular among locals or mainly attracts tourists, and why might this information be useful?", "category": "common-sense"}
+{"question_id": 32, "text": "What are some subtle clues that suggest someone is pretending to understand a topic or conversation when they are actually confused or uninformed?", "category": "common-sense"}
+{"question_id": 33, "text": "Why might someone choose to use a paper map or ask for directions instead of relying on a GPS device or smartphone app?", "category": "common-sense"}
+{"question_id": 34, "text": "How can you determine if a person is genuinely interested in a conversation or simply being polite?", "category": "common-sense"}
+{"question_id": 35, "text": "Why might someone prefer to shop at a small, locally-owned business instead of a large chain store, even if the prices are higher?", "category": "common-sense"}
+{"question_id": 36, "text": "How can you assess the credibility of a source of information, such as a news article or blog post, without relying solely on the reputation of the author or publisher?", "category": "common-sense"}
+{"question_id": 37, "text": "Why do some people enjoy the sensation of being scared, such as by watching horror movies or going on roller coasters, while others avoid these experiences?", "category": "common-sense"}
+{"question_id": 38, "text": "How can observing the behavior of other people in a social situation provide clues about cultural norms and expectations?", "category": "common-sense"}
+{"question_id": 39, "text": "Do we have a moral obligation to explore space, or should we focus on solving Earth's problems first?", "category": "common-sense"}
+{"question_id": 40, "text": "In a world where automation is becoming increasingly prevalent, is it more important to prioritize job creation or technological progress?", "category": "common-sense"}
+{"question_id": 41, "text": "How many times does the average human blink in a lifetime? Try to explain your answer. Your explanation should take the reader through your reasoning step-by-step.", "category": "fermi"}
+{"question_id": 42, "text": "How many atoms are in a grain of salt? Try to explain your answer. Your explanation should take the reader through your reasoning step-by-step.", "category": "fermi"}
+{"question_id": 43, "text": "How many lightning strikes occur on Earth each day? Try to explain your answer. Your explanation should take the reader through your reasoning step-by-step.", "category": "fermi"}
+{"question_id": 44, "text": "How many balloons would it take to lift a house like in the movie \"Up\"? Try to explain your answer. Your explanation should take the reader through your reasoning step-by-step.", "category": "fermi"}
+{"question_id": 45, "text": "How many text messages are sent globally in a minute? Try to explain your answer. Your explanation should take the reader through your reasoning step-by-step.", "category": "fermi"}
+{"question_id": 46, "text": "How many words are spoken daily on Earth? Try to explain your answer. Your explanation should take the reader through your reasoning step-by-step.", "category": "fermi"}
+{"question_id": 47, "text": "How many snowflakes fall during a typical winter? Try to explain your answer. Your explanation should take the reader through your reasoning step-by-step.", "category": "fermi"}
+{"question_id": 48, "text": "How many pages are in all the books ever written? Try to explain your answer. Your explanation should take the reader through your reasoning step-by-step.", "category": "fermi"}
+{"question_id": 49, "text": "How many times has the Earth orbited the Sun since the beginning of life? Try to explain your answer. Your explanation should take the reader through your reasoning step-by-step.", "category": "fermi"}
+{"question_id": 50, "text": "How many songs have been recorded throughout history? Try to explain your answer. Your explanation should take the reader through your reasoning step-by-step.", "category": "fermi"}
+{"question_id": 51, "text": "What if the Internet had been invented during the Renaissance period?", "category": "counterfactual"}
+{"question_id": 52, "text": "What if the Aztecs had successfully repelled the Spanish conquistadors?", "category": "counterfactual"}
+{"question_id": 53, "text": "What if the Black Death had not occurred in the 14th century?", "category": "counterfactual"}
+{"question_id": 54, "text": "What if Isaac Newton had focused on biology instead of physics?", "category": "counterfactual"}
+{"question_id": 55, "text": "What if the Beatles had never formed as a band?", "category": "counterfactual"}
+{"question_id": 56, "text": "What if Alan Turing had not cracked the Enigma code during World War II?", "category": "counterfactual"}
+{"question_id": 57, "text": "What if the Suez Canal had never been constructed?", "category": "counterfactual"}
+{"question_id": 58, "text": "What if the Maya civilization had never mysteriously collapsed?", "category": "counterfactual"}
+{"question_id": 59, "text": "What if Christopher Columbus had not discovered the Americas?", "category": "counterfactual"}
+{"question_id": 60, "text": "What if Vincent van Gogh had been a successful artist during his lifetime?", "category": "counterfactual"}
+{"question_id": 61, "text": "Develop a C++ program that reads a text file line by line and counts the number of occurrences of a specific word in the file.", "category": "coding"}
+{"question_id": 62, "text": "Implement a Python function to find the longest common subsequence of two input strings using dynamic programming.", "category": "coding"}
+{"question_id": 63, "text": "Implement a regular expression in Python to validate an email address.", "category": "coding"}
+{"question_id": 64, "text": "Write a program to find the nth Fibonacci number using dynamic programming.", "category": "coding"}
+{"question_id": 65, "text": "Implement a binary search algorithm to find a specific element in a sorted array.", "category": "coding"}
+{"question_id": 66, "text": "Implement a queue data structure using two stacks in Python.", "category": "coding"}
+{"question_id": 67, "text": "Implement a program to find the common elements in two arrays without using any extra data structures.", "category": "coding"}
+{"question_id": 68, "text": "Given that f(x) = 5x^3 - 2x + 3, find the value of f(2).", "category": "math"}
+{"question_id": 69, "text": "Solve for x in the equation 3x + 10 = 5(x - 2).", "category": "math"}
+{"question_id": 70, "text": "If the endpoints of a line segment are (2, -2) and (10, 4), what is the length of the segment?", "category": "math"}
+{"question_id": 71, "text": "Can you help me write a formal email to a potential business partner proposing a joint venture?", "category": "writing"}
+{"question_id": 72, "text": "Can you help me write a resignation letter to my current employer, while leaving on good terms and expressing gratitude for the opportunities provided?", "category": "writing"}
+{"question_id": 73, "text": "Use an appropriate format to structure a formal letter of recommendation for a student applying to a prestigious graduate program in computer science.", "category": "writing"}
+{"question_id": 74, "text": "Write a compelling product launch announcement email to inform our customers of our new software solution.", "category": "writing"}
+{"question_id": 75, "text": "Draft an apology email to a customer who experienced a delay in their order, and provide reassurance that the issue has been resolved.", "category": "writing"}
+{"question_id": 76, "text": "Write a script for a YouTube video exploring the history and cultural significance of jazz.", "category": "writing"}
+{"question_id": 77, "text": "Compose an engaging travel blog post about a recent trip to Hawaii, highlighting cultural experiences and must-see attractions.", "category": "writing"}
+{"question_id": 78, "text": "Write a captivating movie review for a recently released science fiction film, discussing its plot, characters, and special effects.", "category": "writing"}
+{"question_id": 79, "text": "Structure a podcast script for an episode discussing the influence of streaming platforms on the music industry.", "category": "writing"}
+{"question_id": 80, "text": "Write a symphony concert review, discussing the orchestra's performance and overall audience experience.", "category": "writing"}

evalscope/third_party/longbench_write/README.md

@@ -0,0 +1,118 @@
+
+## Description
+The LongWriter supports 10,000+ Word Generation From Long Context LLMs.
+We can use the benchmark LongBench-Write focuses more on measuring the long output quality as well as the output length.
+
+Refer to https://github.com/THUDM/LongWriter
+
+## Usage
+
+### Installation
+
+```bash
+pip install evalscope[framework]
+```
+
+### Task configuration
+
+There are few ways to configure the task: dict, json and yaml.
+
+1. Configuration with dict:
+
+```python
+task_cfg = dict(stage=['infer', 'eval_l', 'eval_q'],
+                model='ZhipuAI/LongWriter-glm4-9b',
+                input_data_path=None,
+                output_dir='./outputs',
+                openai_api_key=None,
+                openai_gpt_model='gpt-4o-2024-05-13',
+                infer_generation_kwargs={
+                    'max_new_tokens': 32768,
+                    'temperature': 0.5
+                },
+                eval_generation_kwargs={
+                    'max_new_tokens': 1024,
+                    'temperature': 0.5,
+                    'stop': None
+                },
+                proc_num=8)
+
+```
+- Arguments:
+  - `stage`: To run multiple stages, `infer`--run the inference process. `eval_l`--run eval length process. `eval_q`--run eval quality process.
+  - `model`: model id on the ModelScope hub, or local model dir.
+  - `input_data_path`: input data path, default to `None`, it means to use [longbench_write](resources/longbench_write.jsonl)
+  - `output_dir`: output root directory.
+  - `openai_api_key`: openai_api_key when enabling the stage `eval_q` to use `Model-as-Judge`. Default to None if not needed.
+  - `openai_gpt_model`: Judge model name from OpenAI. Default to `gpt-4o-2024-05-13`
+  - `infer_generation_kwargs`: The generation kwargs for models to be evaluated.
+  - `eval_generation_kwargs`: The generation kwargs for judge-models.
+  - `proc_num`: proc num.
+
+
+2. Configuration with json (Optional):
+
+```json
+{
+    "stage": ["infer", "eval_l", "eval_q"],
+    "model": "ZhipuAI/LongWriter-glm4-9b",
+    "input_data_path": null,
+    "output_dir": "./outputs",
+    "openai_api_key": null,
+    "openai_gpt_model": "gpt-4o-2024-05-13",
+    "infer_generation_kwargs": {
+        "max_new_tokens": 32768,
+        "temperature": 0.5
+    },
+    "eval_generation_kwargs": {
+        "max_new_tokens": 1024,
+        "temperature": 0.5,
+        "stop": null
+    },
+    "proc_num": 8
+}
+```
+Refer to [default_task.json](default_task.json) for more details.
+
+
+2. Configuration with yaml (Optional):
+
+```yaml
+stage:
+  - infer
+  - eval_l
+  - eval_q
+model: ZhipuAI/LongWriter-glm4-9b
+input_data_path: null
+output_dir: ./outputs
+openai_api_key: null
+openai_gpt_model: gpt-4o-2024-05-13
+infer_generation_kwargs:
+  max_new_tokens: 32768
+  temperature: 0.5
+eval_generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0.5
+  stop: null
+proc_num: 8
+
+```
+Refer to [default_task.yaml](default_task.yaml) for more details.
+
+
+
+### Run the task
+
+```python
+from evalscope.third_party.longbench_write import run_task
+
+run_task(task_cfg=task_cfg)
+```
+
+
+### Results and metrics
+See `eval_length.jsonl` and `eval_quality.jsonl` in the outputs dir.
+
+- Metrics:
+  - `score_l`: The average score of the length evaluation.
+  - `score_q`: The average score of the quality evaluation.

evalscope/third_party/longbench_write/default_task.json

@@ -0,0 +1,27 @@
+{
+  "stage": ["infer", "eval_l", "eval_q"],
+  "model": "ZhipuAI/LongWriter-glm4-9b",
+  "input_data_path": null,
+  "output_dir": "./outputs",
+  "infer_config": {
+    "openai_api_base": "http://127.0.0.1:8000/v1/chat/completions",
+    "is_chat": true,
+    "verbose": false,
+    "generation_kwargs": {
+      "max_new_tokens": 32768,
+      "temperature": 0.5,
+      "repetition_penalty": 1.0
+    }
+  },
+  "eval_config": {
+    "openai_api_key": "YOUR_OPENAI_API_KEY",
+    "openai_api_base": "https://api.openai.com/v1/chat/completions",
+    "openai_gpt_model": "gpt-4o-2024-05-13",
+    "generation_kwargs": {
+      "max_new_tokens": 1024,
+      "temperature": 0.5,
+      "stop": null
+    },
+    "proc_num": 16
+  }
+}

evalscope/third_party/longbench_write/default_task.yaml

@@ -0,0 +1,24 @@
+stage:
+  - infer
+  - eval_l
+  - eval_q
+model: ZhipuAI/LongWriter-glm4-9b
+input_data_path: null
+output_dir: './outputs'
+infer_config:
+  openai_api_base: 'http://127.0.0.1:8000/v1/chat/completions'
+  is_chat: true
+  verbose: false
+  generation_kwargs:
+    max_new_tokens: 32768
+    temperature: 0.5
+    repetition_penalty: 1.0
+eval_config:
+  openai_api_key: 'YOUR_OPENAI_API_KEY'
+  openai_api_base: 'https://api.openai.com/v1/chat/completions'
+  openai_gpt_model: 'gpt-4o-2024-05-13'
+  generation_kwargs:
+    max_new_tokens: 1024
+    temperature: 0.5
+    stop: null
+  proc_num: 16

evalscope/third_party/toolbench_static/README.md

@@ -0,0 +1,118 @@
+
+## Description
+We evaluate the effectiveness of tool learning benchmark: [ToolBench](https://arxiv.org/pdf/2307.16789) (Qin et al.,2023b). The task involve integrating API calls to accomplish tasks, where the agent must accurately select the appropriate API and compose necessary API requests.
+
+Moreover, we partition the test set of ToolBench into in-domain and out-of-domain based on whether the tools used in the test instances have been seen during training.
+
+This division allows us to evaluate performance in both in-distribution and out-of-distribution scenarios. We call this dataset to be `ToolBench-Static`.
+
+For more details, please refer to: [Small LLMs Are Weak Tool Learners: A Multi-LLM Agent](https://arxiv.org/abs/2401.07324)
+
+## Dataset
+
+- Dataset statistics:
+  - Number of in_domain: 1588
+  - Number of out_domain: 781
+
+## Usage
+
+### Installation
+
+```bash
+pip install evalscope -U
+pip install ms-swift -U
+pip install rouge -U
+```
+
+
+### Download the dataset
+
+```bash
+wget https://modelscope.oss-cn-beijing.aliyuncs.com/open_data/toolbench-static/data.zip
+```
+
+
+### Unzip the dataset
+
+```bash
+unzip data.zip
+# The dataset will be unzipped to the `/path/to/data/toolbench_static` folder
+```
+
+
+### Task configuration
+
+There are two ways to configure the task: dict and yaml.
+
+1. Configuration with dict:
+
+```python
+your_task_config = {
+    'infer_args': {
+        'model_name_or_path': '/path/to/model_dir',
+        'model_type': 'qwen2-7b-instruct',
+        'data_path': 'data/toolbench_static',
+        'output_dir': 'output_res',
+        'deploy_type': 'swift',
+        'max_new_tokens': 2048,
+        'num_infer_samples': None
+    },
+    'eval_args': {
+        'input_path': 'output_res',
+        'output_path': 'output_res'
+    }
+}
+```
+- Arguments:
+  - `model_name_or_path`: The path to the model local directory.
+  - `model_type`: The model type, refer to [模型类型列表](https://github.com/modelscope/swift/blob/main/docs/source/LLM/%E6%94%AF%E6%8C%81%E7%9A%84%E6%A8%A1%E5%9E%8B%E5%92%8C%E6%95%B0%E6%8D%AE%E9%9B%86.md)
+  - `data_path`: The path to the dataset directory contains `in_domain.json` and `out_of_domain.json` files.
+  - `output_dir`: The path to the output directory. Default to `output_res`.
+  - `deploy_type`: The deploy type, default to `swift`.
+  - `max_new_tokens`: The maximum number of tokens to generate.
+  - `num_infer_samples`: The number of samples to infer. Default to `None`, which means infer all samples.
+  - `input_path`: The path to the input directory for evaluation, should be the same as `output_dir` of `infer_args`.
+  - `output_path`: The path to the output directory for evaluation.
+
+
+2. Configuration with yaml:
+
+```yaml
+infer_args:
+  model_name_or_path: /path/to/model_dir      # absolute path is recommended
+  model_type: qwen2-7b-instruct
+  data_path: /path/to/data/toolbench_static   # absolute path is recommended
+  deploy_type: swift
+  max_new_tokens: 2048
+  num_infer_samples: null
+  output_dir: output_res
+eval_args:
+  input_path: output_res
+  output_path: output_res
+```
+refer to [config_default.yaml](config_default.yaml) for more details.
+
+
+### Run the task
+
+```python
+from evalscope.third_party.toolbench_static import run_task
+
+# Run the task with dict configuration
+run_task(task_cfg=your_task_config)
+
+# Run the task with yaml configuration
+run_task(task_cfg='/path/to/your_task_config.yaml')
+```
+
+
+### Results and metrics
+
+- Metrics:
+  - `Plan.EM`: The agent’s planning decisions at each step for using tools invocation, generating answer, or giving up. Exact match score.
+  - `Act.EM`: Action exact match score, including the tool name and arguments.
+  - `HalluRate`（lower is better）: The hallucination rate of the agent's answers at each step.
+  - `Avg.F1`: The average F1 score of the agent's tools calling at each step.
+  - `R-L`: The Rouge-L score of the agent's answers at each step.
+
+Generally, we focus on `Act.EM`, `HalluRate` and `Avg.F1` metrics.

evalscope/third_party/toolbench_static/config_default.json

@@ -0,0 +1,15 @@
+{
+  "infer_args": {
+    "model_name_or_path": "/path/to/model_dir",
+    "model_type": "qwen2-7b-instruct",
+    "data_path": "/path/to/data/toolbench_static",
+    "deploy_type": "swift",
+    "max_new_tokens": 2048,
+    "num_infer_samples": null,
+    "output_dir": "output_res"
+  },
+  "eval_args": {
+    "input_path": "output_res",
+    "output_path": "output_res"
+  }
+}

evalscope/third_party/toolbench_static/config_default.yaml

@@ -0,0 +1,12 @@
+
+infer_args:
+  model_name_or_path: /path/to/model_dir   # Note: need to replace with your model dir, absolute path is recommended
+  model_type: qwen2-7b-instruct           # Note: need to replace with your model type
+  data_path: /path/to/data/toolbench_static   # absolute path is recommended
+  deploy_type: swift
+  max_new_tokens: 2048
+  num_infer_samples: null
+  output_dir: output_res
+eval_args:
+  input_path: output_res
+  output_path: output_res

evalscope/third_party/toolbench_static/requirements.txt

@@ -0,0 +1,2 @@
+ms-swift>=2.1.0
+rouge

evalscope/utils/logger.py

@@ -1,19 +1,19 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-
 import importlib.util as iutil
 import logging
 from typing import Optional
 
 init_loggers = {}
-format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-formatter = logging.Formatter(format)
 
-logging.basicConfig(format=format, level=logging.INFO)
+detailed_format = '%(asctime)s - %(name)s - %(filename)s - %(funcName)s - %(lineno)d - %(levelname)s - %(message)s'
+simple_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+
+detailed_formatter = logging.Formatter(detailed_format)
+simple_formatter = logging.Formatter(simple_format)
+
+logging.basicConfig(format=simple_format, level=logging.INFO)
 
 
-def get_logger(
-    log_file: Optional[str] = None, log_level: int = logging.INFO, file_mode: str = "w"
-):
+def get_logger(log_file: Optional[str] = None, log_level: int = logging.INFO, file_mode: str = 'w'):
     """Get logging logger
 
     Args:
@@ -24,25 +24,23 @@ def get_logger(
             specified (if filemode is unspecified, it defaults to 'w').
     """
 
-    logger_name = __name__.split(".")[0]
+    logger_name = __name__.split('.')[0]
     logger = logging.getLogger(logger_name)
     logger.propagate = False
+
     if logger_name in init_loggers:
-        add_file_handler_if_needed(logger, log_file, file_mode, log_level)
         if logger.level != log_level:
             logger.setLevel(log_level)
+        add_file_handler_if_needed(logger, log_file, file_mode, log_level)
+        for handler in logger.handlers:
+            handler.setLevel(log_level)
+            handler.setFormatter(detailed_formatter if log_level == logging.DEBUG else simple_formatter)
         return logger
 
     # handle duplicate logs to the console
-    # Starting in 1.8.0, PyTorch DDP attaches a StreamHandler <stderr> (NOTSET)
-    # to the root logger. As logger.propagate is True by default, this root
-    # level handler causes logging messages from rank>0 processes to
-    # unexpectedly show up on the console, creating much unwanted clutter.
-    # To fix this issue, we set the root logger's StreamHandler, if any, to log
-    # at the ERROR level.
     torch_dist = False
     is_worker0 = True
-    if iutil.find_spec("torch") is not None:
+    if iutil.find_spec('torch') is not None:
         from modelscope.utils.torch_utils import is_dist, is_master
 
         torch_dist = is_dist()
@@ -61,7 +59,7 @@ def get_logger(
         handlers.append(file_handler)
 
     for handler in handlers:
-        handler.setFormatter(formatter)
+        handler.setFormatter(detailed_formatter if log_level == logging.DEBUG else simple_formatter)
         handler.setLevel(log_level)
         logger.addHandler(handler)
 
@@ -80,7 +78,7 @@ def add_file_handler_if_needed(logger, log_file, file_mode, log_level):
         if isinstance(handler, logging.FileHandler):
             return
 
-    if iutil.find_spec("torch") is not None:
+    if iutil.find_spec('torch') is not None:
         from modelscope.utils.torch_utils import is_master
 
         is_worker0 = is_master()
@@ -89,6 +87,6 @@ def add_file_handler_if_needed(logger, log_file, file_mode, log_level):
 
     if is_worker0 and log_file is not None:
         file_handler = logging.FileHandler(log_file, file_mode)
-        file_handler.setFormatter(formatter)
+        file_handler.setFormatter(detailed_formatter if log_level == logging.DEBUG else simple_formatter)
         file_handler.setLevel(log_level)
         logger.addHandler(file_handler)