evalscope 0.8.0__py3-none-any.whl → 0.8.2__py3-none-any.whl

evalscope/tools/rewrite_eval_results.py

@@ -4,7 +4,7 @@ import time
 
 from evalscope.models.custom import CustomModel
 from evalscope.run import run_task
-from evalscope.utils import yaml_to_dict
+from evalscope.utils.io_utils import yaml_to_dict
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()

evalscope/utils/__init__.py

@@ -1,3 +1,4 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
+from evalscope.constants import *
 from evalscope.utils.utils import *

evalscope/utils/chat_service.py

@@ -43,7 +43,7 @@ class DeltaMessage(BaseModel):
 
 class ChatCompletionRequest(BaseModel):
     model: str
-    messages: List[ChatMessage] | str
+    messages: Union[List[ChatMessage], str]
     temperature: Optional[float] = None
     top_p: Optional[float] = None
     max_tokens: Optional[int] = 2048

evalscope/utils/io_utils.py

@@ -0,0 +1,162 @@
+import json
+import jsonlines as jsonl
+import os
+import yaml
+
+from evalscope.constants import DumpMode
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+class OutputsStructure:
+    LOGS_DIR = 'logs'
+    PREDICTIONS_DIR = 'predictions'
+    REVIEWS_DIR = 'reviews'
+    REPORTS_DIR = 'reports'
+    CONFIGS_DIR = 'configs'
+
+    def __init__(self, outputs_dir: str, is_make=True):
+        self.outputs_dir = outputs_dir
+        self.is_make = is_make
+        self._dirs = {
+            'logs_dir': None,
+            'predictions_dir': None,
+            'reviews_dir': None,
+            'reports_dir': None,
+            'configs_dir': None
+        }
+
+    def _get_dir(self, attr_name, dir_name):
+        if self._dirs[attr_name] is None:
+            dir_path = os.path.join(self.outputs_dir, dir_name)
+            if self.is_make:
+                os.makedirs(dir_path, exist_ok=True)
+            self._dirs[attr_name] = dir_path
+        return self._dirs[attr_name]
+
+    @property
+    def logs_dir(self):
+        return self._get_dir('logs_dir', OutputsStructure.LOGS_DIR)
+
+    @property
+    def predictions_dir(self):
+        return self._get_dir('predictions_dir', OutputsStructure.PREDICTIONS_DIR)
+
+    @property
+    def reviews_dir(self):
+        return self._get_dir('reviews_dir', OutputsStructure.REVIEWS_DIR)
+
+    @property
+    def reports_dir(self):
+        return self._get_dir('reports_dir', OutputsStructure.REPORTS_DIR)
+
+    @property
+    def configs_dir(self):
+        return self._get_dir('configs_dir', OutputsStructure.CONFIGS_DIR)
+
+
+def jsonl_to_list(jsonl_file):
+    """
+    Read jsonl file to list.
+
+    Args:
+        jsonl_file: jsonl file path.
+
+    Returns:
+        list: list of lines. Each line is a dict.
+    """
+    res_list = []
+    with jsonl.open(jsonl_file, mode='r') as reader:
+        for line in reader.iter(type=dict, allow_none=True, skip_invalid=False):
+            res_list.append(line)
+    return res_list
+
+
+def jsonl_to_reader(jsonl_file):
+    """
+    Read jsonl file to reader object.
+
+    Args:
+        jsonl_file: jsonl file path.
+
+    Returns:
+        reader: jsonl reader object.
+    """
+    with jsonl.open(jsonl_file, mode='r') as reader:
+        return reader
+
+
+def dump_jsonl_data(data_list, jsonl_file, dump_mode=DumpMode.OVERWRITE):
+    """
+    Dump data to jsonl file.
+
+    Args:
+        data_list: data list to be dumped.  [{'a': 'aaa'}, ...]
+        jsonl_file: jsonl file path.
+        dump_mode: dump mode. It can be 'overwrite' or 'append'.
+    """
+    if not jsonl_file:
+        raise ValueError('output file must be provided.')
+
+    jsonl_file = os.path.expanduser(jsonl_file)
+
+    if not isinstance(data_list, list):
+        data_list = [data_list]
+
+    if dump_mode == DumpMode.OVERWRITE:
+        dump_mode = 'w'
+    elif dump_mode == DumpMode.APPEND:
+        dump_mode = 'a'
+    with jsonl.open(jsonl_file, mode=dump_mode) as writer:
+        writer.write_all(data_list)
+
+
+def jsonl_to_csv():
+    pass
+
+
+def yaml_to_dict(yaml_file) -> dict:
+    """
+    Read yaml file to dict.
+    """
+    with open(yaml_file, 'r') as f:
+        try:
+            stream = yaml.safe_load(f)
+        except yaml.YAMLError as e:
+            logger.error(f'{e}')
+            raise e
+
+    return stream
+
+
+def dict_to_yaml(d: dict, yaml_file: str):
+    """
+    Dump dict to yaml file.
+    """
+    with open(yaml_file, 'w') as f:
+        yaml.dump(d, f, default_flow_style=False)
+
+
+def json_to_dict(json_file) -> dict:
+    """
+    Read json file to dict.
+    """
+    with open(json_file, 'r') as f:
+        try:
+            stream = json.load(f)
+        except json.JSONDecodeError as e:
+            logger.error(f'{e}')
+            raise e
+
+    return stream
+
+
+def are_paths_same(path1, path2):
+    """
+    Check if two paths are the same.
+    """
+    real_path1 = os.path.realpath(os.path.abspath(os.path.expanduser(path1)))
+    real_path2 = os.path.realpath(os.path.abspath(os.path.expanduser(path2)))
+
+    return real_path1 == real_path2

evalscope/utils/logger.py

@@ -75,6 +75,14 @@ def get_logger(log_file: Optional[str] = None, log_level: int = DEFAULT_LEVEL, f
     return logger
 
 
+def configure_logging(debug: bool, log_file: Optional[str] = None):
+    """Configure logging level based on the debug flag."""
+    if log_file:
+        get_logger(log_file=log_file, force=True)
+    if debug:
+        get_logger(log_level=logging.DEBUG, force=True)
+
+
 def add_file_handler_if_needed(logger, log_file, file_mode, log_level):
     for handler in logger.handlers:
         if isinstance(handler, logging.FileHandler):

evalscope/utils/utils.py

@@ -5,19 +5,13 @@ import functools
 import hashlib
 import importlib
 import importlib.util
-import json
-import jsonlines as jsonl
 import numpy as np
 import os
 import random
 import re
-import sys
 import torch
-import torch.nn.functional as F
-import yaml
 from typing import Any, Dict, List, Tuple, Union
 
-from evalscope.constants import DumpMode
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -36,102 +30,6 @@ def test_level_list():
     return TEST_LEVEL_LIST
 
 
-def jsonl_to_list(jsonl_file):
-    """
-    Read jsonl file to list.
-
-    Args:
-        jsonl_file: jsonl file path.
-
-    Returns:
-        list: list of lines. Each line is a dict.
-    """
-    res_list = []
-    with jsonl.open(jsonl_file, mode='r') as reader:
-        for line in reader.iter(type=dict, allow_none=True, skip_invalid=False):
-            res_list.append(line)
-    return res_list
-
-
-def jsonl_to_reader(jsonl_file):
-    """
-    Read jsonl file to reader object.
-
-    Args:
-        jsonl_file: jsonl file path.
-
-    Returns:
-        reader: jsonl reader object.
-    """
-    with jsonl.open(jsonl_file, mode='r') as reader:
-        return reader
-
-
-def jsonl_to_csv():
-    pass
-
-
-def dump_jsonl_data(data_list, jsonl_file, dump_mode=DumpMode.OVERWRITE):
-    """
-    Dump data to jsonl file.
-
-    Args:
-        data_list: data list to be dumped.  [{'a': 'aaa'}, ...]
-        jsonl_file: jsonl file path.
-        dump_mode: dump mode. It can be 'overwrite' or 'append'.
-    """
-    if not jsonl_file:
-        raise ValueError('output file must be provided.')
-
-    jsonl_file = os.path.expanduser(jsonl_file)
-
-    if not isinstance(data_list, list):
-        data_list = [data_list]
-
-    if dump_mode == DumpMode.OVERWRITE:
-        dump_mode = 'w'
-    elif dump_mode == DumpMode.APPEND:
-        dump_mode = 'a'
-    with jsonl.open(jsonl_file, mode=dump_mode) as writer:
-        writer.write_all(data_list)
-
-
-def yaml_to_dict(yaml_file) -> dict:
-    """
-    Read yaml file to dict.
-    """
-    with open(yaml_file, 'r') as f:
-        try:
-            stream = yaml.safe_load(f)
-        except yaml.YAMLError as e:
-            logger.error(f'{e}')
-            raise e
-
-    return stream
-
-
-def dict_to_yaml(d: dict, yaml_file: str):
-    """
-    Dump dict to yaml file.
-    """
-    with open(yaml_file, 'w') as f:
-        yaml.dump(d, f, default_flow_style=False)
-
-
-def json_to_dict(json_file) -> dict:
-    """
-    Read json file to dict.
-    """
-    with open(json_file, 'r') as f:
-        try:
-            stream = json.load(f)
-        except json.JSONDecodeError as e:
-            logger.error(f'{e}')
-            raise e
-
-    return stream
-
-
 def get_obj_from_cfg(eval_class_ref: Any, *args, **kwargs) -> Any:
     module_name, spliter, cls_name = eval_class_ref.partition(':')
 
@@ -300,18 +198,6 @@ class ResponseParser:
         return ''
 
 
-def make_outputs_dir(root_dir: str, datasets: list, model_id: str, model_revision: str):
-    if not model_id:
-        model_id = 'default'
-    model_id = model_id.replace('/', '_')
-
-    if not model_revision:
-        model_revision = 'default'
-
-    outputs_dir = os.path.join(root_dir, model_id, model_revision, f"eval_{'-'.join(datasets)}")
-
-    return outputs_dir
-
 
 def import_module_util(import_path_prefix: str, module_name: str, members_to_import: list) -> dict:
     """
@@ -355,67 +241,6 @@ def normalize_score(score: Union[float, dict], keep_num: int = 4) -> Union[float
     return score
 
 
-def split_str_parts_by(text: str, delimiters: List[str]):
-    """Split the text field into parts.
-    Args:
-        text: A text to be split.
-        delimiters: The delimiters.
-    Returns:
-        The split text in list of dicts.
-    """
-    all_start_chars = [d[0] for d in delimiters]
-    all_length = [len(d) for d in delimiters]
-
-    text_list = []
-    last_words = ''
-
-    while len(text) > 0:
-        for char_idx, char in enumerate(text):
-            match_index = [idx for idx, start_char in enumerate(all_start_chars) if start_char == char]
-            is_delimiter = False
-            for index in match_index:
-                if text[char_idx:char_idx + all_length[index]] == delimiters[index]:
-                    if last_words:
-                        if text_list:
-                            text_list[-1]['content'] = last_words
-                        else:
-                            text_list.append({'key': '', 'content': last_words})
-                    last_words = ''
-                    text_list.append({'key': delimiters[index]})
-                    text = text[char_idx + all_length[index]:]
-                    is_delimiter = True
-                    break
-            if not is_delimiter:
-                last_words += char
-            else:
-                break
-        if last_words == text:
-            text = ''
-
-    text_list[-1]['content'] = last_words
-    return text_list
-
-
-def get_bucket_sizes(max_length: int) -> List[int]:
-    return [max_length // 4 * (i + 1) for i in range(4)]
-
-
-def _get_closet_bucket(bucket_sizes, data_length):
-    """Select the one from bucket_sizes that is closest in distance to
-    data_length. This is required for TorchAcc.
-    """
-    cloest_length = sys.maxsize
-    for b in bucket_sizes:
-        if b == data_length or ((b < cloest_length) and (b > data_length)):
-            cloest_length = b
-
-    if cloest_length == sys.maxsize:
-        bucket_sizes.append(data_length)
-        cloest_length = data_length
-
-    return cloest_length
-
-
 def is_module_installed(module_name):
     try:
         importlib.import_module(module_name)

evalscope/version.py

@@ -1,4 +1,4 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-__version__ = '0.8.0'
-__release_datetime__ = '2024-12-15 00:00:00'
+__version__ = '0.8.2'
+__release_datetime__ = '2024-12-26 20:00:00'

{evalscope-0.8.0.dist-info → evalscope-0.8.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: evalscope
-Version: 0.8.0
+Version: 0.8.2
 Summary: EvalScope: Lightweight LLMs Evaluation Framework
 Home-page: https://github.com/modelscope/evalscope
 Author: ModelScope team
@@ -84,7 +84,7 @@ Requires-Dist: transformers-stream-generator; extra == "all"
 Requires-Dist: ms-opencompass>=0.1.4; extra == "all"
 Requires-Dist: ms-vlmeval>=0.0.9; extra == "all"
 Requires-Dist: mteb==1.19.4; extra == "all"
-Requires-Dist: ragas==0.2.7; extra == "all"
+Requires-Dist: ragas==0.2.9; extra == "all"
 Requires-Dist: webdataset>0.2.0; extra == "all"
 Requires-Dist: aiohttp; extra == "all"
 Requires-Dist: fastapi; extra == "all"
@@ -129,7 +129,7 @@ Requires-Dist: transformers; extra == "perf"
 Requires-Dist: unicorn; extra == "perf"
 Provides-Extra: rag
 Requires-Dist: mteb==1.19.4; extra == "rag"
-Requires-Dist: ragas==0.2.7; extra == "rag"
+Requires-Dist: ragas==0.2.9; extra == "rag"
 Requires-Dist: webdataset>0.2.0; extra == "rag"
 Provides-Extra: vlmeval
 Requires-Dist: ms-vlmeval>=0.0.9; extra == "vlmeval"
@@ -181,6 +181,8 @@ The framework accommodates multiple evaluation scenarios such as end-to-end RAG
   <br>EvalScope Framework.
 </p>
 
+<details><summary>Framework Description</summary>
+
 The architecture includes the following modules:
 1. **Model Adapter**: The model adapter is used to convert the outputs of specific models into the format required by the framework, supporting both API call models and locally run models.
 2. **Data Adapter**: The data adapter is responsible for converting and processing input data to meet various evaluation needs and formats.
@@ -194,6 +196,16 @@ The architecture includes the following modules:
 5. **Evaluation Report**: The final generated evaluation report summarizes the model's performance, which can be used for decision-making and further model optimization.
 6. **Visualization**: Visualization results help users intuitively understand evaluation results, facilitating analysis and comparison of different model performances.
 
+</details>
+
+## ☎ User Groups
+
+Please scan the QR code below to join our community groups:
+
+[Discord Group](https://discord.com/invite/D27yfEFVz5)              |  WeChat Group | DingTalk Group
+:-------------------------:|:-------------------------:|:-------------------------:
+<img src="docs/asset/discord_qr.jpg" width="160" height="160">  |  <img src="docs/asset/wechat.png" width="160" height="160"> | <img src="docs/asset/dingding.png" width="160" height="160">
+
 
 ## 🎉 News
 - 🔥 **[2024.12.13]** Model evaluation optimization: no need to pass the `--template-type` parameter anymore; supports starting evaluation with `evalscope eval --args`. Refer to the [📖 User Guide](https://evalscope.readthedocs.io/en/latest/get_started/basic_usage.html) for more details.