evalscope 0.7.2__py3-none-any.whl → 0.8.1__py3-none-any.whl

evalscope/utils/model_utils.py

@@ -0,0 +1,11 @@
+from transformers import GenerationConfig
+
+
+def fix_do_sample_warning(generation_config: GenerationConfig) -> None:
+    # Use the default values of temperature/top_p/top_k in generation_config.
+    if generation_config.temperature == 0:
+        generation_config.do_sample = False
+    if generation_config.do_sample is False:
+        generation_config.temperature = 1.
+        generation_config.top_p = 1.
+        generation_config.top_k = 50

evalscope/utils/utils.py

@@ -5,20 +5,13 @@ import functools
 import hashlib
 import importlib
 import importlib.util
+import numpy as np
 import os
 import random
 import re
-import sys
-from typing import Any, Dict, List, Tuple, Union
-
-import json
-import jsonlines as jsonl
-import numpy as np
 import torch
-import torch.nn.functional as F
-import yaml
+from typing import Any, Dict, List, Tuple, Union
 
-from evalscope.constants import DumpMode, OutputsStructure
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -37,101 +30,6 @@ def test_level_list():
     return TEST_LEVEL_LIST
 
 
-def jsonl_to_list(jsonl_file):
-    """
-    Read jsonl file to list.
-
-    Args:
-        jsonl_file: jsonl file path.
-
-    Returns:
-        list: list of lines. Each line is a dict.
-    """
-    res_list = []
-    with jsonl.open(jsonl_file, mode='r') as reader:
-        for line in reader.iter(type=dict, allow_none=True, skip_invalid=False):
-            res_list.append(line)
-    return res_list
-
-
-def jsonl_to_reader(jsonl_file):
-    """
-    Read jsonl file to reader object.
-
-    Args:
-        jsonl_file: jsonl file path.
-
-    Returns:
-        reader: jsonl reader object.
-    """
-    with jsonl.open(jsonl_file, mode='r') as reader:
-        return reader
-
-
-def jsonl_to_csv():
-    pass
-
-
-def dump_jsonl_data(data_list, jsonl_file, dump_mode=DumpMode.OVERWRITE):
-    """
-    Dump data to jsonl file.
-
-    Args:
-        data_list: data list to be dumped.  [{'a': 'aaa'}, ...]
-        jsonl_file: jsonl file path.
-        dump_mode: dump mode. It can be 'overwrite' or 'append'.
-    """
-    if not jsonl_file:
-        raise ValueError('output file must be provided.')
-
-    jsonl_file = os.path.expanduser(jsonl_file)
-
-    if dump_mode == DumpMode.OVERWRITE:
-        dump_mode = 'w'
-    elif dump_mode == DumpMode.APPEND:
-        dump_mode = 'a'
-    with jsonl.open(jsonl_file, mode=dump_mode) as writer:
-        writer.write_all(data_list)
-    logger.info(f'Dump data to {jsonl_file} successfully.')
-
-
-def yaml_to_dict(yaml_file) -> dict:
-    """
-    Read yaml file to dict.
-    """
-    with open(yaml_file, 'r') as f:
-        try:
-            stream = yaml.safe_load(f)
-        except yaml.YAMLError as e:
-            logger.error(f'{e}')
-            raise e
-
-    return stream
-
-
-def dict_to_yaml(d: dict, yaml_file: str):
-    """
-    Dump dict to yaml file.
-    """
-    with open(yaml_file, 'w') as f:
-        yaml.dump(d, f, default_flow_style=False)
-    logger.info(f'Dump data to {yaml_file} successfully.')
-
-
-def json_to_dict(json_file) -> dict:
-    """
-    Read json file to dict.
-    """
-    with open(json_file, 'r') as f:
-        try:
-            stream = json.load(f)
-        except json.JSONDecodeError as e:
-            logger.error(f'{e}')
-            raise e
-
-    return stream
-
-
 def get_obj_from_cfg(eval_class_ref: Any, *args, **kwargs) -> Any:
     module_name, spliter, cls_name = eval_class_ref.partition(':')
 
@@ -148,25 +46,13 @@ def get_obj_from_cfg(eval_class_ref: Any, *args, **kwargs) -> Any:
     return functools.partial(obj_cls, *args, **kwargs)
 
 
-def markdown_table(header_l, data_l):
-    md_str = f'| {" | ".join(header_l)} |'
-    md_str += f'\n| {" | ".join(["---"] * len(header_l))} |'
-    for data in data_l:
-        if isinstance(data, str):
-            data = [data]
-        assert len(data) <= len(header_l)
-        tmp = data + [''] * (len(header_l) - len(data))
-        md_str += f'\n| {" | ".join(tmp)} |'
-    return md_str
-
-
 def random_seeded_choice(seed: Union[int, str, float], choices, **kwargs):
     """Random choice with a (potentially string) seed."""
     return random.Random(seed).choices(choices, k=1, **kwargs)[0]
 
 
-def gen_hash(name: str):
-    return hashlib.md5(name.encode(encoding='UTF-8')).hexdigest()
+def gen_hash(name: str, bits: int = 32):
+    return hashlib.md5(name.encode(encoding='UTF-8')).hexdigest()[:bits]
 
 
 def dict_torch_dtype_to_str(d: Dict[str, Any]) -> dict:
@@ -312,52 +198,6 @@ class ResponseParser:
         return ''
 
 
-def make_outputs_dir(root_dir: str, datasets: list, model_id: str, model_revision: str):
-    # model_revision = model_revision if model_revision is not None else 'none'
-    # now = datetime.datetime.now()
-    # format_time = now.strftime('%Y%m%d_%H%M%S')
-    # outputs_name = format_time + '_' + 'default' + '_' + model_id.replace('/', '_') + '_' + model_revision
-    # outputs_dir = os.path.join(work_dir, outputs_name)
-    # dataset_name = dataset_id.replace('/', '_')
-    # outputs_dir = os.path.join(work_dir, dataset_name)
-
-    if not model_id:
-        model_id = 'default'
-    model_id = model_id.replace('/', '_')
-
-    if not model_revision:
-        model_revision = 'default'
-
-    outputs_dir = os.path.join(root_dir, f"eval_{'-'.join(datasets)}_{model_id}_{model_revision}")
-
-    return outputs_dir
-
-
-def process_outputs_structure(outputs_dir: str, is_make: bool = True) -> dict:
-    logs_dir = os.path.join(outputs_dir, 'logs')
-    predictions_dir = os.path.join(outputs_dir, 'predictions')
-    reviews_dir = os.path.join(outputs_dir, 'reviews')
-    reports_dir = os.path.join(outputs_dir, 'reports')
-    configs_dir = os.path.join(outputs_dir, 'configs')
-
-    if is_make:
-        os.makedirs(outputs_dir, exist_ok=True)
-        os.makedirs(logs_dir, exist_ok=True)
-        os.makedirs(predictions_dir, exist_ok=True)
-        os.makedirs(reviews_dir, exist_ok=True)
-        os.makedirs(reports_dir, exist_ok=True)
-        os.makedirs(configs_dir, exist_ok=True)
-
-    outputs_structure = {
-        OutputsStructure.LOGS_DIR: logs_dir,
-        OutputsStructure.PREDICTIONS_DIR: predictions_dir,
-        OutputsStructure.REVIEWS_DIR: reviews_dir,
-        OutputsStructure.REPORTS_DIR: reports_dir,
-        OutputsStructure.CONFIGS_DIR: configs_dir,
-    }
-
-    return outputs_structure
-
 
 def import_module_util(import_path_prefix: str, module_name: str, members_to_import: list) -> dict:
     """
@@ -401,148 +241,6 @@ def normalize_score(score: Union[float, dict], keep_num: int = 4) -> Union[float
     return score
 
 
-def split_str_parts_by(text: str, delimiters: List[str]):
-    """Split the text field into parts.
-    Args:
-        text: A text to be split.
-        delimiters: The delimiters.
-    Returns:
-        The split text in list of dicts.
-    """
-    all_start_chars = [d[0] for d in delimiters]
-    all_length = [len(d) for d in delimiters]
-
-    text_list = []
-    last_words = ''
-
-    while len(text) > 0:
-        for char_idx, char in enumerate(text):
-            match_index = [idx for idx, start_char in enumerate(all_start_chars) if start_char == char]
-            is_delimiter = False
-            for index in match_index:
-                if text[char_idx:char_idx + all_length[index]] == delimiters[index]:
-                    if last_words:
-                        if text_list:
-                            text_list[-1]['content'] = last_words
-                        else:
-                            text_list.append({'key': '', 'content': last_words})
-                    last_words = ''
-                    text_list.append({'key': delimiters[index]})
-                    text = text[char_idx + all_length[index]:]
-                    is_delimiter = True
-                    break
-            if not is_delimiter:
-                last_words += char
-            else:
-                break
-        if last_words == text:
-            text = ''
-
-    text_list[-1]['content'] = last_words
-    return text_list
-
-
-def calculate_loss_scale(response: str, use_loss_scale=False) -> Tuple[List[str], List[float]]:
-    """Calculate the loss scale by splitting the agent response.
-    This algorithm comes from paper: https://arxiv.org/pdf/2309.00986.pdf
-    Agent response format:
-    ```text
-        Thought: you should always think about what to do
-        Action: the action to take, should be one of the above tools[fire_recognition,
-            fire_alert, call_police, call_fireman]
-        Action Input: the input to the action
-        Observation: the result of the action
-        ... (this Thought/Action/Action Input/Observation can be repeated zero or more times)
-        Thought: I now know the final answer
-        Final Answer: the final answer to the original input question
-    ```
-    Args:
-        response: The response text
-        use_loss_scale: Use weighted loss. With this, some part of the loss will be enhanced to improve performance.
-    Returns:
-        A tuple of agent response parts and their weights.
-    """
-    if 'Action:' in response and 'Observation:' in response and use_loss_scale:
-        agent_keyword = ['Action:', 'Action Input:', 'Thought:', 'Final Answer:', 'Observation:']
-        agent_parts = split_str_parts_by(response, agent_keyword)
-        weights = []
-        agent_content = []
-        for c in agent_parts:
-            if c['key'] in ('Action:', 'Action Input:'):
-                weights += [2.0]
-                weights += [2.0]
-            elif c['key'] in ('Thought:', 'Final Answer:', ''):
-                weights += [1.0]
-                weights += [1.0]
-            elif c['key'] in ('Observation:', ):
-                weights += [2.0]
-                weights += [0.0]
-            agent_content.append(c['key'])
-            agent_content.append(c['content'])
-        return agent_content, weights
-    else:
-        return [response], [1.0]
-
-
-def get_bucket_sizes(max_length: int) -> List[int]:
-    return [max_length // 4 * (i + 1) for i in range(4)]
-
-
-def _get_closet_bucket(bucket_sizes, data_length):
-    """Select the one from bucket_sizes that is closest in distance to
-    data_length. This is required for TorchAcc.
-    """
-    cloest_length = sys.maxsize
-    for b in bucket_sizes:
-        if b == data_length or ((b < cloest_length) and (b > data_length)):
-            cloest_length = b
-
-    if cloest_length == sys.maxsize:
-        bucket_sizes.append(data_length)
-        cloest_length = data_length
-
-    return cloest_length
-
-
-def pad_and_split_batch(padding_to, input_ids, attention_mask, labels, loss_scale, max_length, tokenizer, rank,
-                        world_size):
-    if padding_to is None:
-        longest_len = input_ids.shape[-1]
-        bucket_sizes = get_bucket_sizes(max_length)
-        bucket_data_length = _get_closet_bucket(bucket_sizes, longest_len)
-        padding_length = bucket_data_length - input_ids.shape[1]
-        input_ids = F.pad(input_ids, (0, padding_length), 'constant', tokenizer.pad_token_id)
-        attention_mask = F.pad(attention_mask, (0, padding_length), 'constant', 0)
-        if loss_scale:
-            loss_scale = F.pad(loss_scale, (0, padding_length), 'constant', 0.)
-        labels = F.pad(labels, (0, padding_length), 'constant', -100)
-
-    # manully split the batch to different DP rank.
-    batch_size = input_ids.shape[0] // world_size
-    if batch_size > 0:
-        start = rank * batch_size
-        end = (rank + 1) * batch_size
-        input_ids = input_ids[start:end, :]
-        attention_mask = attention_mask[start:end, :]
-        labels = labels[start:end, :]
-        if loss_scale:
-            loss_scale = loss_scale[start:end, :]
-    return input_ids, attention_mask, labels, loss_scale
-
-
-def get_dist_setting() -> Tuple[int, int, int, int]:
-    """return rank, local_rank, world_size, local_world_size"""
-    rank = int(os.getenv('RANK', -1))
-    local_rank = int(os.getenv('LOCAL_RANK', -1))
-    world_size = int(os.getenv('WORLD_SIZE', 1))
-    local_world_size = int(os.getenv('LOCAL_WORLD_SIZE', 1))
-    return rank, local_rank, world_size, local_world_size
-
-
-def use_torchacc() -> bool:
-    return os.getenv('USE_TORCHACC', '0') == '1'
-
-
 def is_module_installed(module_name):
     try:
         importlib.import_module(module_name)
@@ -576,6 +274,7 @@ def get_valid_list(input_list, candidate_list):
 
 def get_latest_folder_path(work_dir):
     from datetime import datetime
+
     # Get all subdirectories in the work_dir
     folders = [f for f in os.listdir(work_dir) if os.path.isdir(os.path.join(work_dir, f))]
 

evalscope/version.py

@@ -1,4 +1,4 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-__version__ = '0.7.2'
-__release_datetime__ = '2024-12-04 12:00:00'
+__version__ = '0.8.1'
+__release_datetime__ = '2024-12-17 20:00:00'