evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/utils/io_utils.py

@@ -1,7 +1,18 @@
+import base64
+import csv
+import hashlib
+import io
 import json
 import jsonlines as jsonl
 import os
+import re
+import string
+import unicodedata
 import yaml
+from datetime import datetime
+from io import BytesIO
+from PIL import Image
+from typing import Tuple
 
 from evalscope.constants import DumpMode
 from evalscope.utils.logger import get_logger
@@ -27,7 +38,7 @@ class OutputsStructure:
             'configs_dir': None
         }
 
-    def _get_dir(self, attr_name, dir_name):
+    def _get_dir(self, attr_name, dir_name) -> str:
         if self._dirs[attr_name] is None:
             dir_path = os.path.join(self.outputs_dir, dir_name)
             if self.is_make:
@@ -66,10 +77,20 @@ def jsonl_to_list(jsonl_file):
     Returns:
         list: list of lines. Each line is a dict.
     """
-    res_list = []
-    with jsonl.open(jsonl_file, mode='r') as reader:
-        for line in reader.iter(type=dict, allow_none=True, skip_invalid=False):
-            res_list.append(line)
+    try:
+        res_list = []
+        with jsonl.open(jsonl_file, mode='r') as reader:
+            for line in reader.iter(type=dict, allow_none=True, skip_invalid=False):
+                res_list.append(line)
+    except Exception:
+        # Fallback to reading line by line
+        res_list = []
+        with open(jsonl_file, 'r', encoding='utf-8') as f:
+            for line in f:
+                if line.strip():  # Skip empty lines
+                    res_list.append(json.loads(line.strip()))
+    if not res_list:
+        logger.warning(f'No data found in {jsonl_file}.')
     return res_list
 
 
@@ -104,6 +125,9 @@ def dump_jsonl_data(data_list, jsonl_file, dump_mode=DumpMode.OVERWRITE):
     if not isinstance(data_list, list):
         data_list = [data_list]
 
+    # Convert non-serializable types to serializable ones
+    data_list = convert_normal_types(data_list)
+
     if dump_mode == DumpMode.OVERWRITE:
         dump_mode = 'w'
     elif dump_mode == DumpMode.APPEND:
@@ -112,8 +136,76 @@ def dump_jsonl_data(data_list, jsonl_file, dump_mode=DumpMode.OVERWRITE):
         writer.write_all(data_list)
 
 
-def jsonl_to_csv():
-    pass
+def jsonl_to_csv(jsonl_file, csv_file):
+    """
+    Convert jsonl file to csv file.
+
+    Args:
+        jsonl_file: jsonl file path.
+        csv_file: csv file path.
+    """
+    data = jsonl_to_list(jsonl_file)
+    if not data:
+        logger.warning(f'No data found in {jsonl_file}.')
+        return
+
+    with open(csv_file, 'w', newline='', encoding='utf-8') as f:
+        writer = csv.writer(f)
+        writer.writerow(data[0].keys())  # Write header
+        for item in data:
+            writer.writerow(item.values())
+
+
+def csv_to_list(csv_file) -> list:
+    """
+    Read csv file to list.
+
+    Args:
+        csv_file: csv file path.
+
+    Returns:
+        list: list of lines. Each line is a dict.
+    """
+    res_list = []
+    with open(csv_file, 'r', encoding='utf-8') as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            res_list.append(row)
+    return res_list
+
+
+def tsv_to_list(tsv_file) -> list:
+    """
+    Read tsv file to list.
+
+    Args:
+        tsv_file: tsv file path.
+
+    Returns:
+        list: list of lines. Each line is a dict.
+    """
+    res_list = []
+    with open(tsv_file, 'r', encoding='utf-8') as f:
+        reader = csv.DictReader(f, delimiter='\t')
+        for row in reader:
+            res_list.append(row)
+    return res_list
+
+
+def csv_to_jsonl(csv_file, jsonl_file):
+    """
+    Convert csv file to jsonl file.
+
+    Args:
+        csv_file: csv file path.
+        jsonl_file: jsonl file path.
+    """
+    data = csv_to_list(csv_file)
+    if not data:
+        logger.warning(f'No data found in {csv_file}.')
+        return
+
+    dump_jsonl_data(data, jsonl_file, dump_mode=DumpMode.OVERWRITE)
 
 
 def yaml_to_dict(yaml_file) -> dict:
@@ -135,7 +227,7 @@ def dict_to_yaml(d: dict, yaml_file: str):
     Dump dict to yaml file.
     """
     with open(yaml_file, 'w') as f:
-        yaml.dump(d, f, default_flow_style=False)
+        yaml.dump(d, f, default_flow_style=False, allow_unicode=True)
 
 
 def json_to_dict(json_file) -> dict:
@@ -168,3 +260,239 @@ def dict_to_json(d: dict, json_file: str):
     """
     with open(json_file, 'w') as f:
         json.dump(d, f, indent=4, ensure_ascii=False)
+
+
+def get_latest_folder_path(work_dir):
+    from datetime import datetime
+
+    # Get all subdirectories in the work_dir
+    folders = [f for f in os.listdir(work_dir) if os.path.isdir(os.path.join(work_dir, f))]
+
+    # Get the timestamp（YYYYMMDD_HHMMSS）
+    timestamp_pattern = re.compile(r'^\d{8}_\d{6}$')
+
+    # Filter out the folders
+    timestamped_folders = [f for f in folders if timestamp_pattern.match(f)]
+
+    if not timestamped_folders:
+        print(f'>> No timestamped folders found in {work_dir}!')
+        return None
+
+    # timestamp parser
+    def parse_timestamp(folder_name):
+        return datetime.strptime(folder_name, '%Y%m%d_%H%M%S')
+
+    # Find the latest folder
+    latest_folder = max(timestamped_folders, key=parse_timestamp)
+
+    return os.path.join(work_dir, latest_folder)
+
+
+def gen_hash(name: str, bits: int = 32):
+    return hashlib.md5(name.encode(encoding='UTF-8')).hexdigest()[:bits]
+
+
+def get_valid_list(input_list, candidate_list):
+    """
+    Get the valid and invalid list from input_list based on candidate_list.
+    Args:
+        input_list: The input list.
+        candidate_list: The candidate list.
+
+    Returns:
+        valid_list: The valid list.
+        invalid_list: The invalid list.
+    """
+    return [i for i in input_list if i in candidate_list], \
+           [i for i in input_list if i not in candidate_list]
+
+
+def PIL_to_base64(image: Image.Image, format: str = 'JPEG', add_header: bool = False) -> str:
+    """
+    Convert a PIL Image to a base64 encoded string.
+
+    Args:
+        image (Image.Image): The PIL Image to convert.
+        format (str): The format to save the image in. Default is 'JPEG'.
+        add_header (bool): Whether to add the base64 header. Default is False.
+
+    Returns:
+        str: Base64 encoded string of the image.
+    """
+    buffered = BytesIO()
+    image.save(buffered, format=format)
+    img_str = base64.b64encode(buffered.getvalue()).decode('utf-8')
+    if add_header:
+        img_str = f'data:image/{format.lower()};base64,{img_str}'
+    return img_str
+
+
+def bytes_to_base64(bytes_data: bytes, *, format: str = 'png', add_header: bool = False, content_type='image') -> str:
+    """Convert bytes to a base64 encoded string.
+
+    Args:
+        bytes_data (bytes): The bytes to convert.
+        format (str): The format of the image. Default is 'png'.
+        add_header (bool): Whether to add the base64 header. Default is False.
+        content_type (str): The type of the data, 'image' or 'audio'. Default is 'image'.
+
+    Returns:
+        str: Base64 encoded string of the bytes.
+    """
+    base64_str = base64.b64encode(bytes_data).decode('utf-8')
+    if add_header:
+        base64_str = f'data:{content_type}/{format};base64,{base64_str}'
+    return base64_str
+
+
+def base64_to_PIL(base64_str):
+    """Convert a base64 encoded string to a PIL Image.
+
+    Args:
+        base64_str (str): The base64 encoded string.
+
+    Returns:
+        Image.Image: The decoded PIL Image.
+    """
+    # remove header
+    if ',' in base64_str:
+        base64_str = base64_str.split(',', 1)[1]
+
+    # decode
+    img_data = base64.b64decode(base64_str)
+    img_file = io.BytesIO(img_data)
+    img = Image.open(img_file)
+    return img
+
+
+def safe_filename(s: str, max_length: int = 255) -> str:
+    """
+    Convert a string into a safe filename by removing or replacing unsafe characters.
+
+    Args:
+        s (str): The input string to convert
+        max_length (int): Maximum length of the resulting filename (default 255)
+
+    Returns:
+        str: A safe filename string
+
+    Examples:
+        >>> safe_filename("Hello/World?.txt")
+        'Hello_World.txt'
+    """
+    # normalize unicode characters
+    s = unicodedata.normalize('NFKD', s)
+    s = s.encode('ASCII', 'ignore').decode('ASCII')
+
+    # remove or replace unsafe characters
+    # Keep only alphanumeric characters, dots, dashes, and underscores
+    safe_chars = string.ascii_letters + string.digits + '.-_'
+    s = ''.join(c if c in safe_chars else '_' for c in s)
+
+    # remove consecutive underscores
+    s = re.sub(r'_+', '_', s)
+
+    # remove leading/trailing periods and underscores
+    s = s.strip('._')
+
+    # handle empty string case
+    if not s:
+        s = 'untitled'
+
+    # handle starting with a period (hidden files)
+    if s.startswith('.'):
+        s = '_' + s
+
+    # enforce length limit
+    if len(s) > max_length:
+        # If we need to truncate, preserve the file extension if present
+        name, ext = os.path.splitext(s)
+        ext_len = len(ext)
+        if ext_len > 0:
+            max_name_length = max_length - ext_len
+            s = name[:max_name_length] + ext
+        else:
+            s = s[:max_length]
+
+    return s
+
+
+def convert_normal_types(obj):
+    """Recursively convert numpy types and datetime objects to native Python types for JSON serialization."""
+    import numpy as np
+
+    if isinstance(obj, datetime):
+        return obj.isoformat()
+    elif isinstance(obj, np.bool_):
+        return bool(obj)
+    elif isinstance(obj, np.integer):
+        return int(obj)
+    elif isinstance(obj, np.floating):
+        return float(obj)
+    elif isinstance(obj, np.ndarray):
+        return obj.tolist()
+    elif isinstance(obj, dict):
+        return {key: convert_normal_types(value) for key, value in obj.items()}
+    elif isinstance(obj, list):
+        return [convert_normal_types(item) for item in obj]
+    elif isinstance(obj, tuple):
+        return tuple(convert_normal_types(item) for item in obj)
+    elif isinstance(obj, os.PathLike):
+        return str(obj)
+    else:
+        return obj
+
+
+def compress_image_to_limit(image_bytes: bytes, max_bytes: int = 10_000_000) -> Tuple[bytes, str]:
+    """
+    Ensure image bytes are under max_bytes by re-encoding to JPEG with quality reduction
+    and optional downscaling. Returns (processed_bytes, format_str).
+    If the original bytes are already below the limit, returns them as PNG.
+    """
+    if len(image_bytes) <= max_bytes:
+        return image_bytes, 'png'
+
+    try:
+        img = Image.open(BytesIO(image_bytes))
+    except Exception as exc:
+        logger.warning(f'Failed to open image bytes with PIL, sending original image; may exceed API limit: {exc}')
+        return image_bytes, 'png'
+
+    # Convert to RGB for JPEG if needed
+    if img.mode not in ('RGB', 'L'):
+        img = img.convert('RGB')
+
+    def encode_jpeg(source: Image.Image, quality: int) -> bytes:
+        buf = BytesIO()
+        source.save(buf, format='JPEG', quality=quality, optimize=True, progressive=True)
+        return buf.getvalue()
+
+    # Start with moderate quality and reduce
+    quality: int = 85
+    out: bytes = encode_jpeg(img, quality)
+    quality_floor: int = 40
+
+    while len(out) > max_bytes and quality > quality_floor:
+        quality -= 10
+        out = encode_jpeg(img, quality)
+
+    # If still too large, progressively downscale
+    min_side_floor: int = 256
+    scale: float = 0.9
+    while len(out) > max_bytes and min(img.size) > min_side_floor:
+        new_w = max(min_side_floor, int(img.width * scale))
+        new_h = max(min_side_floor, int(img.height * scale))
+        if (new_w, new_h) == img.size:
+            break
+        img = img.resize((new_w, new_h), Image.LANCZOS)
+        out = encode_jpeg(img, quality)
+
+    if len(out) > max_bytes:
+        logger.warning(f'Image remains above limit after compression: size={len(out)} bytes (limit={max_bytes}).')
+    else:
+        logger.info(
+            f'Compressed image from {len(image_bytes)} to {len(out)} bytes; '
+            f'quality={quality}, size={img.width}x{img.height}.'
+        )
+
+    return out, 'jpeg'

evalscope/utils/json_schema.py

@@ -0,0 +1,231 @@
+import types
+import typing
+from copy import deepcopy
+from dataclasses import is_dataclass
+from datetime import date, datetime, time
+from enum import EnumMeta
+from pydantic import BaseModel, Field, field_validator, model_validator
+from typing import (
+    Any,
+    Dict,
+    List,
+    Literal,
+    Optional,
+    Set,
+    Tuple,
+    Type,
+    Union,
+    cast,
+    get_args,
+    get_origin,
+    get_type_hints,
+    is_typeddict,
+)
+
+JSONType = Literal['string', 'integer', 'number', 'boolean', 'array', 'object', 'null']
+"""Valid types within JSON schema."""
+
+
+class JSONSchema(BaseModel):
+    """JSON Schema for type."""
+
+    type: Optional[JSONType] = Field(default=None)
+    """JSON type of tool parameter."""
+
+    format: Optional[str] = Field(default=None)
+    """Format of the parameter (e.g. date-time)."""
+
+    description: Optional[str] = Field(default=None)
+    """Parameter description."""
+
+    default: Any = Field(default=None)
+    """Default value for parameter."""
+
+    enum: Optional[List[Any]] = Field(default=None)
+    """Valid values for enum parameters."""
+
+    items: Optional['JSONSchema'] = Field(default=None)
+    """Valid type for array parameters."""
+
+    properties: Optional[Dict[str, 'JSONSchema']] = Field(default=None)
+    """Valid fields for object parametrs."""
+
+    additionalProperties: Optional[Union['JSONSchema', bool]] = Field(default=None)
+    """Are additional properties allowed?"""
+
+    anyOf: Optional[List['JSONSchema']] = Field(default=None)
+    """Valid types for union parameters."""
+
+    required: Optional[List[str]] = Field(default=None)
+    """Required fields for object parameters."""
+
+    @model_validator(mode='before')
+    def convert_type_before_validation(cls, values):
+        values = deepcopy(values)
+
+        def recursive_convert_type(obj):
+            if isinstance(obj, dict):
+                # Convert 'type' field if it's a string
+                if 'type' in obj and isinstance(obj['type'], str):
+                    try:
+                        obj['type'] = python_type_to_json_type(obj['type'])
+                    except ValueError:
+                        # If conversion fails, leave it as is
+                        pass
+                # Recursively process nested structures
+                for k, v in obj.items():
+                    obj[k] = recursive_convert_type(v)
+            elif isinstance(obj, list):
+                return [recursive_convert_type(item) for item in obj]
+            return obj
+
+        return recursive_convert_type(values)
+
+
+def json_schema(t: Type[Any]) -> JSONSchema:
+    """Provide a JSON Schema for the specified type.
+
+    Schemas can be automatically inferred for a wide variety of
+    Python class types including Pydantic BaseModel, dataclasses,
+    and typed dicts.
+
+    Args:
+        t: Python type
+
+    Returns:
+        JSON Schema for type.
+    """
+    origin = get_origin(t)
+    args = get_args(t)
+
+    if origin is None:
+        if t is int:
+            return JSONSchema(type='integer')
+        elif t is float:
+            return JSONSchema(type='number')
+        elif t is str:
+            return JSONSchema(type='string')
+        elif t is bool:
+            return JSONSchema(type='boolean')
+        elif t is datetime:
+            return JSONSchema(type='string', format='date-time')
+        elif t is date:
+            return JSONSchema(type='string', format='date')
+        elif t is time:
+            return JSONSchema(type='string', format='time')
+        elif t is list or t is set:
+            return JSONSchema(type='array', items=JSONSchema())
+        elif t is dict:
+            return JSONSchema(type='object', additionalProperties=JSONSchema())
+        elif (is_dataclass(t) or is_typeddict(t) or (isinstance(t, type) and issubclass(t, BaseModel))):
+            return cls_json_schema(t)
+        elif isinstance(t, EnumMeta):
+            return JSONSchema(enum=[item.value for item in t])
+        elif t is type(None):
+            return JSONSchema(type='null')
+        else:
+            return JSONSchema()
+    elif (origin is list or origin is List or origin is tuple or origin is Tuple or origin is set or origin is Set):
+        return JSONSchema(type='array', items=json_schema(args[0]) if args else JSONSchema())
+    elif origin is dict or origin is Dict:
+        return JSONSchema(
+            type='object',
+            additionalProperties=json_schema(args[1]) if len(args) > 1 else JSONSchema(),
+        )
+    elif origin is Union or origin is types.UnionType:
+        return JSONSchema(anyOf=[json_schema(arg) for arg in args])
+    elif origin is Optional:
+        return JSONSchema(anyOf=[json_schema(arg) for arg in args] + [JSONSchema(type='null')])
+    elif origin is typing.Literal:
+        return JSONSchema(enum=list(args))
+
+    return JSONSchema()  # Default case if we can't determine the type
+
+
+def cls_json_schema(cls: Type[Any]) -> JSONSchema:
+    properties: Dict[str, JSONSchema] = {}
+    required: List[str] = []
+
+    if is_dataclass(cls):
+        fields = cls.__dataclass_fields__  # type: ignore
+        for name, field in fields.items():
+            properties[name] = json_schema(field.type)  # type: ignore
+            if field.default == field.default_factory:
+                required.append(name)
+    elif isinstance(cls, type) and issubclass(cls, BaseModel):
+        schema = cls.model_json_schema()
+        schema = resolve_schema_references(schema)
+        for name, prop in schema.get('properties', {}).items():
+            properties[name] = JSONSchema(**prop)
+        required = schema.get('required', [])
+    elif is_typeddict(cls):
+        annotations = get_type_hints(cls)
+        for name, type_hint in annotations.items():
+            properties[name] = json_schema(type_hint)
+            if name in cls.__required_keys__:
+                required.append(name)
+
+    return JSONSchema(
+        type='object',
+        properties=properties,
+        required=required if required else None,
+        additionalProperties=False,
+    )
+
+
+def python_type_to_json_type(python_type: Optional[str]) -> JSONType:
+    if python_type is not None and python_type in get_args(JSONType):
+        return python_type
+    if python_type == 'str':
+        return 'string'
+    elif python_type == 'int':
+        return 'integer'
+    elif python_type == 'float':
+        return 'number'
+    elif python_type == 'bool':
+        return 'boolean'
+    elif python_type == 'list':
+        return 'array'
+    elif python_type == 'dict':
+        return 'object'
+    elif python_type == 'None':
+        return 'null'
+    elif python_type is None:
+        # treat 'unknown' as string as anything can be converted to string
+        return 'string'
+    else:
+        raise ValueError(f'Unsupported type: {python_type} for Python to JSON conversion.')
+
+
+def resolve_schema_references(schema: Dict[str, Any]) -> Dict[str, Any]:
+    """Resolves all $ref references in a JSON schema by inlining the definitions."""
+    schema = deepcopy(schema)
+    definitions = schema.pop('$defs', {})
+
+    def _resolve_refs(obj: Any) -> Any:
+        if isinstance(obj, dict):
+            if '$ref' in obj and obj['$ref'].startswith('#/$defs/'):
+                ref_key = obj['$ref'].split('/')[-1]
+                if ref_key in definitions:
+                    # Replace with a deep copy of the definition
+                    resolved = deepcopy(definitions[ref_key])
+                    # Process any nested references in the definition
+                    resolved = _resolve_refs(resolved)
+
+                    # Merge in the current object fields, which should take priority
+                    # This means that if you have e.g.
+                    # {"$ref": "#/$defs/SubType", "description": "subtype of type SubType"},
+                    # and SubType resolves to
+                    # {"description": "The SubType Class", "parameters": {"param1": {"type": "string"}}},
+                    # the final result will be:
+                    # {"description": "subtype of type SubType", "parameters": {"param1": {"type": "string"}}}
+                    return resolved | {k: o for k, o in obj.items() if k != '$ref'}
+
+            # Process all entries in the dictionary
+            return {k: _resolve_refs(v) for k, v in obj.items()}
+        elif isinstance(obj, list):
+            return [_resolve_refs(item) for item in obj]
+        else:
+            return obj
+
+    return cast(Dict[str, Any], _resolve_refs(schema))