evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/api/dataset/loader.py

@@ -0,0 +1,262 @@
+import copy
+import os
+import random
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Callable, Dict, List, Optional, Union
+
+from evalscope.api.dataset.utils import record_to_sample_fn
+from evalscope.constants import DEFAULT_EVALSCOPE_CACHE_DIR, HubType
+from evalscope.utils import get_logger
+from evalscope.utils.io_utils import csv_to_list, gen_hash, jsonl_to_list, safe_filename
+from .dataset import Dataset, FieldSpec, MemoryDataset, Sample
+from .utils import data_to_samples, shuffle_choices_if_requested
+
+logger = get_logger()
+
+
+class DataLoader(ABC):
+    """
+    Abstract base class for data loaders.
+    """
+
+    def __init__(
+        self,
+        data_id_or_path: str,
+        split: str,
+        sample_fields: Union[FieldSpec, Callable] = None,
+        filter_func: Callable = None,
+        subset: str = 'default',
+        version: str = None,
+        limit: Union[int, float] = None,
+        data_source: Optional[str] = None,
+        shuffle: bool = False,
+        shuffle_choices: Optional[Union[bool, int]] = None,
+        seed: Optional[int] = None,
+        auto_id: bool = True,
+        repeats: int = 1,
+        trust_remote: bool = True,
+        **kwargs
+    ):
+        self.data_id_or_path = data_id_or_path
+        self.split = split
+        self.sample_fields = sample_fields
+        self.filter_func = filter_func
+        self.subset = subset
+        self.version = version
+        self.limit = limit
+        self.data_source = data_source
+        self.shuffle = shuffle
+        self.shuffle_choices = shuffle_choices
+        self.seed = seed
+        self.auto_id = auto_id
+        self.repeats = repeats
+        self.trust_remote = trust_remote
+        self.kwargs = kwargs
+
+    @abstractmethod
+    def load(self) -> Dataset:
+        """
+        Load data from the source.
+        """
+        ...
+
+
+class RemoteDataLoader(DataLoader):
+    """
+    Data loader for remote datasets: ModelScope or Huggingface.
+    """
+
+    def load(self) -> Dataset:
+        import datasets
+        from modelscope import MsDataset
+
+        path = self.data_id_or_path
+        # resolve data_to_sample function
+        data_to_sample = record_to_sample_fn(self.sample_fields)
+        # generate a unique cache dir for this dataset
+        dataset_hash = gen_hash(f'{path}{self.split}{self.subset}{self.version}{self.kwargs}')
+        datasets_cache_dir = os.path.join(DEFAULT_EVALSCOPE_CACHE_DIR, 'datasets')
+        dataset_cache_dir = os.path.join(datasets_cache_dir, f'{safe_filename(path)}-{dataset_hash}')
+        if os.path.exists(dataset_cache_dir):
+            dataset = datasets.load_from_disk(dataset_cache_dir)
+        else:
+            logger.info(
+                f'Loading dataset {path} from {self.data_source} > subset: {self.subset} > split: {self.split} ...'
+            )
+            if self.data_source == HubType.MODELSCOPE:
+                dataset = MsDataset.load(
+                    dataset_name=path,
+                    split=self.split,
+                    subset_name=self.subset,
+                    version=self.version,
+                    trust_remote_code=self.trust_remote,
+                    **self.kwargs,
+                )
+                # convert to Huggingface dataset if necessary
+                if not isinstance(dataset, datasets.Dataset):
+                    dataset = dataset.to_hf_dataset()
+            elif self.data_source in [HubType.HUGGINGFACE, HubType.LOCAL]:
+                # remove dataset_infos.json file if exists, since datasets will occur an error if it exists.
+                dataset_infos_path = os.path.join(path, 'dataset_infos.json')
+                if os.path.exists(dataset_infos_path):
+                    logger.info(f'Removing dataset_infos.json file at {dataset_infos_path} to avoid datasets errors.')
+                    os.remove(dataset_infos_path)
+                # load dataset from Huggingface or local path
+                dataset = datasets.load_dataset(
+                    path=path,
+                    name=self.subset if self.subset != 'default' else None,
+                    split=self.split,
+                    revision=self.version,
+                    trust_remote_code=self.trust_remote,
+                    **self.kwargs,
+                )
+
+            # Only save to disk if not loading from local path
+            if self.data_source != HubType.LOCAL:
+                dataset.save_to_disk(dataset_cache_dir)
+
+        # shuffle if requested
+        if self.shuffle:
+            dataset = dataset.shuffle(seed=self.seed)
+
+        # limit if requested
+        if self.limit:
+            if isinstance(self.limit, float):
+                self.limit = int(len(dataset) * self.limit)
+            elif isinstance(self.limit, int) and self.limit < 0:
+                raise ValueError('Limit must be a non-negative integer or a float between 0 and 1.')
+            if len(dataset) > self.limit:
+                dataset = dataset.select(range(self.limit))
+
+        # convert to list
+        dataset = dataset.to_list()
+
+        # repeat k times
+        if self.repeats > 1:
+            dataset = [copy.deepcopy(item) for item in dataset for _ in range(self.repeats)]
+
+        # return the dataset
+        memory_dataset = MemoryDataset(
+            samples=data_to_samples(data=dataset, data_to_sample=data_to_sample),
+            name=Path(path).stem if Path(path).exists() else path,
+            location=path,
+        )
+
+        # Apply filtering if a filter function is provided
+        if self.filter_func is not None:
+            memory_dataset = memory_dataset.filter(self.filter_func)
+
+        # assign ids and group_ids if requested
+        if self.auto_id:
+            memory_dataset.reindex(group_size=self.repeats)
+
+        shuffle_choices_if_requested(memory_dataset, self.shuffle_choices)
+
+        return memory_dataset
+
+
+class LocalDataLoader(DataLoader):
+    """
+    Data loader for local datasets. Reads from JSONL or CSV files.
+    """
+
+    def load(self):
+
+        path = self.data_id_or_path
+        data_to_sample = record_to_sample_fn(self.sample_fields)
+        dataset = []
+
+        # Check for JSONL or CSV files in the specified path
+        for ext, loader in [('.jsonl', jsonl_to_list), ('.csv', csv_to_list)]:
+            # Check if the file exists with the given extension
+            if os.path.isfile(path) and path.endswith(ext):
+                file_paths = [path]
+            else:
+                file_paths = [
+                    os.path.join(path, f'{self.subset}_{self.split}{ext}'),
+                    os.path.join(path, f'{self.subset}{ext}')
+                ]
+            # If the file exists, load it
+            for file_path in file_paths:
+                if os.path.exists(file_path):
+                    dataset = loader(file_path)
+                    break  # Stop checking other extensions once a file is found
+
+        # shuffle if requested
+        if self.shuffle:
+            random.shuffle(dataset, self.seed)
+
+        # limit if requested
+        if self.limit:
+            if isinstance(self.limit, float):
+                self.limit = int(len(dataset) * self.limit)
+            elif isinstance(self.limit, int) and self.limit < 0:
+                raise ValueError('Limit must be a non-negative integer or a float between 0 and 1.')
+            dataset = dataset[:self.limit]
+
+        # repeat k times
+        if self.repeats > 1:
+            dataset = [copy.deepcopy(item) for item in dataset for _ in range(self.repeats)]
+
+        # return the dataset
+        memory_dataset = MemoryDataset(
+            samples=data_to_samples(data=dataset, data_to_sample=data_to_sample),
+            name=Path(path).stem if Path(path).exists() else path,
+            location=path,
+        )
+
+        # Apply filtering if a filter function is provided
+        if self.filter_func is not None:
+            memory_dataset = memory_dataset.filter(self.filter_func)
+
+        # assign ids and group_ids if requested
+        if self.auto_id:
+            memory_dataset.reindex(group_size=self.repeats)
+
+        shuffle_choices_if_requested(memory_dataset, self.shuffle_choices)
+
+        return memory_dataset
+
+
+class DictDataLoader(DataLoader):
+    """Load dataset from a list of dictionaries."""
+
+    def __init__(self, dict_list: list, **kwargs):
+        super().__init__(data_id_or_path='', split='', **kwargs)
+        self.dict_list = dict_list
+
+    def load(self) -> Dataset:
+        data_to_sample = record_to_sample_fn(self.sample_fields)
+        dataset = self.dict_list
+
+        # shuffle if requested
+        if self.shuffle:
+            random.shuffle(dataset, self.seed)
+
+        # limit if requested
+        if self.limit:
+            if isinstance(self.limit, float):
+                self.limit = int(len(dataset) * self.limit)
+            elif isinstance(self.limit, int) and self.limit < 0:
+                raise ValueError('Limit must be a non-negative integer or a float between 0 and 1.')
+            dataset = dataset[:self.limit]
+
+        # repeat k times
+        if self.repeats > 1:
+            dataset = [copy.deepcopy(item) for item in dataset for _ in range(self.repeats)]
+
+        # return the dataset
+        memory_dataset = MemoryDataset(samples=data_to_samples(data=dataset, data_to_sample=data_to_sample), )
+
+        # Apply filtering if a filter function is provided
+        if self.filter_func is not None:
+            memory_dataset = memory_dataset.filter(self.filter_func)
+
+        # assign ids and group_ids if requested
+        if self.auto_id:
+            memory_dataset.reindex(group_size=self.repeats)
+
+        shuffle_choices_if_requested(memory_dataset, self.shuffle_choices)
+
+        return memory_dataset

evalscope/api/dataset/utils.py

@@ -0,0 +1,143 @@
+import json
+from tqdm import tqdm
+from typing import Any, Callable, Dict, Iterable, List, Optional, Union, cast
+
+from .dataset import Dataset, FieldSpec, Sample
+
+
+def record_to_sample_fn(sample_fields: Union[FieldSpec, Callable, None] = None, ) -> Callable:
+    if sample_fields is None:
+        sample_fields = FieldSpec()
+
+    if isinstance(sample_fields, FieldSpec):
+
+        def record_to_sample(record: dict) -> Sample:
+            # collect metadata if specified
+            metadata: Optional[Dict[str, Any]] = None
+            if sample_fields.metadata:
+                if isinstance(sample_fields.metadata, list):
+                    metadata = {}
+                    for name in sample_fields.metadata:
+                        metadata[name] = record.get(name)
+
+            elif 'metadata' in record:
+                metadata_field = record.get('metadata')
+                if isinstance(metadata_field, str):
+                    metadata = json.loads(metadata_field)
+                elif isinstance(metadata_field, dict):
+                    metadata = metadata_field
+                else:
+                    raise ValueError(f"Unexpected type for 'metadata' field: {type(metadata_field)}")
+
+            # return sample
+            return Sample(
+                input=read_input(record.get(sample_fields.input)),
+                target=read_target(record.get(sample_fields.target)),
+                choices=read_choices(record.get(sample_fields.choices)),
+                id=record.get(sample_fields.id, None),
+                metadata=metadata,
+                sandbox=read_sandbox(record.get(sample_fields.sandbox)),
+                files=read_files(record.get(sample_fields.files)),
+                setup=read_setup(record.get(sample_fields.setup)),
+            )
+
+        return record_to_sample
+
+    else:
+        return sample_fields
+
+
+def data_to_samples(data: Iterable[dict], data_to_sample: Callable) -> List[Sample]:
+    samples: List[Sample] = []
+    for record in tqdm(data, desc='Processing records'):
+        record_samples = as_sample_list(data_to_sample(record=record))
+        samples.extend(record_samples)
+    return samples
+
+
+def as_sample_list(samples: Union[Sample, List[Sample]]) -> List[Sample]:
+    if isinstance(samples, list):
+        return samples
+    else:
+        return [samples]
+
+
+def read_input(input_val: Optional[Any]) -> str:
+    if not input_val:
+        raise ValueError('No input in dataset')
+    return str(input_val)
+
+
+def read_target(obj: Optional[Any]) -> Union[str, List[str]]:
+    if obj is not None:
+        return [str(item) for item in obj] if isinstance(obj, list) else str(obj)
+    else:
+        return ''
+
+
+def read_choices(obj: Optional[Any]) -> Optional[List[str]]:
+    if obj is not None:
+        if isinstance(obj, list):
+            return [str(choice) for choice in obj]
+        elif isinstance(obj, str):
+            choices = obj.split(',')
+            if len(choices) == 1:
+                choices = obj.split()
+            return [choice.strip() for choice in choices]
+        else:
+            return [str(obj)]
+    else:
+        return None
+
+
+def read_setup(setup: Optional[Any]) -> Optional[str]:
+    if setup is not None:
+        return str(setup)
+    else:
+        return None
+
+
+def read_sandbox(sandbox: Optional[Any]) -> Optional[str]:
+    if sandbox is not None:
+        if isinstance(sandbox, str):
+            return sandbox
+        elif isinstance(sandbox, dict):
+            return json.dumps(sandbox)
+        else:
+            raise ValueError(f"Unexpected type for 'sandbox' field: {type(sandbox)}")
+    else:
+        return None
+
+
+def read_files(files: Optional[Any]) -> Optional[Dict[str, str]]:
+    if files is not None:
+        if isinstance(files, str):
+            files = json.loads(files)
+        if isinstance(files, dict):
+            if all(isinstance(v, str) for v in files.values()):
+                return cast(Dict[str, str], files)
+
+        # didn't find the right type
+        raise ValueError(f"Unexpected type for 'files' field: {type(files)}")
+    else:
+        return None
+
+
+def shuffle_choices_if_requested(dataset: Dataset, shuffle_choices: Optional[Union[bool, int]]) -> None:
+    """
+    Shuffle the choices in the dataset if requested.
+
+    The `shuffle_choices` parameter passed to `json_dataset`, `csv_dataset`,
+    and `hf_dataset` can be a boolean, an integer, or `None` (default).
+    If it is a boolean, it will shuffle the choices if the value is `True`,
+    and do nothing if it is `False`.
+    If it is an integer, it will shuffle the choices using the integer as the seed.
+    """
+    # Note that `isinstance(x, int)` returns True if x is True or False,
+    # so we need to check for both explicitly
+    if shuffle_choices is True:
+        dataset.shuffle_choices()
+    elif shuffle_choices is False:
+        pass
+    elif isinstance(shuffle_choices, int):
+        dataset.shuffle_choices(seed=shuffle_choices)

evalscope/api/evaluator/__init__.py

@@ -0,0 +1,3 @@
+from .cache import CacheManager, ModelResult, ReviewResult
+from .evaluator import Evaluator
+from .state import Choices, Target, TaskState