evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/arguments.py

@@ -1,7 +1,8 @@
+# flake8: noqa: E501
 import argparse
 import json
 
-from evalscope.constants import EvalBackend, EvalStage, EvalType
+from evalscope.constants import EvalBackend, EvalType, JudgeStrategy, ModelTask
 
 
 class ParseStrArgsAction(argparse.Action):
@@ -9,6 +10,15 @@ class ParseStrArgsAction(argparse.Action):
     def __call__(self, parser, namespace, values, option_string=None):
         assert isinstance(values, str), 'args should be a string.'
 
+        # try json load first
+        try:
+            arg_dict = json.loads(values)
+            setattr(namespace, self.dest, arg_dict)
+            return
+        except (json.JSONDecodeError, ValueError):
+            pass
+
+        # If JSON load fails, fall back to parsing as key=value pairs
         arg_dict = {}
         for arg in values.strip().split(','):
             key, value = map(str.strip, arg.split('=', 1))  # Use maxsplit=1 to handle multiple '='
@@ -35,9 +45,9 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--model', type=str, required=False, help='The model id on modelscope, or local model dir.')
     parser.add_argument('--model-id', type=str, required=False, help='The model id for model name in report.')
     parser.add_argument('--model-args', type=str, action=ParseStrArgsAction, help='The model args, should be a string.')
+    parser.add_argument('--model-task', type=str, default=ModelTask.TEXT_GENERATION, choices=[ModelTask.TEXT_GENERATION, ModelTask.IMAGE_GENERATION], help='The model task for model id.')  # noqa: E501
 
     # Template-related arguments
-    parser.add_argument('--template-type', type=str, required=False, help='Deprecated, will be removed in v1.0.0.')
     parser.add_argument('--chat-template', type=str, required=False, help='The custom jinja template for chat generation.')  # noqa: E501
 
     # Dataset-related arguments
@@ -50,26 +60,39 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--generation-config', type=str, action=ParseStrArgsAction, help='The generation config, should be a string.')  # noqa: E501
 
     # Evaluation-related arguments
-    parser.add_argument('--eval-type', type=str, help='The type for evaluating.',
-                        choices=[EvalType.CHECKPOINT, EvalType.CUSTOM, EvalType.SERVICE])
+    parser.add_argument('--eval-type', type=str, help='The type for evaluating.')
     parser.add_argument('--eval-backend', type=str, help='The evaluation backend to use.',
                         choices=[EvalBackend.NATIVE, EvalBackend.OPEN_COMPASS, EvalBackend.VLM_EVAL_KIT, EvalBackend.RAG_EVAL])  # noqa: E501
     parser.add_argument('--eval-config', type=str, required=False, help='The eval task config file path for evaluation backend.')  # noqa: E501
-    parser.add_argument('--stage', type=str, default='all', help='The stage of evaluation pipeline.',
-                        choices=[EvalStage.ALL, EvalStage.INFER, EvalStage.REVIEW])
-    parser.add_argument('--limit', type=int, default=None, help='Max evaluation samples num for each subset.')
+    parser.add_argument('--eval-batch-size', type=int, default=1, help='The batch size for evaluation.')
+    parser.add_argument('--limit', type=float, default=None, help='Max evaluation samples num for each subset.')
+    parser.add_argument('--repeats', type=int, default=1, help='Number of times to repeat the dataset items for k-metrics.')  # noqa: E501
 
     # Cache and working directory arguments
-    parser.add_argument('--mem-cache', action='store_true', default=False, help='Deprecated, will be removed in v1.0.0.')  # noqa: E501
     parser.add_argument('--use-cache', type=str, help='Path to reuse the cached results.')
+    parser.add_argument('--rerun-review', action='store_true', default=False, help='Rerun the review process when use_cache.')
     parser.add_argument('--work-dir', type=str, help='The root cache dir.')
 
     # Debug and runtime mode arguments
+    parser.add_argument('--ignore-errors', action='store_true', default=False, help='Ignore errors during evaluation.')
     parser.add_argument('--debug', action='store_true', default=False, help='Debug mode, will print information for debugging.')  # noqa: E501
-    parser.add_argument('--dry-run', action='store_true', default=False, help='Dry run in single processing mode.')
     parser.add_argument('--seed', type=int, default=42, help='Random seed for reproducibility.')
     parser.add_argument('--api-key', type=str, default='EMPTY', help='The API key for the remote API model.')
     parser.add_argument('--api-url', type=str, default=None, help='The API url for the remote API model.')
+    parser.add_argument('--timeout', type=float, default=None, help='The timeout for the remote API model.')
+    parser.add_argument('--stream', action='store_true', default=None, help='Stream mode.')  # noqa: E501
+
+    # LLMJudge arguments
+    parser.add_argument('--judge-strategy', type=str, default=JudgeStrategy.AUTO, help='The judge strategy.')
+    parser.add_argument('--judge-model-args', type=json.loads, default='{}', help='The judge model args, should be a json string.')  # noqa: E501
+    parser.add_argument('--judge-worker-num', type=int, default=1, help='The number of workers for the judge model.')
+    parser.add_argument('--analysis-report', action='store_true', default=False, help='Generate analysis report for the evaluation results using judge model.')  # noqa: E501
+
+    # Sandbox-related arguments
+    parser.add_argument('--use-sandbox', action='store_true', default=False, help='Whether to use sandbox for model evaluation.')  # noqa: E501
+    parser.add_argument('--sandbox-type', type=str, default='docker', help='The sandbox type to use.')  # noqa: E501
+    parser.add_argument('--sandbox-config', type=json.loads, default='{}', help='The sandbox config, should be a json string.')  # noqa: E501
+    parser.add_argument('--sandbox-manager-config', type=json.loads, default='{}', help='The sandbox manager config, should be a json string.')  # noqa: E501
     # yapf: enable
 
 

evalscope/backend/opencompass/api_meta_template.py

@@ -49,7 +49,8 @@ register_template(
         reserved_roles=[
             dict(role='SYSTEM', api_role='SYSTEM'),
         ],
-    ))
+    )
+)
 
 if __name__ == '__main__':
     res = MetaTemplateType.get_template_name_list()

evalscope/backend/opencompass/backend_manager.py

@@ -1,4 +1,5 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
+import os
 import subprocess
 import tempfile
 from dataclasses import asdict
@@ -7,7 +8,8 @@ from typing import Optional, Union
 
 from evalscope.backend.base import BackendManager
 from evalscope.backend.opencompass.api_meta_template import get_template
-from evalscope.utils import get_module_path, get_valid_list, is_module_installed
+from evalscope.utils.import_utils import get_module_path, is_module_installed
+from evalscope.utils.io_utils import get_valid_list
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -45,7 +47,6 @@ class OpenCompassBackendManager(BackendManager):
                     datasets: list, the datasets.
                     models: list, the models.
                     work_dir (Optional): str, the working directory. Default to None, which means the current directory.
-                    dry_run (Optional): bool, the dry-run flag. Default to False.
                     debug (Optional): bool, the debug flag. Default to False.
                     reuse (Optional): str, reuse previous outputs & results. Default to None.
                     generation_kwargs (Optional): dict, the generation config. Default to {}.
@@ -138,7 +139,6 @@ class OpenCompassBackendManager(BackendManager):
             cmd_str = f'python -m run_oc ' \
                       f'--models {" ".join(self.args.models)} ' \
                       f'--datasets {" ".join(self.args.datasets)} ' \
-                      f'{self.get_restore_arg("dry-run", self.args.dry_run)} ' \
                       f'{self.get_arg_with_default("work-dir", self.args.work_dir)}'
 
         elif cmd_mode == CmdMode.SCRIPT:
@@ -180,8 +180,10 @@ class OpenCompassBackendManager(BackendManager):
             else:
                 valid_dataset_names, invalid_dataset_names = get_valid_list(dataset_names, dataset_names_all)
                 if len(invalid_dataset_names) > 0:
-                    logger.error(f'Invalid datasets: {invalid_dataset_names}, '
-                                 f'refer to the following list to get proper dataset name: {dataset_names_all}')
+                    logger.error(
+                        f'Invalid datasets: {invalid_dataset_names}, '
+                        f'refer to the following list to get proper dataset name: {dataset_names_all}'
+                    )
                 assert len(valid_dataset_names) > 0, f'No valid datasets. ' \
                                                      f'To get the valid datasets, please refer to {dataset_names_all}'
 
@@ -204,7 +206,7 @@ class OpenCompassBackendManager(BackendManager):
                     model_d['meta_template'] = get_template(model_d['meta_template'])
 
                 # set the 'abbr' as the 'path' if 'abbr' is not specified
-                model_d['abbr'] = model_d['path']
+                model_d['abbr'] = os.path.basename(model_d['path'])
 
                 model_config = ApiModelConfig(**model_d)
                 models.append(asdict(model_config))
@@ -250,7 +252,8 @@ if __name__ == '__main__':
                 'openai_api_base': 'http://127.0.0.1:8000/v1/chat/completions'
             }],
             'limit': 5
-        })
+        }
+    )
     all_datasets = OpenCompassBackendManager.list_datasets()
     print(f'all_datasets: {all_datasets}')
     oc_backend_manager.run()

evalscope/backend/rag_eval/__init__.py

@@ -1,4 +1,4 @@
-from evalscope.backend.rag_eval.backend_manager import RAGEvalBackendManager
+from evalscope.backend.rag_eval.backend_manager import RAGEvalBackendManager, Tools
 from evalscope.backend.rag_eval.utils.clip import VisionModel
 from evalscope.backend.rag_eval.utils.embedding import EmbeddingModel
 from evalscope.backend.rag_eval.utils.llm import LLM, ChatOpenAI, LocalLLM

evalscope/backend/rag_eval/backend_manager.py

@@ -2,12 +2,19 @@ import os
 from typing import Optional, Union
 
 from evalscope.backend.base import BackendManager
-from evalscope.utils import get_valid_list, is_module_installed
+from evalscope.utils.import_utils import is_module_installed
+from evalscope.utils.io_utils import get_valid_list
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
 
+class Tools:
+    MTEB = 'mteb'
+    RAGAS = 'ragas'
+    CLIP_BENCHMARK = 'clip_benchmark'
+
+
 class RAGEvalBackendManager(BackendManager):
 
     def __init__(self, config: Union[str, dict], **kwargs):
@@ -47,9 +54,19 @@ class RAGEvalBackendManager(BackendManager):
         from evalscope.backend.rag_eval.ragas.tasks import generate_testset
 
         if testset_args is not None:
-            generate_testset(TestsetGenerationArguments(**testset_args))
+            if isinstance(testset_args, dict):
+                generate_testset(TestsetGenerationArguments(**testset_args))
+            elif isinstance(testset_args, TestsetGenerationArguments):
+                generate_testset(testset_args)
+            else:
+                raise ValueError('Please provide the testset generation arguments.')
         if eval_args is not None:
-            rag_eval(EvaluationArguments(**eval_args))
+            if isinstance(eval_args, dict):
+                rag_eval(EvaluationArguments(**eval_args))
+            elif isinstance(eval_args, EvaluationArguments):
+                rag_eval(eval_args)
+            else:
+                raise ValueError('Please provide the evaluation arguments.')
 
     @staticmethod
     def run_clip_benchmark(args):
@@ -59,17 +76,17 @@ class RAGEvalBackendManager(BackendManager):
 
     def run(self, *args, **kwargs):
         tool = self.config_d.pop('tool')
-        if tool.lower() == 'mteb':
+        if tool.lower() == Tools.MTEB:
             self._check_env('mteb')
             model_args = self.config_d['model']
             eval_args = self.config_d['eval']
             self.run_mteb(model_args, eval_args)
-        elif tool.lower() == 'ragas':
+        elif tool.lower() == Tools.RAGAS:
             self._check_env('ragas')
             testset_args = self.config_d.get('testset_generation', None)
             eval_args = self.config_d.get('eval', None)
             self.run_ragas(testset_args, eval_args)
-        elif tool.lower() == 'clip_benchmark':
+        elif tool.lower() == Tools.CLIP_BENCHMARK:
             self._check_env('webdataset')
             self.run_clip_benchmark(self.config_d['eval'])
         else:

evalscope/backend/rag_eval/clip_benchmark/dataset_builder.py

@@ -1,4 +1,5 @@
 import os
+import posixpath  # For URL path handling
 import torch
 from torch.utils.data import DataLoader
 from torch.utils.data import Dataset as TorchDataset
@@ -99,16 +100,16 @@ class DatasetWrapper(TorchDataset):
 
 def get_dataset_default_task(dataset):
     if dataset in (
-            'custom',
-            'muge',
-            'flickr30k',
-            'flickr8k',
-            'mscoco_captions',
-            'mscoco_captions2017',
-            'multilingual_mscoco_captions',
-            'flickr30k-200',
-            'crossmodal3600',
-            'xtd200',
+        'custom',
+        'muge',
+        'flickr30k',
+        'flickr8k',
+        'mscoco_captions',
+        'mscoco_captions2017',
+        'multilingual_mscoco_captions',
+        'flickr30k-200',
+        'crossmodal3600',
+        'xtd200',
     ):
         return 'zeroshot_retrieval'
     else:
@@ -186,42 +187,53 @@ def build_wds_dataset(dataset_name, transform, split='test', data_dir='root', ca
 
     Set `cache_dir` to a path to cache the dataset, otherwise, no caching will occur.
     """
+    import requests
     import webdataset as wds
 
     def read_txt(fname):
-        if '://' in fname:
-            stream = os.popen("curl -L -s --fail '%s'" % fname, 'r')
-            value = stream.read()
-            if stream.close():
-                raise FileNotFoundError('Failed to retreive data')
+        if fname.startswith(('http://', 'https://')):
+            try:
+                response = requests.get(fname)
+                response.raise_for_status()  # Ensure the HTTP request was successful
+                return response.text
+            except requests.exceptions.RequestException as e:
+                raise FileNotFoundError(f'Failed to read {fname}: {e}')
         else:
             with open(fname, 'r') as file:
-                value = file.read()
-        return value
+                return file.read()
+
+    def url_path_join(*parts):
+        """Join URL path parts with forward slashes regardless of platform"""
+        return posixpath.join(*parts)
 
     if not data_dir:
         data_dir = f'https://modelscope.cn/datasets/clip-benchmark/wds_{dataset_name}/resolve/master'
 
     # Git LFS files have a different file path to access the raw data than other files
-    if data_dir.startswith('https://modelscope.cn/datasets'):
+    is_url = data_dir.startswith(('http://', 'https://'))
+    if is_url and data_dir.startswith('https://modelscope.cn/datasets'):
         *split_url_head, _, url_path = data_dir.split('/', 7)
         url_head = '/'.join(split_url_head)
         metadata_dir = '/'.join([url_head, 'resolve', url_path])
         tardata_dir = '/'.join([url_head, 'resolve', url_path])
     else:
         metadata_dir = tardata_dir = data_dir
+
+    # Use appropriate path joining function based on whether we're dealing with a URL
+    path_join = url_path_join if is_url else os.path.join
+
     # Get number of shards
-    nshards_fname = os.path.join(metadata_dir, split, 'nshards.txt')
+    nshards_fname = path_join(metadata_dir, split, 'nshards.txt')
     nshards = int(read_txt(nshards_fname))  # Do not catch FileNotFound, nshards.txt should be mandatory
 
     # Get dataset type (classification or retrieval)
-    type_fname = os.path.join(metadata_dir, 'dataset_type.txt')
+    type_fname = path_join(metadata_dir, 'dataset_type.txt')
     try:
         dataset_type = read_txt(type_fname).strip().lower()
     except FileNotFoundError:
         dataset_type = 'classification'
 
-    filepattern = os.path.join(tardata_dir, split, '{0..%d}.tar' % (nshards - 1))
+    filepattern = path_join(tardata_dir, split, '{0..%d}.tar' % (nshards - 1))
     # Load webdataset (support WEBP, PNG, and JPG for now)
     if not cache_dir or not isinstance(cache_dir, str):
         cache_dir = None

evalscope/backend/rag_eval/clip_benchmark/task_template.py

@@ -4,8 +4,11 @@ import torch
 from itertools import product
 
 from evalscope.backend.rag_eval.clip_benchmark.arguments import Arguments
-from evalscope.backend.rag_eval.clip_benchmark.dataset_builder import (build_dataset, get_dataloader,
-                                                                       get_dataset_default_task)
+from evalscope.backend.rag_eval.clip_benchmark.dataset_builder import (
+    build_dataset,
+    get_dataloader,
+    get_dataset_default_task,
+)
 from evalscope.backend.rag_eval.clip_benchmark.tasks import image_caption, zeroshot_classification, zeroshot_retrieval
 from evalscope.backend.rag_eval.utils.clip import VisionModel
 from evalscope.utils.logger import get_logger
@@ -66,8 +69,9 @@ def evaluate(args: Arguments):
             if verbose:
                 logger.info(f'Zero-shot templates: {zeroshot_templates}')
             classnames = dataset.classes if hasattr(dataset, 'classes') else None
-            assert (zeroshot_templates is not None
-                    and classnames is not None), 'Dataset does not support classification'
+            assert (
+                zeroshot_templates is not None and classnames is not None
+            ), 'Dataset does not support classification'
             metrics = zeroshot_classification.evaluate(
                 model,
                 dataloader,

evalscope/backend/rag_eval/cmteb/arguments.py

@@ -11,7 +11,9 @@ class ModelArguments:
     pooling_mode: Optional[str] = None
     max_seq_length: int = 512  # max sequence length
     # prompt for llm based model
-    prompt: str = ''
+    prompt: Optional[str] = None
+    # prompts dictionary for different tasks, if prompt is not set
+    prompts: Optional[Dict[str, str]] = None
     # model kwargs
     model_kwargs: dict = field(default_factory=dict)
     # config kwargs
@@ -20,6 +22,12 @@ class ModelArguments:
     encode_kwargs: dict = field(default_factory=lambda: {'show_progress_bar': True, 'batch_size': 32})
     hub: str = 'modelscope'  # modelscope or huggingface
 
+    # for API embedding model
+    model_name: Optional[str] = None
+    api_base: Optional[str] = None
+    api_key: Optional[str] = None
+    dimensions: Optional[int] = None
+
     def to_dict(self) -> Dict[str, Any]:
         return {
             'model_name_or_path': self.model_name_or_path,
@@ -27,10 +35,15 @@ class ModelArguments:
             'pooling_mode': self.pooling_mode,
             'max_seq_length': self.max_seq_length,
             'prompt': self.prompt,
+            'prompts': self.prompts,
             'model_kwargs': self.model_kwargs,
             'config_kwargs': self.config_kwargs,
             'encode_kwargs': self.encode_kwargs,
             'hub': self.hub,
+            'model_name': self.model_name,
+            'api_base': self.api_base,
+            'api_key': self.api_key,
+            'dimensions': self.dimensions,
         }
 
 

evalscope/backend/rag_eval/cmteb/task_template.py

@@ -1,6 +1,6 @@
 import mteb
 import os
-from mteb.task_selection import results_to_dataframe
+from tabulate import tabulate
 
 from evalscope.backend.rag_eval import EmbeddingModel, cmteb
 from evalscope.utils.logger import get_logger
@@ -12,14 +12,27 @@ def show_results(output_folder, model, results):
     model_name = model.mteb_model_meta.model_name_as_path()
     revision = model.mteb_model_meta.revision
 
-    results_df = results_to_dataframe({model_name: {revision: results}})
+    data = []
+    for model_res in results:
+        main_res = model_res.only_main_score()
+        for split, score in main_res.scores.items():
+            for sub_score in score:
+                data.append({
+                    'Model': model_name.replace('eval__', ''),
+                    'Revision': revision,
+                    'Task Type': main_res.task_type,
+                    'Task': main_res.task_name,
+                    'Split': split,
+                    'Subset': sub_score['hf_subset'],
+                    'Main Score': sub_score['main_score'],
+                })
 
     save_path = os.path.join(
         output_folder,
         model_name,
         revision,
     )
-    logger.info(f'Evaluation results:\n{results_df.to_markdown()}')
+    logger.info(f'Evaluation results:\n{tabulate(data, headers="keys", tablefmt="grid")}')
     logger.info(f'Evaluation results saved in {os.path.abspath(save_path)}')
 
 
@@ -34,6 +47,7 @@ def one_stage_eval(
     tasks = cmteb.TaskBase.get_tasks(task_names=eval_args['tasks'], dataset_path=custom_dataset_path)
     evaluation = mteb.MTEB(tasks=tasks)
 
+    eval_args['encode_kwargs'] = model_args.get('encode_kwargs', {})
     # run evaluation
     results = evaluation.run(model, **eval_args)
 
@@ -66,6 +80,7 @@ def two_stage_eval(
             overwrite_results=True,
             hub=eval_args['hub'],
             limits=eval_args['limits'],
+            encode_kwargs=model1_args.get('encode_kwargs', {}),
         )
         # stage 2: run cross encoder
         results = evaluation.run(
@@ -77,6 +92,7 @@ def two_stage_eval(
             overwrite_results=True,
             hub=eval_args['hub'],
             limits=eval_args['limits'],
+            encode_kwargs=model2_args.get('encode_kwargs', {}),
         )
 
         # save and log results

evalscope/backend/rag_eval/cmteb/tasks/CustomTask.py

@@ -9,7 +9,6 @@ class CustomRetrieval(AbsTaskRetrieval):
     ignore_identical_ids: bool = True
 
     def __init__(self, dataset_path: Optional[str] = 'custom_eval/text/retrieval', **kwargs):
-        super().__init__(**kwargs)
         self.metadata = TaskMetadata(
             name='CustomRetrieval',
             description='CustomRetrieval Task',
@@ -34,6 +33,7 @@ class CustomRetrieval(AbsTaskRetrieval):
             bibtex_citation='',
             descriptive_stats={},
         )
+        super().__init__(**kwargs)
 
     def load_data(self, **kwargs):
         if self.data_loaded:

evalscope/backend/rag_eval/ragas/arguments.py

@@ -21,7 +21,6 @@ class TestsetGenerationArguments:
     """
     generator_llm: Dict = field(default_factory=dict)
     embeddings: Dict = field(default_factory=dict)
-    distribution: str = field(default_factory=lambda: {'simple': 0.5, 'multi_context': 0.4, 'reasoning': 0.1})
     # For LLM based evaluation
     # available: ['english', 'hindi', 'marathi', 'chinese', 'spanish', 'amharic', 'arabic',
     # 'armenian', 'bulgarian', 'urdu', 'russian', 'polish', 'persian', 'dutch', 'danish',

evalscope/backend/rag_eval/ragas/task_template.py

@@ -34,7 +34,8 @@ def rag_eval(args: EvaluationArguments, ) -> None:
             target_lang=args.language,
             llm=LangchainLLMWrapper(llm),
             adapt_instruction=True,
-        ))
+        )
+    )
     # load dataset
     dataset = Dataset.from_json(args.testset_file)
 

evalscope/backend/rag_eval/ragas/tasks/build_distribution.py

@@ -27,7 +27,8 @@ def default_query_distribution(llm: BaseRagasLLM, kg: KnowledgeGraph, language:
             target_lang=language,
             llm=llm,
             adapt_instruction=True,
-        ))
+        )
+    )
 
     default_queries = [
         single_hop,

evalscope/backend/rag_eval/ragas/tasks/build_transform.py

@@ -44,8 +44,9 @@ def default_transforms(
         return bins
 
     def filter_doc_with_num_tokens(node, min_num_tokens=500):
-        return (node.type == NodeType.DOCUMENT
-                and num_tokens_from_string(node.properties['page_content']) > min_num_tokens)
+        return (
+            node.type == NodeType.DOCUMENT and num_tokens_from_string(node.properties['page_content']) > min_num_tokens
+        )
 
     def filter_docs(node):
         return node.type == NodeType.DOCUMENT
@@ -90,7 +91,8 @@ def default_transforms(
                 target_lang=language,
                 llm=llm,
                 adapt_instruction=True,
-            ))
+            )
+        )
 
         transforms = [
             headline_extractor,
@@ -121,7 +123,8 @@ def default_transforms(
                 target_lang=language,
                 llm=llm,
                 adapt_instruction=True,
-            ))
+            )
+        )
 
         transforms = [
             summary_extractor,

evalscope/backend/rag_eval/ragas/tasks/testset_generation.py

@@ -67,9 +67,14 @@ def get_persona(llm, kg, language):
 
 
 def load_data(file_path):
-    from langchain_community.document_loaders import UnstructuredFileLoader
+    import nltk
+    from langchain_unstructured import UnstructuredLoader
 
-    loader = UnstructuredFileLoader(file_path, mode='single')
+    if nltk.data.find('taggers/averaged_perceptron_tagger_eng') is False:
+        # need to download nltk data for the first time
+        nltk.download('averaged_perceptron_tagger_eng')
+
+    loader = UnstructuredLoader(file_path)
     data = loader.load()
     return data
 
@@ -108,7 +113,8 @@ def generate_testset(args: TestsetGenerationArguments) -> None:
 
     # generate testset
     generator = TestsetGenerator(
-        llm=wrapped_llm, embedding_model=wrapped_embeddings, knowledge_graph=knowledge_graph, persona_list=persona_list)
+        llm=wrapped_llm, embedding_model=wrapped_embeddings, knowledge_graph=knowledge_graph, persona_list=persona_list
+    )
 
     testset = generator.generate(
         testset_size=args.test_size,

evalscope/backend/rag_eval/ragas/tasks/translate_prompt.py

@@ -2,7 +2,6 @@ import asyncio
 import os
 from ragas.llms import BaseRagasLLM
 from ragas.prompt import PromptMixin, PydanticPrompt
-from ragas.utils import RAGAS_SUPPORTED_LANGUAGE_CODES
 from typing import List
 
 from evalscope.utils.logger import get_logger
@@ -16,10 +15,6 @@ async def translate_prompt(
     llm: BaseRagasLLM,
     adapt_instruction: bool = False,
 ):
-    if target_lang not in RAGAS_SUPPORTED_LANGUAGE_CODES:
-        logger.warning(f'{target_lang} is not in supported language: {list(RAGAS_SUPPORTED_LANGUAGE_CODES)}')
-        return
-
     if not issubclass(type(prompt_user), PromptMixin):
         logger.info(f"{prompt_user} is not a PromptMixin, don't translate it")
         return
@@ -39,7 +34,8 @@ async def translate_prompt(
 
     logger.info(f'Translating prompts to {target_lang}')
     adapted_prompts = await prompt_user.adapt_prompts(
-        language=target_lang, llm=llm, adapt_instruction=adapt_instruction)
+        language=target_lang, llm=llm, adapt_instruction=adapt_instruction
+    )
     prompt_user.set_prompts(**adapted_prompts)
     try:
         prompt_user.save_prompts(prompt_dir)