evalscope 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/wmt/wmt24_adapter.py

@@ -0,0 +1,294 @@
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser, ContentText
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.import_utils import check_import
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+PROMPT_TEMPLATE = """
+Translate the following {source_language} sentence into {target_language}:
+
+{source_language}: {source_text}
+{target_language}:
+""".strip()
+
+LANGUAGE_PAIRS = [
+    'en-ar_eg',
+    'en-ar_sa',
+    'en-bg_bg',
+    'en-bn_in',
+    'en-ca_es',
+    'en-cs_cz',
+    'en-da_dk',
+    'en-de_de',
+    'en-el_gr',
+    'en-es_mx',
+    'en-et_ee',
+    'en-fa_ir',
+    'en-fi_fi',
+    'en-fil_ph',
+    'en-fr_ca',
+    'en-fr_fr',
+    'en-gu_in',
+    'en-he_il',
+    'en-hi_in',
+    'en-hr_hr',
+    'en-hu_hu',
+    'en-id_id',
+    'en-is_is',
+    'en-it_it',
+    'en-ja_jp',
+    'en-kn_in',
+    'en-ko_kr',
+    'en-lt_lt',
+    'en-lv_lv',
+    'en-ml_in',
+    'en-mr_in',
+    'en-nl_nl',
+    'en-no_no',
+    'en-pa_in',
+    'en-pl_pl',
+    'en-pt_br',
+    'en-pt_pt',
+    'en-ro_ro',
+    'en-ru_ru',
+    'en-sk_sk',
+    'en-sl_si',
+    'en-sr_rs',
+    'en-sv_se',
+    'en-sw_ke',
+    'en-sw_tz',
+    'en-ta_in',
+    'en-te_in',
+    'en-th_th',
+    'en-tr_tr',
+    'en-uk_ua',
+    'en-ur_pk',
+    'en-vi_vn',
+    'en-zh_cn',
+    'en-zh_tw',
+    'en-zu_za',
+]
+
+LANGUAGE_BY_CODE = {
+    'ar_eg': 'arabic',
+    'ar_sa': 'arabic',
+    'bg_bg': 'bulgarian',
+    'bn_bd': 'bengali',
+    'bn_in': 'bengali',
+    'ca_es': 'catalan',
+    'cs_cz': 'czech',
+    'da_dk': 'danish',
+    'de_de': 'german',
+    'el_gr': 'greek',
+    'es_mx': 'spanish',
+    'et_ee': 'estonian',
+    'fa_ir': 'farsi',
+    'fi_fi': 'finnish',
+    'fil_ph': 'filipino',
+    'fr_ca': 'french',
+    'fr_fr': 'french',
+    'gu_in': 'gujarati',
+    'he_il': 'hebrew',
+    'hi_in': 'hindi',
+    'hr_hr': 'croatian',
+    'hu_hu': 'hungarian',
+    'id_id': 'indonesian',
+    'is_is': 'icelandic',
+    'it_it': 'italian',
+    'ja_jp': 'japanese',
+    'kn_in': 'kannada',
+    'ko_kr': 'korean',
+    'lt_lt': 'lithuanian',
+    'lv_lv': 'latvian',
+    'ml_in': 'malayalam',
+    'mr_in': 'marathi',
+    'nl_nl': 'dutch',
+    'no_no': 'norwegian',
+    'pa_in': 'punjabi',
+    'pl_pl': 'polish',
+    'pt_br': 'portuguese',
+    'pt_pt': 'portuguese',
+    'ro_ro': 'romanian',
+    'ru_ru': 'russian',
+    'sk_sk': 'slovak',
+    'sl_si': 'slovenian',
+    'sr_rs': 'serbian',
+    'sv_se': 'swedish',
+    'sw_ke': 'swahili',
+    'sw_tz': 'swahili',
+    'ta_in': 'tamil',
+    'te_in': 'telugu',
+    'th_th': 'thai',
+    'tr_tr': 'turkish',
+    'uk_ua': 'ukrainian',
+    'ur_pk': 'urdu',
+    'vi_vn': 'vietnamese',
+    'zh_cn': 'mandarin',
+    'zh_tw': 'mandarin',
+    'zu_za': 'zulu',
+    'en': 'english',
+}
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='wmt24pp',
+        pretty_name='WMT2024++',
+        dataset_id='extraordinarylab/wmt24pp',
+        tags=[Tags.MULTI_LINGUAL, Tags.MT],
+        description=(
+            'WMT2024 news translation benchmark supporting multiple language pairs. '
+            'Each subset represents a specific translation direction'
+        ),
+        subset_list=LANGUAGE_PAIRS,
+        eval_split='test',
+        metric_list={
+            'bleu': {},
+            'bert_score': {
+                'model_id_or_path': 'AI-ModelScope/xlm-roberta-large',
+                'model_type': 'xlm-roberta-large'
+            },
+            'comet': {
+                'model_id_or_path': 'evalscope/wmt22-comet-da',
+            }
+        },
+        few_shot_num=0,
+        prompt_template=PROMPT_TEMPLATE,
+    )
+)
+class WMT24PPAdapter(DefaultDataAdapter):
+
+    def __init__(self, **kwargs: Any) -> None:
+        """Initialize adapter and configure dataset subsets."""
+        super().__init__(**kwargs)
+        self.reformat_subset = True
+        self.use_batch_scoring = True  # Enable batch scoring
+
+        if 'comet' in self.metric_list:
+            check_import('comet', 'unbabel-comet', raise_error=True, feature_name='COMETScore Metric')
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a data record to a Sample object.
+        """
+        source_text = str(record['source'])
+        target_text = str(record['target'])
+        language_pair = str(record['language_pair'])
+        source_language, target_language = language_pair.split('-')
+
+        # Format the generation prompt with the text
+        input_prompt = self.prompt_template.format(
+            source_text=source_text,
+            source_language=LANGUAGE_BY_CODE[source_language],
+            target_language=LANGUAGE_BY_CODE[target_language],
+        )
+
+        # Create content list for the input
+        content_list = [ContentText(text=input_prompt)]
+
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            target=target_text,
+            subset_key=language_pair,
+            metadata={
+                'source_text': source_text,
+                'target_text': target_text,
+                'source_language': source_language,
+                'target_language': target_language,
+            },
+        )
+
+    def match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        """Compute per-sample translation metrics."""
+        # Create a Score object for the current sample
+        score = Score(
+            prediction=original_prediction,
+            extracted_prediction=filtered_prediction,
+            value={},
+        )
+
+        # ---- BLEU ----
+        if 'bleu' in self.metric_list:
+            try:
+                from evalscope.metrics import bleu_ngram_one_sample
+
+                bleu_results = bleu_ngram_one_sample(filtered_prediction, reference)
+                score.value.update(bleu_results)
+            except Exception as e:
+                logger.warning(f'[WMT24PPAdapter] BLEU single-sample calculation failed: {e}')
+        return score
+
+    def batch_match_score(
+        self,
+        original_predictions: List[str],
+        filtered_predictions: List[str],
+        references: List[str],
+        task_states: List[TaskState],
+    ) -> List[Score]:
+        """Compute batched translation metrics (BLEU, BERTScore, COMET)."""
+        scores: List[Score] = []
+        for i in range(len(original_predictions)):
+            score = Score(
+                extracted_prediction=filtered_predictions[i],
+                prediction=original_predictions[i],
+                value={},
+            )
+            scores.append(score)
+
+        # ---- BLEU (per-sample within batch) ----
+        if 'bleu' in self.metric_list:
+            try:
+                from evalscope.metrics import bleu_ngram_one_sample
+
+                for i in range(len(scores)):
+                    bleu_results = bleu_ngram_one_sample(filtered_predictions[i], references[i])
+                    scores[i].value.update(bleu_results)
+            except Exception as e:
+                logger.warning(f'[WMT24PPAdapter] BLEU batch calculation failed: {e}')
+
+        # ---- BERTScore ----
+        if 'bert_score' in self.metric_list:
+            try:
+                from evalscope.metrics.metric import BertScore
+
+                score_args = self.metric_list.get('bert_score', {})
+                bert_scorer = BertScore(**score_args)
+                bert_score_f1 = bert_scorer.apply(filtered_predictions, references)
+                for i in range(len(scores)):
+                    scores[i].value.update({'bert_score': bert_score_f1[i]})
+            except Exception as e:
+                logger.warning(f'[WMT24PPAdapter] BERTScore batch calculation failed: {e}')
+
+        # ---- COMET ----
+        if 'comet' in self.metric_list:
+            try:
+                from evalscope.metrics.metric import COMETScore
+
+                score_args = self.metric_list.get('comet', {})
+                comet_scorer = COMETScore(**score_args)
+                data = [{
+                    'src': st.metadata.get('source_text'),
+                    'mt': pred,
+                    'ref': ref
+                } for pred, ref, st in zip(filtered_predictions, references, task_states)]
+                comet_scores = comet_scorer.apply(data)
+                for i in range(len(scores)):
+                    scores[i].value.update({'comet': comet_scores[i]})
+            except Exception as e:
+                logger.warning(f'[WMT24PPAdapter] COMET batch calculation failed: {e}')
+
+        return scores

evalscope/benchmarks/zerobench/zerobench_adapter.py

@@ -0,0 +1,64 @@
+# flake8: noqa: E501
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64, compress_image_to_limit
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+# 定义提示模板
+PROMPT_TEMPLATE = """{question}
+\n\n\nLet's think step by step and give the final answer in curly braces,
+like this: {{final answer}}"
+"""
+
+SUBSET_LIST = ['default']
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='zerobench',
+        pretty_name='ZeroBench',
+        dataset_id='evalscope/zerobench',
+        tags=[Tags.KNOWLEDGE, Tags.QA, Tags.MULTI_MODAL],
+        description=
+        'ZeroBench is a challenging visual reasoning benchmark for Large Multimodal Models (LMMs). It consists of a main set of 100 high-quality, manually curated questions covering numerous domains, reasoning types and image type. Questions in ZeroBench have been designed and calibrated to be beyond the capabilities of current frontier models. As such, none of the evaluated models achieves a non-zero pass@1 (with greedy decoding) or 5/5 reliability score.',
+        subset_list=SUBSET_LIST,
+        metric_list=['acc'],
+        eval_split='zerobench',
+        train_split='zerobench_subquestions',
+        prompt_template=PROMPT_TEMPLATE,
+    )
+)
+class ZeroBenchAdapter(VisionLanguageAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self._use_llm_judge = True
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        question = record['question_text']
+        content_list: List[Content] = [ContentText(text=self.prompt_template.format(question=question))]
+        image = record['question_images_decoded']
+        if len(image) > 0:
+            for img in image:
+                # Ensure image is under OpenAI's 10MB data-URI limit by compressing if needed
+                processed_bytes, fmt = compress_image_to_limit(img['bytes'], 10_000_000)
+                image_base64 = bytes_to_base64(processed_bytes, format=fmt, add_header=True)
+                content_list.append(ContentImage(image=image_base64))
+
+        metadata = {
+            'question_id': record['question_id'],
+            'question_images': record['question_images'],
+            'image_attribution': record['image_attribution']
+        }
+
+        return Sample(
+            input=[ChatMessageUser(content=content_list)], target=record['question_answer'], metadata=metadata
+        )

evalscope/cli/start_app.py

@@ -28,6 +28,12 @@ class StartAppCMD(CLICommand):
         parser.set_defaults(func=subparser_func)
 
     def execute(self):
-        from evalscope.app import create_app
+        try:
+            from evalscope.app import create_app
+        except ImportError as e:
+            raise ImportError(
+                f'Failed to import create_app from evalscope.app, due to {e}. '
+                "Please run `pip install 'evalscope[app]'`."
+            )
 
         create_app(self.args)

evalscope/cli/start_perf.py

@@ -28,6 +28,12 @@ class PerfBenchCMD(CLICommand):
         parser.set_defaults(func=subparser_func)
 
     def execute(self):
-        from evalscope.perf.main import run_perf_benchmark
+        try:
+            from evalscope.perf.main import run_perf_benchmark
+        except ImportError as e:
+            raise ImportError(
+                f'Failed to import run_perf_benchmark from evalscope.perf.main, due to {e}. '
+                "Please run `pip install 'evalscope[perf]'`."
+            )
 
         run_perf_benchmark(self.args)

evalscope/config.py

@@ -6,7 +6,7 @@ from argparse import Namespace
 from dataclasses import dataclass, field
 from typing import Dict, List, Optional, Union
 
-from evalscope.api.model import GenerateConfig
+from evalscope.api.model import GenerateConfig, Model, ModelAPI
 from evalscope.constants import (
     DEFAULT_DATASET_CACHE_DIR,
     DEFAULT_WORK_DIR,
@@ -15,12 +15,13 @@ from evalscope.constants import (
     HubType,
     JudgeStrategy,
     ModelTask,
-    OutputType,
 )
 from evalscope.utils.argument_utils import BaseArgument, parse_int_or_float
 from evalscope.utils.deprecation_utils import deprecated_warning
+from evalscope.utils.import_utils import check_import
 from evalscope.utils.io_utils import dict_to_yaml, gen_hash, safe_filename
 from evalscope.utils.logger import get_logger
+from evalscope.version import __version__ as evalscope_version
 
 logger = get_logger()
 
@@ -28,51 +29,118 @@ logger = get_logger()
 @dataclass
 class TaskConfig(BaseArgument):
     # Model-related arguments
-    model: Optional[str] = None
+    model: Optional[Union[str, Model, ModelAPI]] = None
+    """The model to be evaluated. Can be a string path, Model object, or ModelAPI object."""
+
     model_id: Optional[str] = None
+    """Unique identifier for the model. Auto-generated from model name if not provided."""
+
     model_args: Dict = field(default_factory=dict)
+    """Additional arguments to pass to the model during initialization."""
+
     model_task: str = ModelTask.TEXT_GENERATION
+    """The type of task the model performs (e.g., text generation, image generation)."""
 
     # Template-related arguments
     chat_template: Optional[str] = None
+    """Chat template to use for formatting conversations with the model."""
 
     # Dataset-related arguments
     datasets: List[str] = field(default_factory=list)
+    """List of dataset names to evaluate the model on."""
+
     dataset_args: Dict = field(default_factory=dict)
+    """Additional arguments to pass to datasets during loading."""
+
     dataset_dir: str = DEFAULT_DATASET_CACHE_DIR
+    """Directory where datasets are cached locally."""
+
     dataset_hub: str = HubType.MODELSCOPE
-    repeats: int = 1  # Number of times to repeat the dataset items for k-metrics
+    """Hub platform to download datasets from (e.g., ModelScope, HuggingFace)."""
+
+    repeats: int = 1
+    """Number of times to repeat the dataset items for k-metrics evaluation."""
 
     # Generation configuration arguments
     generation_config: Union[Dict, GenerateConfig] = field(default_factory=dict)
+    """Configuration parameters for text/image generation."""
 
     # Evaluation-related arguments
     eval_type: str = EvalType.CHECKPOINT
+    """Type of evaluation: checkpoint, service, or mock."""
+
     eval_backend: str = EvalBackend.NATIVE
+    """Backend framework to use for evaluation."""
+
     eval_config: Union[str, Dict, None] = None
+    """Additional evaluation configuration parameters."""
+
     limit: Optional[Union[int, float]] = None
+    """Maximum number of samples to evaluate. Can be int (count) or float (fraction)."""
+
     eval_batch_size: int = 1
+    """Batch size for evaluation processing."""
 
     # Cache and working directory arguments
     use_cache: Optional[str] = None
+    """Whether to use cached results and which cache strategy to apply."""
+
     rerun_review: bool = False
+    """Whether to rerun the review process even if results exist."""
+
     work_dir: str = DEFAULT_WORK_DIR
+    """Working directory for storing evaluation results and temporary files."""
 
     # Debug and runtime mode arguments
     ignore_errors: bool = False
+    """Whether to continue evaluation when encountering errors."""
+
     debug: bool = False
-    dry_run: bool = False
+    """Enable debug mode for detailed logging and error reporting."""
+
     seed: Optional[int] = 42
-    api_url: Optional[str] = None  # Only used for server model
-    api_key: Optional[str] = 'EMPTY'  # Only used for server model
-    timeout: Optional[float] = None  # Only used for server model
-    stream: Optional[bool] = None  # Only used for server model
+    """Random seed for reproducible results."""
+
+    api_url: Optional[str] = None
+    """API endpoint URL for server-based model evaluation."""
+
+    api_key: Optional[str] = 'EMPTY'
+    """API key for authenticating with server-based models."""
+
+    timeout: Optional[float] = None
+    """Request timeout in seconds for server-based models."""
+
+    stream: Optional[bool] = None
+    """Whether to use streaming responses for server-based models."""
 
     # LLMJudge arguments
     judge_strategy: str = JudgeStrategy.AUTO
+    """Strategy for LLM-based judgment (auto, single, pairwise)."""
+
     judge_worker_num: int = 1
+    """Number of worker processes for parallel LLM judging."""
+
     judge_model_args: Optional[Dict] = field(default_factory=dict)
+    """Additional arguments for the judge model configuration."""
+
     analysis_report: bool = False
+    """Whether to generate detailed analysis reports after evaluation."""
+
+    # Sandbox configuration arguments
+    use_sandbox: bool = False
+    """Whether to execute code in a sandboxed environment."""
+
+    sandbox_type: Optional[str] = 'docker'
+    """Type of sandbox environment for code execution (e.g., docker). Default is 'docker'."""
+
+    sandbox_manager_config: Optional[Dict] = field(default_factory=dict)
+    """Configuration for the sandbox manager. Default is local manager. If url is provided, it will use remote manager."""
+
+    sandbox_config: Optional[Dict] = field(default_factory=dict)
+    """Configuration for sandboxed code execution environments."""
+
+    evalscope_version: Optional[str] = evalscope_version
+    """EvalScope version used for the evaluation."""
 
     def __post_init__(self):
         self.__init_model_and_id()
@@ -82,20 +150,22 @@ class TaskConfig(BaseArgument):
         # Set default generation_config and model_args
         self.__init_default_generation_config()
         self.__init_default_model_args()
+        self.__init_default_sandbox_config()
 
     def __init_model_and_id(self):
         # Set model to DummyCustomModel if not provided
         if self.model is None:
             self.model = self.model_task
             self.eval_type = EvalType.MOCK_LLM
-        else:
-            if self.model_task == ModelTask.IMAGE_GENERATION:
-                self.eval_type = EvalType.TEXT2IMAGE
 
         # Set model_id if not provided
         if not self.model_id:
-            if self.model:
+            if isinstance(self.model, str):
                 self.model_id = safe_filename(os.path.basename(self.model))
+            elif isinstance(self.model, Model):
+                self.model_id = safe_filename(self.model.name)
+            elif isinstance(self.model, ModelAPI):
+                self.model_id = safe_filename(self.model.model_name)
             else:
                 self.model_id = 'dummy_model'
 
@@ -113,6 +183,11 @@ class TaskConfig(BaseArgument):
                     'num_inference_steps': 50,
                     'guidance_scale': 9.0,
                 }
+                if self.eval_batch_size != 1:
+                    logger.warning(
+                        'For image generation task, we only support eval_batch_size=1 for now, changed to 1.'
+                    )
+                    self.eval_batch_size = 1
             elif self.model_task == ModelTask.TEXT_GENERATION:
                 if self.eval_type == EvalType.CHECKPOINT:
                     self.generation_config = {
@@ -125,7 +200,6 @@ class TaskConfig(BaseArgument):
                     }
                 elif self.eval_type == EvalType.SERVICE:
                     self.generation_config = {
-                        'max_tokens': 2048,
                         'temperature': 0.0,
                     }
         if isinstance(self.generation_config, dict):
@@ -138,14 +212,14 @@ class TaskConfig(BaseArgument):
         if self.timeout is not None:
             deprecated_warning(
                 logger,
-                'The `timeout` parameter is deprecated and will be removed in v1.1.0. Use `generation_config.timeout` instead.'
+                'The `timeout` parameter is deprecated and will be removed in v2.0.0. Use `generation_config.timeout` instead.'
             )
             self.generation_config.timeout = self.timeout
 
         if self.stream is not None:
             deprecated_warning(
                 logger,
-                'The `stream` parameter is deprecated and will be removed in v1.1.0. Use `generation_config.stream` instead.'
+                'The `stream` parameter is deprecated and will be removed in v2.0.0. Use `generation_config.stream` instead.'
             )
             self.generation_config.stream = self.stream
 
@@ -154,7 +228,7 @@ class TaskConfig(BaseArgument):
             self.generation_config.n = 1
             deprecated_warning(
                 logger,
-                'The `n` parameter in generation_config is deprecated and will be removed in v1.1.0. Use `TaskConfig.repeats` instead.'
+                'The `n` parameter in generation_config is deprecated and will be removed in v2.0.0. Use `TaskConfig.repeats` instead.'
             )
 
     def __init_default_model_args(self):
@@ -167,6 +241,14 @@ class TaskConfig(BaseArgument):
                     'precision': 'torch.float16',
                 }
 
+    def __init_default_sandbox_config(self):
+        if not self.use_sandbox:
+            return
+        check_import('ms_enclave', 'ms_enclave[docker]', raise_error=True)
+
+        if not self.sandbox_type:
+            self.sandbox_type = 'docker'
+
     def update(self, other: Union['TaskConfig', dict]):
         if isinstance(other, TaskConfig):
             other = other.to_dict()
@@ -182,9 +264,12 @@ class TaskConfig(BaseArgument):
             logger.warning(f'Failed to dump overall task config: {e}')
 
     def to_dict(self):
-        result = copy.deepcopy(self.__dict__)
+        result = copy.copy(self.__dict__)
         del result['api_key']  # Do not expose api_key in the config
 
+        if isinstance(self.model, (Model, ModelAPI)):
+            result['model'] = self.model.__class__.__name__
+
         if isinstance(self.generation_config, GenerateConfig):
             result['generation_config'] = self.generation_config.model_dump(exclude_unset=True)
         return result

evalscope/constants.py

@@ -15,6 +15,8 @@ DEFAULT_ROOT_CACHE_DIR = DEFAULT_DATASET_CACHE_DIR  # compatible with old versio
 DEFAULT_EVALSCOPE_CACHE_DIR = os.path.expanduser(
     os.getenv('EVALSCOPE_CACHE', '~/.cache/evalscope')
 )  # ~/.cache/evalscope
+IS_BUILD_DOC = os.getenv('BUILD_DOC', '0') == '1'  # To avoid some heavy dependencies when building doc
+HEARTBEAT_INTERVAL_SEC = 60  # 60 seconds
 
 
 class HubType:
@@ -70,6 +72,7 @@ class EvalType:
     CHECKPOINT = 'llm_ckpt'  # native model checkpoint
     SERVICE = 'openai_api'  # model service
     TEXT2IMAGE = 'text2image'  # image generation service
+    IMAGE_EDITING = 'image_editing'  # image editing service
 
 
 class OutputType:
@@ -119,6 +122,7 @@ class Tags:
     CHINESE = 'Chinese'
     COMMONSENSE = 'Commonsense'
     QA = 'QA'
+    NER = 'NER'
     READING_COMPREHENSION = 'ReadingComprehension'
     CUSTOM = 'Custom'
     INSTRUCTION_FOLLOWING = 'InstructionFollowing'
@@ -127,3 +131,17 @@ class Tags:
     RETRIEVAL = 'Retrieval'
     FUNCTION_CALLING = 'FunctionCalling'
     TEXT_TO_IMAGE = 'TextToImage'
+    IMAGE_EDITING = 'ImageEditing'
+    MULTI_MODAL = 'MultiModal'
+    MULTI_LINGUAL = 'MultiLingual'
+    MULTI_TURN = 'MultiTurn'
+    YES_NO = 'Yes/No'
+    HALLUCINATION = 'Hallucination'
+    MEDICAL = 'Medical'
+    AGENT = 'Agent'
+    MT = 'MachineTranslation'
+
+
+class FileConstants:
+    IMAGE_PATH = 'image_path'
+    ID = 'id'