evalscope 0.5.0__py3-none-any.whl

evalscope/utils/logger.py

@@ -0,0 +1,64 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import logging
+from typing import Optional
+
+init_loggers = {}
+
+formatter = logging.Formatter(
+    '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+
+def get_logger(log_file: Optional[str] = None,
+               log_level: int = logging.INFO,
+               file_mode: str = 'w'):
+    """ Get logging logger
+
+    Args:
+        log_file: Log filename, if specified, file handler will be added to
+            logger
+        log_level: Logging level.
+        file_mode: Specifies the mode to open the file, if filename is
+            specified (if filemode is unspecified, it defaults to 'w').
+    """
+
+    logger_name = __name__.split('.')[0]
+    logger = logging.getLogger(logger_name)
+
+    if logger_name in init_loggers:
+        add_file_handler_if_needed(logger, log_file, file_mode, log_level)
+        return logger
+
+    for handler in logger.root.handlers:
+        if type(handler) is logging.StreamHandler:
+            handler.setLevel(logging.ERROR)
+
+    stream_handler = logging.StreamHandler()
+    handlers = [stream_handler]
+
+    if log_file is not None:
+        file_handler = logging.FileHandler(log_file, file_mode)
+        handlers.append(file_handler)
+
+    for handler in handlers:
+        handler.setFormatter(formatter)
+        handler.setLevel(log_level)
+        logger.addHandler(handler)
+
+    logger.setLevel(log_level)
+
+    init_loggers[logger_name] = True
+
+    return logger
+
+
+def add_file_handler_if_needed(logger, log_file, file_mode, log_level):
+    for handler in logger.handlers:
+        if isinstance(handler, logging.FileHandler):
+            return
+
+    if log_file is not None:
+        file_handler = logging.FileHandler(log_file, file_mode)
+        file_handler.setFormatter(formatter)
+        file_handler.setLevel(log_level)
+        logger.addHandler(file_handler)

evalscope/utils/task_cfg_parser.py

@@ -0,0 +1,10 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+"""
+This file is used to parse the eval-task config file.
+"""
+
+import os
+import json
+
+

evalscope/utils/task_utils.py

@@ -0,0 +1,19 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from enum import Enum
+
+
+class EvalBackend(Enum):
+    # Use native evaluation pipeline of Eval-Scope
+    NATIVE = 'Native'
+
+    # Use OpenCompass framework as the evaluation backend
+    OPEN_COMPASS = 'OpenCompass'
+    
+    # Use VLM Eval Kit as the multi-modal model evaluation backend
+    VLM_EVAL_KIT = 'VLMEvalKit'
+
+    # Use third-party evaluation backend/modules
+    THIRD_PARTY = 'ThirdParty'
+    
+    
+