paddlex 2.0.0rc4__py3-none-any.whl → 3.0.0b2__py3-none-any.whl

paddlex/utils/logging.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,47 +12,146 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import time
-import os
+
+import inspect
+import logging
 import sys
-import colorama
-from colorama import init
-import paddlex
-import paddle
 
-init(autoreset=True)
-levels = {0: 'ERROR', 1: 'WARNING', 2: 'INFO', 3: 'DEBUG'}
+import colorlog
+
+from .flags import DEBUG
+
+__all__ = ["debug", "info", "warning", "error", "critical", "setup_logging"]
+
+LOGGER_NAME = "paddlex"
+_LOG_CONFIG = {
+    "DEBUG": {"color": "purple"},
+    "INFO": {"color": "green"},
+    "WARNING": {"color": "yellow"},
+    "ERROR": {"color": "red"},
+    "CRITICAL": {"color": "bold_red"},
+}
+
+_logger = logging.getLogger(LOGGER_NAME)
+
+
+def debug(msg, *args, **kwargs):
+    """debug"""
+    if DEBUG:
+        frame = inspect.currentframe()
+        caller_frame = frame.f_back
+        caller_func_name = caller_frame.f_code.co_name
+
+        if "self" in caller_frame.f_locals:
+            caller_class_name = caller_frame.f_locals["self"].__class__.__name__
+        elif "cls" in caller_frame.f_locals:
+            caller_class_name = caller_frame.f_locals["cls"].__name__
+        else:
+            caller_class_name = None
+
+        if caller_class_name:
+            caller_info = f"{caller_class_name}::{caller_func_name}"
+        else:
+            caller_info = f"{caller_func_name}"
+        msg = f"【{caller_info}】{msg}"
+
+    _logger.debug(msg, *args, **kwargs)
+
+
+def info(msg, *args, **kwargs):
+    """info"""
+    _logger.info(msg, *args, **kwargs)
+
+
+def warning(msg, *args, **kwargs):
+    """warning"""
+    _logger.warning(msg, *args, **kwargs)
+
+
+def error(msg, *args, **kwargs):
+    """error"""
+    _logger.error(msg, *args, **kwargs)
+
+
+def critical(msg, *args, **kwargs):
+    """critical"""
+    _logger.critical(msg, *args, **kwargs)
+
+
+def exception(msg, *args, **kwargs):
+    """exception"""
+    _logger.exception(msg, *args, **kwargs)
+
+
+def setup_logging(verbosity: str = None):
+    """setup logging level
+
+    Args:
+        verbosity (str, optional): the logging level, `DEBUG`, `INFO`, `WARNING`. Defaults to None.
+    """
+    if verbosity is None:
+        if DEBUG:
+            verbosity = "DEBUG"
+        else:
+            verbosity = "INFO"
+
+    if verbosity is not None:
+        _configure_logger(_logger, verbosity.upper())
 
 
-def log(level=2, message="", use_color=False):
-    if paddle.distributed.get_rank() == 0:
-        current_time = time.time()
-        time_array = time.localtime(current_time)
-        current_time = time.strftime("%Y-%m-%d %H:%M:%S", time_array)
-        if paddlex.log_level >= level:
-            if use_color:
-                print("\033[1;31;40m{} [{}]\t{}\033[0m".format(
-                    current_time, levels[level], message).encode("utf-8")
-                      .decode("latin1"))
-            else:
-                print("{} [{}]\t{}".format(current_time, levels[
-                    level], message).encode("utf-8").decode("latin1"))
-            sys.stdout.flush()
+def _configure_logger(logger, verbosity):
+    """_configure_logger"""
+    if verbosity == "DEBUG":
+        _logger.setLevel(logging.DEBUG)
+    elif verbosity == "INFO":
+        _logger.setLevel(logging.INFO)
+    elif verbosity == "WARNING":
+        _logger.setLevel(logging.WARNING)
+    logger.propagate = False
+    if not logger.hasHandlers():
+        _add_handler(logger)
 
 
-def debug(message="", use_color=False):
-    log(level=3, message=message, use_color=use_color)
+def _add_handler(logger):
+    """_add_handler"""
+    format = colorlog.ColoredFormatter(
+        "%(log_color)s%(message)s",
+        log_colors={key: conf["color"] for key, conf in _LOG_CONFIG.items()},
+    )
 
+    handler = logging.StreamHandler(sys.stderr)
+    handler.setFormatter(format)
+    logger.addHandler(handler)
 
-def info(message="", use_color=False):
-    log(level=2, message=message, use_color=use_color)
 
+def advertise():
+    """
+    Show the advertising message like the following:
 
-def warning(message="", use_color=True):
-    log(level=1, message=message, use_color=use_color)
+    ===========================================================
+    ==         PaddleX is powered by PaddlePaddle !          ==
+    ===========================================================
+    ==                                                       ==
+    ==   For more info please go to the following website.   ==
+    ==                                                       ==
+    ==        https://github.com/PaddlePaddle/PaddleX        ==
+    ===========================================================
 
+    """
+    copyright = "PaddleX is powered by PaddlePaddle !"
+    ad = "For more info please go to the following website."
+    website = "https://github.com/PaddlePaddle/PaddleX"
+    AD_LEN = 6 + len(max([copyright, ad, website], key=len))
 
-def error(message="", use_color=True, exit=True):
-    log(level=0, message=message, use_color=use_color)
-    if exit:
-        sys.exit(-1)
+    info(
+        "\n{0}\n{1}\n{2}\n{3}\n{4}\n{5}\n{6}\n{7}\n".format(
+            "=" * (AD_LEN + 4),
+            "=={}==".format(copyright.center(AD_LEN)),
+            "=" * (AD_LEN + 4),
+            "=={}==".format(" " * AD_LEN),
+            "=={}==".format(ad.center(AD_LEN)),
+            "=={}==".format(" " * AD_LEN),
+            "=={}==".format(website.center(AD_LEN)),
+            "=" * (AD_LEN + 4),
+        )
+    )

paddlex/utils/misc.py

@@ -0,0 +1,201 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import threading
+import numpy as np
+from abc import ABCMeta
+from .errors import (
+    raise_class_not_found_error,
+    raise_no_entity_registered_error,
+    DuplicateRegistrationError,
+)
+from .logging import *
+
+
+def convert_and_remove_types(data):
+    if isinstance(data, dict):
+        return {
+            k: convert_and_remove_types(v)
+            for k, v in data.items()
+            if not isinstance(v, type)
+        }
+    elif isinstance(data, list):
+        return [convert_and_remove_types(v) for v in data]
+    elif isinstance(data, np.ndarray):
+        return data.tolist()
+    elif isinstance(data, (np.float32, np.float64)):
+        return float(data)
+    elif isinstance(data, (np.int32, np.int64)):
+        return int(data)
+    elif isinstance(data, np.bool_):
+        return bool(data)
+    elif isinstance(data, (np.str_, np.unicode_)):
+        return str(data)
+    return data
+
+
+def abspath(path: str):
+    """get absolute path
+
+    Args:
+        path (str): the relative path
+
+    Returns:
+        str: the absolute path
+    """
+    return os.path.abspath(path)
+
+
+class CachedProperty(object):
+    """
+    A property that is only computed once per instance and then replaces itself
+    with an ordinary attribute.
+
+    The implementation refers to
+    https://github.com/pydanny/cached-property/blob/master/cached_property.py .
+
+    Note that this implementation does NOT work in multi-thread or coroutine
+    senarios.
+    """
+
+    def __init__(self, func):
+        super().__init__()
+        self.func = func
+        self.__doc__ = getattr(func, "__doc__", "")
+
+    def __get__(self, obj, cls):
+        if obj is None:
+            return self
+        val = self.func(obj)
+        # Hack __dict__ of obj to inject the value
+        # Note that this is only executed once
+        obj.__dict__[self.func.__name__] = val
+        return val
+
+
+class Constant(object):
+    """Constant"""
+
+    def __init__(self, val):
+        super().__init__()
+        self.val = val
+
+    def __get__(self, obj, type_=None):
+        return self.val
+
+    def __set__(self, obj, val):
+        raise AttributeError("The value of a constant cannot be modified!")
+
+
+class Singleton(type):
+    """singleton meta class
+
+    Args:
+        type (class): type
+
+    Returns:
+        class: meta class
+    """
+
+    _insts = {}
+    _lock = threading.Lock()
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._insts:
+            with cls._lock:
+                if cls not in cls._insts:
+                    cls._insts[cls] = super().__call__(*args, **kwargs)
+        return cls._insts[cls]
+
+
+# TODO(gaotingquan): has been mv to subclass_register.py
+class AutoRegisterMetaClass(type):
+    """meta class that automatically registry subclass to its baseclass
+
+    Args:
+        type (class): type
+
+    Returns:
+        class: meta class
+    """
+
+    __model_type_attr_name = "entities"
+    __base_class_flag = "__is_base"
+    __registered_map_name = "__registered_map"
+
+    def __new__(mcs, name, bases, attrs):
+        cls = super().__new__(mcs, name, bases, attrs)
+        mcs.__register_model_entity(bases, cls, attrs)
+        return cls
+
+    @classmethod
+    def __register_model_entity(mcs, bases, cls, attrs):
+        if bases:
+            for base in bases:
+                base_cls = mcs.__find_base_class(base)
+                if base_cls:
+                    mcs.__register_to_base_class(base_cls, cls)
+
+    @classmethod
+    def __find_base_class(mcs, cls):
+        is_base_flag = mcs.__base_class_flag
+        if is_base_flag.startswith("__"):
+            is_base_flag = f"_{cls.__name__}" + is_base_flag
+        if getattr(cls, is_base_flag, False):
+            return cls
+        for base in cls.__bases__:
+            base_cls = mcs.__find_base_class(base)
+            if base_cls:
+                return base_cls
+        return None
+
+    @classmethod
+    def __register_to_base_class(mcs, base, cls):
+        cls_entity_name = getattr(cls, mcs.__model_type_attr_name, cls.__name__)
+        if isinstance(cls_entity_name, str):
+            cls_entity_name = [cls_entity_name]
+
+        records = getattr(base, mcs.__registered_map_name, {})
+        for name in cls_entity_name:
+            if name in records and records[name] is not cls:
+                raise DuplicateRegistrationError(
+                    f"The name(`{name}`) duplicated registration! The class entities are: `{cls.__name__}` and \
+`{records[name].__name__}`."
+                )
+            records[name] = cls
+            debug(
+                f"The class entity({cls.__name__}) has been register as name(`{name}`)."
+            )
+        setattr(base, mcs.__registered_map_name, records)
+
+    def all(cls):
+        """get all subclass"""
+        if not hasattr(cls, type(cls).__registered_map_name):
+            raise_no_entity_registered_error(cls)
+        return getattr(cls, type(cls).__registered_map_name)
+
+    def get(cls, name: str):
+        """get the registried class by name"""
+        all_entities = cls.all()
+        if name not in all_entities:
+            raise_class_not_found_error(name, cls, all_entities)
+        return all_entities[name]
+
+
+class AutoRegisterABCMetaClass(ABCMeta, AutoRegisterMetaClass):
+    """AutoRegisterABCMetaClass"""
+
+    pass

paddlex/utils/result_saver.py

@@ -0,0 +1,59 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import sys
+import json
+import traceback
+from .flags import DEBUG
+from .file_interface import custom_open, write_json_file
+
+
+def try_except_decorator(func):
+    """try-except"""
+
+    def wrap(self, *args, **kwargs):
+        try:
+            result = func(self, *args, **kwargs)
+            if result:
+                save_result(True, self._mode, self._output, result_dict=result)
+        except Exception as e:
+            exc_type, exc_value, exc_tb = sys.exc_info()
+            save_result(
+                False,
+                self._mode,
+                self._output,
+                err_type=str(exc_type),
+                err_msg=str(exc_value),
+            )
+            traceback.print_exception(exc_type, exc_value, exc_tb)
+            sys.exit(1)
+
+    return wrap
+
+
+def save_result(run_pass, mode, output, result_dict=None, err_type=None, err_msg=None):
+    """format, build and save result"""
+    json_data = {"done_flag": run_pass}
+    if not run_pass:
+        assert result_dict is None and err_type is not None and err_msg is not None
+        json_data.update({"err_type": err_type, "err_msg": err_msg})
+    else:
+        assert result_dict is not None and err_type is None and err_msg is None
+        json_data.update(result_dict)
+    if not os.path.exists(output):
+        os.makedirs(output, exist_ok=True)
+
+    write_json_file(json_data, os.path.join(output, f"{mode}_result.json"), indent=2)

paddlex/utils/subclass_register.py

@@ -0,0 +1,101 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABCMeta
+
+from . import logging
+from .errors import (
+    raise_class_not_found_error,
+    raise_no_entity_registered_error,
+    DuplicateRegistrationError,
+)
+
+
+class AutoRegisterMetaClass(type):
+    """meta class that automatically registry subclass to its baseclass
+
+    Args:
+        type (class): type
+
+    Returns:
+        class: meta class
+    """
+
+    __model_type_attr_name = "entities"
+    __base_class_flag = "__is_base"
+    __registered_map_name = "__registered_map"
+
+    def __new__(mcs, name, bases, attrs):
+        cls = super().__new__(mcs, name, bases, attrs)
+        mcs.__register_model_entity(bases, cls, attrs)
+        return cls
+
+    @classmethod
+    def __register_model_entity(mcs, bases, cls, attrs):
+        if bases:
+            for base in bases:
+                base_cls = mcs.__find_base_class(base)
+                if base_cls:
+                    mcs.__register_to_base_class(base_cls, cls)
+
+    @classmethod
+    def __find_base_class(mcs, cls):
+        is_base_flag = mcs.__base_class_flag
+        if is_base_flag.startswith("__"):
+            is_base_flag = f"_{cls.__name__}" + is_base_flag
+        if getattr(cls, is_base_flag, False):
+            return cls
+        for base in cls.__bases__:
+            base_cls = mcs.__find_base_class(base)
+            if base_cls:
+                return base_cls
+        return None
+
+    @classmethod
+    def __register_to_base_class(mcs, base, cls):
+        cls_entity_name = getattr(cls, mcs.__model_type_attr_name, cls.__name__)
+        if isinstance(cls_entity_name, str):
+            cls_entity_name = [cls_entity_name]
+
+        records = getattr(base, mcs.__registered_map_name, {})
+        for name in cls_entity_name:
+            if name in records and records[name] is not cls:
+                raise DuplicateRegistrationError(
+                    f"The name(`{name}`) duplicated registration! The class entities are: `{cls.__name__}` and \
+`{records[name].__name__}`."
+                )
+            records[name] = cls
+            logging.debug(
+                f"The class entity({cls.__name__}) has been register as name(`{name}`)."
+            )
+        setattr(base, mcs.__registered_map_name, records)
+
+    def all(cls):
+        """get all subclass"""
+        if not hasattr(cls, type(cls).__registered_map_name):
+            raise_no_entity_registered_error(cls)
+        return getattr(cls, type(cls).__registered_map_name)
+
+    def get(cls, name: str):
+        """get the registried class by name"""
+        all_entities = cls.all()
+        if name not in all_entities:
+            raise_class_not_found_error(name, cls, all_entities)
+        return all_entities[name]
+
+
+class AutoRegisterABCMetaClass(ABCMeta, AutoRegisterMetaClass):
+    """AutoRegisterABCMetaClass"""
+
+    pass

paddlex/version.py

@@ -0,0 +1,54 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+
+__all__ = ["get_pdx_version", "get_version_dict", "show_versions"]
+
+
+def get_pdx_version():
+    """get_pdx_version"""
+    with open(
+        os.path.join(os.path.dirname(__file__), ".version"), "r", encoding="ascii"
+    ) as fv:
+        ver = fv.read().rstrip()
+    return ver
+
+
+def get_version_dict():
+    """get_version_dict"""
+    import paddle
+    from . import repo_manager
+
+    ver_dict = dict()
+    ver_dict["pdx"] = get_pdx_version()
+    ver_dict["paddle"] = paddle.__version__
+    ver_dict["devkits"] = repo_manager.get_versions()
+    return ver_dict
+
+
+def show_versions():
+    """show_versions"""
+    ver_dict = get_version_dict()
+    pdx_ver = f"PDX version: {ver_dict['pdx']}\n"
+    paddle_ver = f"PaddlePaddle version: {ver_dict['paddle']}\n"
+    repo_vers = []
+    for repo_name, vers in ver_dict["devkits"].items():
+        sta_ver = vers[0]
+        commit = vers[1]
+        repo_vers.append(f"{repo_name}:\nversion: {sta_ver}\ncommit id: {commit}\n")
+    all_vers = [pdx_ver, paddle_ver, *repo_vers]
+    ver_str = "\n".join(all_vers)
+    print(ver_str)