paddlex 2.1.0__py3-none-any.whl → 3.0.0__py3-none-any.whl

paddlex/utils/cache.py

@@ -0,0 +1,146 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import atexit
+import functools
+import hashlib
+import inspect
+import os
+import os.path as osp
+import pickle
+import tempfile
+from pathlib import Path
+
+import filelock
+
+DEFAULT_CACHE_DIR = osp.abspath(osp.join(os.path.expanduser("~"), ".paddlex"))
+CACHE_DIR = os.environ.get("PADDLE_PDX_CACHE_HOME", DEFAULT_CACHE_DIR)
+FUNC_CACHE_DIR = osp.join(CACHE_DIR, "func_ret")
+FILE_LOCK_DIR = osp.join(CACHE_DIR, "locks")
+TEMP_DIR = osp.join(CACHE_DIR, "temp")
+
+
+def create_cache_dir(*args, **kwargs):
+    """create cache dir"""
+    # `args` and `kwargs` reserved for extension
+    os.makedirs(CACHE_DIR, exist_ok=True)
+    os.makedirs(FUNC_CACHE_DIR, exist_ok=True)
+    os.makedirs(FILE_LOCK_DIR, exist_ok=True)
+    # TODO: Ensure permission
+
+
+def get_cache_dir(*args, **kwargs):
+    """get cache dir"""
+    # `args` and `kwargs` reserved for extension
+    return CACHE_DIR
+
+
+def persist(cond=None):
+    """persist"""
+    # FIXME: Current implementation creates files in cache dir and we do
+    # not set a limit on number of files
+    # TODO: Faster implementation and support more arg types
+    FILENAME_PATTERN = "persist_{key}.pkl"
+    SUPPORTED_ARG_TYPES = (str, int, float)
+
+    if cond is None:
+        cond = lambda ret: ret is not None
+
+    def _to_bytes(obj):
+        return str(obj).encode("utf-8")
+
+    def _make_key(func, bnd_args):
+        # Use MD5 algorithm to make deterministic hashing
+        # Note that the object-to-bytes conversion should be deterministic,
+        # we ensure this by restricting types of arguments.
+        m = hashlib.md5()
+        m.update(_to_bytes(osp.realpath(inspect.getsourcefile(func))))
+        m.update(_to_bytes(func.__name__))
+        for k, v in bnd_args.arguments.items():
+            if not isinstance(v, SUPPORTED_ARG_TYPES):
+                raise TypeError(
+                    f"{repr(k)}: {v}, {type(v)} is unhashable or not a supported type."
+                )
+            m.update(_to_bytes(k))
+            m.update(_to_bytes(v))
+        hash_ = m.hexdigest()
+        return hash_
+
+    def _deco(func):
+        @functools.wraps(func)
+        def _wrapper(*args, **kwargs):
+            sig = inspect.signature(func)
+            bnd_args = sig.bind(*args, **kwargs)
+            bnd_args.apply_defaults()
+            key = _make_key(func, bnd_args)
+            cache_file_path = osp.join(
+                FUNC_CACHE_DIR, FILENAME_PATTERN.format(key=str(key))
+            )
+            lock = filelock.FileLock(osp.join(FILE_LOCK_DIR, f"{key}.lock"))
+            with lock:
+                if osp.exists(cache_file_path):
+                    with open(cache_file_path, "rb") as f:
+                        ret = pickle.load(f)
+                else:
+                    ret = func(*args, **kwargs)
+                    if cond(ret):
+                        with open(cache_file_path, "wb") as f:
+                            pickle.dump(ret, f)
+            return ret
+
+        return _wrapper
+
+    return _deco
+
+
+class TempFileManager:
+    def __init__(self):
+        self.temp_files = []
+        Path(TEMP_DIR).mkdir(parents=True, exist_ok=True)
+        atexit.register(self.cleanup)
+
+    def create_temp_file(self, **kwargs):
+        temp_file = tempfile.NamedTemporaryFile(delete=False, dir=TEMP_DIR, **kwargs)
+        self.temp_files.append(temp_file)
+        return temp_file
+
+    def cleanup(self):
+        for temp_file in self.temp_files:
+            try:
+                temp_file.close()
+                os.remove(temp_file.name)
+            except FileNotFoundError:
+                pass
+        self.temp_files = []
+
+    class TempFileContextManager:
+        def __init__(self, manager, **kwargs):
+            self.manager = manager
+            self.kwargs = kwargs
+            self.temp_file = None
+
+        def __enter__(self):
+            self.temp_file = self.manager.create_temp_file(**self.kwargs)
+            return self.temp_file
+
+        def __exit__(self, exc_type, exc_value, traceback):
+            if self.temp_file:
+                self.temp_file.close()
+
+    def temp_file_context(self, **kwargs):
+        return self.TempFileContextManager(self, **kwargs)
+
+
+temp_file_manager = TempFileManager()

paddlex/utils/config.py

@@ -0,0 +1,216 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import copy
+import os
+
+import yaml
+
+from . import logging
+from .file_interface import custom_open
+
+__all__ = ["get_config"]
+
+
+class AttrDict(dict):
+    """Attr Dict"""
+
+    def __getattr__(self, key):
+        if key in self:
+            return self[key]
+        else:
+            raise AttributeError(key)
+
+    def __setattr__(self, key, value):
+        if key in self.__dict__:
+            self.__dict__[key] = value
+        else:
+            self[key] = value
+
+    def __deepcopy__(self, content):
+        return copy.deepcopy(dict(self))
+
+
+def create_attr_dict(yaml_config):
+    """create attr dict"""
+    from ast import literal_eval
+
+    for key, value in yaml_config.items():
+        if type(value) is dict:
+            yaml_config[key] = value = AttrDict(value)
+        if isinstance(value, str):
+            try:
+                value = literal_eval(value)
+            except BaseException:
+                pass
+        if isinstance(value, AttrDict):
+            create_attr_dict(yaml_config[key])
+        else:
+            yaml_config[key] = value
+
+
+def parse_config(cfg_file):
+    """Load a config file into AttrDict"""
+    with custom_open(cfg_file, "r") as fopen:
+        yaml_config = AttrDict(yaml.load(fopen, Loader=yaml.SafeLoader))
+    create_attr_dict(yaml_config)
+    return yaml_config
+
+
+def print_dict(d, delimiter=0):
+    """
+    Recursively visualize a dict and
+    indenting according by the relationship of keys.
+    """
+    placeholder = "-" * 60
+    for k, v in sorted(d.items()):
+        if isinstance(v, dict):
+            logging.info("{}{} : ".format(delimiter * " ", k))
+            print_dict(v, delimiter + 4)
+        elif isinstance(v, list) and len(v) >= 1 and isinstance(v[0], dict):
+            logging.info("{}{} : ".format(delimiter * " ", k))
+            for value in v:
+                print_dict(value, delimiter + 4)
+        else:
+            logging.info("{}{} : {}".format(delimiter * " ", k, v))
+        if k.isupper():
+            logging.info(placeholder)
+
+
+def print_config(config):
+    """
+    visualize configs
+    Arguments:
+        config: configs
+    """
+    logging.advertise()
+    print_dict(config)
+
+
+def override(dl, ks, v):
+    """
+    Recursively replace dict of list
+    Args:
+        dl(dict or list): dict or list to be replaced
+        ks(list): list of keys
+        v(str): value to be replaced
+    """
+
+    def parse_str(s):
+        """convert str type value
+        to None type if it is "None",
+        to bool type if it means True or False,
+        to int type if it can be eval().
+        """
+        if s in ("None"):
+            return None
+        elif s in ("TRUE", "True", "true", "T", "t"):
+            return True
+        elif s in ("FALSE", "False", "false", "F", "f"):
+            return False
+        try:
+            return eval(v)
+        except Exception:
+            return s
+
+    assert isinstance(dl, (list, dict)), "{} should be a list or a dict"
+    assert len(ks) > 0, "length of keys should larger than 0"
+    if isinstance(dl, list):
+        k = parse_str(ks[0])
+        if len(ks) == 1:
+            assert k < len(dl), "index({}) out of range({})".format(k, dl)
+            dl[k] = parse_str(v)
+        else:
+            override(dl[k], ks[1:], v)
+    else:
+        if len(ks) == 1:
+            # assert ks[0] in dl, ('{} is not exist in {}'.format(ks[0], dl))
+            if not ks[0] in dl:
+                logging.warning(f"A new field ({ks[0]}) detected!")
+            dl[ks[0]] = parse_str(v)
+        else:
+            if ks[0] not in dl.keys():
+                dl[ks[0]] = {}
+                logging.warning(f"A new Series field ({ks[0]}) detected!")
+            override(dl[ks[0]], ks[1:], v)
+
+
+def override_config(config, options=None):
+    """
+    Recursively override the config
+    Args:
+        config(dict): dict to be replaced
+        options(list): list of pairs(key0.key1.idx.key2=value)
+            such as: [
+                'topk=2',
+                'VALID.transforms.1.ResizeImage.resize_short=300'
+            ]
+    Returns:
+        config(dict): replaced config
+    """
+    if options is not None:
+        for opt in options:
+            assert isinstance(opt, str), "option({}) should be a str".format(opt)
+            assert "=" in opt, (
+                "option({}) should contain a ="
+                "to distinguish between key and value".format(opt)
+            )
+            pair = opt.split("=")
+            assert len(pair) == 2, "there can be only a = in the option"
+            key, value = pair
+            keys = key.split(".")
+            override(config, keys, value)
+    return config
+
+
+def get_config(fname, overrides=None, show=False):
+    """
+    Read config from file
+    """
+    assert os.path.exists(fname), "config file({}) is not exist".format(fname)
+    config = parse_config(fname)
+    override_config(config, overrides)
+    if show:
+        print_config(config)
+    # check_config(config)
+    return config
+
+
+def parse_args():
+    """parse args"""
+    parser = argparse.ArgumentParser("PaddleX script")
+    parser.add_argument(
+        "-c",
+        "--config",
+        type=str,
+        default="configs/config.yaml",
+        help="config file path",
+    )
+    parser.add_argument(
+        "-o",
+        "--override",
+        action="append",
+        default=[],
+        help="config options to be overridden",
+    )
+    parser.add_argument(
+        "-p",
+        "--profiler_options",
+        type=str,
+        default=None,
+        help='The option of profiler, which should be in format "key1=value1;key2=value2;key3=value3".',
+    )
+    args = parser.parse_args()
+    return args

paddlex/utils/custom_device_list.py

@@ -0,0 +1,311 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DCU_WHITELIST = [
+    "ResNet18",
+    "ResNet34",
+    "ResNet50",
+    "ResNet101",
+    "ResNet152",
+    "PP-LCNet_x1_0",
+    "PP-HGNetV2-B0_ML",
+    "PP-HGNetV2-B4_ML",
+    "PP-HGNetV2-B6_ML",
+    "CLIP_vit_base_patch16_224_ML",
+    "PP-ShiTuV2_rec_CLIP_vit_base",
+    "PP-YOLOE_plus-L",
+    "PP-YOLOE_plus-M",
+    "PP-YOLOE_plus-S",
+    "RT-DETR-R18",
+    "PicoDet-L",
+    "PicoDet-M",
+    "PicoDet-S",
+    "PicoDet-XS",
+    "FCOS-ResNet50",
+    "YOLOX-N",
+    "FasterRCNN-ResNet34-FPN",
+    "YOLOv3-DarkNet53",
+    "Cascade-FasterRCNN-ResNet50-FPN",
+    "PP-YOLOE_plus_SOD-S",
+    "PP-YOLOE_plus_SOD-L",
+    "PP-YOLOE_plus_SOD-largesize-L",
+    "STFPM",
+    "Deeplabv3_Plus-R50",
+    "Deeplabv3_Plus-R101",
+    "PP-LiteSeg-T",
+    "PP-OCRv4_server_rec",
+    "PP-OCRv4_server_det",
+    "PP-OCRv4_mobile_det",
+    "DLinear",
+    "RLinear",
+    "NLinear",
+    "PicoDet_LCNet_x2_5_face",
+]
+
+MLU_WHITELIST = [
+    "ResNet18",
+    "ResNet34",
+    "ResNet50",
+    "ResNet101",
+    "ResNet152",
+    "ResNet18_vd",
+    "ResNet34_vd",
+    "ResNet50_vd",
+    "ResNet101_vd",
+    "ResNet152_vd",
+    "ResNet200_vd",
+    "PP-LCNet_x0_25",
+    "PP-LCNet_x0_35",
+    "PP-LCNet_x0_5",
+    "PP-LCNet_x0_75",
+    "PP-LCNet_x1_0",
+    "PP-LCNet_x1_5",
+    "PP-LCNet_x2_5",
+    "PP-LCNet_x2_0",
+    "MobileNetV3_large_x0_35",
+    "MobileNetV3_large_x0_5",
+    "MobileNetV3_large_x0_75",
+    "MobileNetV3_large_x1_0",
+    "MobileNetV3_large_x1_25",
+    "MobileNetV3_small_x0_35",
+    "MobileNetV3_small_x0_5",
+    "MobileNetV3_small_x0_75",
+    "MobileNetV3_small_x1_0",
+    "MobileNetV3_small_x1_25",
+    "PP-HGNet_small",
+    "PP-HGNet_tiny",
+    "PP-HGNet_base",
+    "PP-ShiTuV2_rec_CLIP_vit_base",
+    "PP-ShiTuV2_rec_CLIP_vit_large",
+    "PP-YOLOE_plus-X",
+    "PP-YOLOE_plus-L",
+    "PP-YOLOE_plus-M",
+    "PP-YOLOE_plus-S",
+    "PicoDet-L",
+    "PicoDet-M",
+    "PicoDet-S",
+    "PicoDet-XS",
+    "STFPM",
+    "PP-LiteSeg-T",
+    "PP-OCRv4_server_rec",
+    "PP-OCRv4_mobile_rec",
+    "PP-OCRv4_server_det",
+    "PP-OCRv4_mobile_det",
+    "PicoDet_layout_1x",
+    "DLinear",
+    "RLinear",
+    "NLinear",
+    "PicoDet_LCNet_x2_5_face",
+]
+
+NPU_BLACKLIST = [
+    "PP-LiteSeg-B",
+    "MaskFormer_tiny",
+    "MaskFormer_small",
+    "PP-YOLOE_plus-S_face",
+    "BlazeFace",
+    "BlazeFace-FPN-SSH",
+    "MobileFaceNet",
+    "ResNet50_face",
+    "PP-TSM-R50_8frames_uniform",
+    "PP-TSMv2-LCNetV2_8frames_uniform",
+    "PP-TSMv2-LCNetV2_16frames_uniform",
+    "YOWO",
+    "PP-YOLOE-R-L",
+    "SAM-H-BOX",
+    "SAM-H-point",
+    "GroundingDINO-T",
+    "PP-TinyPose128x96",
+    "PP-TinyPose256x192",
+    "BEVFusion",
+    "whisper_large",
+    "whisper_medium",
+    "whisper_small",
+    "whisper_base",
+    "whisper_tiny",
+    "UniMERNet",
+]
+
+XPU_WHITELIST = [
+    "ResNet18",
+    "ResNet34",
+    "ResNet50",
+    "ResNet101",
+    "ResNet152",
+    "ResNet18_vd",
+    "ResNet34_vd",
+    "ResNet50_vd",
+    "ResNet101_vd",
+    "ResNet152_vd",
+    "ResNet200_vd",
+    "PP-LCNet_x0_25",
+    "PP-LCNet_x0_35",
+    "PP-LCNet_x0_5",
+    "PP-LCNet_x0_75",
+    "PP-LCNet_x1_0",
+    "PP-LCNet_x1_5",
+    "PP-LCNet_x2_5",
+    "PP-LCNet_x2_0",
+    "MobileNetV3_large_x0_35",
+    "MobileNetV3_large_x0_5",
+    "MobileNetV3_large_x0_75",
+    "MobileNetV3_large_x1_0",
+    "MobileNetV3_large_x1_25",
+    "MobileNetV3_small_x0_35",
+    "MobileNetV3_small_x0_5",
+    "MobileNetV3_small_x0_75",
+    "MobileNetV3_small_x1_0",
+    "MobileNetV3_small_x1_25",
+    "PP-HGNet_small",
+    "PP-HGNet_tiny",
+    "PP-HGNet_base",
+    "PP-YOLOE_plus-X",
+    "PP-YOLOE_plus-L",
+    "PP-YOLOE_plus-M",
+    "PP-YOLOE_plus-S",
+    "PicoDet-L",
+    "PicoDet-M",
+    "PicoDet-S",
+    "PicoDet-XS",
+    "STFPM",
+    "PP-LiteSeg-T",
+    "PP-OCRv4_server_rec",
+    "PP-OCRv4_mobile_rec",
+    "PP-OCRv4_server_det",
+    "PP-OCRv4_mobile_det",
+    "PicoDet_layout_1x",
+    "DLinear",
+    "RLinear",
+    "NLinear",
+    "PicoDet_LCNet_x2_5_face",
+    "PP-LCNet_x1_0_doc_ori",
+    "UVDoc",
+    "PP-DocLayout-L",
+    "PP-DocLayout-M",
+    "PP-DocLayout-S",
+    "PP-DocLayout_plus-L",
+    "PP-DocBlockLayout",
+    "SLANeXt_wired",
+    "SLANeXt_wireless",
+    "PP-LCNet_x1_0_table_cls",
+    "RT-DETR-L_wired_table_cell_det",
+    "RT-DETR-L_wireless_table_cell_det",
+    "PP-OCRv4_server_seal_det",
+    "PP-OCRv4_mobile_seal_det",
+    "PP-OCRv5_server_rec",
+    "PP-OCRv5_mobile_rec",
+    "PPDocBee-2B",
+    "PPDocBee-7B",
+    "PP-FormulaNet-S",
+    "PP-FormulaNet-L",
+    "PP-FormulaNet-M",
+    "PP-FormulaNet_plus-M",
+    "PP-FormulaNet_plus-L",
+    "PP-OCRv5_server_det",
+    "PP-OCRv5_mobile_det",
+    "PP-Chart2Table",
+]
+
+GCU_WHITELIST = [
+    "ConvNeXt_base_224",
+    "ConvNeXt_base_384",
+    "ConvNeXt_large_224",
+    "ConvNeXt_large_384",
+    "ConvNeXt_small",
+    "ConvNeXt_tiny",
+    "FasterNet-L",
+    "FasterNet-M",
+    "FasterNet-S",
+    "FasterNet-T0",
+    "FasterNet-T1",
+    "FasterNet-T2",
+    "MobileNetV1_x0_25",
+    "MobileNetV1_x0_5",
+    "MobileNetV1_x0_75",
+    "MobileNetV1_x1_0",
+    "MobileNetV2_x0_25",
+    "MobileNetV2_x0_5",
+    "MobileNetV2_x1_0",
+    "MobileNetV2_x1_5",
+    "MobileNetV2_x2_0",
+    "MobileNetV3_large_x0_35",
+    "MobileNetV3_large_x0_5",
+    "MobileNetV3_large_x0_75",
+    "MobileNetV3_large_x1_0",
+    "MobileNetV3_large_x1_25",
+    "MobileNetV3_small_x0_35",
+    "MobileNetV3_small_x0_5",
+    "MobileNetV3_small_x0_75",
+    "MobileNetV3_small_x1_0",
+    "MobileNetV3_small_x1_25",
+    "MobileNetV4_conv_large",
+    "MobileNetV4_conv_medium",
+    "MobileNetV4_conv_small",
+    "PP-HGNet_base",
+    "PP-HGNet_small",
+    "PP-HGNet_tiny",
+    "PP-HGNetV2-B0",
+    "PP-HGNetV2-B1",
+    "PP-HGNetV2-B2",
+    "PP-HGNetV2-B3",
+    "PP-HGNetV2-B4",
+    "PP-HGNetV2-B5",
+    "PP-HGNetV2-B6",
+    "PP-LCNet_x0_25",
+    "PP-LCNet_x0_35",
+    "PP-LCNet_x0_5",
+    "PP-LCNet_x0_75",
+    "PP-LCNet_x1_0",
+    "PP-LCNet_x1_5",
+    "PP-LCNet_x2_0",
+    "PP-LCNet_x2_5",
+    "PP-LCNetV2_base",
+    "PP-LCNetV2_large",
+    "PP-LCNetV2_small",
+    "ResNet18_vd",
+    "ResNet18",
+    "ResNet34_vd",
+    "ResNet34",
+    "ResNet50_vd",
+    "ResNet50",
+    "ResNet101_vd",
+    "ResNet101",
+    "ResNet152_vd",
+    "ResNet152",
+    "ResNet200_vd",
+    "StarNet-S1",
+    "StarNet-S2",
+    "StarNet-S3",
+    "StarNet-S4",
+    "FCOS-ResNet50",
+    "PicoDet-L",
+    "PicoDet-M",
+    "PicoDet-S",
+    "PicoDet-XS",
+    "PP-YOLOE_plus-L",
+    "PP-YOLOE_plus-M",
+    "PP-YOLOE_plus-S",
+    "PP-YOLOE_plus-X",
+    "RT-DETR-H",
+    "RT-DETR-L",
+    "RT-DETR-R18",
+    "RT-DETR-R50",
+    "RT-DETR-X",
+    "PP-YOLOE-L_human",
+    "PP-YOLOE-S_human",
+    "PP-OCRv4_mobile_det",
+    "PP-OCRv4_server_det",
+    "PP-OCRv4_mobile_rec",
+    "PP-OCRv4_server_rec",
+]