paddlex 2.0.0rc4__py3-none-any.whl → 3.0.0b2__py3-none-any.whl

paddlex/utils/errors/others.py

@@ -0,0 +1,152 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import signal
+from typing import Union
+from pathlib import Path
+
+__all__ = [
+    "UnsupportedAPIError",
+    "UnsupportedParamError",
+    "CalledProcessError",
+    "raise_unsupported_api_error",
+    "raise_key_not_found_error",
+    "raise_class_not_found_error",
+    "raise_no_entity_registered_error",
+    "raise_unsupported_device_error",
+    "raise_model_not_found_error",
+    "DuplicateRegistrationError",
+]
+
+
+class UnsupportedAPIError(Exception):
+    """UnsupportedAPIError"""
+
+    pass
+
+
+class UnsupportedParamError(Exception):
+    """UnsupportedParamError"""
+
+    pass
+
+
+class KeyNotFoundError(Exception):
+    """KeyNotFoundError"""
+
+    pass
+
+
+class ClassNotFoundException(Exception):
+    """ClassNotFoundException"""
+
+    pass
+
+
+class NoEntityRegisteredException(Exception):
+    """NoEntityRegisteredException"""
+
+    pass
+
+
+class UnsupportedDeviceError(Exception):
+    """UnsupportedDeviceError"""
+
+    pass
+
+
+class CalledProcessError(Exception):
+    """CalledProcessError"""
+
+    def __init__(self, returncode, cmd, output=None, stderr=None):
+        super().__init__()
+        self.returncode = returncode
+        self.cmd = cmd
+        self.output = output
+        self.stderr = stderr
+
+    def __str__(self):
+        if self.returncode and self.returncode < 0:
+            try:
+                return f"Command {repr(self.cmd)} died with {repr(signal.Signals(-self.returncode))}."
+            except ValueError:
+                return f"Command {repr(self.cmd)} died with unknown signal {-self.returncode}."
+        else:
+            return f"Command {repr(self.cmd)} returned non-zero exit status {self.returncode}."
+
+
+class DuplicateRegistrationError(Exception):
+    """DuplicateRegistrationError"""
+
+    pass
+
+
+class ModelNotFoundError(Exception):
+    """Model Not Found Error"""
+
+
+def raise_unsupported_api_error(api_name, cls=None):
+    """raise unsupported api error"""
+    # TODO: Automatically extract `api_name` and `cls` from stack frame
+    if cls is not None:
+        name = f"{cls.__name__}.{api_name}"
+    else:
+        name = api_name
+    raise UnsupportedAPIError(f"The API `{name}` is not supported.")
+
+
+def raise_key_not_found_error(key, config=None):
+    """raise key not found error"""
+    msg = f"`{key}` not found in config."
+    if config:
+        config_str = json.dumps(config, indent=4, ensure_ascii=False)
+        msg += f"\nThe content of config:\n{config_str}"
+    raise KeyNotFoundError(msg)
+
+
+def raise_class_not_found_error(cls_name, base_cls, all_entities=None):
+    """raise class not found error"""
+    base_cls_name = base_cls.__name__
+    msg = f"`{cls_name}` is not registered on {base_cls_name}."
+    if all_entities is not None:
+        all_entities_str = ",  ".join(all_entities)
+        msg += f"\nThe registied entities: [{all_entities_str}]"
+    raise ClassNotFoundException(msg)
+
+
+def raise_no_entity_registered_error(base_cls):
+    """raise no entity registered error"""
+    base_cls_name = base_cls.__name__
+    msg = f"There no entity register on {base_cls_name}. Hint: Maybe the subclass is not imported."
+    raise NoEntityRegisteredException(msg)
+
+
+def raise_unsupported_device_error(device, supported_device=None):
+    """raise_unsupported_device_error"""
+    msg = f"The device `{device}` is not supported! "
+    if supported_device is not None:
+        supported_device_str = ", ".join(supported_device)
+        msg += f"The supported device types are: [{supported_device_str}]."
+    raise UnsupportedDeviceError(msg)
+
+
+def raise_model_not_found_error(model_path: Union[str, Path]):
+    """raise ModelNotFoundError
+
+    Args:
+        model_path (str|Path): the path to model file.
+    """
+    msg = f"The model file(s)(`{model_path}`) is not found."
+    raise ModelNotFoundError(msg)

paddlex/utils/file_interface.py

@@ -0,0 +1,212 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import logging
+import os
+from threading import ThreadError
+
+from filelock import FileLock
+import yaml
+import ruamel.yaml
+import chardet
+
+try:
+    import ujson as json
+except:
+    logging.warning("failed to import ujson, using json instead")
+    import json
+
+from contextlib import contextmanager
+
+
+@contextmanager
+def custom_open(file_path, mode):
+    """
+    自定义打开文件函数
+
+    Args:
+        file_path (str): 文件路径
+        mode (str): 文件打开模式，'r'，'w' 或 'a'
+
+    Returns:
+        Any: 返回文件对象
+
+    Raises:
+        FileNotFoundError: 当文件不存在时，raise FileNotFoundError
+        ValueError: 当 mode 参数不是 'r'， 'w' 和 'a' 时，raise ValueError
+    """
+    if mode == "r":
+        if not os.path.exists(file_path):
+            raise FileNotFoundError("file {} not found".format(file_path))
+        file = open(file_path, "r", encoding="utf-8")
+        try:
+            file.read()
+            file.seek(0)
+            yield file
+        except UnicodeDecodeError:
+            file = open(file_path, "r", encoding="gbk")
+            try:
+                file.read()
+                file.seek(0)
+                yield file
+            except UnicodeDecodeError:
+                with open(file_path, "rb") as f:
+                    encoding = chardet.detect(f.read())["encoding"]
+                file = open(file_path, "r", encoding=encoding)
+                yield file
+        finally:
+            file.close()
+    elif mode == "w":
+        file = open(file_path, "w", encoding="utf-8")
+        yield file
+        file.close()
+    elif mode == "a":
+        encoding = "utf-8"
+        if os.path.exists(file_path):
+            file = open(file_path, "r", encoding=encoding)
+            try:
+                file.read()
+                file.seek(0)
+            except UnicodeDecodeError:
+                encoding = "gbk"
+                file = open(file_path, "r", encoding=encoding)
+                try:
+                    file.read()
+                    file.seek(0)
+                except UnicodeDecodeError:
+                    with open(file_path, "rb") as f:
+                        encoding = chardet.detect(f.read())["encoding"]
+            finally:
+                file.close()
+
+        file = open(file_path, "a", encoding=encoding)
+        yield file
+        file.close()
+    else:
+        raise ValueError("mode must be 'r', 'w' or 'a', but got {}".format(mode))
+
+
+# --------------- yaml ---------------
+
+
+def read_yaml_file(yaml_path: str, to_dict=True):
+    """read from yaml file"""
+    try:
+        with open(yaml_path, "r", encoding="utf-8") as file:
+            yaml_content = yaml.full_load(file)
+    except UnicodeDecodeError:
+        with open(yaml_path, "r", encoding="gbk") as file:
+            yaml_content = yaml.full_load(file)
+    yaml_content = dict(yaml_content) if to_dict else yaml_content
+    return yaml_content
+
+
+def write_config_file(yaml_dict: dict, yaml_path: str):
+    """write to config yaml file"""
+    yaml = ruamel.yaml.YAML()
+    lock = FileLock(yaml_path + ".lock")
+    with lock:
+        with open(yaml_path, "w", encoding="utf-8") as file:
+            # yaml.safe_dump(yaml_dict, file, sort_keys=False)
+            yaml.dump(yaml_dict, file)
+
+
+def update_yaml_file_with_dict(yaml_path, key_values: dict):
+    """update yaml file with key_values
+    key_values is a dict
+    """
+    yaml_dict = read_yaml_file(yaml_path)
+    yaml_dict.update(key_values)
+    write_config_file(yaml_dict, yaml_path)
+
+
+def get_yaml_keys(yaml_path):
+    """get all keys of yaml file"""
+    yaml_dict = read_yaml_file(yaml_path)
+    return yaml_dict.keys()
+
+
+# --------------- markdown ---------------
+
+
+def generate_markdown_from_dict(metrics):
+    """generate_markdown_from_dict"""
+    mk = ""
+    keys = metrics.keys()
+    mk += "| ".join(keys())
+    mk += os.linesep
+    mk += "|".join([" :----: "])
+
+
+# ------------------- jsonl ---------------------
+
+
+def read_jsonl_file(jsonl_path: str):
+    """read from jsonl file"""
+    with custom_open(jsonl_path, "r") as file:
+        jsonl_content = [json.loads(line) for line in file]
+    return jsonl_content
+
+
+def write_json_file(content, jsonl_path: str, ensure_ascii=False, **kwargs):
+    """write to json file"""
+    with custom_open(jsonl_path, "w") as file:
+        json.dump(content, file, ensure_ascii=ensure_ascii, **kwargs)
+
+
+# --------------- check webui yaml -----------------
+
+
+def check_dict_keys(to_checked_dict, standard_dict, escape_list=None):
+    """check if all keys of to_checked_dict is the same as standard_dict, and the value is the same type
+    Args:
+        escape_list: if set, will not check the keys in white_list
+    """
+    escape_list = [] if escape_list is None else escape_list
+    for key in standard_dict.keys():
+        if key not in to_checked_dict:
+            logging.error(f"key {key} not in yaml file")
+            return False
+        if not isinstance(standard_dict[key], type(to_checked_dict[key])):
+            logging.error(
+                f"value type of key {key} is not the same as standard: "
+                f"{type(standard_dict[key])}, {type(to_checked_dict[key])}"
+            )
+            return False
+
+        if (
+            isinstance(standard_dict[key], dict)
+            and isinstance(to_checked_dict[key], dict)
+            and key not in escape_list
+        ):
+            return check_dict_keys(
+                to_checked_dict[key], standard_dict[key], escape_list
+            )
+    if len(to_checked_dict.keys()) != len(standard_dict.keys()):
+        logging.error(f"yaml file has extra keys")
+        return False
+
+    return True
+
+
+def check_dataset_valid(path_list):
+    """check if dataset valid in path_list for datset_ui"""
+    if path_list is not None and len(path_list) > 0:
+        for path in path_list:
+            if not os.path.exists(path):
+                return False
+        return True
+    else:
+        return False

paddlex/utils/flags.py

@@ -0,0 +1,61 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+
+__all__ = [
+    "DEBUG",
+    "DRY_RUN",
+    "CHECK_OPTS",
+    "EAGER_INITIALIZATION",
+    "INFER_BENCHMARK",
+    "INFER_BENCHMARK_ITER",
+    "INFER_BENCHMARK_WARMUP",
+    "INFER_BENCHMARK_OUTPUT",
+    "INFER_BENCHMARK_DATA_SIZE",
+    "FLAGS_json_format_model",
+]
+
+
+def get_flag_from_env_var(name, default, format_func=str):
+    """get_flag_from_env_var"""
+    env_var = os.environ.get(name, default)
+    if env_var in (True, "True", "true", "TRUE", "1"):
+        return True
+    elif env_var in (False, "False", "false", "FALSE", "0"):
+        return False
+    elif env_var in (None, "None", "none", "Null", "null"):
+        return None
+    return format_func(env_var)
+
+
+DEBUG = get_flag_from_env_var("PADDLE_PDX_DEBUG", False)
+DRY_RUN = get_flag_from_env_var("PADDLE_PDX_DRY_RUN", False)
+CHECK_OPTS = get_flag_from_env_var("PADDLE_PDX_CHECK_OPTS", False)
+EAGER_INITIALIZATION = get_flag_from_env_var("PADDLE_PDX_EAGER_INIT", True)
+FLAGS_json_format_model = get_flag_from_env_var("FLAGS_json_format_model", None)
+
+# Inference Benchmark
+INFER_BENCHMARK = get_flag_from_env_var("PADDLE_PDX_INFER_BENCHMARK", None)
+INFER_BENCHMARK_WARMUP = get_flag_from_env_var(
+    "PADDLE_PDX_INFER_BENCHMARK_WARMUP", 0, int
+)
+INFER_BENCHMARK_OUTPUT = get_flag_from_env_var(
+    "PADDLE_PDX_INFER_BENCHMARK_OUTPUT", None
+)
+INFER_BENCHMARK_ITER = get_flag_from_env_var("PADDLE_PDX_INFER_BENCHMARK_ITER", 10, int)
+INFER_BENCHMARK_DATA_SIZE = get_flag_from_env_var(
+    "PADDLE_PDX_INFER_BENCHMARK_DATA_SIZE", 1024
+)

paddlex/utils/fonts/__init__.py

@@ -0,0 +1,24 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from pathlib import Path
+
+
+def get_pingfang_file_path():
+    """get pingfang font file path"""
+    return (Path(__file__).parent / "PingFang-SC-Regular.ttf").resolve().as_posix()
+
+
+PINGFANG_FONT_FILE_PATH = get_pingfang_file_path()

paddlex/utils/func_register.py

@@ -0,0 +1,41 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from functools import wraps
+
+from . import logging
+
+
+class FuncRegister(object):
+    def __init__(self, register_map):
+        assert isinstance(register_map, dict)
+        self._register_map = register_map
+
+    def __call__(self, key=None):
+        """register the decoratored func as key in dict"""
+
+        def decorator(func):
+            actual_key = key if key is not None else func.__name__
+            self._register_map[actual_key] = func
+            logging.debug(
+                f"The func ({func.__name__}) has been registered as key ({actual_key})."
+            )
+
+            @wraps(func)
+            def wrapper(*args, **kwargs):
+                return func(*args, **kwargs)
+
+            return wrapper
+
+        return decorator

paddlex/utils/interactive_get_pipeline.py

@@ -0,0 +1,55 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pathlib import Path
+import shutil
+
+from ..utils import logging
+from ..inference.utils.get_pipeline_path import get_pipeline_path
+
+
+def interactive_get_pipeline(pipeline, save_path):
+    file_path = get_pipeline_path(pipeline)
+    file_name = Path(file_path).name
+
+    if save_path is None:
+        logging.info(
+            "Please enter the path that you want to save the pipeline config file: (default `./`)"
+        )
+        target_path = input() or "."
+    else:
+        target_path = save_path
+    target_path = Path(target_path)
+
+    if target_path.suffix not in (".yaml", ".yml"):
+        target_path /= file_name
+
+    if not target_path.parent.exists():
+        try:
+            target_path.parent.mkdir(parents=True, exist_ok=True)
+        except Exception as e:
+            logging.error(f"Failed to create directory: {e}")
+            return
+
+    if target_path.exists():
+        logging.info(f"The file({target_path}) already exists. Is it covered? (y/N):")
+        overwrite = input().lower()
+        if overwrite != "y":
+            logging.warning("Exit!")
+            return
+    try:
+        shutil.copy2(file_path, target_path)
+        logging.info(f"The pipeline config has been saved to: {target_path}")
+    except Exception as e:
+        logging.error(f"File saving failed: {e}")

paddlex/utils/lazy_loader.py

@@ -0,0 +1,66 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Code copied from https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/util/lazy_loader.py
+import importlib
+import types
+import os
+
+from . import logging
+from .flags import FLAGS_json_format_model
+
+
+def disable_pir_bydefault():
+    # FLAGS_json_format_model not set
+    if not FLAGS_json_format_model:
+        os.environ["FLAGS_json_format_model"] = "0"
+        os.environ["FLAGS_enable_pir_api"] = "0"
+        logging.debug("FLAGS_enable_pir_api has been set 0")
+
+
+class LazyLoader(types.ModuleType):
+    """Lazily import a module, mainly to avoid pulling in large dependencies."""
+
+    def __init__(self, local_name, parent_module_globals, name):
+        self._local_name = local_name
+        self._parent_module_globals = parent_module_globals
+        self._module = None
+
+        super(LazyLoader, self).__init__(name)
+
+    @property
+    def loaded(self):
+        return self._module is not None
+
+    def _load(self):
+        # TODO(gaotingquan): disable PIR using Flag
+        if self.__name__ == "paddle":
+            disable_pir_bydefault()
+        module = importlib.import_module(self.__name__)
+        self._parent_module_globals[self._local_name] = module
+        self._module = module
+
+    def __getattr__(self, item):
+        if not self.loaded:
+            # HACK: For circumventing shared library symbol conflicts when
+            # importing paddlex_hpi
+            if item in ("__file__",):
+                raise AttributeError
+            self._load()
+        return getattr(self._module, item)
+
+    def __dir__(self):
+        if not self.loaded:
+            self._load()
+        return dir(self._module)