paddlex 2.0.0rc4__py3-none-any.whl → 3.0.0b2__py3-none-any.whl

paddlex/modules/base/dataset_checker/dataset_checker.py

@@ -0,0 +1,164 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+from abc import ABC, abstractmethod
+
+from .utils import build_res_dict
+from ....utils.misc import AutoRegisterABCMetaClass
+from ....utils.config import AttrDict
+from ....utils.logging import info
+
+
+def build_dataset_checker(config: AttrDict) -> "BaseDatasetChecker":
+    """build dataset checker
+
+    Args:
+        config (AttrDict): PaddleX pipeline config, which is loaded from pipeline yaml file.
+
+    Returns:
+        BaseDatasetChecker: the dataset checker, which is subclass of BaseDatasetChecker.
+    """
+    model_name = config.Global.model
+    return BaseDatasetChecker.get(model_name)(config)
+
+
+class BaseDatasetChecker(ABC, metaclass=AutoRegisterABCMetaClass):
+    """Base Dataset Checker"""
+
+    __is_base = True
+
+    def __init__(self, config):
+        """Initialize the instance.
+
+        Args:
+            config (AttrDict): PaddleX pipeline config, which is loaded from pipeline yaml file.
+        """
+        super().__init__()
+        self.global_config = config.Global
+        self.check_dataset_config = config.CheckDataset
+        self.output = os.path.join(self.global_config.output, "check_dataset")
+
+    def check(self) -> dict:
+        """execute dataset checking
+
+        Returns:
+            dict: the dataset checking result.
+        """
+        dataset_dir = self.get_dataset_root(self.global_config.dataset_dir)
+
+        if not os.path.exists(self.output):
+            os.makedirs(self.output)
+
+        if self.check_dataset_config.get("convert", None):
+            if self.check_dataset_config.convert.get("enable", False):
+                self.convert_dataset(dataset_dir)
+                info("Convert dataset successfully !")
+
+        if self.check_dataset_config.get("split", None):
+            if self.check_dataset_config.split.get("enable", False):
+                self.split_dataset(dataset_dir)
+                info("Split dataset successfully !")
+
+        attrs = self.check_dataset(dataset_dir)
+        analysis = self.analyse(dataset_dir)
+        check_result = build_res_dict(True)
+        check_result["attributes"] = attrs
+        check_result["analysis"] = analysis
+        check_result["dataset_path"] = self.global_config.dataset_dir
+        check_result["show_type"] = self.get_show_type()
+        check_result["dataset_type"] = self.get_dataset_type()
+        info("Check dataset passed !")
+        return check_result
+
+    def get_dataset_root(self, dataset_dir: str) -> str:
+        """find the dataset root dir
+
+        Args:
+            dataset_dir (str): the directory that contain dataset.
+
+        Returns:
+            str: the root directory of dataset.
+        """
+        # XXX: forward compatible
+        # dataset_dir = [d for d in Path(dataset_dir).iterdir() if d.is_dir()]
+        # assert len(dataset_dir) == 1
+        # return dataset_dir[0].as_posix()
+        return dataset_dir
+
+    @abstractmethod
+    def check_dataset(self, dataset_dir: str):
+        """check if the dataset meets the specifications and get dataset summary
+
+        Args:
+            dataset_dir (str): the root directory of dataset.
+
+        Raises:
+            NotImplementedError
+        """
+        raise NotImplementedError
+
+    def convert_dataset(self, src_dataset_dir: str) -> str:
+        """convert the dataset from other type to specified type
+
+        Args:
+            src_dataset_dir (str): the root directory of dataset.
+
+        Returns:
+            str: the root directory of converted dataset.
+        """
+        dst_dataset_dir = src_dataset_dir
+        return dst_dataset_dir
+
+    def split_dataset(self, src_dataset_dir: str) -> str:
+        """repartition the train and validation dataset
+
+        Args:
+            src_dataset_dir (str): the root directory of dataset.
+
+        Returns:
+            str: the root directory of splited dataset.
+        """
+        dst_dataset_dir = src_dataset_dir
+        return dst_dataset_dir
+
+    def analyse(self, dataset_dir: str) -> dict:
+        """deep analyse dataset
+
+        Args:
+            dataset_dir (str): the root directory of dataset.
+
+        Returns:
+            dict: the deep analysis results.
+        """
+        return {}
+
+    @abstractmethod
+    def get_show_type(self):
+        """return the dataset show type
+
+        Raises:
+            NotImplementedError
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_dataset_type(self):
+        """return the dataset type
+
+        Raises:
+            NotImplementedError
+        """
+        raise NotImplementedError

paddlex/modules/base/dataset_checker/utils.py

@@ -0,0 +1,110 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+
+__all__ = [
+    "build_res_dict",
+    "CheckFailedError",
+    "ConvertFailedError",
+    "SplitFailedError",
+    "AnalyseFailedError",
+    "DatasetFileNotFoundError",
+]
+
+
+def build_res_dict(check_pass, err_type=None, err_msg=None, **kwargs):
+    """build the dataset checking result to return"""
+    if check_pass:
+        if err_type is not None:
+            raise ValueError(
+                f"`check_pass` is {check_pass}, but `err_type` is not None."
+            )
+        if err_msg is not None:
+            raise ValueError(
+                f"`check_pass` is {check_pass}, but `err_msg` is not None."
+            )
+        return dict(check_pass=check_pass, **kwargs)
+    else:
+        if err_type is None:
+            raise ValueError(f"`check_pass` is {check_pass}, but `err_type` is None.")
+        if err_msg is None:
+            if _is_known_error_type(err_type):
+                err_msg = ""
+            else:
+                raise ValueError(
+                    f"{err_type} is not a known error type, in which case `err_msg` must be specified to a value other \
+than None."
+                )
+        return dict(check_pass=check_pass, err_type=err_type, err_msg=err_msg, **kwargs)
+
+
+class FailedError(Exception):
+    """base error class"""
+
+    def __init__(self, err_info=None, solution=None, message=None):
+        if message is None:
+            message = self._construct_message(err_info, solution)
+        super().__init__(message)
+
+    def _construct_message(self, err_info, solution):
+        if err_info is None:
+            return ""
+        else:
+            msg = (
+                f"{self.mode} failed. We encountered the following error:\n  {err_info}"
+            )
+            if solution is not None:
+                msg += f"\nPlease try to resolve the issue as follows:\n  {solution}"
+            return msg
+
+
+class CheckFailedError(FailedError):
+    """check dataset error"""
+
+    mode = "Check dataset"
+
+
+class ConvertFailedError(FailedError):
+    """convert dataset error"""
+
+    mode = "Convert dataset"
+
+
+class SplitFailedError(FailedError):
+    """split dataset error"""
+
+    mode = "Split dataset"
+
+
+class AnalyseFailedError(FailedError):
+    """analyse dataset error"""
+
+    mode = "Analyse dataset"
+
+
+class DatasetFileNotFoundError(CheckFailedError):
+    """dataset file not found error"""
+
+    def __init__(self, file_path=None, err_info=None, solution=None, message=None):
+        if err_info is None:
+            if file_path is not None:
+                err_info = f"{file_path} does not exist."
+        super().__init__(err_info, solution, message)
+
+
+def _is_known_error_type(err_type):
+    """is known error type"""
+    return isinstance(err_type, CheckFailedError)

paddlex/modules/base/evaluator.py

@@ -0,0 +1,154 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from pathlib import Path
+from abc import ABC, abstractmethod
+
+from .build_model import build_model
+from ...utils.device import update_device_num, set_env_for_device
+from ...utils.misc import AutoRegisterABCMetaClass
+from ...utils.config import AttrDict
+from ...utils.logging import *
+
+
+def build_evaluater(config: AttrDict) -> "BaseEvaluator":
+    """build model evaluater
+
+    Args:
+        config (AttrDict): PaddleX pipeline config, which is loaded from pipeline yaml file.
+
+    Returns:
+        BaseEvaluator: the evaluater, which is subclass of BaseEvaluator.
+    """
+    model_name = config.Global.model
+    return BaseEvaluator.get(model_name)(config)
+
+
+class BaseEvaluator(ABC, metaclass=AutoRegisterABCMetaClass):
+    """Base Model Evaluator"""
+
+    __is_base = True
+
+    def __init__(self, config):
+        """Initialize the instance.
+
+        Args:
+            config (AttrDict):  PaddleX pipeline config, which is loaded from pipeline yaml file.
+        """
+        super().__init__()
+        self.global_config = config.Global
+        self.eval_config = config.Evaluate
+
+        config_path = self.get_config_path(self.eval_config.weight_path)
+
+        self.pdx_config, self.pdx_model = build_model(
+            self.global_config.model, config_path=config_path
+        )
+
+    def get_config_path(self, weight_path):
+        """
+        get config path
+
+        Args:
+            weight_path (str): The path to the weight
+
+        Returns:
+            config_path (str): The path to the config
+
+        """
+
+        config_path = Path(weight_path).parent / "config.yaml"
+        if not config_path.exists():
+            config_path = config_path.parent.parent / "config.yaml"
+
+        if not config_path.exists():
+            warning(
+                f"The config file (`{config_path}`) related to weight file (`{weight_path}`) does not exist. Using default instead."
+            )
+            config_path = None
+        return config_path
+
+    def check_return(self, metrics: dict) -> bool:
+        """check evaluation metrics
+
+        Args:
+            metrics (dict): evaluation output metrics
+
+        Returns:
+            bool: whether the format of evaluation metrics is legal
+        """
+        if not isinstance(metrics, dict):
+            return False
+        for metric in metrics:
+            val = metrics[metric]
+            if not isinstance(val, (float, int)):
+                return False
+        return True
+
+    def evaluate(self) -> dict:
+        """execute model evaluating
+
+        Returns:
+            dict: the evaluation metrics
+        """
+        self.update_config()
+        # self.dump_config()
+        evaluate_result = self.pdx_model.evaluate(**self.get_eval_kwargs())
+        assert (
+            evaluate_result.returncode == 0
+        ), f"Encountered an unexpected error({evaluate_result.returncode}) in \
+evaling!"
+
+        metrics = evaluate_result.metrics
+        assert self.check_return(
+            metrics
+        ), f"The return value({metrics}) of Evaluator.eval() is illegal!"
+        return {"metrics": metrics}
+
+    def dump_config(self, config_file_path=None):
+        """dump the config
+
+        Args:
+            config_file_path (str, optional): the path to save dumped config.
+                Defaults to None, means that save in `Global.output` as `config.yaml`.
+        """
+        if config_file_path is None:
+            config_file_path = os.path.join(self.global_config.output, "config.yaml")
+        self.pdx_config.dump(config_file_path)
+
+    def get_device(self, using_device_number: int = None) -> str:
+        """get device setting from config
+
+        Args:
+            using_device_number (int, optional): specify device number to use.
+                Defaults to None, means that base on config setting.
+
+        Returns:
+            str: device setting, such as: `gpu:0,1`, `npu:0,1`, `cpu`.
+        """
+        set_env_for_device(self.global_config.device)
+        if using_device_number:
+            return update_device_num(self.global_config.device, using_device_number)
+        return self.global_config.device
+
+    @abstractmethod
+    def update_config(self):
+        """update evalution config"""
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_eval_kwargs(self):
+        """get key-value arguments of model evalution function"""
+        raise NotImplementedError

paddlex/modules/base/exportor.py

@@ -0,0 +1,121 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from pathlib import Path
+from abc import ABC, abstractmethod
+
+from .build_model import build_model
+from ...utils.device import update_device_num, set_env_for_device
+from ...utils.misc import AutoRegisterABCMetaClass
+from ...utils.config import AttrDict
+from ...utils import logging
+
+
+def build_exportor(config: AttrDict) -> "BaseExportor":
+    """build model exportor
+
+    Args:
+        config (AttrDict): PaddleX pipeline config, which is loaded from pipeline yaml file.
+
+    Returns:
+        BaseExportor: the exportor, which is subclass of BaseExportor.
+    """
+    model_name = config.Global.model
+    return BaseExportor.get(model_name)(config)
+
+
+class BaseExportor(ABC, metaclass=AutoRegisterABCMetaClass):
+    """Base Model Exportor"""
+
+    __is_base = True
+
+    def __init__(self, config):
+        """Initialize the instance.
+
+        Args:
+            config (AttrDict):  PaddleX pipeline config, which is loaded from pipeline yaml file.
+        """
+        super().__init__()
+        self.global_config = config.Global
+        self.export_config = config.Export
+
+        config_path = self.get_config_path(self.export_config.weight_path)
+
+        self.pdx_config, self.pdx_model = build_model(
+            self.global_config.model, config_path=config_path
+        )
+
+    def get_config_path(self, weight_path):
+        """
+        get config path
+
+        Args:
+            weight_path (str): The path to the weight
+
+        Returns:
+            config_path (str): The path to the config
+
+        """
+
+        config_path = Path(weight_path).parent / "config.yaml"
+        if not config_path.exists():
+            logging.warning(
+                f"The config file(`{config_path}`) related to weight file(`{weight_path}`) is not exist, use default instead."
+            )
+            config_path = None
+
+        return config_path
+
+    def export(self) -> dict:
+        """execute model exporting
+
+        Returns:
+            dict: the export metrics
+        """
+        self.update_config()
+        export_result = self.pdx_model.export(**self.get_export_kwargs())
+        assert (
+            export_result.returncode == 0
+        ), f"Encountered an unexpected error({export_result.returncode}) in \
+exporting!"
+
+        return None
+
+    def get_device(self, using_device_number: int = None) -> str:
+        """get device setting from config
+
+        Args:
+            using_device_number (int, optional): specify device number to use.
+                Defaults to None, means that base on config setting.
+
+        Returns:
+            str: device setting, such as: `gpu:0,1`, `npu:0,1`, `cpu`.
+        """
+        set_env_for_device(self.global_config.device)
+        if using_device_number:
+            return update_device_num(self.global_config.device, using_device_number)
+        return self.global_config.device
+
+    def update_config(self):
+        """update export config"""
+        pass
+
+    def get_export_kwargs(self):
+        """get key-value arguments of model export function"""
+        return {
+            "weight_path": self.export_config.weight_path,
+            "save_dir": self.global_config.output,
+            "device": self.get_device(1),
+        }

paddlex/modules/base/trainer.py

@@ -0,0 +1,111 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from abc import ABC, abstractmethod
+from pathlib import Path
+from .build_model import build_model
+from ...utils.device import update_device_num, set_env_for_device
+from ...utils.misc import AutoRegisterABCMetaClass
+from ...utils.config import AttrDict
+
+
+def build_trainer(config: AttrDict) -> "BaseTrainer":
+    """build model trainer
+
+    Args:
+        config (AttrDict): PaddleX pipeline config, which is loaded from pipeline yaml file.
+
+    Returns:
+        BaseTrainer: the trainer, which is subclass of BaseTrainer.
+    """
+    model_name = config.Global.model
+    return BaseTrainer.get(model_name)(config)
+
+
+class BaseTrainer(ABC, metaclass=AutoRegisterABCMetaClass):
+    """Base Model Trainer"""
+
+    __is_base = True
+
+    def __init__(self, config: AttrDict):
+        """Initialize the instance.
+
+        Args:
+            config (AttrDict):  PaddleX pipeline config, which is loaded from pipeline yaml file.
+        """
+        super().__init__()
+        self.config = config
+        self.global_config = config.Global
+        self.train_config = config.Train
+        self.benchmark_config = config.get("Benchmark", None)
+
+        self.pdx_config, self.pdx_model = build_model(self.global_config.model)
+
+    def train(self, *args, **kwargs):
+        """execute model training"""
+        os.makedirs(self.global_config.output, exist_ok=True)
+        self.update_config()
+        self.dump_config()
+        train_args = self.get_train_kwargs()
+        if self.benchmark_config is not None:
+            train_args.update({"benchmark": self.benchmark_config})
+        train_args.update(
+            {
+                "uniform_output_enabled": self.train_config.get(
+                    "uniform_output_enabled", True
+                )
+            }
+        )
+        train_result = self.pdx_model.train(**train_args)
+        assert (
+            train_result.returncode == 0
+        ), f"Encountered an unexpected error({train_result.returncode}) in \
+training!"
+
+    def dump_config(self, config_file_path: str = None):
+        """dump the config
+
+        Args:
+            config_file_path (str, optional): the path to save dumped config. Defaults to None,
+                means that save in `Global.output` as `config.yaml`.
+        """
+        if config_file_path is None:
+            config_file_path = os.path.join(self.global_config.output, "config.yaml")
+        self.pdx_config.dump(config_file_path)
+
+    def get_device(self, using_device_number: int = None) -> str:
+        """get device setting from config
+
+        Args:
+            using_device_number (int, optional): specify device number to use. Defaults to None,
+                means that base on config setting.
+
+        Returns:
+            str: device setting, such as: `gpu:0,1`, `npu:0,1` `cpu`.
+        """
+        set_env_for_device(self.global_config.device)
+        if using_device_number:
+            return update_device_num(self.global_config.device, using_device_number)
+        return self.global_config.device
+
+    @abstractmethod
+    def update_config(self):
+        """update training config"""
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_train_kwargs(self):
+        """get key-value arguments of model training function"""
+        raise NotImplementedError

paddlex/modules/face_recognition/__init__.py

@@ -0,0 +1,18 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .trainer import FaceRecTrainer
+from .dataset_checker import FaceRecDatasetChecker
+from .evaluator import FaceRecEvaluator
+from .exportor import FaceRecExportor