paddlex 2.0.0rc4__py3-none-any.whl → 3.0.0__py3-none-any.whl

paddlex/modules/anomaly_detection/dataset_checker/dataset_src/utils/visualizer.py

@@ -0,0 +1,76 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import numpy as np
+
+from ......utils.deps import function_requires_deps, is_dep_available
+
+if is_dep_available("opencv-contrib-python"):
+    import cv2
+
+
+def get_color_map_list(length):
+    """Returns the color map for visualizing the segmentation mask"""
+    length += 1
+    color_map = length * [0, 0, 0]
+    for i in range(0, length):
+        j = 0
+        lab = i
+        while lab:
+            color_map[i * 3] |= ((lab >> 0) & 1) << (7 - j)
+            color_map[i * 3 + 1] |= ((lab >> 1) & 1) << (7 - j)
+            color_map[i * 3 + 2] |= ((lab >> 2) & 1) << (7 - j)
+            j += 1
+            lab >>= 3
+    color_map = color_map[3:]
+    return color_map
+
+
+@function_requires_deps("opencv-contrib-python")
+def visualize(image, result, weight=0.6, use_multilabel=False):
+    """Convert predict result to color image, and save added image."""
+    color_map = get_color_map_list(256)
+    color_map = [color_map[i : i + 3] for i in range(0, len(color_map), 3)]
+    color_map = np.array(color_map).astype("uint8")
+
+    if not use_multilabel:
+        # Use OpenCV LUT for color mapping
+        c1 = cv2.LUT(result, color_map[:, 0])
+        c2 = cv2.LUT(result, color_map[:, 1])
+        c3 = cv2.LUT(result, color_map[:, 2])
+        pseudo_img = np.dstack((c3, c2, c1))
+
+        vis_result = cv2.addWeighted(image, weight, pseudo_img, 1 - weight, 0)
+    else:
+        vis_result = image.copy()
+        for i in range(result.shape[0]):
+            mask = result[i]
+            c1 = np.where(mask, color_map[i, 0], vis_result[..., 0])
+            c2 = np.where(mask, color_map[i, 1], vis_result[..., 1])
+            c3 = np.where(mask, color_map[i, 2], vis_result[..., 2])
+            pseudo_img = np.dstack((c3, c2, c1)).astype("uint8")
+
+            contour, _ = cv2.findContours(
+                mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE
+            )
+            vis_result = cv2.addWeighted(vis_result, weight, pseudo_img, 1 - weight, 0)
+            contour_color = (
+                int(color_map[i, 0]),
+                int(color_map[i, 1]),
+                int(color_map[i, 2]),
+            )
+            vis_result = cv2.drawContours(vis_result, contour, -1, contour_color, 1)
+
+    return vis_result

paddlex/modules/anomaly_detection/evaluator.py

@@ -0,0 +1,58 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+from pathlib import Path
+
+from ..base import BaseEvaluator
+from .model_list import MODELS
+
+
+class UadEvaluator(BaseEvaluator):
+    """Semantic Segmentation Model Evaluator"""
+
+    entities = MODELS
+
+    def update_config(self):
+        """update evaluation config"""
+        self.pdx_config.update_dataset(self.global_config.dataset_dir, "SegDataset")
+        self.pdx_config.update_pretrained_weights(None, is_backbone=True)
+
+    def get_config_path(self, weight_path):
+        """
+        get config path
+
+        Args:
+            weight_path (str): The path to the weight
+
+        Returns:
+            config_path (str): The path to the config
+
+        """
+
+        config_path = Path(weight_path).parent.parent / "config.yaml"
+
+        return config_path
+
+    def get_eval_kwargs(self) -> dict:
+        """get key-value arguments of model evaluation function
+
+        Returns:
+            dict: the arguments of evaluation function.
+        """
+        device = self.get_device()
+        # XXX:
+        os.environ.pop("FLAGS_npu_jit_compile", None)
+        return {"weight_path": self.eval_config.weight_path, "device": device}

paddlex/modules/anomaly_detection/exportor.py

@@ -0,0 +1,22 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ..base import BaseExportor
+from .model_list import MODELS
+
+
+class UadExportor(BaseExportor):
+    """Semantic Segmentation Model Exportor"""
+
+    entities = MODELS

paddlex/modules/anomaly_detection/model_list.py

@@ -0,0 +1,16 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+MODELS = ["STFPM"]

paddlex/modules/anomaly_detection/trainer.py

@@ -0,0 +1,70 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+
+from ..base import BaseTrainer
+from .model_list import MODELS
+
+
+class UadTrainer(BaseTrainer):
+    """Uad Model Trainer"""
+
+    entities = MODELS
+
+    def update_config(self):
+        """update training config"""
+        self.pdx_config.update_dataset(self.global_config.dataset_dir, "SegDataset")
+        if self.train_config.num_classes is not None:
+            self.pdx_config.update_num_classes(self.train_config.num_classes)
+        if (
+            self.train_config.pretrain_weight_path
+            and self.train_config.pretrain_weight_path != ""
+        ):
+            self.pdx_config.update_pretrained_weights(
+                self.train_config.pretrain_weight_path, is_backbone=True
+            )
+
+    def get_train_kwargs(self) -> dict:
+        """get key-value arguments of model training function
+
+        Returns:
+            dict: the arguments of training function.
+        """
+        train_args = {"device": self.get_device()}
+        # XXX:
+        os.environ.pop("FLAGS_npu_jit_compile", None)
+        if self.train_config.batch_size is not None:
+            train_args["batch_size"] = self.train_config.batch_size
+        if self.train_config.learning_rate is not None:
+            train_args["learning_rate"] = self.train_config.learning_rate
+        if self.train_config.epochs_iters is not None:
+            train_args["epochs_iters"] = self.train_config.epochs_iters
+        if (
+            self.train_config.resume_path is not None
+            and self.train_config.resume_path != ""
+        ):
+            train_args["resume_path"] = self.train_config.resume_path
+        if self.global_config.output is not None:
+            train_args["save_dir"] = self.global_config.output
+        if self.train_config.log_interval:
+            train_args["log_iters"] = self.train_config.log_interval
+        if self.train_config.eval_interval:
+            train_args["do_eval"] = True
+            train_args["save_interval"] = self.train_config.eval_interval
+        train_args["dy2st"] = self.train_config.get("dy2st", False)
+        # amp support 'O1', 'O2', 'OFF'
+        train_args["amp"] = self.train_config.get("amp", "OFF")
+        return train_args

paddlex/modules/base/__init__.py

@@ -0,0 +1,18 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .dataset_checker import BaseDatasetChecker, build_dataset_checker
+from .evaluator import BaseEvaluator, build_evaluator
+from .exportor import BaseExportor, build_exportor
+from .trainer import BaseTrainer, build_trainer

paddlex/modules/base/build_model.py

@@ -0,0 +1,33 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from ...repo_apis.base import Config, PaddleModel
+
+
+def build_model(model_name: str, config_path: str = None) -> tuple:
+    """build Config and PaddleModel
+
+    Args:
+        model_name (str): model name
+        device (str): device, such as gpu, cpu, npu, xpu, mlu, gcu
+        config_path (str, optional): path to the PaddleX config yaml file.
+            Defaults to None, i.e. using the default config file.
+
+    Returns:
+        tuple(Config, PaddleModel): the Config and PaddleModel
+    """
+    config = Config(model_name, config_path)
+    model = PaddleModel(config=config)
+    return config, model

paddlex/modules/base/dataset_checker/__init__.py

@@ -0,0 +1,16 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from .dataset_checker import BaseDatasetChecker, build_dataset_checker

paddlex/modules/base/dataset_checker/dataset_checker.py

@@ -0,0 +1,169 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+from abc import ABC, abstractmethod
+
+from ....utils.config import AttrDict
+from ....utils.logging import info
+from ....utils.misc import AutoRegisterABCMetaClass
+from .utils import build_res_dict
+
+
+def build_dataset_checker(config: AttrDict) -> "BaseDatasetChecker":
+    """build dataset checker
+
+    Args:
+        config (AttrDict): PaddleX pipeline config, which is loaded from pipeline yaml file.
+
+    Returns:
+        BaseDatasetChecker: the dataset checker, which is subclass of BaseDatasetChecker.
+    """
+    model_name = config.Global.model
+    try:
+        pass
+    except ModuleNotFoundError:
+        pass
+
+    return BaseDatasetChecker.get(model_name)(config)
+
+
+class BaseDatasetChecker(ABC, metaclass=AutoRegisterABCMetaClass):
+    """Base Dataset Checker"""
+
+    __is_base = True
+
+    def __init__(self, config):
+        """Initialize the instance.
+
+        Args:
+            config (AttrDict): PaddleX pipeline config, which is loaded from pipeline yaml file.
+        """
+        super().__init__()
+        self.global_config = config.Global
+        self.check_dataset_config = config.CheckDataset
+        self.output = os.path.join(self.global_config.output, "check_dataset")
+
+    def check(self) -> dict:
+        """execute dataset checking
+
+        Returns:
+            dict: the dataset checking result.
+        """
+        dataset_dir = self.get_dataset_root(self.global_config.dataset_dir)
+
+        if not os.path.exists(self.output):
+            os.makedirs(self.output)
+
+        if self.check_dataset_config.get("convert", None):
+            if self.check_dataset_config.convert.get("enable", False):
+                self.convert_dataset(dataset_dir)
+                info("Convert dataset successfully !")
+
+        if self.check_dataset_config.get("split", None):
+            if self.check_dataset_config.split.get("enable", False):
+                self.split_dataset(dataset_dir)
+                info("Split dataset successfully !")
+
+        attrs = self.check_dataset(dataset_dir)
+        analysis = self.analyse(dataset_dir)
+        check_result = build_res_dict(True)
+        check_result["attributes"] = attrs
+        check_result["analysis"] = analysis
+        check_result["dataset_path"] = os.path.basename(dataset_dir)
+        check_result["show_type"] = self.get_show_type()
+        check_result["dataset_type"] = self.get_dataset_type()
+        info("Check dataset passed !")
+        return check_result
+
+    def get_dataset_root(self, dataset_dir: str) -> str:
+        """find the dataset root dir
+
+        Args:
+            dataset_dir (str): the directory that contain dataset.
+
+        Returns:
+            str: the root directory of dataset.
+        """
+        # XXX: forward compatible
+        # dataset_dir = [d for d in Path(dataset_dir).iterdir() if d.is_dir()]
+        # assert len(dataset_dir) == 1
+        # return dataset_dir[0].as_posix()
+        return dataset_dir
+
+    @abstractmethod
+    def check_dataset(self, dataset_dir: str):
+        """check if the dataset meets the specifications and get dataset summary
+
+        Args:
+            dataset_dir (str): the root directory of dataset.
+
+        Raises:
+            NotImplementedError
+        """
+        raise NotImplementedError
+
+    def convert_dataset(self, src_dataset_dir: str) -> str:
+        """convert the dataset from other type to specified type
+
+        Args:
+            src_dataset_dir (str): the root directory of dataset.
+
+        Returns:
+            str: the root directory of converted dataset.
+        """
+        dst_dataset_dir = src_dataset_dir
+        return dst_dataset_dir
+
+    def split_dataset(self, src_dataset_dir: str) -> str:
+        """repartition the train and validation dataset
+
+        Args:
+            src_dataset_dir (str): the root directory of dataset.
+
+        Returns:
+            str: the root directory of splited dataset.
+        """
+        dst_dataset_dir = src_dataset_dir
+        return dst_dataset_dir
+
+    def analyse(self, dataset_dir: str) -> dict:
+        """deep analyse dataset
+
+        Args:
+            dataset_dir (str): the root directory of dataset.
+
+        Returns:
+            dict: the deep analysis results.
+        """
+        return {}
+
+    @abstractmethod
+    def get_show_type(self):
+        """return the dataset show type
+
+        Raises:
+            NotImplementedError
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_dataset_type(self):
+        """return the dataset type
+
+        Raises:
+            NotImplementedError
+        """
+        raise NotImplementedError

paddlex/modules/base/dataset_checker/utils.py

@@ -0,0 +1,108 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+__all__ = [
+    "build_res_dict",
+    "CheckFailedError",
+    "ConvertFailedError",
+    "SplitFailedError",
+    "AnalyseFailedError",
+    "DatasetFileNotFoundError",
+]
+
+
+def build_res_dict(check_pass, err_type=None, err_msg=None, **kwargs):
+    """build the dataset checking result to return"""
+    if check_pass:
+        if err_type is not None:
+            raise ValueError(
+                f"`check_pass` is {check_pass}, but `err_type` is not None."
+            )
+        if err_msg is not None:
+            raise ValueError(
+                f"`check_pass` is {check_pass}, but `err_msg` is not None."
+            )
+        return dict(check_pass=check_pass, **kwargs)
+    else:
+        if err_type is None:
+            raise ValueError(f"`check_pass` is {check_pass}, but `err_type` is None.")
+        if err_msg is None:
+            if _is_known_error_type(err_type):
+                err_msg = ""
+            else:
+                raise ValueError(
+                    f"{err_type} is not a known error type, in which case `err_msg` must be specified to a value other \
+than None."
+                )
+        return dict(check_pass=check_pass, err_type=err_type, err_msg=err_msg, **kwargs)
+
+
+class FailedError(Exception):
+    """base error class"""
+
+    def __init__(self, err_info=None, solution=None, message=None):
+        if message is None:
+            message = self._construct_message(err_info, solution)
+        super().__init__(message)
+
+    def _construct_message(self, err_info, solution):
+        if err_info is None:
+            return ""
+        else:
+            msg = (
+                f"{self.mode} failed. We encountered the following error:\n  {err_info}"
+            )
+            if solution is not None:
+                msg += f"\nPlease try to resolve the issue as follows:\n  {solution}"
+            return msg
+
+
+class CheckFailedError(FailedError):
+    """check dataset error"""
+
+    mode = "Check dataset"
+
+
+class ConvertFailedError(FailedError):
+    """convert dataset error"""
+
+    mode = "Convert dataset"
+
+
+class SplitFailedError(FailedError):
+    """split dataset error"""
+
+    mode = "Split dataset"
+
+
+class AnalyseFailedError(FailedError):
+    """analyse dataset error"""
+
+    mode = "Analyse dataset"
+
+
+class DatasetFileNotFoundError(CheckFailedError):
+    """dataset file not found error"""
+
+    def __init__(self, file_path=None, err_info=None, solution=None, message=None):
+        if err_info is None:
+            if file_path is not None:
+                err_info = f"{file_path} does not exist."
+        super().__init__(err_info, solution, message)
+
+
+def _is_known_error_type(err_type):
+    """is known error type"""
+    return isinstance(err_type, CheckFailedError)