paddlex 2.1.0__py3-none-any.whl → 3.0.0__py3-none-any.whl

paddlex/modules/anomaly_detection/dataset_checker/__init__.py

@@ -0,0 +1,94 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import os.path as osp
+
+from ...base import BaseDatasetChecker
+from ..model_list import MODELS
+from .dataset_src import anaylse_dataset, check_dataset, convert_dataset, split_dataset
+
+
+class UadDatasetChecker(BaseDatasetChecker):
+    """Dataset Checker for Semantic Segmentation Model"""
+
+    entities = MODELS
+    sample_num = 10
+
+    def convert_dataset(self, src_dataset_dir: str) -> str:
+        """convert the dataset from other type to specified type
+
+        Args:
+            src_dataset_dir (str): the root directory of dataset.
+
+        Returns:
+            str: the root directory of converted dataset.
+        """
+        return convert_dataset(
+            self.check_dataset_config.convert.src_dataset_type, src_dataset_dir
+        )
+
+    def split_dataset(self, src_dataset_dir: str) -> str:
+        """repartition the train and validation dataset
+
+        Args:
+            src_dataset_dir (str): the root directory of dataset.
+
+        Returns:
+            str: the root directory of splited dataset.
+        """
+        return split_dataset(
+            src_dataset_dir,
+            self.check_dataset_config.split.train_percent,
+            self.check_dataset_config.split.val_percent,
+        )
+
+    def check_dataset(self, dataset_dir: str, sample_num: int = sample_num) -> dict:
+        """check if the dataset meets the specifications and get dataset summary
+
+        Args:
+            dataset_dir (str): the root directory of dataset.
+            sample_num (int): the number to be sampled.
+        Returns:
+            dict: dataset summary.
+        """
+        return check_dataset(dataset_dir, self.output, sample_num)
+
+    def analyse(self, dataset_dir: str) -> dict:
+        """deep analyse dataset
+
+        Args:
+            dataset_dir (str): the root directory of dataset.
+
+        Returns:
+            dict: the deep analysis results.
+        """
+        return anaylse_dataset(dataset_dir, self.output)
+
+    def get_show_type(self) -> str:
+        """get the show type of dataset
+
+        Returns:
+            str: show type
+        """
+        return "image"
+
+    def get_dataset_type(self) -> str:
+        """return the dataset type
+
+        Returns:
+            str: dataset type
+        """
+        return "SegDataset"

paddlex/modules/anomaly_detection/dataset_checker/dataset_src/__init__.py

@@ -0,0 +1,19 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from .analyse_dataset import anaylse_dataset
+from .check_dataset import check_dataset
+from .convert_dataset import convert_dataset
+from .split_dataset import split_dataset

paddlex/modules/anomaly_detection/dataset_checker/dataset_src/analyse_dataset.py

@@ -0,0 +1,82 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import os.path as osp
+
+import numpy as np
+from PIL import Image, ImageOps
+
+from .....utils.deps import function_requires_deps, is_dep_available
+from .....utils.file_interface import custom_open
+
+if is_dep_available("matplotlib"):
+    import matplotlib.pyplot as plt
+
+
+@function_requires_deps("matplotlib")
+def anaylse_dataset(dataset_dir, output):
+    """class analysis for dataset"""
+
+    split_tags = ["train", "val"]
+    label2count = {tag: dict() for tag in split_tags}
+    for tag in split_tags:
+        mapping_file = osp.join(dataset_dir, f"{tag}.txt")
+        with custom_open(mapping_file, "r") as fp:
+            lines = filter(None, (line.strip() for line in fp.readlines()))
+            for i, line in enumerate(lines):
+                _, ann_file = line.split(" ")
+                ann_file = osp.join(dataset_dir, ann_file)
+                ann = np.array(ImageOps.exif_transpose(Image.open(ann_file)), "uint8")
+                if tag == "train":
+                    if 0 not in label2count[tag]:
+                        label2count[tag][0] = 1
+                    else:
+                        label2count[tag][0] += 1
+
+                if tag == "val":
+                    for idx in set(ann.reshape([-1]).tolist()):
+                        if idx == 255:
+                            continue
+                        if idx not in label2count[tag]:
+                            label2count[tag][idx] = 1
+                        else:
+                            label2count[tag][idx] += 1
+            if label2count[tag].get(0, None) is None:
+                label2count[tag][0] = 0
+
+    train_label_idx = np.array(list(label2count["train"].keys()))
+    val_label_idx = np.array(list(label2count["val"].keys()))
+    label_idx = np.array(list(set(train_label_idx) | set(val_label_idx)))
+    x = np.arange(len(label_idx))
+    train_list = []
+    val_list = []
+    for i in range(len(label_idx)):
+        train_list.append(label2count["train"].get(i, 0))
+        val_list.append(label2count["val"].get(i, 0))
+    fig, ax = plt.subplots(figsize=(max(8, int(len(label_idx) / 5)), 5), dpi=120)
+
+    width = (0.5,)
+    ax.bar(x, train_list, width=width, label="train")
+    ax.bar(x + width, val_list, width=width, label="val")
+
+    plt.xticks(x + 0.25, label_idx)
+    ax.set_xlabel("Label Index")
+    ax.set_ylabel("Sample Counts")
+    plt.legend()
+    fig.tight_layout()
+    fig_path = os.path.join(output, "histogram.png")
+    fig.savefig(fig_path)
+    return {"histogram": os.path.join("check_dataset", "histogram.png")}

paddlex/modules/anomaly_detection/dataset_checker/dataset_src/check_dataset.py

@@ -0,0 +1,91 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import os.path as osp
+
+import numpy as np
+from PIL import Image
+
+from .....utils.deps import function_requires_deps, is_dep_available
+from .....utils.errors import DatasetFileNotFoundError
+from .....utils.file_interface import custom_open
+from .....utils.logging import info
+from .utils.visualizer import visualize
+
+if is_dep_available("opencv-contrib-python"):
+    import cv2
+
+
+@function_requires_deps("opencv-contrib-python")
+def check_dataset(dataset_dir, output, sample_num=10):
+    """check dataset"""
+    dataset_dir = osp.abspath(dataset_dir)
+    if not osp.exists(dataset_dir) or not osp.isdir(dataset_dir):
+        raise DatasetFileNotFoundError(file_path=dataset_dir)
+    vis_save_dir = osp.join(output, "demo_img")
+    if not osp.exists(vis_save_dir):
+        os.makedirs(vis_save_dir)
+    split_tags = ["train", "val"]
+    attrs = dict()
+    class_ids = set()
+    for tag in split_tags:
+        mapping_file = osp.join(dataset_dir, f"{tag}.txt")
+        if not osp.exists(mapping_file):
+            info(f"The mapping file ({mapping_file}) doesn't exist, ignored.")
+            info(
+                "If you are using MVTec_AD dataset, add args below in your training commands:"
+            )
+            info("-o CheckDataset.convert.enable=True")
+            info("-o CheckDataset.convert.src_dataset_type=MVTec_AD")
+            continue
+        with custom_open(mapping_file, "r") as fp:
+            lines = filter(None, (line.strip() for line in fp.readlines()))
+            for i, line in enumerate(lines):
+                img_file, ann_file = line.split(" ")
+                img_file = osp.join(dataset_dir, img_file)
+                ann_file = osp.join(dataset_dir, ann_file)
+                assert osp.exists(img_file), FileNotFoundError(
+                    f"{img_file} not exist, please check!"
+                )
+                assert osp.exists(ann_file), FileNotFoundError(
+                    f"{ann_file} not exist, please check!"
+                )
+                img = np.array(cv2.imread(img_file), "uint8")
+                ann = np.array(cv2.imread(ann_file), "uint8")[:, :, 0]
+                assert img.shape[:2] == ann.shape, ValueError(
+                    f"The shape of {img_file}:{img.shape[:2]} and "
+                    f"{ann_file}:{ann.shape} must be the same!"
+                )
+                if tag == "val":
+                    class_ids = class_ids | set(ann.reshape([-1]).tolist())
+                if i < sample_num:
+                    vis_img = visualize(img, ann)
+                    vis_img = Image.fromarray(vis_img)
+                    vis_save_path = osp.join(vis_save_dir, osp.basename(img_file))
+                    vis_img.save(vis_save_path)
+                    vis_save_path = osp.join(
+                        "check_dataset", os.path.relpath(vis_save_path, output)
+                    )
+                    if f"{tag}_sample_paths" not in attrs:
+                        attrs[f"{tag}_sample_paths"] = [vis_save_path]
+                    else:
+                        attrs[f"{tag}_sample_paths"].append(vis_save_path)
+            if f"{tag}_samples" not in attrs:
+                attrs[f"{tag}_samples"] = i + 1
+    if 255 in class_ids:
+        class_ids.remove(255)
+    attrs["num_classes"] = len(class_ids)
+    return attrs

paddlex/modules/anomaly_detection/dataset_checker/dataset_src/convert_dataset.py

@@ -0,0 +1,233 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import json
+import os
+import os.path as osp
+import shutil
+
+import numpy as np
+from PIL import Image, ImageDraw
+
+from .....utils import logging
+from .....utils.deps import function_requires_deps, is_dep_available
+from .....utils.file_interface import custom_open
+from .....utils.logging import info
+
+if is_dep_available("opencv-contrib-python"):
+    import cv2
+
+
+def convert_dataset(dataset_type, input_dir):
+    """convert to paddlex official format"""
+    if dataset_type == "LabelMe":
+        return convert_labelme_dataset(input_dir)
+    elif dataset_type == "MVTec_AD":
+        return convert_mvtec_dataset(input_dir)
+    else:
+        raise NotImplementedError(dataset_type)
+
+
+@function_requires_deps("opencv-contrib-python")
+def convert_labelme_dataset(input_dir):
+    """convert labelme format to paddlex official format"""
+    bg_name = "_background_"
+    ignore_name = "__ignore__"
+
+    # prepare dir
+    output_img_dir = osp.join(input_dir, "images")
+    output_annot_dir = osp.join(input_dir, "annotations")
+    if not osp.exists(output_img_dir):
+        os.makedirs(output_img_dir)
+    if not osp.exists(output_annot_dir):
+        os.makedirs(output_annot_dir)
+
+    # collect class_names and set class_name_to_id
+    class_names = []
+    class_name_to_id = {}
+    split_tags = ["train", "val"]
+    for tag in split_tags:
+        mapping_file = osp.join(input_dir, f"{tag}_anno_list.txt")
+        with open(mapping_file, "r") as f:
+            label_files = [
+                osp.join(input_dir, line.strip("\n")) for line in f.readlines()
+            ]
+        for label_file in label_files:
+            with custom_open(label_file, "r") as fp:
+                data = json.load(fp)
+                for shape in data["shapes"]:
+                    cls_name = shape["label"]
+                    if cls_name not in class_names:
+                        class_names.append(cls_name)
+
+        if ignore_name in class_names:
+            class_name_to_id[ignore_name] = 255
+            class_names.remove(ignore_name)
+        if bg_name in class_names:
+            class_names.remove(bg_name)
+        class_name_to_id[bg_name] = 0
+        for i, name in enumerate(class_names):
+            class_name_to_id[name] = i + 1
+
+        if len(class_names) > 256:
+            raise ValueError(
+                f"There are {len(class_names)} categories in the annotation file, "
+                f"exceeding 256, Not compliant with paddlex official format!"
+            )
+
+        # create annotated images and copy origin images
+        color_map = get_color_map_list(256)
+        img_file_list = []
+        label_file_list = []
+        for i, label_file in enumerate(label_files):
+            filename = osp.splitext(osp.basename(label_file))[0]
+            annotated_img_path = osp.join(output_annot_dir, filename + ".png")
+            with custom_open(label_file, "r") as f:
+                data = json.load(f)
+                img_path = osp.join(osp.dirname(label_file), data["imagePath"])
+                if not os.path.exists(img_path):
+                    logging.info("%s is not existed, skip this image" % img_path)
+                    continue
+                img_name = img_path.split("/")[-1]
+                img_file_list.append(f"images/{img_name}")
+                label_img_name = annotated_img_path.split("/")[-1]
+                label_file_list.append(f"annotations/{label_img_name}")
+
+                img = np.asarray(cv2.imread(img_path))
+                lbl = shape2label(
+                    img_size=img.shape,
+                    shapes=data["shapes"],
+                    class_name_mapping=class_name_to_id,
+                )
+                lbl_pil = Image.fromarray(lbl.astype(np.uint8), mode="P")
+                lbl_pil.putpalette(color_map)
+                lbl_pil.save(annotated_img_path)
+
+                shutil.copy(img_path, output_img_dir)
+        with custom_open(osp.join(input_dir, f"{tag}.txt"), "w") as fp:
+            for img_path, lbl_path in zip(img_file_list, label_file_list):
+                fp.write(f"{img_path} {lbl_path}\n")
+
+    with custom_open(osp.join(input_dir, "class_name.txt"), "w") as fp:
+        for name in class_names:
+            fp.write(f"{name}{os.linesep}")
+    with custom_open(osp.join(input_dir, "class_name_to_id.txt"), "w") as fp:
+        for key, val in class_name_to_id.items():
+            fp.write(f"{val}: {key}{os.linesep}")
+
+    return input_dir
+
+
+def get_color_map_list(num_classes):
+    """get color map list"""
+    num_classes += 1
+    color_map = num_classes * [0, 0, 0]
+    for i in range(0, num_classes):
+        j = 0
+        lab = i
+        while lab:
+            color_map[i * 3] |= ((lab >> 0) & 1) << (7 - j)
+            color_map[i * 3 + 1] |= ((lab >> 1) & 1) << (7 - j)
+            color_map[i * 3 + 2] |= ((lab >> 2) & 1) << (7 - j)
+            j += 1
+            lab >>= 3
+    color_map = color_map[3:]
+    return color_map
+
+
+def shape2label(img_size, shapes, class_name_mapping):
+    """根据输入的形状列表，将图像的标签矩阵填充为对应形状的类别编号"""
+    label = np.zeros(img_size[:2], dtype=np.int32)
+    for shape in shapes:
+        points = shape["points"]
+        class_name = shape["label"]
+        label_mask = polygon2mask(img_size[:2], points)
+        label[label_mask] = class_name_mapping[class_name]
+    return label
+
+
+def polygon2mask(img_size, points):
+    """将给定形状的点转换成对应的掩膜"""
+    label_mask = Image.fromarray(np.zeros(img_size[:2], dtype=np.uint8))
+    image_draw = ImageDraw.Draw(label_mask)
+    points_list = [tuple(point) for point in points]
+    assert len(points_list) > 2, ValueError("Polygon must have points more than 2")
+    image_draw.polygon(xy=points_list, outline=1, fill=1)
+    return np.array(label_mask, dtype=bool)
+
+
+def save_item_to_txt(items, file_path):
+    try:
+        with open(file_path, "a") as file:
+            file.write(items)
+        file.close()
+    except Exception as e:
+        print(f"Saving_error: {e}")
+
+
+def save_training_txt(cls_root, mode, cat):
+    imgs = os.listdir(os.path.join(cls_root, mode, cat))
+    imgs.sort()
+    for img in imgs:
+        if mode == "train":
+            item = os.path.join(cls_root, mode, cat, img)
+            items = item + " " + item + "\n"
+            save_item_to_txt(items, os.path.join(cls_root, "train.txt"))
+        elif mode == "test" and cat != "good":
+            item1 = os.path.join(cls_root, mode, cat, img)
+            item2 = os.path.join(
+                cls_root, "ground_truth", cat, img.split(".")[0] + "_mask.png"
+            )
+            items = item1 + " " + item2 + "\n"
+            save_item_to_txt(items, os.path.join(cls_root, "val.txt"))
+
+
+def check_old_txt(cls_pth, mode):
+    set_name = "train.txt" if mode == "train" else "val.txt"
+    pth = os.path.join(cls_pth, set_name)
+    if os.path.exists(pth):
+        os.remove(pth)
+
+
+def convert_mvtec_dataset(input_dir):
+    classes = [
+        "bottle",
+        "cable",
+        "capsule",
+        "hazelnut",
+        "metal_nut",
+        "pill",
+        "screw",
+        "toothbrush",
+        "transistor",
+        "zipper",
+        "carpet",
+        "grid",
+        "leather",
+        "tile",
+        "wood",
+    ]
+    clas = os.path.split(input_dir)[-1]
+    assert clas in classes, info(
+        f"Make sure your class: '{clas}' in your dataset root in\n {classes}"
+    )
+    modes = ["train", "test"]
+    cls_root = input_dir
+    for mode in modes:
+        check_old_txt(cls_root, mode)
+        cats = os.listdir(os.path.join(cls_root, mode))
+        for cat in cats:
+            save_training_txt(cls_root, mode, cat)
+    info(f"Add train.txt/val.txt successfully for {input_dir}")

paddlex/modules/anomaly_detection/dataset_checker/dataset_src/split_dataset.py

@@ -0,0 +1,87 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import os.path as osp
+import random
+import shutil
+
+from .....utils import logging
+from .....utils.file_interface import custom_open
+
+
+def split_dataset(root_dir, train_percent, val_percent):
+    """split dataset"""
+    assert train_percent > 0, ValueError(
+        f"The train_percent({train_percent}) must greater than 0!"
+    )
+    assert val_percent > 0, ValueError(
+        f"The val_percent({val_percent}) must greater than 0!"
+    )
+    if train_percent + val_percent != 100:
+        raise ValueError(
+            f"The sum of train_percent({train_percent})and val_percent({val_percent}) should be 100!"
+        )
+
+    img_dir = osp.join(root_dir, "images")
+    assert osp.exists(img_dir), FileNotFoundError(
+        f"The dir of images ({img_dir}) doesn't exist, please check!"
+    )
+    ann_dir = osp.join(root_dir, "annotations")
+    assert osp.exists(ann_dir), FileNotFoundError(
+        f"The dir of annotations ({ann_dir}) doesn't exist, please check!"
+    )
+
+    img_file_list = [osp.join("images", img_name) for img_name in os.listdir(img_dir)]
+    img_num = len(img_file_list)
+    ann_file_list = [
+        osp.join("annotations", ann_name) for ann_name in os.listdir(ann_dir)
+    ]
+    ann_num = len(ann_file_list)
+    assert img_num == ann_num, ValueError(
+        "The number of images and annotations must be equal!"
+    )
+
+    split_tags = ["train", "val"]
+    mapping_line_list = []
+    for tag in split_tags:
+        mapping_file = osp.join(root_dir, f"{tag}.txt")
+        if not osp.exists(mapping_file):
+            logging.info(f"The mapping file ({mapping_file}) doesn't exist, ignored.")
+            continue
+        with custom_open(mapping_file, "r") as fp:
+            lines = filter(None, (line.strip() for line in fp.readlines()))
+            mapping_line_list.extend(lines)
+
+    sample_num = len(mapping_line_list)
+    random.shuffle(mapping_line_list)
+    split_percents = [train_percent, val_percent]
+    start_idx = 0
+    for tag, percent in zip(split_tags, split_percents):
+        if tag == "test" and percent == 0:
+            continue
+        end_idx = start_idx + round(sample_num * percent / 100)
+        end_idx = min(end_idx, sample_num)
+        mapping_file = osp.join(root_dir, f"{tag}.txt")
+        if os.path.exists(mapping_file):
+            shutil.move(mapping_file, mapping_file + ".bak")
+            logging.info(
+                f"The original mapping file ({mapping_file}) "
+                f"has been backed up to ({mapping_file}.bak)"
+            )
+        with custom_open(mapping_file, "w") as fp:
+            fp.write("\n".join(mapping_line_list[start_idx:end_idx]))
+        start_idx = end_idx
+    return root_dir

paddlex/modules/anomaly_detection/dataset_checker/dataset_src/utils/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.