paddlex 3.0.0b2__py3-none-any.whl → 3.0.0rc1__py3-none-any.whl

paddlex/modules/video_detection/dataset_checker/dataset_src/check_dataset.py

@@ -0,0 +1,132 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import os.path as osp
+import random
+from collections import defaultdict
+
+from .....utils.errors import CheckFailedError, DatasetFileNotFoundError
+from .....utils.file_interface import custom_open
+
+
+def check(dataset_dir, output, sample_num=10):
+    """check dataset"""
+    dataset_dir = osp.abspath(dataset_dir)
+    # Custom dataset
+    if not osp.exists(dataset_dir) or not osp.isdir(dataset_dir):
+        raise DatasetFileNotFoundError(file_path=dataset_dir)
+
+    tags = ["train", "val"]
+    valid_num_parts = 5
+
+    sample_cnts = dict()
+    label_map_dict = dict()
+    sample_paths = defaultdict(list)
+    labels = []
+    image_dir = osp.join(dataset_dir, "rgb-images")
+    label_dir = osp.join(dataset_dir, "labels")
+    if not osp.exists(image_dir):
+        raise DatasetFileNotFoundError(file_path=image_dir)
+    if not osp.exists(label_dir):
+        raise DatasetFileNotFoundError(file_path=label_dir)
+
+    label_map_file = osp.join(dataset_dir, "label_map.txt")
+    if not osp.exists(label_map_file):
+        raise DatasetFileNotFoundError(
+            file_path=label_map_file,
+            solution=f"Ensure that `label_map.txt` exist in {dataset_dir}",
+        )
+    with open(label_map_file, "r", encoding="utf-8") as f:
+        all_lines = f.readlines()
+        for line in all_lines:
+            substr = line.strip("\n").split(" ", 1)
+            try:
+                label_idx = int(substr[1])
+                labels.append(label_idx)
+                label_map_dict[label_idx] = str(substr[0])
+            except:
+                raise CheckFailedError(
+                    f"Ensure that the second number in each line in {label_map_file} should be int."
+                )
+    if min(labels) != 1:
+        raise CheckFailedError(
+            f"Ensure that the index starts from 1 in `{label_map_file}`."
+        )
+
+    for tag in tags:
+        file_list = osp.join(dataset_dir, f"{tag}.txt")
+        if not osp.exists(file_list):
+            if tag in ("train", "val"):
+                # train and val file lists must exist
+                raise DatasetFileNotFoundError(
+                    file_path=file_list,
+                    solution=f"Ensure that both `train.txt` and `val.txt` exist in {dataset_dir}",
+                )
+            else:
+                # tag == 'test'
+                continue
+        else:
+            with open(file_list, "r", encoding="utf-8") as f:
+                all_lines = f.readlines()
+                random.seed(123)
+                random.shuffle(all_lines)
+                sample_cnts[tag] = len(all_lines)
+
+                for line in all_lines:
+                    substr = line.strip("\n")
+                    label_path = osp.join(dataset_dir, substr)
+                    img_path = (
+                        osp.join(dataset_dir, substr)
+                        .replace("labels", "rgb-images")
+                        .replace("txt", "jpg")
+                    )
+
+                    if not osp.exists(img_path):
+                        raise DatasetFileNotFoundError(file_path=img_path)
+                    if not osp.exists(label_path):
+                        raise DatasetFileNotFoundError(file_path=label_path)
+                    with custom_open(label_path, "r") as f:
+                        label_lines = f.readlines()
+                        for label_line in label_lines:
+                            label_info = label_line.strip().split(" ")
+                            try:
+                                int(label_info[0])
+                            except (ValueError, TypeError) as e:
+                                raise CheckFailedError(
+                                    f"Ensure that the first number in each line in {label_info} should be int."
+                                ) from e
+                                if len(label_info) != valid_num_parts:
+                                    raise CheckFailedError(
+                                        f"Ensure that each line in {label_line} has exactly two numbers."
+                                    )
+
+                    if len(sample_paths[tag]) < sample_num:
+                        sample_path = osp.join(
+                            "check_dataset", os.path.relpath(img_path, output)
+                        )
+                        sample_paths[tag].append(sample_path)
+
+    num_classes = max(labels)
+
+    attrs = {}
+    attrs["label_file"] = osp.relpath(label_map_file, output)
+    attrs["num_classes"] = num_classes
+    attrs["train_samples"] = sample_cnts["train"]
+    attrs["train_sample_paths"] = sample_paths["train"]
+
+    attrs["val_samples"] = sample_cnts["val"]
+    attrs["val_sample_paths"] = sample_paths["val"]
+
+    return attrs

paddlex/modules/video_detection/evaluator.py

@@ -0,0 +1,42 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ..base import BaseEvaluator
+from .model_list import MODELS
+
+
+class VideoDetEvaluator(BaseEvaluator):
+    """Image Classification Model Evaluator"""
+
+    entities = MODELS
+
+    def update_config(self):
+        """update evalution config"""
+        if self.eval_config.log_interval:
+            self.pdx_config.update_log_interval(self.eval_config.log_interval)
+        self.pdx_config.update_dataset(
+            self.global_config.dataset_dir, "VideoDetDataset"
+        )
+        self.pdx_config.update_pretrained_weights(self.eval_config.weight_path)
+
+    def get_eval_kwargs(self) -> dict:
+        """get key-value arguments of model evalution function
+
+        Returns:
+            dict: the arguments of evaluation function.
+        """
+        return {
+            "weight_path": self.eval_config.weight_path,
+            "device": self.get_device(using_device_number=1),
+        }

paddlex/modules/video_detection/exportor.py

@@ -0,0 +1,22 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ..base import BaseExportor
+from .model_list import MODELS
+
+
+class VideoDetExportor(BaseExportor):
+    """Image Classification Model Exportor"""
+
+    entities = MODELS

paddlex/modules/video_detection/model_list.py

@@ -0,0 +1,15 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+MODELS = ["YOWO"]

paddlex/modules/video_detection/trainer.py

@@ -0,0 +1,82 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import shutil
+from pathlib import Path
+
+from ..base import BaseTrainer
+from .model_list import MODELS
+
+
+class VideoDetTrainer(BaseTrainer):
+    """Image Classification Model Trainer"""
+
+    entities = MODELS
+
+    def dump_label_dict(self, src_label_dict_path: str):
+        """dump label dict config
+
+        Args:
+            src_label_dict_path (str): path to label dict file to be saved.
+        """
+        dst_label_dict_path = Path(self.global_config.output).joinpath("label_map.txt")
+        shutil.copyfile(src_label_dict_path, dst_label_dict_path)
+
+    def update_config(self):
+        """update training config"""
+        if self.train_config.log_interval:
+            self.pdx_config.update_log_interval(self.train_config.log_interval)
+        if self.train_config.eval_interval:
+            self.pdx_config.update_eval_interval(self.train_config.eval_interval)
+        if self.train_config.save_interval:
+            self.pdx_config.update_save_interval(self.train_config.save_interval)
+        if self.train_config.num_classes is not None:
+            self.pdx_config.update_num_classes(self.train_config.num_classes)
+
+        self.pdx_config.update_dataset(
+            self.global_config.dataset_dir, "VideoDetDataset"
+        )
+        if self.train_config.pretrain_weight_path != "":
+            self.pdx_config.update_pretrained_weights(
+                self.train_config.pretrain_weight_path
+            )
+
+        label_dict_path = Path(self.global_config.dataset_dir).joinpath("label_map.txt")
+        if label_dict_path.exists():
+            self.pdx_config.update_label_list(label_dict_path)
+        if self.train_config.batch_size is not None:
+            self.pdx_config.update_batch_size(self.train_config.batch_size)
+        if self.train_config.learning_rate is not None:
+            self.pdx_config.update_learning_rate(self.train_config.learning_rate)
+        if self.train_config.epochs_iters is not None:
+            self.pdx_config._update_epochs(self.train_config.epochs_iters)
+        if self.global_config.output is not None:
+            self.pdx_config._update_output_dir(self.global_config.output)
+
+    def get_train_kwargs(self) -> dict:
+        """get key-value arguments of model training function
+
+        Returns:
+            dict: the arguments of training function.
+        """
+        train_args = {"device": self.get_device()}
+        if (
+            self.train_config.resume_path is not None
+            and self.train_config.resume_path != ""
+        ):
+            train_args["resume_path"] = self.train_config.resume_path
+        train_args["dy2st"] = self.train_config.get("dy2st", False)
+        # amp support 'O1', 'O2', 'OFF'
+        train_args["amp"] = self.train_config.get("amp", "OFF")
+        return train_args

paddlex/ops/__init__.py

@@ -0,0 +1,152 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib
+import inspect
+import os
+import sys
+from types import ModuleType
+
+import filelock
+
+from ..utils import logging
+from ..utils.deps import class_requires_deps
+
+
+def get_user_home() -> str:
+    return os.path.expanduser("~")
+
+
+def get_pprndr_home() -> str:
+    return os.path.join(get_user_home(), ".pprndr")
+
+
+def get_sub_home(directory: str) -> str:
+    home = os.path.join(get_pprndr_home(), directory)
+    os.makedirs(home, exist_ok=True)
+    return home
+
+
+TMP_HOME = get_sub_home("tmp")
+
+custom_ops = {
+    "voxelize": {
+        "sources": ["voxel/voxelize_op.cc", "voxel/voxelize_op.cu"],
+        "version": "0.1.0",
+    },
+    "iou3d_nms": {
+        "sources": [
+            "iou3d_nms/iou3d_cpu.cpp",
+            "iou3d_nms/iou3d_nms_api.cpp",
+            "iou3d_nms/iou3d_nms.cpp",
+            "iou3d_nms/iou3d_nms_kernel.cu",
+        ],
+        "version": "0.1.0",
+    },
+}
+
+
+class CustomOpNotFoundException(Exception):
+    def __init__(self, op_name):
+        self.op_name = op_name
+
+    def __str__(self):
+        return "Couldn't Found custom op {}".format(self.op_name)
+
+
+class CustomOperatorPathFinder:
+    def find_module(self, fullname: str, path: str = None):
+        if not fullname.startswith("paddlex.ops"):
+            return None
+
+        return CustomOperatorPathLoader()
+
+
+class CustomOperatorPathLoader:
+    def load_module(self, fullname: str):
+        modulename = fullname.split(".")[-1]
+
+        if modulename not in custom_ops:
+            raise CustomOpNotFoundException(modulename)
+
+        if fullname not in sys.modules:
+            try:
+                sys.modules[fullname] = importlib.import_module(modulename)
+            except ImportError:
+                sys.modules[fullname] = PaddleXCustomOperatorModule(
+                    modulename, fullname
+                )
+        return sys.modules[fullname]
+
+
+class PaddleXCustomOperatorModule(ModuleType):
+    def __init__(self, modulename: str, fullname: str):
+        self.fullname = fullname
+        self.modulename = modulename
+        self.module = None
+        super().__init__(modulename)
+
+    def jit_build(self):
+        from paddle.utils.cpp_extension import load as paddle_jit_load
+
+        try:
+            lockfile = "paddlex.ops.{}".format(self.modulename)
+            lockfile = os.path.join(TMP_HOME, lockfile)
+            file = inspect.getabsfile(sys.modules["paddlex.ops"])
+            rootdir = os.path.split(file)[0]
+
+            args = custom_ops[self.modulename].copy()
+            sources = args.pop("sources")
+            sources = [os.path.join(rootdir, file) for file in sources]
+
+            args.pop("version")
+            with filelock.FileLock(lockfile):
+                return paddle_jit_load(name=self.modulename, sources=sources, **args)
+        except:
+            logging.error("{} builded fail!".format(self.modulename))
+            raise
+
+    def _load_module(self):
+        if self.module is None:
+            try:
+                self.module = importlib.import_module(self.modulename)
+            except ImportError:
+                logging.warning(
+                    "No custom op {} found, try JIT build".format(self.modulename)
+                )
+                self.module = self.jit_build()
+                logging.info("{} builded success!".format(self.modulename))
+
+            # refresh
+            sys.modules[self.fullname] = self.module
+        return self.module
+
+    def __getattr__(self, attr: str):
+        if attr in ["__path__", "__file__"]:
+            return None
+
+        if attr in ["__loader__", "__package__", "__name__", "__spec__"]:
+            return super().__getattr__(attr)
+
+        module = self._load_module()
+        if not hasattr(module, attr):
+            raise ImportError(
+                "cannot import name '{}' from '{}' ({})".format(
+                    attr, self.modulename, module.__file__
+                )
+            )
+        return getattr(module, attr)
+
+
+sys.meta_path.insert(0, CustomOperatorPathFinder())