paddlex 3.0.0b2__py3-none-any.whl → 3.0.0rc1__py3-none-any.whl

paddlex/repo_apis/PaddleVideo_api/video_det/model.py

@@ -0,0 +1,298 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from ....utils import logging
+from ....utils.errors import raise_unsupported_api_error
+from ....utils.misc import abspath
+from ...base import BaseModel
+from ...base.utils.arg import CLIArgument
+from ...base.utils.subprocess import CompletedProcess
+
+
+class VideoDetModel(BaseModel):
+    """Video Det Model"""
+
+    def train(
+        self,
+        batch_size: int = None,
+        learning_rate: float = None,
+        epochs_iters: int = None,
+        ips: str = None,
+        device: str = "gpu",
+        resume_path: str = None,
+        dy2st: bool = False,
+        amp: str = "OFF",
+        num_workers: int = None,
+        use_vdl: bool = True,
+        save_dir: str = None,
+        **kwargs,
+    ) -> CompletedProcess:
+        """train self
+
+        Args:
+            batch_size (int, optional): the train batch size value. Defaults to None.
+            learning_rate (float, optional): the train learning rate value. Defaults to None.
+            epochs_iters (int, optional): the train epochs value. Defaults to None.
+            ips (str, optional): the ip addresses of nodes when using distribution. Defaults to None.
+            device (str, optional): the running device. Defaults to 'gpu'.
+            resume_path (str, optional): the checkpoint file path to resume training. Train from scratch if it is set
+                to None. Defaults to None.
+            dy2st (bool, optional): Enable dynamic to static. Defaults to False.
+            amp (str, optional): the amp settings. Defaults to 'OFF'.
+            num_workers (int, optional): the workers number. Defaults to None.
+            use_vdl (bool, optional): enable VisualDL. Defaults to True.
+            save_dir (str, optional): the directory path to save train output. Defaults to None.
+
+        Returns:
+           CompletedProcess: the result of training subprocess execution.
+        """
+        if resume_path is not None:
+            resume_path = abspath(resume_path)
+
+        with self._create_new_config_file() as config_path:
+            # Update YAML config file
+            config = self.config.copy()
+            config.update_device(device)
+            config._update_to_static(dy2st)
+            config._update_use_vdl(use_vdl)
+
+            if batch_size is not None:
+                config.update_batch_size(batch_size)
+            if learning_rate is not None:
+                config.update_learning_rate(learning_rate)
+            if epochs_iters is not None:
+                config._update_epochs(epochs_iters)
+            config._update_checkpoints(resume_path)
+            if save_dir is not None:
+                save_dir = abspath(save_dir)
+            else:
+                # `save_dir` is None
+                save_dir = abspath(config.get_train_save_dir())
+            config._update_output_dir(save_dir)
+            if num_workers is not None:
+                config.update_num_workers(num_workers)
+
+            cli_args = []
+            do_eval = kwargs.pop("do_eval", True)
+            profile = kwargs.pop("profile", None)
+            if profile is not None:
+                cli_args.append(CLIArgument("--profiler_options", profile))
+
+            # Benchmarking mode settings
+            benchmark = kwargs.pop("benchmark", None)
+            if benchmark is not None:
+                envs = benchmark.get("env", None)
+                seed = benchmark.get("seed", None)
+                do_eval = benchmark.get("do_eval", False)
+                num_workers = benchmark.get("num_workers", None)
+                config.update_log_ranks(device)
+                config._update_amp(benchmark.get("amp", None))
+                config.update_dali(benchmark.get("dali", False))
+                config.update_shuffle(benchmark.get("shuffle", False))
+                config.update_shared_memory(benchmark.get("shared_memory", True))
+                config.update_print_mem_info(benchmark.get("print_mem_info", True))
+                if num_workers is not None:
+                    config.update_num_workers(num_workers)
+                if seed is not None:
+                    config.update_seed(seed)
+                if envs is not None:
+                    for env_name, env_value in envs.items():
+                        os.environ[env_name] = str(env_value)
+            else:
+                config._update_amp(amp)
+            # PDX related settings
+            device_type = device.split(":")[0]
+            uniform_output_enabled = kwargs.pop("uniform_output_enabled", True)
+            config.update({"Global.uniform_output_enabled": uniform_output_enabled})
+            config.update({"Global.pdx_model_name": self.name})
+
+            config.dump(config_path)
+            self._assert_empty_kwargs(kwargs)
+            return self.runner.train(
+                config_path, cli_args, device, ips, save_dir, do_eval=do_eval
+            )
+
+    def evaluate(
+        self,
+        weight_path: str,
+        batch_size: int = None,
+        ips: str = None,
+        device: str = "gpu",
+        amp: str = "OFF",
+        num_workers: int = None,
+        **kwargs,
+    ) -> CompletedProcess:
+        """evaluate self using specified weight
+
+        Args:
+            weight_path (str): the path of model weight file to be evaluated.
+            batch_size (int, optional): the batch size value in evaluating. Defaults to None.
+            ips (str, optional): the ip addresses of nodes when using distribution. Defaults to None.
+            device (str, optional): the running device. Defaults to 'gpu'.
+            amp (str, optional): the AMP setting. Defaults to 'OFF'.
+            num_workers (int, optional): the workers number in evaluating. Defaults to None.
+
+        Returns:
+            CompletedProcess: the result of evaluating subprocess execution.
+        """
+
+        with self._create_new_config_file() as config_path:
+            # Update YAML config file
+            config = self.config.copy()
+            config._update_amp(amp)
+            config.update_device(device)
+            config.update_pretrained_weights(weight_path)
+            if batch_size is not None:
+                config.update_batch_size(batch_size)
+            if num_workers is not None:
+                config.update_num_workers(num_workers)
+
+            config.dump(config_path)
+
+            self._assert_empty_kwargs(kwargs)
+
+            cp = self.runner.evaluate(config_path, [], device, ips)
+            return cp
+
+    def predict(
+        self,
+        weight_path: str,
+        input_path: str,
+        input_list_path: str = None,
+        device: str = "gpu",
+        save_dir: str = None,
+        **kwargs,
+    ) -> CompletedProcess:
+        """predict using specified weight
+
+        Args:
+            weight_path (str): the path of model weight file used to predict.
+            input_path (str): the path of image file to be predicted.
+            input_list_path (str, optional): the paths of images to be predicted if is not None. Defaults to None.
+            device (str, optional): the running device. Defaults to 'gpu'.
+            save_dir (str, optional): the directory path to save predict output. Defaults to None.
+
+        Returns:
+            CompletedProcess: the result of predicting subprocess execution.
+        """
+        input_path = abspath(input_path)
+        if input_list_path:
+            input_list_path = abspath(input_list_path)
+
+        with self._create_new_config_file() as config_path:
+            # Update YAML config file
+            config = self.config.copy()
+            config.update_pretrained_weights(weight_path)
+            config._update_predict_img(input_path, input_list_path)
+            config.update_device(device)
+            config._update_save_predict_result(save_dir)
+
+            config.dump(config_path)
+
+            self._assert_empty_kwargs(kwargs)
+
+            return self.runner.predict(config_path, [], device)
+
+    def export(self, weight_path: str, save_dir: str, **kwargs) -> CompletedProcess:
+        """export the dynamic model to static model
+
+        Args:
+            weight_path (str): the model weight file path that used to export.
+            save_dir (str): the directory path to save export output.
+
+        Returns:
+            CompletedProcess: the result of exporting subprocess execution.
+        """
+        if not weight_path.startswith(("http://", "https://")):
+            weight_path = abspath(weight_path)
+        save_dir = abspath(save_dir)
+
+        with self._create_new_config_file() as config_path:
+            # Update YAML config file
+            config = self.config.copy()
+            config.update_pretrained_weights(weight_path)
+            config._update_save_inference_dir(save_dir)
+            device = kwargs.pop("device", None)
+            if device:
+                config.update_device(device)
+            # PDX related settings
+            uniform_output_enabled = kwargs.pop("uniform_output_enabled", True)
+            config.update({"Global.uniform_output_enabled": uniform_output_enabled})
+            config.update({"Global.pdx_model_name": self.name})
+
+            config.dump(config_path)
+
+            self._assert_empty_kwargs(kwargs)
+
+            return self.runner.export(config_path, [], None, save_dir)
+
+    def infer(
+        self,
+        model_dir: str,
+        input_path: str,
+        device: str = "gpu",
+        save_dir: str = None,
+        dict_path: str = None,
+        **kwargs,
+    ) -> CompletedProcess:
+        """predict image using infernece model
+
+        Args:
+            model_dir (str): the directory path of inference model files that would use to predict.
+            input_path (str): the path of image that would be predict.
+            device (str, optional): the running device. Defaults to 'gpu'.
+            save_dir (str, optional): the directory path to save output. Defaults to None.
+            dict_path (str, optional): the label dict file path. Defaults to None.
+
+        Returns:
+            CompletedProcess: the result of infering subprocess execution.
+        """
+        model_dir = abspath(model_dir)
+        input_path = abspath(input_path)
+        if save_dir is not None:
+            logging.warning("`save_dir` will not be used.")
+        config_path = os.path.join(model_dir, "inference.yml")
+        config = self.config.copy()
+        config.load(config_path)
+        config._update_inference_model_dir(model_dir)
+        config._update_infer_img(input_path)
+        config._update_infer_device(device)
+        if dict_path is not None:
+            dict_path = abspath(dict_path)
+            config.update_label_dict_path(dict_path)
+        if "enable_mkldnn" in kwargs:
+            config._update_enable_mkldnn(kwargs.pop("enable_mkldnn"))
+
+        with self._create_new_config_file() as config_path:
+            config.dump(config_path)
+
+            self._assert_empty_kwargs(kwargs)
+
+            return self.runner.infer(config_path, [], device)
+
+    def compression(
+        self,
+        weight_path: str,
+        batch_size: int = None,
+        learning_rate: float = None,
+        epochs_iters: int = None,
+        device: str = "gpu",
+        use_vdl: bool = True,
+        save_dir: str = None,
+        **kwargs,
+    ):
+        """compression model"""
+        raise_unsupported_api_error("compression", self.__class__)

paddlex/repo_apis/PaddleVideo_api/video_det/register.py

@@ -0,0 +1,44 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import os.path as osp
+
+from ...base.register import register_model_info, register_suite_info
+from .config import VideoDetConfig
+from .model import VideoDetModel
+from .runner import VideoDetRunner
+
+REPO_ROOT_PATH = os.environ.get("PADDLE_PDX_PADDLEVIDEO_PATH")
+PDX_CONFIG_DIR = osp.abspath(osp.join(osp.dirname(__file__), "..", "configs"))
+
+register_suite_info(
+    {
+        "suite_name": "VideoDet",
+        "model": VideoDetModel,
+        "runner": VideoDetRunner,
+        "config": VideoDetConfig,
+        "runner_root_path": REPO_ROOT_PATH,
+    }
+)
+
+################ Models Using Universal Config ################
+register_model_info(
+    {
+        "model_name": "YOWO",
+        "suite": "VideoDet",
+        "config_path": osp.join(PDX_CONFIG_DIR, "YOWO.yaml"),
+        "supported_apis": ["train", "evaluate", "predict", "export", "infer"],
+    }
+)

paddlex/repo_apis/PaddleVideo_api/video_det/runner.py

@@ -0,0 +1,199 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from ...base import BaseRunner
+from ...base.utils.subprocess import CompletedProcess
+
+
+class VideoDetRunner(BaseRunner):
+    """Cls Runner"""
+
+    def train(
+        self,
+        config_path: str,
+        cli_args: list,
+        device: str,
+        ips: str,
+        save_dir: str,
+        do_eval=True,
+    ) -> CompletedProcess:
+        """train model
+
+        Args:
+            config_path (str): the config file path used to train.
+            cli_args (list): the additional parameters.
+            device (str): the training device.
+            ips (str): the ip addresses of nodes when using distribution.
+            save_dir (str): the directory path to save training output.
+            do_eval (bool, optional): whether or not to evaluate model during training. Defaults to True.
+
+        Returns:
+            CompletedProcess: the result of training subprocess execution.
+        """
+        args, env = self.distributed(device, ips, log_dir=save_dir)
+        cmd = [*args, "main.py", "--validate", "-c", config_path, *cli_args]
+        cmd.extend(["-o", f"Global.eval_during_train={do_eval}"])
+        return self.run_cmd(
+            cmd,
+            env=env,
+            switch_wdir=True,
+            echo=True,
+            silent=False,
+            capture_output=True,
+            log_path=self._get_train_log_path(save_dir),
+        )
+
+    def evaluate(
+        self, config_path: str, cli_args: list, device: str, ips: str
+    ) -> CompletedProcess:
+        """run model evaluating
+
+        Args:
+            config_path (str): the config file path used to evaluate.
+            cli_args (list): the additional parameters.
+            device (str): the evaluating device.
+            ips (str): the ip addresses of nodes when using distribution.
+
+        Returns:
+            CompletedProcess: the result of evaluating subprocess execution.
+        """
+        args, env = self.distributed(device, ips)
+        cmd = [*args, "main.py", "--test", "-c", config_path, *cli_args]
+        cp = self.run_cmd(
+            cmd, env=env, switch_wdir=True, echo=True, silent=False, capture_output=True
+        )
+
+        if cp.returncode == 0:
+            metric_dict = _extract_eval_metrics(cp.stdout)
+            cp.metrics = metric_dict
+        return cp
+
+    def predict(
+        self, config_path: str, cli_args: list, device: str
+    ) -> CompletedProcess:
+        """run predicting using dynamic mode
+
+        Args:
+            config_path (str): the config file path used to predict.
+            cli_args (list): the additional parameters.
+            device (str): unused.
+
+        Returns:
+            CompletedProcess: the result of predicting subprocess execution.
+        """
+        # `device` unused
+        cmd = [self.python, "tools/infer.py", "-c", config_path, *cli_args]
+        return self.run_cmd(cmd, switch_wdir=True, echo=True, silent=False)
+
+    def export(
+        self, config_path: str, cli_args: list, device: str, save_dir: str = None
+    ) -> CompletedProcess:
+        """run exporting
+
+        Args:
+            config_path (str): the path of config file used to export.
+            cli_args (list): the additional parameters.
+            device (str): unused.
+            save_dir (str, optional): the directory path to save exporting output. Defaults to None.
+
+        Returns:
+            CompletedProcess: the result of exporting subprocess execution.
+        """
+        # `device` unused
+
+        cmd = [
+            self.python,
+            "tools/export_model.py",
+            "-c",
+            config_path,
+            *cli_args,
+            "-o",
+            save_dir,
+        ]
+
+        cp = self.run_cmd(cmd, switch_wdir=True, echo=True, silent=False)
+        return cp
+
+    def infer(self, config_path: str, cli_args: list, device: str) -> CompletedProcess:
+        """run predicting using inference model
+
+        Args:
+            config_path (str): the path of config file used to predict.
+            cli_args (list): the additional parameters.
+            device (str): unused.
+
+        Returns:
+            CompletedProcess: the result of infering subprocess execution.
+        """
+        # `device` unused
+        cmd = [self.python, "python/predict_cls.py", "-c", config_path, *cli_args]
+        return self.run_cmd(cmd, switch_wdir="deploy", echo=True, silent=False)
+
+    def compression(
+        self,
+        config_path: str,
+        train_cli_args: list,
+        export_cli_args: list,
+        device: str,
+        train_save_dir: str,
+    ) -> CompletedProcess:
+        """run compression model
+
+        Args:
+            config_path (str): the path of config file used to predict.
+            train_cli_args (list): the additional training parameters.
+            export_cli_args (list): the additional exporting parameters.
+            device (str): the running device.
+            train_save_dir (str): the directory path to save output.
+
+        Returns:
+            CompletedProcess: the result of compression subprocess execution.
+        """
+        # Step 1: Train model
+        cp_train = self.train(config_path, train_cli_args, device, None, train_save_dir)
+
+        # Step 2: Export model
+        weight_path = os.path.join(train_save_dir, "best_model", "model")
+        export_cli_args = [
+            *export_cli_args,
+            "-o",
+            f"Global.pretrained_model={weight_path}",
+        ]
+        cp_export = self.export(config_path, export_cli_args, device)
+
+        return cp_train, cp_export
+
+
+def _extract_eval_metrics(stdout: str) -> dict:
+    """extract evaluation metrics from training log
+
+    Args:
+        stdout (str): the training log
+
+    Returns:
+        dict: the training metric
+    """
+    import re
+
+    pattern = r"mAP:\s*([\d.]+)"
+    compiled_pattern = re.compile(pattern)
+    metric_dict = {}
+    for line in stdout.splitlines():
+        match = compiled_pattern.search(line)
+        if match:
+            fscore_avg_value = float(match.group(1))
+            metric_dict["mAP"] = fscore_avg_value
+    return metric_dict

paddlex/repo_apis/__init__.py

@@ -1,4 +1,4 @@
-# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

paddlex/repo_apis/base/__init__.py

@@ -1,4 +1,4 @@
-# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,11 +13,10 @@
 # limitations under the License.
 
 
-from .config import Config, BaseConfig
-from .model import PaddleModel, BaseModel
-from .runner import BaseRunner, InferOnlyRunner
-
 # Init cache
 from ...utils.cache import create_cache_dir
+from .config import BaseConfig, Config
+from .model import BaseModel, PaddleModel
+from .runner import BaseRunner, InferOnlyRunner
 
 create_cache_dir()

paddlex/repo_apis/base/config.py

@@ -1,4 +1,4 @@
-# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,8 +17,8 @@ import abc
 import collections.abc
 from collections import OrderedDict
 
-from .register import get_registered_model_info, get_registered_suite_info
 from ...utils.errors import UnsupportedParamError
+from .register import get_registered_model_info, get_registered_suite_info
 
 __all__ = ["Config", "BaseConfig"]
 
@@ -132,7 +132,6 @@ class BaseConfig(_Config, metaclass=abc.ABCMeta):
 
     def update_device(self, device):
         """Update the device"""
-        pass
 
     @abc.abstractmethod
     def load(self, config_path):

paddlex/repo_apis/base/model.py

@@ -1,4 +1,4 @@
-# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,32 +13,24 @@
 # limitations under the License.
 
 
-import os
 import abc
-import inspect
-import functools
 import contextlib
+import functools
+import inspect
+import os
 import tempfile
-import hashlib
-import base64
-from datetime import datetime, timedelta
 
+from ...utils import flags, logging
+from ...utils.cache import get_cache_dir
+from ...utils.device import parse_device
+from ...utils.errors import UnsupportedAPIError, UnsupportedParamError
+from ...utils.misc import CachedProperty as cached_property
 from .config import Config
 from .register import (
-    get_registered_model_info,
-    build_runner_from_model_info,
     build_model_from_model_info,
+    build_runner_from_model_info,
+    get_registered_model_info,
 )
-from ...utils import flags
-from ...utils import logging
-from ...utils.errors import (
-    UnsupportedAPIError,
-    UnsupportedParamError,
-    raise_unsupported_api_error,
-)
-from ...utils.device import parse_device
-from ...utils.misc import CachedProperty as cached_property
-from ...utils.cache import get_cache_dir
 
 __all__ = ["PaddleModel", "BaseModel"]
 

paddlex/repo_apis/base/register.py

@@ -1,4 +1,4 @@
-# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.