paddlex 2.0.0rc4__py3-none-any.whl → 3.0.0b2__py3-none-any.whl

paddlex/repo_apis/PaddleTS_api/ts_base/model.py

@@ -0,0 +1,272 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from ...base import BaseModel
+from ...base.utils.arg import CLIArgument
+from ...base.utils.subprocess import CompletedProcess
+from ....utils.device import parse_device
+from ....utils.misc import abspath
+from ....utils.errors import raise_unsupported_api_error
+
+
+class TSModel(BaseModel):
+    """TS Model"""
+
+    def train(
+        self,
+        batch_size: int = None,
+        learning_rate: float = None,
+        epochs_iters: int = None,
+        ips: str = None,
+        device: str = "gpu",
+        resume_path: str = None,
+        dy2st: bool = False,
+        amp: str = "OFF",
+        num_workers: int = None,
+        use_vdl: bool = False,
+        save_dir: str = None,
+        **kwargs,
+    ) -> CompletedProcess:
+        """train self
+
+        Args:
+            batch_size (int, optional): the train batch size value. Defaults to None.
+            learning_rate (float, optional): the train learning rate value. Defaults to None.
+            epochs_iters (int, optional): the train epochs value. Defaults to None.
+            ips (str, optional): the ip addresses of nodes when using distribution. Defaults to None.
+            device (str, optional): the running device. Defaults to 'gpu'.
+            resume_path (str, optional): the checkpoint file path to resume training. Train from scratch if it is set
+                to None. Defaults to None.
+            dy2st (bool, optional): Enable dynamic to static. Defaults to False.
+            amp (str, optional): the amp settings. Defaults to 'OFF'.
+            num_workers (int, optional): the workers number. Defaults to None.
+            use_vdl (bool, optional): enable VisualDL. Defaults to False.
+            save_dir (str, optional): the directory path to save train output. Defaults to None.
+
+        Returns:
+           CompletedProcess: the result of training subprocess execution.
+        """
+        config = self.config.copy()
+        cli_args = []
+        if batch_size is not None:
+            cli_args.append(CLIArgument("--batch_size", batch_size))
+
+        if learning_rate is not None:
+            cli_args.append(CLIArgument("--learning_rate", learning_rate))
+
+        if epochs_iters is not None:
+            cli_args.append(CLIArgument("--epoch", epochs_iters))
+
+        if resume_path:
+            raise ValueError("`resume_path` is not supported.")
+        # No need to handle `ips`
+        if amp is not None and amp != "OFF":
+            raise ValueError(f"`amp`={amp} is not supported.")
+
+        if dy2st:
+            raise ValueError(f"`dy2st`={dy2st} is not supported.")
+
+        if use_vdl:
+            raise ValueError(f"`use_vdl`={use_vdl} is not supported.")
+
+        if device is not None:
+            device_type, _ = parse_device(device)
+            cli_args.append(CLIArgument("--device", device_type))
+
+        if save_dir is not None:
+            save_dir = abspath(save_dir)
+        else:
+            # `save_dir` is None
+            save_dir = abspath(os.path.join("output", "train"))
+        cli_args.append(CLIArgument("--save_dir", save_dir))
+
+        # Benchmarking mode settings
+        benchmark = kwargs.pop("benchmark", None)
+        if benchmark is not None:
+            envs = benchmark.get("env", None)
+            num_workers = benchmark.get("num_workers", None)
+            config.update_log_ranks(device)
+            config.update_print_mem_info(benchmark.get("print_mem_info", True))
+            if num_workers is not None:
+                assert isinstance(num_workers, int), "num_workers must be an integer"
+                cli_args.append(CLIArgument("--num_workers", num_workers))
+            if envs is not None:
+                for env_name, env_value in envs.items():
+                    os.environ[env_name] = str(env_value)
+        else:
+            if num_workers is not None:
+                cli_args.append(CLIArgument("--num_workers", num_workers))
+        # PDX related settings
+        uniform_output_enabled = kwargs.pop("uniform_output_enabled", True)
+        config.update({"uniform_output_enabled": uniform_output_enabled})
+        config.update({"pdx_model_name": self.name})
+
+        self._assert_empty_kwargs(kwargs)
+
+        with self._create_new_config_file() as config_path:
+            config.dump(config_path)
+            return self.runner.train(config_path, cli_args, device, ips, save_dir)
+
+    def evaluate(
+        self,
+        weight_path: str,
+        batch_size: int = None,
+        ips: str = None,
+        device: str = "gpu",
+        amp: str = "OFF",
+        num_workers: int = None,
+        **kwargs,
+    ) -> CompletedProcess:
+        """evaluate self using specified weight
+
+        Args:
+            weight_path (str): the path of model weight file to be evaluated.
+            batch_size (int, optional): the batch size value in evaluating. Defaults to None.
+            ips (str, optional): the ip addresses of nodes when using distribution. Defaults to None.
+            device (str, optional): the running device. Defaults to 'gpu'.
+            amp (str, optional): the AMP setting. Defaults to 'OFF'.
+            num_workers (int, optional): the workers number in evaluating. Defaults to None.
+
+        Returns:
+            CompletedProcess: the result of evaluating subprocess execution.
+        """
+        config = self.config.copy()
+        cli_args = []
+
+        weight_path = abspath(weight_path)
+        cli_args.append(CLIArgument("--checkpoints", weight_path))
+
+        if batch_size is not None:
+            if batch_size != 1:
+                raise ValueError("Batch size other than 1 is not supported.")
+
+        # No need to handle `ips`
+        if device is not None:
+            device_type, _ = parse_device(device)
+            cli_args.append(CLIArgument("--device", device_type))
+
+        if amp is not None:
+            if amp != "OFF":
+                raise ValueError(f"`amp`={amp} is not supported.")
+
+        if num_workers is not None:
+            cli_args.append(CLIArgument("--num_workers", num_workers))
+
+        self._assert_empty_kwargs(kwargs)
+
+        with self._create_new_config_file() as config_path:
+            config.dump(config_path)
+            cp = self.runner.evaluate(config_path, cli_args, device, ips)
+            return cp
+
+    def predict(
+        self,
+        weight_path: str,
+        input_path: str,
+        device: str = "gpu",
+        save_dir: str = None,
+        **kwargs,
+    ) -> CompletedProcess:
+        """predict using specified weight
+
+        Args:
+            weight_path (str): the path of model weight file used to predict.
+            input_path (str): the path of image file to be predicted.
+            device (str, optional): the running device. Defaults to 'gpu'.
+            save_dir (str, optional): the directory path to save predict output. Defaults to None.
+
+        Returns:
+            CompletedProcess: the result of predicting subprocess execution.
+        """
+        config = self.config.copy()
+        cli_args = []
+
+        weight_path = abspath(weight_path)
+        cli_args.append(CLIArgument("--checkpoints", weight_path))
+
+        input_path = abspath(input_path)
+        cli_args.append(CLIArgument("--csv_path", input_path))
+
+        if device is not None:
+            device_type, _ = parse_device(device)
+            cli_args.append(CLIArgument("--device", device_type))
+
+        if save_dir is not None:
+            save_dir = abspath(save_dir)
+        else:
+            # `save_dir` is None
+            save_dir = abspath(os.path.join("output", "predict"))
+        cli_args.append(CLIArgument("--save_dir", save_dir))
+
+        self._assert_empty_kwargs(kwargs)
+
+        with self._create_new_config_file() as config_path:
+            config.dump(config_path)
+            return self.runner.predict(config_path, cli_args, device)
+
+    def export(
+        self, weight_path: str, save_dir: str = None, device: str = "gpu", **kwargs
+    ):
+        """export"""
+        if not weight_path.startswith(("http://", "https://")):
+            weight_path = abspath(weight_path)
+        save_dir = abspath(save_dir)
+        cli_args = []
+
+        cli_args.append(CLIArgument("--checkpoints", weight_path))
+        if save_dir is not None:
+            save_dir = abspath(save_dir)
+        else:
+            save_dir = abspath(os.path.join("output", "inference"))
+        cli_args.append(CLIArgument("--save_dir", save_dir))
+        if device is not None:
+            device_type, _ = parse_device(device)
+            cli_args.append(CLIArgument("--device", device_type))
+
+        self._assert_empty_kwargs(kwargs)
+        with self._create_new_config_file() as config_path:
+            # Update YAML config file
+            config = self.config.copy()
+            config.update_pretrained_weights(weight_path)
+            config.update({"pdx_model_name": self.name})
+            config.dump(config_path)
+
+            return self.runner.export(config_path, cli_args, device)
+
+    def infer(
+        self,
+        model_dir: str,
+        input_path: str,
+        device: str = "gpu",
+        save_dir: str = None,
+        **kwargs,
+    ):
+        """infer"""
+        raise_unsupported_api_error("infer", self.__class__)
+
+    def compression(
+        self,
+        weight_path: str,
+        batch_size=None,
+        learning_rate=None,
+        epochs_iters=None,
+        device: str = "gpu",
+        use_vdl=True,
+        save_dir=None,
+        **kwargs,
+    ):
+        """compression"""
+        raise_unsupported_api_error("compression", self.__class__)

paddlex/repo_apis/PaddleTS_api/ts_base/runner.py

@@ -0,0 +1,158 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from ...base import BaseRunner
+from ...base.utils.arg import gather_opts_args
+from ...base.utils.subprocess import CompletedProcess
+from ....utils.errors import raise_unsupported_api_error
+
+
+class TSRunner(BaseRunner):
+    """TS Runner"""
+
+    def train(
+        self,
+        config_path: str,
+        cli_args: list,
+        device: str,
+        ips: str,
+        save_dir: str,
+        do_eval=True,
+    ) -> CompletedProcess:
+        """train model
+
+        Args:
+            config_path (str): the config file path used to train.
+            cli_args (list): the additional parameters.
+            device (str): the training device.
+            ips (str): the ip addresses of nodes when using distribution.
+            save_dir (str): the directory path to save training output.
+            do_eval (bool, optional): whether or not to evaluate model during training. Defaults to True.
+
+        Returns:
+            CompletedProcess: the result of training subprocess execution.
+        """
+        args, env = self.distributed(device, ips, log_dir=save_dir)
+        cli_args = self._gather_opts_args(cli_args)
+        cmd = [*args, "tools/train.py", "--config", config_path, *cli_args]
+        return self.run_cmd(
+            cmd,
+            env=env,
+            switch_wdir=True,
+            echo=True,
+            silent=False,
+            capture_output=True,
+            log_path=self._get_train_log_path(save_dir),
+        )
+
+    def evaluate(
+        self, config_path: str, cli_args: list, device: str, ips: str
+    ) -> CompletedProcess:
+        """run model evaluating
+
+        Args:
+            config_path (str): the config file path used to evaluate.
+            cli_args (list): the additional parameters.
+            device (str): the evaluating device.
+            ips (str): the ip addresses of nodes when using distribution.
+
+        Returns:
+            CompletedProcess: the result of evaluating subprocess execution.
+        """
+        args, env = self.distributed(device, ips)
+        cli_args = self._gather_opts_args(cli_args)
+        cmd = [*args, "tools/val.py", "--config", config_path, *cli_args]
+
+        cp = self.run_cmd(
+            cmd, env=env, switch_wdir=True, echo=True, silent=False, capture_output=True
+        )
+        if cp.returncode == 0:
+            metric_dict = _extract_eval_metrics(cp.stderr)
+            cp.metrics = metric_dict
+        return cp
+
+    def predict(
+        self, config_path: str, cli_args: list, device: str
+    ) -> CompletedProcess:
+        """run predicting using dynamic mode
+
+        Args:
+            config_path (str): the config file path used to predict.
+            cli_args (list): the additional parameters.
+            device (str): unused.
+
+        Returns:
+            CompletedProcess: the result of predicting subprocess execution.
+        """
+        # `device` unused
+        cli_args = self._gather_opts_args(cli_args)
+        cmd = [self.python, "tools/predict.py", "--config", config_path, *cli_args]
+        return self.run_cmd(cmd, switch_wdir=True, echo=True, silent=False)
+
+    def export(self, config_path, cli_args, device):
+        """export"""
+        cmd = [
+            self.python,
+            "tools/export.py",
+            "--config",
+            config_path,
+            *cli_args,
+        ]
+        cp = self.run_cmd(cmd, switch_wdir=True, echo=True, silent=False)
+        return cp
+
+    def infer(self, config_path, cli_args, device):
+        """infer"""
+        raise_unsupported_api_error("infer", self.__class__)
+
+    def compression(
+        self, config_path, train_cli_args, export_cli_args, device, train_save_dir
+    ):
+        """compression"""
+        raise_unsupported_api_error("compression", self.__class__)
+
+    def _gather_opts_args(self, args):
+        # Since `--opts` in PaddleSeg does not use `action='append'`
+        # We collect and arrange all opts args here
+        # e.g.: python tools/train.py --config xxx --opts a=1 c=3 --opts b=2
+        # => python tools/train.py --config xxx c=3 --opts a=1 b=2
+        return gather_opts_args(args, "--opts")
+
+
+def _extract_eval_metrics(stdout: str) -> dict:
+    """extract evaluation metrics from training log
+
+    Args:
+        stdout (str): the training log
+
+    Returns:
+        dict: the training metric
+    """
+    import re
+
+    pattern = r"\'mse\':\s+(\d+\.\d+),+[\s|\n]+\'mae\':\s+(\d+\.\d+)"
+    keys = ["mse", "mae"]
+
+    metric_dict = dict()
+    pattern = re.compile(pattern)
+
+    lines = stdout.splitlines()
+    for line in lines:
+        match = pattern.search(line)
+        if match:
+            for k, v in zip(keys, map(float, match.groups())):
+                metric_dict[k] = v
+
+    return metric_dict

paddlex/repo_apis/PaddleTS_api/ts_cls/__init__.py

@@ -0,0 +1,16 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from . import register

paddlex/repo_apis/PaddleTS_api/ts_cls/config.py

@@ -0,0 +1,73 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import yaml
+
+from ..ts_base.config import BaseTSConfig
+from ....utils.misc import abspath
+
+
+class TSClassifyConfig(BaseTSConfig):
+    """TS Classify Config"""
+
+    def update_dataset(self, dataset_dir: str, dataset_type: str = None):
+        """
+        upadte the dataset
+
+        Args:
+            dataset_dir (str): dataset root path
+            dataset_type (str, optional): type to set for dataset. Default='TSDataset'
+        """
+        if dataset_type is None:
+            dataset_type = "TSCLSDataset"
+        dataset_dir = abspath(dataset_dir)
+        ds_cfg = self._make_custom_dataset_config(dataset_dir)
+        self.update(ds_cfg)
+
+    def update_basic_info(self, info_params: dict):
+        """
+        update basic info including time_col, freq, target_cols.
+
+        Args:
+            info_params (dict): upadte basic info
+
+        Raises:
+            TypeError: if info_params is not dict, raising TypeError
+        """
+        if isinstance(info_params, dict):
+            self.update({"info_params": info_params})
+        else:
+            raise TypeError("`info_params` must be a dict.")
+
+    def _make_custom_dataset_config(self, dataset_root_path: str):
+        """construct the dataset config that meets the format requirements
+
+        Args:
+            dataset_root_path (str): the root directory of dataset.
+
+        Returns:
+            dict: the dataset config.
+        """
+        ds_cfg = {
+            "dataset": {
+                "name": "TSCLSDataset",
+                "dataset_root": dataset_root_path,
+                "train_path": os.path.join(dataset_root_path, "train.csv"),
+                "val_path": os.path.join(dataset_root_path, "val.csv"),
+            },
+        }
+
+        return ds_cfg

paddlex/repo_apis/PaddleTS_api/ts_cls/register.py

@@ -0,0 +1,59 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import os.path as osp
+
+from ...base.register import register_model_info, register_suite_info
+from ..ts_base.model import TSModel
+from .runner import TSCLSRunner
+from .config import TSClassifyConfig
+
+REPO_ROOT_PATH = os.environ.get("PADDLE_PDX_PADDLETS_PATH")
+PDX_CONFIG_DIR = osp.abspath(osp.join(osp.dirname(__file__), "..", "configs"))
+
+register_suite_info(
+    {
+        "suite_name": "TSClassify",
+        "model": TSModel,
+        "runner": TSCLSRunner,
+        "config": TSClassifyConfig,
+        "runner_root_path": REPO_ROOT_PATH,
+    }
+)
+
+################ Models Using Universal Config ################
+
+# timesnet
+TimesNetCLS_CFG_PATH = osp.join(PDX_CONFIG_DIR, "TimesNet_cls.yaml")
+register_model_info(
+    {
+        "model_name": "TimesNet_cls",
+        "suite": "TSClassify",
+        "config_path": TimesNetCLS_CFG_PATH,
+        "supported_apis": ["train", "evaluate", "predict", "export"],
+        "supported_train_opts": {
+            "device": ["cpu", "gpu_n1cx", "xpu", "npu", "mlu"],
+            "dy2st": False,
+            "amp": [],
+        },
+        "supported_evaluate_opts": {
+            "device": ["cpu", "gpu_n1cx", "xpu", "npu", "mlu"],
+            "amp": [],
+        },
+        "supported_predict_opts": {"device": ["cpu", "gpu", "xpu", "npu", "mlu"]},
+        "supported_infer_opts": {"device": ["cpu", "gpu", "xpu", "npu", "mlu"]},
+    }
+)