paddlex 2.0.0rc4__py3-none-any.whl → 3.0.0b2__py3-none-any.whl

paddlex/repo_apis/PaddleOCR_api/text_rec/runner.py

@@ -0,0 +1,240 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import tempfile
+
+from ...base import BaseRunner
+from ...base.utils.subprocess import CompletedProcess
+
+
+class TextRecRunner(BaseRunner):
+    """Text Recognition Runner"""
+
+    def train(
+        self,
+        config_path: str,
+        cli_args: list,
+        device: str,
+        ips: str,
+        save_dir: str,
+        do_eval=True,
+    ) -> CompletedProcess:
+        """train model
+
+        Args:
+            config_path (str): the config file path used to train.
+            cli_args (list): the additional parameters.
+            device (str): the training device.
+            ips (str): the ip addresses of nodes when using distribution.
+            save_dir (str): the directory path to save training output.
+            do_eval (bool, optional): whether or not to evaluate model during training. Defaults to True.
+
+        Returns:
+            CompletedProcess: the result of training subprocess execution.
+        """
+        args, env = self.distributed(device, ips, log_dir=save_dir)
+        cmd = [*args, "tools/train.py", "-c", config_path, *cli_args]
+        if do_eval:
+            # We simply pass here because in PaddleOCR periodic evaluation cannot be switched off
+            pass
+        else:
+            inf = int(1.0e11)
+            cmd.extend(["-o", f"Global.eval_batch_step={inf}"])
+        return self.run_cmd(
+            cmd,
+            env=env,
+            switch_wdir=True,
+            echo=True,
+            silent=False,
+            capture_output=True,
+            log_path=self._get_train_log_path(save_dir),
+        )
+
+    def evaluate(
+        self, config_path: str, cli_args: list, device: str, ips: str
+    ) -> CompletedProcess:
+        """run model evaluating
+
+        Args:
+            config_path (str): the config file path used to evaluate.
+            cli_args (list): the additional parameters.
+            device (str): the evaluating device.
+            ips (str): the ip addresses of nodes when using distribution.
+
+        Returns:
+            CompletedProcess: the result of evaluating subprocess execution.
+        """
+        args, env = self.distributed(device, ips)
+        cmd = [*args, "tools/eval.py", "-c", config_path]
+
+        cp = self.run_cmd(
+            cmd, env=env, switch_wdir=True, echo=True, silent=False, capture_output=True
+        )
+        if cp.returncode == 0:
+            metric_dict = _extract_eval_metrics(cp.stdout)
+            cp.metrics = metric_dict
+        return cp
+
+    def predict(
+        self, config_path: str, cli_args: list, device: str
+    ) -> CompletedProcess:
+        """run predicting using dynamic mode
+
+        Args:
+            config_path (str): the config file path used to predict.
+            cli_args (list): the additional parameters.
+            device (str): unused.
+
+        Returns:
+            CompletedProcess: the result of predicting subprocess execution.
+        """
+        cmd = [self.python, "tools/infer_rec.py", "-c", config_path]
+        return self.run_cmd(cmd, switch_wdir=True, echo=True, silent=False)
+
+    def export(
+        self, config_path: str, cli_args: list, device: str, save_dir: str = None
+    ) -> CompletedProcess:
+        """run exporting
+
+        Args:
+            config_path (str): the path of config file used to export.
+            cli_args (list): the additional parameters.
+            device (str): unused.
+            save_dir (str, optional): the directory path to save exporting output. Defaults to None.
+
+        Returns:
+            CompletedProcess: the result of exporting subprocess execution.
+        """
+        # `device` unused
+        cmd = [self.python, "tools/export_model.py", "-c", config_path]
+        cp = self.run_cmd(cmd, switch_wdir=True, echo=True, silent=False)
+        return cp
+
+    def infer(self, config_path: str, cli_args: list, device: str) -> CompletedProcess:
+        """run predicting using inference model
+
+        Args:
+            config_path (str): the path of config file used to predict.
+            cli_args (list): the additional parameters.
+            device (str): unused.
+
+        Returns:
+            CompletedProcess: the result of infering subprocess execution.
+        """
+        cmd = [self.python, "tools/infer/predict_rec.py", *cli_args]
+        return self.run_cmd(cmd, switch_wdir=True, echo=True, silent=False)
+
+    def compression(
+        self,
+        config_path: str,
+        train_cli_args: list,
+        export_cli_args: list,
+        device: str,
+        train_save_dir: str,
+    ) -> CompletedProcess:
+        """run compression model
+
+        Args:
+            config_path (str): the path of config file used to predict.
+            train_cli_args (list): the additional training parameters.
+            export_cli_args (list): the additional exporting parameters.
+            device (str): the running device.
+            train_save_dir (str): the directory path to save output.
+
+        Returns:
+            CompletedProcess: the result of compression subprocess execution.
+        """
+        # Step 1: Train model
+        args, env = self.distributed(device, log_dir=train_save_dir)
+        cmd = [*args, "deploy/slim/quantization/quant.py", "-c", config_path]
+        cp_train = self.run_cmd(
+            cmd,
+            env=env,
+            switch_wdir=True,
+            echo=True,
+            silent=False,
+            capture_output=True,
+            log_path=self._get_train_log_path(train_save_dir),
+        )
+
+        # Step 2: Export model
+        export_cli_args = [
+            *export_cli_args,
+            "-o",
+            f"Global.checkpoints={train_save_dir}/latest",
+        ]
+        cmd = [
+            self.python,
+            "deploy/slim/quantization/export_model.py",
+            "-c",
+            config_path,
+            *export_cli_args,
+        ]
+        cp_export = self.run_cmd(cmd, switch_wdir=True, echo=True, silent=False)
+
+        return cp_train, cp_export
+
+
+def _extract_eval_metrics(stdout: str) -> dict:
+    """extract evaluation metrics from training log
+
+    Args:
+        stdout (str): the training log
+
+    Returns:
+        dict: the training metric
+    """
+    import re
+
+    def _lazy_split_lines(s):
+        prev_idx = 0
+        while True:
+            curr_idx = s.find(os.linesep, prev_idx)
+            if curr_idx == -1:
+                curr_idx = len(s)
+            yield s[prev_idx:curr_idx]
+            prev_idx = curr_idx + len(os.linesep)
+            if prev_idx >= len(s):
+                break
+
+    _DP = r"[-+]?[0-9]*\.?[0-9]+(?:[eE][-+]?[0-9]+)?"
+    pattern_key_pairs = [
+        (re.compile(r"acc:(_dp)$".replace("_dp", _DP)), "acc"),
+        (re.compile(r"norm_edit_dis:(_dp)$".replace("_dp", _DP)), "norm_edit_dis"),
+        (re.compile(r"Teacher_acc:(_dp)$".replace("_dp", _DP)), "teacher_acc"),
+        (
+            re.compile(r"Teacher_norm_edit_dis:(_dp)$".replace("_dp", _DP)),
+            "teacher_norm_edit_dis",
+        ),
+        (re.compile(r"precision:(_dp)$".replace("_dp", _DP)), "precision"),
+        (re.compile(r"recall:(_dp)$".replace("_dp", _DP)), "recall"),
+        (re.compile(r"hmean:(_dp)$".replace("_dp", _DP)), "hmean"),
+        (re.compile(r"exp_rate:(_dp)$".replace("_dp", _DP)), "exp_rate"),
+    ]
+
+    metric_dict = dict()
+    start_match = False
+    for line in _lazy_split_lines(stdout):
+        if "metric eval" in line:
+            start_match = True
+        if start_match:
+            for pattern, key in pattern_key_pairs:
+                match = pattern.search(line)
+                if match:
+                    assert len(match.groups()) == 1
+                    # Newer overwrites older
+                    metric_dict[key] = float(match.group(1))
+    return metric_dict

paddlex/repo_apis/PaddleSeg_api/__init__.py

@@ -0,0 +1,16 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from .seg import register

paddlex/repo_apis/PaddleSeg_api/base_seg_config.py

@@ -0,0 +1,134 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from urllib.parse import urlparse
+
+import yaml
+
+from ..base import BaseConfig
+from ...utils.misc import abspath
+
+
+class BaseSegConfig(BaseConfig):
+    """BaseSegConfig"""
+
+    def update(self, dict_like_obj):
+        """update"""
+        from paddleseg.cvlibs.config import merge_config_dicts
+
+        dict_ = merge_config_dicts(dict_like_obj, self.dict)
+        self.reset_from_dict(dict_)
+
+    def load(self, config_path):
+        """load"""
+        from paddleseg.cvlibs.config import parse_from_yaml
+
+        dict_ = parse_from_yaml(config_path)
+        if not isinstance(dict_, dict):
+            raise TypeError
+        self.reset_from_dict(dict_)
+
+    def dump(self, config_path):
+        """dump"""
+        from paddleseg.utils import NoAliasDumper
+
+        with open(config_path, "w", encoding="utf-8") as f:
+            yaml.dump(self.dict, f, Dumper=NoAliasDumper)
+
+    def update_learning_rate(self, learning_rate):
+        """update_learning_rate"""
+        if "lr_scheduler" not in self:
+            raise RuntimeError(
+                "Not able to update learning rate, because no LR scheduler config was found."
+            )
+        self.lr_scheduler["learning_rate"] = learning_rate
+
+    def update_batch_size(self, batch_size, mode="train"):
+        """update_batch_size"""
+        if mode == "train":
+            self.set_val("batch_size", batch_size)
+        else:
+            raise ValueError(
+                f"Setting `batch_size` in {repr(mode)} mode is not supported."
+            )
+
+    def update_log_ranks(self, device):
+        """update log ranks
+
+        Args:
+            device (str): the running device to set
+        """
+        log_ranks = device.split(":")[1]
+        self.set_val("log_ranks", log_ranks)
+
+    def update_print_mem_info(self, print_mem_info: bool):
+        """setting print memory info"""
+        assert isinstance(print_mem_info, bool), "print_mem_info should be a bool"
+        self.set_val("print_mem_info", print_mem_info)
+
+    def update_shuffle(self, shuffle: bool):
+        """setting print memory info"""
+        assert isinstance(shuffle, bool), "shuffle should be a bool"
+        self.set_val("shuffle", shuffle)
+
+    def update_pretrained_weights(self, weight_path, is_backbone=False):
+        """update_pretrained_weights"""
+        if "model" not in self:
+            raise RuntimeError(
+                "Not able to update pretrained weight path, because no model config was found."
+            )
+        if isinstance(weight_path, str):
+            if urlparse(weight_path).scheme == "":
+                # If `weight_path` is a string but not URL (with scheme present),
+                # it will be recognized as a local file path.
+                weight_path = abspath(weight_path)
+        else:
+            if weight_path is not None:
+                raise TypeError("`weight_path` must be string or None.")
+        if is_backbone:
+            if "backbone" not in self.model:
+                raise RuntimeError(
+                    "Not able to update pretrained weight path of backbone, because no backbone config was found."
+                )
+            self.model["backbone"]["pretrained"] = weight_path
+        else:
+            self.model["pretrained"] = weight_path
+
+    def update_dy2st(self, dy2st):
+        """update_dy2st"""
+        self.set_val("to_static_training", dy2st)
+
+    def update_dataset(self, dataset_dir, dataset_type=None):
+        """update_dataset"""
+        raise NotImplementedError
+
+    def get_epochs_iters(self):
+        """get_epochs_iters"""
+        raise NotImplementedError
+
+    def get_learning_rate(self):
+        """get_learning_rate"""
+        raise NotImplementedError
+
+    def get_batch_size(self, mode="train"):
+        """get_batch_size"""
+        raise NotImplementedError
+
+    def get_qat_epochs_iters(self):
+        """get_qat_epochs_iters"""
+        return self.get_epochs_iters() // 2
+
+    def get_qat_learning_rate(self):
+        """get_qat_learning_rate"""
+        return self.get_learning_rate() / 2

paddlex/repo_apis/PaddleSeg_api/seg/__init__.py

@@ -0,0 +1,16 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from . import register

paddlex/repo_apis/PaddleSeg_api/seg/config.py

@@ -0,0 +1,177 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+from functools import lru_cache
+
+import yaml
+from typing import Union
+
+from ..base_seg_config import BaseSegConfig
+from ....utils.misc import abspath
+from ....utils import logging
+
+
+class SegConfig(BaseSegConfig):
+    """Semantic Segmentation Config"""
+
+    def update_dataset(self, dataset_path: str, dataset_type: str = None):
+        """update dataset settings
+
+        Args:
+            dataset_path (str): the root path of dataset.
+            dataset_type (str, optional): dataset type. Defaults to None.
+
+        Raises:
+            ValueError: the dataset_type error.
+        """
+        dataset_dir = abspath(dataset_path)
+        if dataset_type is None:
+            dataset_type = "SegDataset"
+        if dataset_type == "SegDataset":
+            # TODO: Prune extra keys
+            ds_cfg = self._make_custom_dataset_config(dataset_dir)
+            self.update(ds_cfg)
+        elif dataset_type == "_dummy":
+            # XXX: A special dataset type to tease PaddleSeg val dataset checkers
+            self.update(
+                {
+                    "val_dataset": {
+                        "type": "SegDataset",
+                        "dataset_root": dataset_dir,
+                        "val_path": os.path.join(dataset_dir, "val.txt"),
+                        "mode": "val",
+                    },
+                }
+            )
+        else:
+            raise ValueError(f"{repr(dataset_type)} is not supported.")
+
+    def update_num_classes(self, num_classes: int):
+        """update classes number
+
+        Args:
+            num_classes (int): the classes number value to set.
+        """
+        if "train_dataset" in self:
+            self.train_dataset["num_classes"] = num_classes
+        if "val_dataset" in self:
+            self.val_dataset["num_classes"] = num_classes
+        if "model" in self:
+            self.model["num_classes"] = num_classes
+
+    def update_train_crop_size(self, crop_size: Union[int, list]):
+        """update the image cropping size of training preprocessing
+
+        Args:
+            crop_size (int | list): the size of image to be cropped.
+
+        Raises:
+            ValueError: the `crop_size` error.
+        """
+        # XXX: This method is highly coupled to PaddleSeg's internal functions
+        if isinstance(crop_size, int):
+            crop_size = [crop_size, crop_size]
+        else:
+            crop_size = list(crop_size)
+            if len(crop_size) != 2:
+                raise ValueError
+            crop_size = [int(crop_size[0]), int(crop_size[1])]
+
+        tf_cfg_list = self.train_dataset["transforms"]
+        modified = False
+        for tf_cfg in tf_cfg_list:
+            if tf_cfg["type"] == "RandomPaddingCrop":
+                tf_cfg["crop_size"] = crop_size
+                modified = True
+        if not modified:
+            logging.warning(
+                "Could not find configuration item of image cropping transformation operator. "
+                "Therefore, the crop size was not updated."
+            )
+
+    def get_epochs_iters(self) -> int:
+        """get epochs
+
+        Returns:
+            int: the epochs value, i.e., `Global.epochs` in config.
+        """
+        if "iters" in self:
+            return self.iters
+        else:
+            # Default iters
+            return 1000
+
+    def get_learning_rate(self) -> float:
+        """get learning rate
+
+        Returns:
+            float: the learning rate value, i.e., `Optimizer.lr.learning_rate` in config.
+        """
+        if "lr_scheduler" not in self or "learning_rate" not in self.lr_scheduler:
+            # Default lr
+            return 0.0001
+        else:
+            return self.lr_scheduler["learning_rate"]
+
+    def get_batch_size(self, mode="train") -> int:
+        """get batch size
+
+        Args:
+            mode (str, optional): the mode that to be get batch size value, must be one of 'train', 'eval', 'test'.
+                Defaults to 'train'.
+
+        Raises:
+            ValueError: the `mode` error. `train` is supported only.
+
+        Returns:
+            int: the batch size value of `mode`, i.e., `DataLoader.{mode}.sampler.batch_size` in config.
+        """
+        if mode == "train":
+            if "batch_size" in self:
+                return self.batch_size
+            else:
+                # Default batch size
+                return 4
+        else:
+            raise ValueError(
+                f"Getting `batch_size` in {repr(mode)} mode is not supported."
+            )
+
+    def _make_custom_dataset_config(self, dataset_root_path: str) -> dict:
+        """construct the dataset config that meets the format requirements
+
+        Args:
+            dataset_root_path (str): the root directory of dataset.
+
+        Returns:
+            dict: the dataset config.
+        """
+        ds_cfg = {
+            "train_dataset": {
+                "type": "SegDataset",
+                "dataset_root": dataset_root_path,
+                "train_path": os.path.join(dataset_root_path, "train.txt"),
+                "mode": "train",
+            },
+            "val_dataset": {
+                "type": "SegDataset",
+                "dataset_root": dataset_root_path,
+                "val_path": os.path.join(dataset_root_path, "val.txt"),
+                "mode": "val",
+            },
+        }
+
+        return ds_cfg