paddlex 2.0.0rc4__py3-none-any.whl → 3.0.0b2__py3-none-any.whl

paddlex/repo_apis/PaddleDetection_api/object_det/runner.py

@@ -0,0 +1,226 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import tempfile
+
+from ...base import BaseRunner
+from ...base.utils.arg import CLIArgument, gather_opts_args
+from ...base.utils.subprocess import CompletedProcess
+
+
+class DetRunner(BaseRunner):
+    """DetRunner"""
+
+    def train(
+        self,
+        config_path: str,
+        cli_args: list,
+        device: str,
+        ips: str,
+        save_dir: str,
+        do_eval=True,
+    ) -> CompletedProcess:
+        """train model
+
+        Args:
+            config_path (str): the config file path used to train.
+            cli_args (list): the additional parameters.
+            device (str): the training device.
+            ips (str): the ip addresses of nodes when using distribution.
+            save_dir (str): the directory path to save training output.
+            do_eval (bool, optional): whether or not to evaluate model during training. Defaults to True.
+
+        Returns:
+            CompletedProcess: the result of training subprocess execution.
+        """
+        args, env = self.distributed(device, ips, log_dir=save_dir)
+        cli_args = self._gather_opts_args(cli_args)
+        cmd = [*args, "tools/train.py"]
+        if do_eval:
+            cmd.append("--eval")
+        cmd.extend(["--config", config_path, *cli_args])
+        return self.run_cmd(
+            cmd,
+            env=env,
+            switch_wdir=True,
+            echo=True,
+            silent=False,
+            capture_output=True,
+            log_path=self._get_train_log_path(save_dir),
+        )
+
+    def evaluate(
+        self, config_path: str, cli_args: list, device: str, ips: str
+    ) -> CompletedProcess:
+        """run model evaluating
+
+        Args:
+            config_path (str): the config file path used to evaluate.
+            cli_args (list): the additional parameters.
+            device (str): the evaluating device.
+            ips (str): the ip addresses of nodes when using distribution.
+
+        Returns:
+            CompletedProcess: the result of evaluating subprocess execution.
+        """
+        args, env = self.distributed(device, ips)
+        cli_args = self._gather_opts_args(cli_args)
+        cmd = [*args, "tools/eval.py", "--config", config_path, *cli_args]
+        cp = self.run_cmd(
+            cmd, env=env, switch_wdir=True, echo=True, silent=False, capture_output=True
+        )
+        if cp.returncode == 0:
+            metric_dict = _extract_eval_metrics(cp.stdout)
+            cp.metrics = metric_dict
+        return cp
+
+    def predict(
+        self, config_path: str, cli_args: list, device: str
+    ) -> CompletedProcess:
+        """run predicting using dynamic mode
+
+        Args:
+            config_path (str): the config file path used to predict.
+            cli_args (list): the additional parameters.
+            device (str): unused.
+
+        Returns:
+            CompletedProcess: the result of predicting subprocess execution.
+        """
+        # `device` unused
+        cli_args = self._gather_opts_args(cli_args)
+        cmd = [self.python, "tools/infer.py", "-c", config_path, *cli_args]
+        return self.run_cmd(cmd, switch_wdir=True, echo=True, silent=False)
+
+    def export(self, config_path: str, cli_args: list, device: str) -> CompletedProcess:
+        """run exporting
+
+        Args:
+            config_path (str): the path of config file used to export.
+            cli_args (list): the additional parameters.
+            device (str): unused.
+            save_dir (str, optional): the directory path to save exporting output. Defaults to None.
+
+        Returns:
+            CompletedProcess: the result of exporting subprocess execution.
+        """
+        # `device` unused
+        cli_args = self._gather_opts_args(cli_args)
+        cmd = [
+            self.python,
+            "tools/export_model.py",
+            "--for_fd",
+            "-c",
+            config_path,
+            *cli_args,
+        ]
+
+        cp = self.run_cmd(cmd, switch_wdir=True, echo=True, silent=False)
+        return cp
+
+    def infer(self, cli_args: list, device: str) -> CompletedProcess:
+        """run predicting using inference model
+
+        Args:
+            cli_args (list): the additional parameters.
+            device (str): unused.
+
+        Returns:
+            CompletedProcess: the result of infering subprocess execution.
+        """
+        # `device` unused
+        cmd = [self.python, "deploy/python/infer.py", "--use_fd_format", *cli_args]
+        return self.run_cmd(cmd, switch_wdir=True, echo=True, silent=False)
+
+    def compression(
+        self,
+        config_path: str,
+        train_cli_args: list,
+        export_cli_args: list,
+        device: str,
+        train_save_dir: str,
+    ) -> CompletedProcess:
+        """run compression model
+
+        Args:
+            config_path (str): the path of config file used to predict.
+            train_cli_args (list): the additional training parameters.
+            export_cli_args (list): the additional exporting parameters.
+            device (str): the running device.
+            train_save_dir (str): the directory path to save output.
+
+        Returns:
+            CompletedProcess: the result of compression subprocess execution.
+        """
+        args, env = self.distributed(device, log_dir=train_save_dir)
+        train_cli_args = self._gather_opts_args(train_cli_args)
+        cmd = [*args, "tools/train.py", "-c", config_path, *train_cli_args]
+        cp_train = self.run_cmd(
+            cmd,
+            env=env,
+            switch_wdir=True,
+            echo=True,
+            silent=False,
+            capture_output=True,
+            log_path=self._get_train_log_path(train_save_dir),
+        )
+
+        cps_weight_path = os.path.join(train_save_dir, "model_final")
+
+        export_cli_args.append(CLIArgument("-o", f"weights={cps_weight_path}"))
+        export_cli_args = self._gather_opts_args(export_cli_args)
+        cmd = [
+            self.python,
+            "tools/export_model.py",
+            "--for_fd",
+            "-c",
+            config_path,
+            *export_cli_args,
+        ]
+        cp_export = self.run_cmd(cmd, switch_wdir=True, echo=True, silent=False)
+
+        return cp_train, cp_export
+
+    def _gather_opts_args(self, args):
+        """_gather_opts_args"""
+        return gather_opts_args(args, "-o")
+
+
+def _extract_eval_metrics(stdout):
+    """extract evaluation metrics from training log
+
+    Args:
+        stdout (str): the training log
+
+    Returns:
+        dict: the training metric
+    """
+    import re
+
+    pattern = r".*\(AP\)\s*@\[\s*IoU=0\.50:0\.95\s*\|\s*area=\s*all\s\|\smaxDets=\s*\d+\s\]\s*=\s*[0-1]?\.[0-9]{3}$"
+    key = "AP"
+
+    metric_dict = dict()
+    pattern = re.compile(pattern)
+    # TODO: Use lazy version to make it more efficient
+    lines = stdout.splitlines()
+    metric_dict[key] = 0
+    for line in lines:
+        match = pattern.search(line)
+        if match:
+            metric_dict[key] = float(match.group(0)[-5:])
+
+    return metric_dict

paddlex/repo_apis/PaddleNLP_api/__init__.py

@@ -0,0 +1,13 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

paddlex/repo_apis/PaddleOCR_api/__init__.py

@@ -0,0 +1,20 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Register models and architectures
+# task
+from .text_det import register
+from .text_rec import register
+from .table_rec import register

paddlex/repo_apis/PaddleOCR_api/config_utils.py

@@ -0,0 +1,53 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+
+import yaml
+
+
+def load_config(file_path):
+    """load_config"""
+    # Refer to https://github.com/PaddlePaddle/PaddleOCR/blob/366ad29d6c202a79bad103c72556c1186915c9c8/tools/program.py#L75
+    _, ext = os.path.splitext(file_path)
+    assert ext in [".yml", ".yaml"], "only support yaml files for now"
+    config = yaml.load(open(file_path, "rb"), Loader=yaml.Loader)
+    return config
+
+
+def merge_config(config, opts):
+    """merge_config"""
+    # Refer to https://github.com/PaddlePaddle/PaddleOCR/blob/366ad29d6c202a79bad103c72556c1186915c9c8/tools/program.py#L88
+    for key, value in opts.items():
+        if "." not in key:
+            if isinstance(value, dict) and key in config:
+                config[key].update(value)
+            else:
+                config[key] = value
+        else:
+            sub_keys = key.split(".")
+            assert sub_keys[0] in config, (
+                "the sub_keys can only be one of global_config: {}, but get: "
+                "{}, please check your running command".format(
+                    config.keys(), sub_keys[0]
+                )
+            )
+            cur = config[sub_keys[0]]
+            for idx, sub_key in enumerate(sub_keys[1:]):
+                if idx == len(sub_keys) - 2:
+                    cur[sub_key] = value
+                else:
+                    cur = cur[sub_key]
+    return config

paddlex/repo_apis/PaddleOCR_api/table_rec/__init__.py

@@ -0,0 +1,16 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from . import register

paddlex/repo_apis/PaddleOCR_api/table_rec/config.py

@@ -0,0 +1,64 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+
+from ....utils.misc import abspath
+from ..text_rec.config import TextRecConfig
+
+
+class TableRecConfig(TextRecConfig):
+    """Table Recognition Config"""
+
+    def update_dataset(self, dataset_path: str, dataset_type: str = None):
+        """update dataset settings
+
+        Args:
+            dataset_path (str): the root path of dataset.
+            dataset_type (str, optional): dataset type. Defaults to None.
+
+        Raises:
+            ValueError: the dataset_type error.
+        """
+        dataset_path = abspath(dataset_path)
+        if dataset_type is None:
+            dataset_type = "PubTabTableRecDataset"
+        if dataset_type == "PubTabTableRecDataset":
+            _cfg = {
+                "Train.dataset.name": dataset_type,
+                "Train.dataset.data_dir": dataset_path,
+                "Train.dataset.label_file_list": [
+                    os.path.join(dataset_path, "train.txt")
+                ],
+                "Eval.dataset.name": dataset_type,
+                "Eval.dataset.data_dir": dataset_path,
+                "Eval.dataset.label_file_list": [os.path.join(dataset_path, "val.txt")],
+            }
+            self.update(_cfg)
+        else:
+            raise ValueError(f"{repr(dataset_type)} is not supported.")
+
+    def _get_infer_shape(self):
+        """get resize scale of ResizeImg operation in the evaluation
+
+        Returns:
+            str: resize scale, i.e. `Eval.dataset.transforms.ResizeTableImage.max_len`
+        """
+        size = None
+        transforms = self.dict["Eval"]["dataset"]["transforms"]
+        for op in transforms:
+            if list(op)[0] == "ResizeTableImage":
+                size = op["ResizeTableImage"]["max_len"]
+        return size

paddlex/repo_apis/PaddleOCR_api/table_rec/model.py

@@ -0,0 +1,126 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from ....utils import logging
+from ...base.utils.arg import CLIArgument
+from ...base.utils.subprocess import CompletedProcess
+from ....utils.device import parse_device
+from ....utils.misc import abspath
+from ..text_rec.model import TextRecModel
+
+
+class TableRecModel(TextRecModel):
+    """Table Recognition Model"""
+
+    METRICS = ["acc"]
+
+    def predict(
+        self,
+        weight_path: str,
+        input_path: str,
+        device: str = "gpu",
+        save_dir: str = None,
+        **kwargs
+    ) -> CompletedProcess:
+        """predict using specified weight
+
+        Args:
+            weight_path (str): the path of model weight file used to predict.
+            input_path (str): the path of image file to be predicted.
+            device (str, optional): the running device. Defaults to 'gpu'.
+            save_dir (str, optional): the directory path to save predict output. Defaults to None.
+
+        Returns:
+            CompletedProcess: the result of predicting subprocess execution.
+        """
+        config = self.config.copy()
+        weight_path = abspath(weight_path)
+        config.update_pretrained_weights(weight_path)
+
+        input_path = abspath(input_path)
+        config._update_infer_img(input_path)
+
+        # TODO: Handle `device`
+        logging.warning("`device` will not be used.")
+
+        if save_dir is not None:
+            save_dir = abspath(save_dir)
+        else:
+            save_dir = abspath(config.get_predict_save_dir())
+        config._update_save_res_path(save_dir)
+
+        self._assert_empty_kwargs(kwargs)
+
+        with self._create_new_config_file() as config_path:
+            config.dump(config_path)
+            return self.runner.predict(config_path, [], device)
+
+    def infer(
+        self,
+        model_dir: str,
+        input_path: str,
+        device: str = "gpu",
+        save_dir: str = None,
+        **kwargs
+    ) -> CompletedProcess:
+        """predict image using infernece model
+
+        Args:
+            model_dir (str): the directory path of inference model files that would use to predict.
+            input_path (str): the path of image that would be predict.
+            device (str, optional): the running device. Defaults to 'gpu'.
+            save_dir (str, optional): the directory path to save output. Defaults to None.
+
+        Returns:
+            CompletedProcess: the result of infering subprocess execution.
+        """
+        config = self.config.copy()
+        cli_args = []
+
+        model_dir = abspath(model_dir)
+        cli_args.append(CLIArgument("--table_model_dir", model_dir))
+
+        input_path = abspath(input_path)
+        cli_args.append(CLIArgument("--image_dir", input_path))
+
+        device_type, _ = parse_device(device)
+        cli_args.append(CLIArgument("--use_gpu", str(device_type == "gpu")))
+
+        if save_dir is not None:
+            save_dir = abspath(save_dir)
+        else:
+            # `save_dir` is None
+            save_dir = abspath(os.path.join("output", "infer"))
+        cli_args.append(CLIArgument("--output", save_dir))
+
+        dict_path = kwargs.pop("dict_path", None)
+        if dict_path is not None:
+            dict_path = abspath(dict_path)
+        else:
+            dict_path = config.get_label_dict_path()
+        cli_args.append(CLIArgument("--table_char_dict_path", dict_path))
+
+        model_type = config._get_model_type()
+        cli_args.append(CLIArgument("--table_algorithm", model_type))
+        infer_shape = config._get_infer_shape()
+        if infer_shape is not None:
+            cli_args.append(CLIArgument("--table_max_len", infer_shape))
+
+        self._assert_empty_kwargs(kwargs)
+
+        with self._create_new_config_file() as config_path:
+            config.dump(config_path)
+            return self.runner.infer(config_path, cli_args, device)

paddlex/repo_apis/PaddleOCR_api/table_rec/register.py

@@ -0,0 +1,53 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import os.path as osp
+from pathlib import Path
+
+from ...base.register import register_model_info, register_suite_info
+from .model import TableRecModel
+from .runner import TableRecRunner
+from .config import TableRecConfig
+
+REPO_ROOT_PATH = os.environ.get("PADDLE_PDX_PADDLEOCR_PATH")
+PDX_CONFIG_DIR = osp.abspath(osp.join(osp.dirname(__file__), "..", "configs"))
+
+register_suite_info(
+    {
+        "suite_name": "TableRec",
+        "model": TableRecModel,
+        "runner": TableRecRunner,
+        "config": TableRecConfig,
+        "runner_root_path": REPO_ROOT_PATH,
+    }
+)
+
+register_model_info(
+    {
+        "model_name": "SLANet",
+        "suite": "TableRec",
+        "config_path": osp.join(PDX_CONFIG_DIR, "SLANet.yaml"),
+        "supported_apis": ["train", "evaluate", "predict", "export"],
+    }
+)
+
+register_model_info(
+    {
+        "model_name": "SLANet_plus",
+        "suite": "TableRec",
+        "config_path": osp.join(PDX_CONFIG_DIR, "SLANet_plus.yaml"),
+        "supported_apis": ["train", "evaluate", "predict", "export"],
+    }
+)

paddlex/repo_apis/PaddleOCR_api/table_rec/runner.py

@@ -0,0 +1,51 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from ..text_rec.runner import TextRecRunner
+from ...base.utils.subprocess import CompletedProcess
+
+
+class TableRecRunner(TextRecRunner):
+    """Table Recognition Runner"""
+
+    def predict(
+        self, config_path: str, cli_args: list, device: str
+    ) -> CompletedProcess:
+        """run predicting using dynamic mode
+
+        Args:
+            config_path (str): the config file path used to predict.
+            cli_args (list): the additional parameters.
+            device (str): unused.
+
+        Returns:
+            CompletedProcess: the result of predicting subprocess execution.
+        """
+        cmd = [self.python, "tools/infer_table.py", "-c", config_path]
+        return self.run_cmd(cmd, switch_wdir=True, echo=True, silent=False)
+
+    def infer(self, config_path: str, cli_args: list, device: str) -> CompletedProcess:
+        """run predicting using inference model
+
+        Args:
+            config_path (str): the path of config file used to predict.
+            cli_args (list): the additional parameters.
+            device (str): unused.
+
+        Returns:
+            CompletedProcess: the result of infering subprocess execution.
+        """
+        cmd = [self.python, "ppstructure/table/predict_structure.py", *cli_args]
+        return self.run_cmd(cmd, switch_wdir=True, echo=True, silent=False)

paddlex/repo_apis/PaddleOCR_api/text_det/__init__.py

@@ -0,0 +1,16 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from . import register