paddlex 3.0.0b2__py3-none-any.whl → 3.0.0rc0__py3-none-any.whl

paddlex/repo_apis/PaddleOCR_api/formula_rec/runner.py

@@ -0,0 +1,240 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import tempfile
+
+from ...base import BaseRunner
+from ...base.utils.subprocess import CompletedProcess
+
+
+class FormulaRecRunner(BaseRunner):
+    """Formula Recognition Runner"""
+
+    def train(
+        self,
+        config_path: str,
+        cli_args: list,
+        device: str,
+        ips: str,
+        save_dir: str,
+        do_eval=True,
+    ) -> CompletedProcess:
+        """train model
+
+        Args:
+            config_path (str): the config file path used to train.
+            cli_args (list): the additional parameters.
+            device (str): the training device.
+            ips (str): the ip addresses of nodes when using distribution.
+            save_dir (str): the directory path to save training output.
+            do_eval (bool, optional): whether or not to evaluate model during training. Defaults to True.
+
+        Returns:
+            CompletedProcess: the result of training subprocess execution.
+        """
+        args, env = self.distributed(device, ips, log_dir=save_dir)
+        cmd = [*args, "tools/train.py", "-c", config_path, *cli_args]
+        if do_eval:
+            # We simply pass here because in PaddleOCR periodic evaluation cannot be switched off
+            pass
+        else:
+            inf = int(1.0e11)
+            cmd.extend(["-o", f"Global.eval_batch_step={inf}"])
+        return self.run_cmd(
+            cmd,
+            env=env,
+            switch_wdir=True,
+            echo=True,
+            silent=False,
+            capture_output=True,
+            log_path=self._get_train_log_path(save_dir),
+        )
+
+    def evaluate(
+        self, config_path: str, cli_args: list, device: str, ips: str
+    ) -> CompletedProcess:
+        """run model evaluating
+
+        Args:
+            config_path (str): the config file path used to evaluate.
+            cli_args (list): the additional parameters.
+            device (str): the evaluating device.
+            ips (str): the ip addresses of nodes when using distribution.
+
+        Returns:
+            CompletedProcess: the result of evaluating subprocess execution.
+        """
+        args, env = self.distributed(device, ips)
+        cmd = [*args, "tools/eval.py", "-c", config_path]
+
+        cp = self.run_cmd(
+            cmd, env=env, switch_wdir=True, echo=True, silent=False, capture_output=True
+        )
+        if cp.returncode == 0:
+            metric_dict = _extract_eval_metrics(cp.stdout)
+            cp.metrics = metric_dict
+        return cp
+
+    def predict(
+        self, config_path: str, cli_args: list, device: str
+    ) -> CompletedProcess:
+        """run predicting using dynamic mode
+
+        Args:
+            config_path (str): the config file path used to predict.
+            cli_args (list): the additional parameters.
+            device (str): unused.
+
+        Returns:
+            CompletedProcess: the result of predicting subprocess execution.
+        """
+        cmd = [self.python, "tools/infer_rec.py", "-c", config_path]
+        return self.run_cmd(cmd, switch_wdir=True, echo=True, silent=False)
+
+    def export(
+        self, config_path: str, cli_args: list, device: str, save_dir: str = None
+    ) -> CompletedProcess:
+        """run exporting
+
+        Args:
+            config_path (str): the path of config file used to export.
+            cli_args (list): the additional parameters.
+            device (str): unused.
+            save_dir (str, optional): the directory path to save exporting output. Defaults to None.
+
+        Returns:
+            CompletedProcess: the result of exporting subprocess execution.
+        """
+        # `device` unused
+        cmd = [self.python, "tools/export_model.py", "-c", config_path]
+        cp = self.run_cmd(cmd, switch_wdir=True, echo=True, silent=False)
+        return cp
+
+    def infer(self, config_path: str, cli_args: list, device: str) -> CompletedProcess:
+        """run predicting using inference model
+
+        Args:
+            config_path (str): the path of config file used to predict.
+            cli_args (list): the additional parameters.
+            device (str): unused.
+
+        Returns:
+            CompletedProcess: the result of infering subprocess execution.
+        """
+        cmd = [self.python, "tools/infer/predict_rec.py", *cli_args]
+        return self.run_cmd(cmd, switch_wdir=True, echo=True, silent=False)
+
+    def compression(
+        self,
+        config_path: str,
+        train_cli_args: list,
+        export_cli_args: list,
+        device: str,
+        train_save_dir: str,
+    ) -> CompletedProcess:
+        """run compression model
+
+        Args:
+            config_path (str): the path of config file used to predict.
+            train_cli_args (list): the additional training parameters.
+            export_cli_args (list): the additional exporting parameters.
+            device (str): the running device.
+            train_save_dir (str): the directory path to save output.
+
+        Returns:
+            CompletedProcess: the result of compression subprocess execution.
+        """
+        # Step 1: Train model
+        args, env = self.distributed(device, log_dir=train_save_dir)
+        cmd = [*args, "deploy/slim/quantization/quant.py", "-c", config_path]
+        cp_train = self.run_cmd(
+            cmd,
+            env=env,
+            switch_wdir=True,
+            echo=True,
+            silent=False,
+            capture_output=True,
+            log_path=self._get_train_log_path(train_save_dir),
+        )
+
+        # Step 2: Export model
+        export_cli_args = [
+            *export_cli_args,
+            "-o",
+            f"Global.checkpoints={train_save_dir}/latest",
+        ]
+        cmd = [
+            self.python,
+            "deploy/slim/quantization/export_model.py",
+            "-c",
+            config_path,
+            *export_cli_args,
+        ]
+        cp_export = self.run_cmd(cmd, switch_wdir=True, echo=True, silent=False)
+
+        return cp_train, cp_export
+
+
+def _extract_eval_metrics(stdout: str) -> dict:
+    """extract evaluation metrics from training log
+
+    Args:
+        stdout (str): the training log
+
+    Returns:
+        dict: the training metric
+    """
+    import re
+
+    def _lazy_split_lines(s):
+        prev_idx = 0
+        while True:
+            curr_idx = s.find(os.linesep, prev_idx)
+            if curr_idx == -1:
+                curr_idx = len(s)
+            yield s[prev_idx:curr_idx]
+            prev_idx = curr_idx + len(os.linesep)
+            if prev_idx >= len(s):
+                break
+
+    _DP = r"[-+]?[0-9]*\.?[0-9]+(?:[eE][-+]?[0-9]+)?"
+    pattern_key_pairs = [
+        (re.compile(r"acc:(_dp)$".replace("_dp", _DP)), "acc"),
+        (re.compile(r"norm_edit_dis:(_dp)$".replace("_dp", _DP)), "norm_edit_dis"),
+        (re.compile(r"Teacher_acc:(_dp)$".replace("_dp", _DP)), "teacher_acc"),
+        (
+            re.compile(r"Teacher_norm_edit_dis:(_dp)$".replace("_dp", _DP)),
+            "teacher_norm_edit_dis",
+        ),
+        (re.compile(r"precision:(_dp)$".replace("_dp", _DP)), "precision"),
+        (re.compile(r"recall:(_dp)$".replace("_dp", _DP)), "recall"),
+        (re.compile(r"hmean:(_dp)$".replace("_dp", _DP)), "hmean"),
+        (re.compile(r"exp_rate:(_dp)$".replace("_dp", _DP)), "exp_rate"),
+    ]
+
+    metric_dict = dict()
+    start_match = False
+    for line in _lazy_split_lines(stdout):
+        if "metric eval" in line:
+            start_match = True
+        if start_match:
+            for pattern, key in pattern_key_pairs:
+                match = pattern.search(line)
+                if match:
+                    assert len(match.groups()) == 1
+                    # Newer overwrites older
+                    metric_dict[key] = float(match.group(1))
+    return metric_dict

paddlex/repo_apis/PaddleOCR_api/table_rec/register.py

@@ -51,3 +51,21 @@ register_model_info(
         "supported_apis": ["train", "evaluate", "predict", "export"],
     }
 )
+
+register_model_info(
+    {
+        "model_name": "SLANeXt_wired",
+        "suite": "TableRec",
+        "config_path": osp.join(PDX_CONFIG_DIR, "SLANeXt_wired.yaml"),
+        "supported_apis": ["train", "evaluate", "predict", "export"],
+    }
+)
+
+register_model_info(
+    {
+        "model_name": "SLANeXt_wireless",
+        "suite": "TableRec",
+        "config_path": osp.join(PDX_CONFIG_DIR, "SLANeXt_wireless.yaml"),
+        "supported_apis": ["train", "evaluate", "predict", "export"],
+    }
+)

paddlex/repo_apis/PaddleOCR_api/text_det/register.py

@@ -70,3 +70,21 @@ register_model_info(
         "supported_apis": ["train", "evaluate", "predict", "export"],
     }
 )
+
+register_model_info(
+    {
+        "model_name": "PP-OCRv3_mobile_det",
+        "suite": "TextDet",
+        "config_path": osp.join(PDX_CONFIG_DIR, "PP-OCRv3_mobile_det.yaml"),
+        "supported_apis": ["train", "evaluate", "predict", "export"],
+    }
+)
+
+register_model_info(
+    {
+        "model_name": "PP-OCRv3_server_det",
+        "suite": "TextDet",
+        "config_path": osp.join(PDX_CONFIG_DIR, "PP-OCRv3_server_det.yaml"),
+        "supported_apis": ["train", "evaluate", "predict", "export"],
+    }
+)

paddlex/repo_apis/PaddleOCR_api/text_rec/config.py

@@ -91,6 +91,17 @@ class TextRecConfig(BaseConfig):
                 "Global.character_dict_path": os.path.join(dataset_path, "dict.txt"),
             }
             self.update(_cfg)
+        elif dataset_type == "SimpleDataSet":
+            _cfg = {
+                "Train.dataset.name": dataset_type,
+                "Train.dataset.data_dir": dataset_path,
+                "Train.dataset.label_file_list": [train_list_path],
+                "Eval.dataset.name": "SimpleDataSet",
+                "Eval.dataset.data_dir": dataset_path,
+                "Eval.dataset.label_file_list": [os.path.join(dataset_path, "val.txt")],
+                "Global.character_dict_path": os.path.join(dataset_path, "dict.txt"),
+            }
+            self.update(_cfg)
         elif dataset_type == "LaTeXOCRDataSet":
             _cfg = {
                 "Train.dataset.name": dataset_type,
@@ -107,6 +118,14 @@ class TextRecConfig(BaseConfig):
         else:
             raise ValueError(f"{repr(dataset_type)} is not supported.")
 
+    def update_dataset_by_list(self, label_file_list, ratio_list):
+        _cfg = {
+            "Train.dataset.name": "MSTextRecDataset",
+            "Train.dataset.label_file_list": label_file_list,
+            "Train.dataset.ratio_list": ratio_list,
+        }
+        self.update(_cfg)
+
     def update_batch_size(self, batch_size: int, mode: str = "train"):
         """update batch size setting
 
@@ -228,6 +247,7 @@ class TextRecConfig(BaseConfig):
             "Global.use_xpu": False,
             "Global.use_npu": False,
             "Global.use_mlu": False,
+            "Global.use_gcu": False,
         }
 
         device_cfg = {
@@ -236,6 +256,7 @@ class TextRecConfig(BaseConfig):
             "xpu": {"Global.use_xpu": True},
             "mlu": {"Global.use_mlu": True},
             "npu": {"Global.use_npu": True},
+            "gcu": {"Global.use_gcu": True},
         }
         default_cfg.update(device_cfg[device])
         self.update(default_cfg)

paddlex/repo_apis/PaddleOCR_api/text_rec/model.py

@@ -135,8 +135,11 @@ class TextRecModel(BaseModel):
         # PDX related settings
         device_type = device.split(":")[0]
         uniform_output_enabled = kwargs.pop("uniform_output_enabled", True)
+        export_with_pir = kwargs.pop("export_with_pir", False)
         config.update({"Global.uniform_output_enabled": uniform_output_enabled})
         config.update({"Global.pdx_model_name": self.name})
+        if export_with_pir:
+            config.update({"Global.export_with_pir": export_with_pir})
 
         self._assert_empty_kwargs(kwargs)
 
@@ -269,8 +272,11 @@ class TextRecModel(BaseModel):
 
         # PDX related settings
         uniform_output_enabled = kwargs.pop("uniform_output_enabled", True)
+        export_with_pir = kwargs.pop("export_with_pir", False)
         config.update({"Global.uniform_output_enabled": uniform_output_enabled})
         config.update({"Global.pdx_model_name": self.name})
+        if export_with_pir:
+            config.update({"Global.export_with_pir": export_with_pir})
 
         self._assert_empty_kwargs(kwargs)
 

paddlex/repo_apis/PaddleOCR_api/text_rec/register.py

@@ -34,6 +34,116 @@ register_suite_info(
     }
 )
 
+register_model_info(
+    {
+        "model_name": "PP-OCRv3_mobile_rec",
+        "suite": "TextRec",
+        "config_path": osp.join(PDX_CONFIG_DIR, "PP-OCRv3_mobile_rec.yaml"),
+        "supported_apis": ["train", "evaluate", "predict", "export"],
+    }
+)
+
+register_model_info(
+    {
+        "model_name": "en_PP-OCRv3_mobile_rec",
+        "suite": "TextRec",
+        "config_path": osp.join(PDX_CONFIG_DIR, "en_PP-OCRv3_mobile_rec.yaml"),
+        "supported_apis": ["train", "evaluate", "predict", "export"],
+    }
+)
+
+
+register_model_info(
+    {
+        "model_name": "korean_PP-OCRv3_mobile_rec",
+        "suite": "TextRec",
+        "config_path": osp.join(PDX_CONFIG_DIR, "korean_PP-OCRv3_mobile_rec.yaml"),
+        "supported_apis": ["train", "evaluate", "predict", "export"],
+    }
+)
+
+register_model_info(
+    {
+        "model_name": "japan_PP-OCRv3_mobile_rec",
+        "suite": "TextRec",
+        "config_path": osp.join(PDX_CONFIG_DIR, "japan_PP-OCRv3_mobile_rec.yaml"),
+        "supported_apis": ["train", "evaluate", "predict", "export"],
+    }
+)
+
+register_model_info(
+    {
+        "model_name": "chinese_cht_PP-OCRv3_mobile_rec",
+        "suite": "TextRec",
+        "config_path": osp.join(PDX_CONFIG_DIR, "chinese_cht_PP-OCRv3_mobile_rec.yaml"),
+        "supported_apis": ["train", "evaluate", "predict", "export"],
+    }
+)
+
+register_model_info(
+    {
+        "model_name": "te_PP-OCRv3_mobile_rec",
+        "suite": "TextRec",
+        "config_path": osp.join(PDX_CONFIG_DIR, "te_PP-OCRv3_mobile_rec.yaml"),
+        "supported_apis": ["train", "evaluate", "predict", "export"],
+    }
+)
+
+register_model_info(
+    {
+        "model_name": "ka_PP-OCRv3_mobile_rec",
+        "suite": "TextRec",
+        "config_path": osp.join(PDX_CONFIG_DIR, "ka_PP-OCRv3_mobile_rec.yaml"),
+        "supported_apis": ["train", "evaluate", "predict", "export"],
+    }
+)
+
+register_model_info(
+    {
+        "model_name": "ta_PP-OCRv3_mobile_rec",
+        "suite": "TextRec",
+        "config_path": osp.join(PDX_CONFIG_DIR, "ta_PP-OCRv3_mobile_rec.yaml"),
+        "supported_apis": ["train", "evaluate", "predict", "export"],
+    }
+)
+
+register_model_info(
+    {
+        "model_name": "latin_PP-OCRv3_mobile_rec",
+        "suite": "TextRec",
+        "config_path": osp.join(PDX_CONFIG_DIR, "latin_PP-OCRv3_mobile_rec.yaml"),
+        "supported_apis": ["train", "evaluate", "predict", "export"],
+    }
+)
+
+register_model_info(
+    {
+        "model_name": "arabic_PP-OCRv3_mobile_rec",
+        "suite": "TextRec",
+        "config_path": osp.join(PDX_CONFIG_DIR, "arabic_PP-OCRv3_mobile_rec.yaml"),
+        "supported_apis": ["train", "evaluate", "predict", "export"],
+    }
+)
+
+register_model_info(
+    {
+        "model_name": "cyrillic_PP-OCRv3_mobile_rec",
+        "suite": "TextRec",
+        "config_path": osp.join(PDX_CONFIG_DIR, "cyrillic_PP-OCRv3_mobile_rec.yaml"),
+        "supported_apis": ["train", "evaluate", "predict", "export"],
+    }
+)
+
+register_model_info(
+    {
+        "model_name": "devanagari_PP-OCRv3_mobile_rec",
+        "suite": "TextRec",
+        "config_path": osp.join(PDX_CONFIG_DIR, "devanagari_PP-OCRv3_mobile_rec.yaml"),
+        "supported_apis": ["train", "evaluate", "predict", "export"],
+    }
+)
+
+
 register_model_info(
     {
         "model_name": "PP-OCRv4_mobile_rec",
@@ -54,27 +164,36 @@ register_model_info(
 
 register_model_info(
     {
-        "model_name": "ch_SVTRv2_rec",
+        "model_name": "en_PP-OCRv4_mobile_rec",
         "suite": "TextRec",
-        "config_path": osp.join(PDX_CONFIG_DIR, "ch_SVTRv2_rec.yaml"),
-        "supported_apis": ["train", "evaluate", "predict", "export", "infer"],
+        "config_path": osp.join(PDX_CONFIG_DIR, "en_PP-OCRv4_mobile_rec.yaml"),
+        "supported_apis": ["train", "evaluate", "predict", "export"],
     }
 )
 
 register_model_info(
     {
-        "model_name": "ch_RepSVTR_rec",
+        "model_name": "PP-OCRv4_server_rec_doc",
         "suite": "TextRec",
-        "config_path": osp.join(PDX_CONFIG_DIR, "ch_RepSVTR_rec.yaml"),
+        "config_path": osp.join(PDX_CONFIG_DIR, "PP-OCRv4_server_rec_doc.yaml"),
+        "supported_apis": ["train", "evaluate", "predict", "export"],
+    }
+)
+
+register_model_info(
+    {
+        "model_name": "ch_SVTRv2_rec",
+        "suite": "TextRec",
+        "config_path": osp.join(PDX_CONFIG_DIR, "ch_SVTRv2_rec.yaml"),
         "supported_apis": ["train", "evaluate", "predict", "export", "infer"],
     }
 )
 
 register_model_info(
     {
-        "model_name": "LaTeX_OCR_rec",
+        "model_name": "ch_RepSVTR_rec",
         "suite": "TextRec",
-        "config_path": osp.join(PDX_CONFIG_DIR, "LaTeX_OCR_rec.yml"),
+        "config_path": osp.join(PDX_CONFIG_DIR, "ch_RepSVTR_rec.yaml"),
         "supported_apis": ["train", "evaluate", "predict", "export", "infer"],
     }
 )

paddlex/repo_apis/PaddleSeg_api/seg/config.py

@@ -72,6 +72,15 @@ class SegConfig(BaseSegConfig):
         if "model" in self:
             self.model["num_classes"] = num_classes
 
+        if self.model_name in ["MaskFormer_tiny", "MaskFormer_small"]:
+            for tf_cfg in self.train_dataset["transforms"]:
+                if tf_cfg["type"] == "GenerateInstanceTargets":
+                    tf_cfg["num_classes"] = num_classes
+
+            losses = self.loss["types"]
+            for loss_cfg in losses:
+                loss_cfg["num_classes"] = num_classes
+
     def update_train_crop_size(self, crop_size: Union[int, list]):
         """update the image cropping size of training preprocessing
 

paddlex/repo_apis/PaddleSeg_api/seg/model.py

@@ -126,6 +126,10 @@ class SegModel(BaseModel):
         if log_iters is not None:
             cli_args.append(CLIArgument("--log_iters", log_iters))
 
+        input_shape = kwargs.pop("input_shape", None)
+        if input_shape is not None:
+            cli_args.append(CLIArgument("--input_shape", *input_shape))
+
         # Benchmarking mode settings
         benchmark = kwargs.pop("benchmark", None)
         if benchmark is not None:
@@ -167,8 +171,11 @@ class SegModel(BaseModel):
 
         # PDX related settings
         uniform_output_enabled = kwargs.pop("uniform_output_enabled", True)
+        export_with_pir = kwargs.pop("export_with_pir", False)
         config.set_val("uniform_output_enabled", uniform_output_enabled)
         config.set_val("pdx_model_name", self.name)
+        if export_with_pir:
+            config.set_val("export_with_pir", export_with_pir)
 
         self._assert_empty_kwargs(kwargs)
 
@@ -352,8 +359,11 @@ class SegModel(BaseModel):
 
         # PDX related settings
         uniform_output_enabled = kwargs.pop("uniform_output_enabled", True)
+        export_with_pir = kwargs.pop("export_with_pir", False)
         config.set_val("uniform_output_enabled", uniform_output_enabled)
         config.set_val("pdx_model_name", self.name)
+        if export_with_pir:
+            config.set_val("export_with_pir", export_with_pir)
 
         self._assert_empty_kwargs(kwargs)
 

paddlex/repo_apis/PaddleSeg_api/seg/register.py

@@ -238,6 +238,26 @@ register_model_info(
 )
 
 
+register_model_info(
+    {
+        "model_name": "MaskFormer_tiny",
+        "suite": "Seg",
+        "config_path": osp.join(PDX_CONFIG_DIR, "MaskFormer_tiny.yaml"),
+        "supported_apis": ["train", "evaluate", "predict", "export"],
+    }
+)
+
+
+register_model_info(
+    {
+        "model_name": "MaskFormer_small",
+        "suite": "Seg",
+        "config_path": osp.join(PDX_CONFIG_DIR, "MaskFormer_small.yaml"),
+        "supported_apis": ["train", "evaluate", "predict", "export"],
+    }
+)
+
+
 # For compatibility
 def _set_alias(model_name, alias):
     from ...base.register import get_registered_model_info

paddlex/repo_apis/PaddleTS_api/ts_base/config.py

@@ -57,6 +57,9 @@ class BaseTSConfig(BaseConfig):
         Args:
             config_file_path (str): the path to save self as yaml file.
         """
+        output_dir = os.path.dirname(config_file_path)
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
         yaml = ruamel.yaml.YAML()
         with open(config_file_path, "w", encoding="utf-8") as f:
             dict_to_dump = self.dict
@@ -71,6 +74,26 @@ class BaseTSConfig(BaseConfig):
         """
         self.update({"epoch": epochs})
 
+    def update_to_static(self, dy2st: bool):
+        """update config to set dynamic to static mode
+
+        Args:
+            dy2st (bool): whether or not to use the dynamic to static mode.
+        """
+        self.update({"to_static_train": dy2st})
+
+    def update_amp(self, amp: str = "O2"):
+        """update AMP settings
+
+        Args:
+            amp (None | str): the AMP level if it is not None or `OFF`.
+        """
+        _cfg = {
+            "use_amp": True if amp is not None else False,
+            "amp_level": amp,
+        }
+        self.update(_cfg)
+
     def update_weights(self, weight_path: str):
         """update weight path
 
@@ -161,6 +184,7 @@ class BaseTSConfig(BaseConfig):
         """
         self.update({"log_interval": log_interval})
 
+
     def update_dataset(self, dataset_dir: str, dataset_type: str = None):
         """update dataset settings"""
         raise NotImplementedError