paddlex 2.0.0rc4__py3-none-any.whl → 3.0.0b2__py3-none-any.whl

paddlex/modules/text_recognition/trainer.py

@@ -0,0 +1,105 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import shutil
+from pathlib import Path
+
+from ..base import BaseTrainer
+from ...utils.config import AttrDict
+from .model_list import MODELS
+from ..formula_recognition.model_list import MODELS as MODELS_LaTeX
+
+MODELS = MODELS + MODELS_LaTeX
+
+
+class TextRecTrainer(BaseTrainer):
+    """Text Recognition Model Trainer"""
+
+    entities = MODELS
+
+    def dump_label_dict(self, src_label_dict_path: str):
+        """dump label dict config
+
+        Args:
+            src_label_dict_path (str): path to label dict file to be saved.
+        """
+        dst_label_dict_path = Path(self.global_config.output).joinpath("label_dict.txt")
+        shutil.copyfile(src_label_dict_path, dst_label_dict_path)
+
+    def update_config(self):
+        """update training config"""
+        if self.train_config.log_interval:
+            self.pdx_config.update_log_interval(self.train_config.log_interval)
+        if self.train_config.eval_interval:
+            self.pdx_config._update_eval_interval_by_epoch(
+                self.train_config.eval_interval
+            )
+        if self.train_config.save_interval:
+            self.pdx_config.update_save_interval(self.train_config.save_interval)
+
+        if self.global_config["model"] == "LaTeX_OCR_rec":
+            self.pdx_config.update_dataset(
+                self.global_config.dataset_dir, "LaTeXOCRDataSet"
+            )
+        else:
+            self.pdx_config.update_dataset(
+                self.global_config.dataset_dir, "MSTextRecDataset"
+            )
+
+        label_dict_path = Path(self.global_config.dataset_dir).joinpath("dict.txt")
+        if label_dict_path.exists():
+            self.pdx_config.update_label_dict_path(label_dict_path)
+            self.dump_label_dict(label_dict_path)
+
+        if self.train_config.pretrain_weight_path:
+            self.pdx_config.update_pretrained_weights(
+                self.train_config.pretrain_weight_path
+            )
+
+        if self.global_config["model"] == "LaTeX_OCR_rec":
+            if (
+                self.train_config.batch_size_train is not None
+                and self.train_config.batch_size_val
+            ):
+                self.pdx_config.update_batch_size_pair(
+                    self.train_config.batch_size_train, self.train_config.batch_size_val
+                )
+        else:
+            if self.train_config.batch_size is not None:
+                self.pdx_config.update_batch_size(self.train_config.batch_size)
+
+        if self.train_config.learning_rate is not None:
+            self.pdx_config.update_learning_rate(self.train_config.learning_rate)
+        if self.train_config.epochs_iters is not None:
+            self.pdx_config._update_epochs(self.train_config.epochs_iters)
+        if (
+            self.train_config.resume_path is not None
+            and self.train_config.resume_path != ""
+        ):
+            self.pdx_config._update_checkpoints(self.train_config.resume_path)
+        if self.global_config.output is not None:
+            self.pdx_config._update_output_dir(self.global_config.output)
+
+    def get_train_kwargs(self) -> dict:
+        """get key-value arguments of model training function
+
+        Returns:
+            dict: the arguments of training function.
+        """
+        return {
+            "device": self.get_device(),
+            "dy2st": self.train_config.get("dy2st", False),
+        }

paddlex/modules/ts_anomaly_detection/__init__.py

@@ -0,0 +1,19 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from .dataset_checker import TSADDatasetChecker
+from .trainer import TSADTrainer
+from .evaluator import TSADEvaluator
+from .exportor import TSADExportor

paddlex/modules/ts_anomaly_detection/dataset_checker/__init__.py

@@ -0,0 +1,97 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import os.path as osp
+from collections import defaultdict, Counter
+
+from PIL import Image
+import json
+
+from ...base import BaseDatasetChecker
+from .dataset_src import check, convert, split_dataset, deep_analyse
+
+from ..model_list import MODELS
+
+
+class TSADDatasetChecker(BaseDatasetChecker):
+    """Dataset Checker for TS Anomaly Detection Model"""
+
+    entities = MODELS
+    sample_num = 10
+
+    def convert_dataset(self, src_dataset_dir: str) -> str:
+        """convert the dataset from other type to specified type
+
+        Args:
+            src_dataset_dir (str): the root directory of dataset.
+
+        Returns:
+            str: the root directory of converted dataset.
+        """
+        return convert(src_dataset_dir)
+
+    def split_dataset(self, src_dataset_dir: str) -> str:
+        """repartition the train and validation dataset
+
+        Args:
+            src_dataset_dir (str): the root directory of dataset.
+
+        Returns:
+            str: the root directory of splited dataset.
+        """
+        return split_dataset(
+            src_dataset_dir,
+            self.check_dataset_config.split.train_percent,
+            self.check_dataset_config.split.val_percent,
+        )
+
+    def check_dataset(self, dataset_dir: str, sample_num: int = sample_num) -> dict:
+        """check if the dataset meets the specifications and get dataset summary
+
+        Args:
+            dataset_dir (str): the root directory of dataset.
+            sample_num (int): the number to be sampled.
+        Returns:
+            dict: dataset summary.
+        """
+        return check(dataset_dir, self.output)
+
+    def analyse(self, dataset_dir: str) -> dict:
+        """deep analyse dataset
+
+        Args:
+            dataset_dir (str): the root directory of dataset.
+
+        Returns:
+            dict: the deep analysis results.
+        """
+        return deep_analyse(dataset_dir, self.output)
+
+    def get_show_type(self) -> str:
+        """get the show type of dataset
+
+        Returns:
+            str: show type
+        """
+        return "csv"
+
+    def get_dataset_type(self) -> str:
+        """return the dataset type
+
+        Returns:
+            str: dataset type
+        """
+        return "TSADDataset"

paddlex/modules/ts_anomaly_detection/dataset_checker/dataset_src/__init__.py

@@ -0,0 +1,19 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from .check_dataset import check
+from .convert_dataset import convert
+from .split_dataset import split_dataset
+from .analyse_dataset import deep_analyse

paddlex/modules/ts_anomaly_detection/dataset_checker/dataset_src/analyse_dataset.py

@@ -0,0 +1,27 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import platform
+from pathlib import Path
+
+import pandas as pd
+import numpy as np
+
+
+def deep_analyse(dataset_dir, output):
+    """class analysis for dataset"""
+
+    return {"histogram": ""}

paddlex/modules/ts_anomaly_detection/dataset_checker/dataset_src/check_dataset.py

@@ -0,0 +1,64 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import os.path as osp
+from collections import defaultdict
+from pathlib import Path
+import pandas as pd
+
+from .....utils.errors import DatasetFileNotFoundError
+
+
+def check(dataset_dir, output, sample_num=10):
+    """check dataset"""
+    dataset_dir = osp.abspath(dataset_dir)
+    if not osp.exists(dataset_dir) or not osp.isdir(dataset_dir):
+        raise DatasetFileNotFoundError(file_path=dataset_dir)
+
+    sample_cnts = dict()
+    tables = defaultdict(list)
+    vis_save_dir = osp.join(output, "demo_data")
+
+    tags = ["train", "val"]
+    for _, tag in enumerate(tags):
+        file_list = osp.join(dataset_dir, f"{tag}.csv")
+        if not osp.exists(file_list):
+            if tag in ("train", "val"):
+                # train and val file lists must exist
+                raise DatasetFileNotFoundError(
+                    file_path=file_list,
+                    solution=f"Ensure that both `train.csv` and `val.csv` exist in \
+{dataset_dir}",
+                )
+            else:
+                continue
+        else:
+            df = pd.read_csv(file_list)
+            sample_cnts[tag] = len(df)
+            vis_path = osp.join(vis_save_dir, f"{tag}.csv")
+            Path(vis_path).parent.mkdir(parents=True, exist_ok=True)
+            vis_df = df.iloc[:sample_num, :]
+            vis_df.to_csv(vis_path, index=False)
+            header_list = df.columns.to_list()
+            data_list = df.head(10).values.tolist()
+            tables[tag] = [header_list] + data_list
+
+    attrs = {}
+    attrs["train_samples"] = sample_cnts["train"]
+    attrs["train_table"] = tables["train"]
+    attrs["val_samples"] = sample_cnts["val"]
+    attrs["val_table"] = tables["val"]
+    return attrs

paddlex/modules/ts_anomaly_detection/dataset_checker/dataset_src/convert_dataset.py

@@ -0,0 +1,78 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import os.path as osp
+import shutil
+import json
+import random
+import numpy as np
+import pandas as pd
+from tqdm import tqdm
+
+from .....utils.errors import ConvertFailedError
+
+
+def check_src_dataset(root_dir):
+    """check src dataset format validity"""
+
+    err_msg_prefix = f"Convert Failed! Only support '.xlsx/.xls' format files."
+
+    for dst_anno, src_anno in [("train.xlsx", "train.xls"), ("val.xlsx", "val.xls")]:
+        src_anno_path = os.path.join(root_dir, src_anno)
+        dst_anno_path = os.path.join(root_dir, dst_anno)
+        if not os.path.exists(src_anno_path) and not os.path.exists(dst_anno_path):
+            if "train" in dst_anno:
+                raise ConvertFailedError(
+                    message=f"{err_msg_prefix} Ensure {src_anno_path} or {dst_anno_path} exist."
+                )
+            continue
+
+
+def convert_excel_dataset(input_dir):
+    """
+    将excel标注的数据集转换为PaddleX需要的格式
+
+    Args:
+        input_dir (str): 输入的目录，包含多个json格式的Labelme标注文件
+
+    Returns:
+        str: 返回一个字符串表示转换的结果，“转换成功”表示转换没有问题。
+
+    Raises:
+        该函数目前没有特定的异常抛出。
+
+    """
+    # read excel file
+    for dst_anno, src_anno in [("train.xlsx", "train.xls"), ("val.xlsx", "val.xls")]:
+        src_anno_path = os.path.join(input_dir, src_anno)
+        dst_anno_path = os.path.join(input_dir, dst_anno)
+
+        if os.path.exists(src_anno_path):
+            excel_file = pd.read_excel(src_anno_path)
+            output_csv_dir = osp.join(input_dir, src_anno.replace(".xls", ".csv"))
+            excel_file.to_csv(output_csv_dir, index=False)
+        elif os.path.exists(dst_anno_path):
+            excel_file = pd.read_excel(dst_anno_path)
+            output_csv_dir = osp.join(input_dir, dst_anno.replace(".xlsx", ".csv"))
+            excel_file.to_csv(output_csv_dir, index=False)
+
+
+def convert(input_dir):
+    """convert dataset to coco format"""
+    # check format validity
+    check_src_dataset(input_dir)
+    convert_excel_dataset(input_dir)
+    return input_dir

paddlex/modules/ts_anomaly_detection/dataset_checker/dataset_src/split_dataset.py

@@ -0,0 +1,63 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import os.path as osp
+import shutil
+import random
+import math
+import pandas as pd
+from tqdm import tqdm
+from .....utils.logging import info
+
+
+def split_dataset(root_dir, train_rate, val_rate):
+    """split dataset"""
+    assert (
+        train_rate + val_rate == 100
+    ), f"The sum of train_rate({train_rate}) and val_rate({val_rate}) should equal 100!"
+    assert (
+        train_rate > 0 and val_rate > 0
+    ), f"The train_rate({train_rate}) and val_rate({val_rate}) should be greater than 0!"
+
+    tags = ["train.csv", "val.csv"]
+    df = pd.DataFrame()
+    for tag in tags:
+        if os.path.exists(osp.join(root_dir, tag)):
+            df_one = pd.read_csv(osp.join(root_dir, tag))
+        if df.empty:
+            df = df_one
+        else:
+            df = pd.concat([df, df_one], axis=0, join="inner")
+    df = df.drop_duplicates(keep="first")
+    df_len = df.shape[0]
+    point_train = math.floor((df_len * train_rate / 100))
+    point_val = math.floor((df_len * (train_rate + val_rate) / 100))
+
+    train_df = df.iloc[:point_train, :]
+    val_df = df.iloc[point_train:point_val, :]
+
+    df_dict = {"train.csv": train_df, "val.csv": val_df}
+    for tag in df_dict.keys():
+        save_path = osp.join(root_dir, tag)
+        if os.path.exists(save_path):
+            bak_path = save_path + ".bak"
+            shutil.move(save_path, bak_path)
+            info(
+                f"The original annotation file {tag} has been backed up to {bak_path}."
+            )
+        df_dict[tag].to_csv(save_path, index=False)
+
+    return root_dir

paddlex/modules/ts_anomaly_detection/evaluator.py

@@ -0,0 +1,67 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import tarfile
+from pathlib import Path
+
+from ..base import BaseEvaluator
+from .model_list import MODELS
+
+
+class TSADEvaluator(BaseEvaluator):
+    """TS Anomaly Detection Model Evaluator"""
+
+    entities = MODELS
+
+    def get_config_path(self, weight_path):
+        """
+        get config path
+
+        Args:
+            weight_path (str): The path to the weight
+
+        Returns:
+            config_path (str): The path to the config
+
+        """
+        self.uncompress_tar_file()
+        config_path = Path(self.eval_config.weight_path).parent.parent / "config.yaml"
+        return config_path
+
+    def update_config(self):
+        """update evalution config"""
+        self.pdx_config.update_dataset(self.global_config.dataset_dir, "TSADDataset")
+        self.pdx_config.update_weights(self.eval_config.weight_path)
+
+    def uncompress_tar_file(self):
+        """unpackage the tar file containing training outputs and update weight path"""
+        if tarfile.is_tarfile(self.eval_config.weight_path):
+            dest_path = Path(self.eval_config.weight_path).parent
+            with tarfile.open(self.eval_config.weight_path, "r") as tar:
+                tar.extractall(path=dest_path)
+            self.eval_config.weight_path = dest_path.joinpath(
+                "best_accuracy.pdparams/best_model/model.pdparams"
+            )
+
+    def get_eval_kwargs(self) -> dict:
+        """get key-value arguments of model evalution function
+
+        Returns:
+            dict: the arguments of evaluation function.
+        """
+        return {
+            "weight_path": self.eval_config.weight_path,
+            "device": self.get_device(using_device_number=1),
+        }

paddlex/modules/ts_anomaly_detection/exportor.py

@@ -0,0 +1,45 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import tarfile
+from pathlib import Path
+
+from ..base import BaseExportor
+from .model_list import MODELS
+from ...utils import logging
+
+
+class TSADExportor(BaseExportor):
+    """Image Classification Model Exportor"""
+
+    entities = MODELS
+
+    def get_config_path(self, weight_path):
+        """
+        get config path
+
+        Args:
+            weight_path (str): The path to the weight
+
+        Returns:
+            config_path (str): The path to the config
+
+        """
+        config_path = Path(self.export_config.weight_path).parent.parent / "config.yaml"
+        if not config_path.exists():
+            logging.warning(
+                f"The config file(`{config_path}`) related to weight file(`{weight_path}`) is not exist, use default instead."
+            )
+            config_path = None
+        return config_path

paddlex/modules/ts_anomaly_detection/model_list.py

@@ -0,0 +1,22 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+MODELS = [
+    "AutoEncoder_ad",
+    "DLinear_ad",
+    "Nonstationary_ad",
+    "PatchTST_ad",
+    "TimesNet_ad",
+]