paddlex 2.1.0__py3-none-any.whl → 3.0.0b2__py3-none-any.whl

paddlex/modules/ts_classification/trainer.py

@@ -0,0 +1,92 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import json
+import time
+import tarfile
+from pathlib import Path
+
+from ..base import BaseTrainer
+from ...utils.config import AttrDict
+from .model_list import MODELS
+
+
+class TSCLSTrainer(BaseTrainer):
+    """TS Classification Model Trainer"""
+
+    entities = MODELS
+
+    def train(self):
+        """firstly, update and dump train config, then train model"""
+        # XXX: using super().train() instead when the train_hook() is supported.
+        os.makedirs(self.global_config.output, exist_ok=True)
+        self.update_config()
+        self.dump_config()
+        train_result = self.pdx_model.train(**self.get_train_kwargs())
+        assert (
+            train_result.returncode == 0
+        ), f"Encountered an unexpected error({train_result.returncode}) in \
+training!"
+
+        self.make_tar_file()
+
+    def make_tar_file(self):
+        """make tar file to package the training outputs"""
+        tar_path = Path(self.global_config.output) / "best_accuracy.pdparams.tar"
+        with tarfile.open(tar_path, "w") as tar:
+            tar.add(self.global_config.output, arcname="best_accuracy.pdparams")
+
+    def update_config(self):
+        """update training config"""
+        self.pdx_config.update_dataset(self.global_config.dataset_dir, "TSCLSDataset")
+        if self.train_config.time_col is not None:
+            self.pdx_config.update_basic_info({"time_col": self.train_config.time_col})
+        if self.train_config.target_cols is not None:
+            self.pdx_config.update_basic_info(
+                {"target_cols": self.train_config.target_cols.split(",")}
+            )
+        if self.train_config.group_id is not None:
+            self.pdx_config.update_basic_info({"group_id": self.train_config.group_id})
+        if self.train_config.static_cov_cols is not None:
+            self.pdx_config.update_basic_info(
+                {"static_cov_cols": self.train_config.static_cov_cols}
+            )
+        if self.train_config.freq is not None:
+            try:
+                self.train_config.freq = int(self.train_config.freq)
+            except ValueError:
+                pass
+            self.pdx_config.update_basic_info({"freq": self.train_config.freq})
+        if self.train_config.batch_size is not None:
+            self.pdx_config.update_batch_size(self.train_config.batch_size)
+        if self.train_config.learning_rate is not None:
+            self.pdx_config.update_learning_rate(self.train_config.learning_rate)
+        if self.train_config.epochs_iters is not None:
+            self.pdx_config.update_epochs(self.train_config.epochs_iters)
+        if self.train_config.log_interval is not None:
+            self.pdx_config.update_log_interval(self.train_config.log_interval)
+        if self.global_config.output is not None:
+            self.pdx_config.update_save_dir(self.global_config.output)
+
+    def get_train_kwargs(self) -> dict:
+        """get key-value arguments of model training function
+
+        Returns:
+            dict: the arguments of training function.
+        """
+        train_args = {"device": self.get_device(using_device_number=1)}
+        if self.global_config.output is not None:
+            train_args["save_dir"] = self.global_config.output
+        return train_args

paddlex/modules/ts_forecast/__init__.py

@@ -0,0 +1,19 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from .dataset_checker import TSFCDatasetChecker
+from .trainer import TSFCTrainer
+from .evaluator import TSFCEvaluator
+from .exportor import TSFCExportor

paddlex/modules/ts_forecast/dataset_checker/__init__.py

@@ -0,0 +1,97 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import os.path as osp
+from collections import defaultdict, Counter
+
+from PIL import Image
+import json
+
+from ...base import BaseDatasetChecker
+from .dataset_src import check, convert, split_dataset, deep_analyse
+
+from ..model_list import MODELS
+
+
+class TSFCDatasetChecker(BaseDatasetChecker):
+    """Dataset Checker for TS Forecast Model"""
+
+    entities = MODELS
+    sample_num = 10
+
+    def convert_dataset(self, src_dataset_dir: str) -> str:
+        """convert the dataset from other type to specified type
+
+        Args:
+            src_dataset_dir (str): the root directory of dataset.
+
+        Returns:
+            str: the root directory of converted dataset.
+        """
+        return convert(src_dataset_dir)
+
+    def split_dataset(self, src_dataset_dir: str) -> str:
+        """repartition the train and validation dataset
+
+        Args:
+            src_dataset_dir (str): the root directory of dataset.
+
+        Returns:
+            str: the root directory of splited dataset.
+        """
+        return split_dataset(
+            src_dataset_dir,
+            self.check_dataset_config.split.train_percent,
+            self.check_dataset_config.split.val_percent,
+        )
+
+    def check_dataset(self, dataset_dir: str, sample_num: int = sample_num) -> dict:
+        """check if the dataset meets the specifications and get dataset summary
+
+        Args:
+            dataset_dir (str): the root directory of dataset.
+            sample_num (int): the number to be sampled.
+        Returns:
+            dict: dataset summary.
+        """
+        return check(dataset_dir, self.output)
+
+    def analyse(self, dataset_dir: str) -> dict:
+        """deep analyse dataset
+
+        Args:
+            dataset_dir (str): the root directory of dataset.
+
+        Returns:
+            dict: the deep analysis results.
+        """
+        return deep_analyse(dataset_dir, self.output)
+
+    def get_show_type(self) -> str:
+        """get the show type of dataset
+
+        Returns:
+            str: show type
+        """
+        return "csv"
+
+    def get_dataset_type(self) -> str:
+        """return the dataset type
+
+        Returns:
+            str: dataset type
+        """
+        return "TSDataset"

paddlex/modules/ts_forecast/dataset_checker/dataset_src/__init__.py

@@ -0,0 +1,19 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from .check_dataset import check
+from .convert_dataset import convert
+from .split_dataset import split_dataset
+from .analyse_dataset import deep_analyse

paddlex/modules/ts_forecast/dataset_checker/dataset_src/analyse_dataset.py

@@ -0,0 +1,27 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import platform
+from pathlib import Path
+
+import pandas as pd
+import numpy as np
+
+
+def deep_analyse(dataset_dir, output):
+    """class analysis for dataset"""
+
+    return {"histogram": ""}

paddlex/modules/ts_forecast/dataset_checker/dataset_src/check_dataset.py

@@ -0,0 +1,64 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import os.path as osp
+from collections import defaultdict
+from pathlib import Path
+import pandas as pd
+
+from .....utils.errors import DatasetFileNotFoundError
+
+
+def check(dataset_dir, output, sample_num=10):
+    """check dataset"""
+    dataset_dir = osp.abspath(dataset_dir)
+    if not osp.exists(dataset_dir) or not osp.isdir(dataset_dir):
+        raise DatasetFileNotFoundError(file_path=dataset_dir)
+
+    sample_cnts = dict()
+    tables = defaultdict(list)
+    vis_save_dir = osp.join(output, "demo_data")
+
+    tags = ["train", "val"]
+    for _, tag in enumerate(tags):
+        file_list = osp.join(dataset_dir, f"{tag}.csv")
+        if not osp.exists(file_list):
+            if tag in ("train", "val"):
+                # train and val file lists must exist
+                raise DatasetFileNotFoundError(
+                    file_path=file_list,
+                    solution=f"Ensure that both `train.csv` and `val.csv` exist in \
+{dataset_dir}",
+                )
+            else:
+                continue
+        else:
+            df = pd.read_csv(file_list)
+            sample_cnts[tag] = len(df)
+            vis_path = osp.join(vis_save_dir, f"{tag}.csv")
+            Path(vis_path).parent.mkdir(parents=True, exist_ok=True)
+            vis_df = df.iloc[:sample_num, :]
+            vis_df.to_csv(vis_path, index=False)
+            header_list = df.columns.to_list()
+            data_list = df.head(10).values.tolist()
+            tables[tag] = [header_list] + data_list
+
+    attrs = {}
+    attrs["train_samples"] = sample_cnts["train"]
+    attrs["train_table"] = tables["train"]
+    attrs["val_samples"] = sample_cnts["val"]
+    attrs["val_table"] = tables["val"]
+    return attrs

paddlex/modules/ts_forecast/dataset_checker/dataset_src/convert_dataset.py

@@ -0,0 +1,77 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import shutil
+import json
+import random
+import numpy as np
+import pandas as pd
+from tqdm import tqdm
+
+from .....utils.errors import ConvertFailedError
+
+
+def check_src_dataset(root_dir):
+    """check src dataset format validity"""
+
+    err_msg_prefix = f"数据格式转换失败！当前仅支持后续为'.xlsx/.xls'格式的数据转换。"
+
+    for dst_anno, src_anno in [("train.xlsx", "train.xls"), ("val.xlsx", "val.xls")]:
+        src_anno_path = os.path.join(root_dir, src_anno)
+        dst_anno_path = os.path.join(root_dir, dst_anno)
+        if not os.path.exists(src_anno_path) and not os.path.exists(dst_anno_path):
+            if "train" in dst_anno:
+                raise ConvertFailedError(
+                    message=f"{err_msg_prefix}保证{src_anno_path}或{dst_anno_path}文件存在。"
+                )
+            continue
+
+
+def convert_excel_dataset(input_dir):
+    """
+    将excel标注的数据集转换为PaddleX需要的格式
+
+    Args:
+        input_dir (str): 输入的目录，包含多个json格式的Labelme标注文件
+
+    Returns:
+        str: 返回一个字符串表示转换的结果，“转换成功”表示转换没有问题。
+
+    Raises:
+        该函数目前没有特定的异常抛出。
+
+    """
+    # read excel file
+    for dst_anno, src_anno in [("train.xlsx", "train.xls"), ("val.xlsx", "val.xls")]:
+        src_anno_path = os.path.join(input_dir, src_anno)
+        dst_anno_path = os.path.join(input_dir, dst_anno)
+
+        if os.path.exists(src_anno_path):
+            excel_file = pd.read_excel(src_anno_path)
+            output_csv_dir = os.path.join(input_dir, src_anno.replace(".xlsx", ".csv"))
+            excel_file.to_csv(output_csv_dir, index=False)
+        if os.path.exists(dst_anno_path):
+            excel_file = pd.read_excel(dst_anno_path)
+            output_csv_dir = os.path.join(input_dir, dst_anno.replace(".xlsx", ".csv"))
+            excel_file.to_csv(output_csv_dir, index=False)
+
+
+def convert(input_dir):
+    """convert dataset to coco format"""
+    # check format validity
+    check_src_dataset(input_dir)
+    convert_excel_dataset(input_dir)
+    return input_dir

paddlex/modules/ts_forecast/dataset_checker/dataset_src/split_dataset.py

@@ -0,0 +1,63 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import os.path as osp
+import shutil
+import random
+import math
+import pandas as pd
+from tqdm import tqdm
+from .....utils.logging import info
+
+
+def split_dataset(root_dir, train_rate, val_rate):
+    """split dataset"""
+    assert (
+        train_rate + val_rate == 100
+    ), f"The sum of train_rate({train_rate}) and val_rate({val_rate}) should equal 100!"
+    assert (
+        train_rate > 0 and val_rate > 0
+    ), f"The train_rate({train_rate}) and val_rate({val_rate}) should be greater than 0!"
+
+    tags = ["train.csv", "val.csv"]
+    df = pd.DataFrame()
+    for tag in tags:
+        if os.path.exists(osp.join(root_dir, tag)):
+            df_one = pd.read_csv(osp.join(root_dir, tag))
+        if df.empty:
+            df = df_one
+        else:
+            df = pd.concat([df, df_one], axis=0, join="inner")
+    df = df.drop_duplicates(keep="first")
+    df_len = df.shape[0]
+    point_train = math.floor((df_len * train_rate / 100))
+    point_val = math.floor((df_len * (train_rate + val_rate) / 100))
+
+    train_df = df.iloc[:point_train, :]
+    val_df = df.iloc[point_train:point_val, :]
+
+    df_dict = {"train.csv": train_df, "val.csv": val_df}
+    for tag in df_dict.keys():
+        save_path = osp.join(root_dir, tag)
+        if os.path.exists(save_path):
+            bak_path = save_path + ".bak"
+            shutil.move(save_path, bak_path)
+            info(
+                f"The original annotation file {tag} has been backed up to {bak_path}."
+            )
+        df_dict[tag].to_csv(save_path, index=False)
+
+    return root_dir

paddlex/modules/ts_forecast/evaluator.py

@@ -0,0 +1,66 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import tarfile
+from pathlib import Path
+
+from ..base import BaseEvaluator
+from .model_list import MODELS
+
+
+class TSFCEvaluator(BaseEvaluator):
+    """TS Forecast Model Evaluator"""
+
+    entities = MODELS
+
+    def get_config_path(self, weight_path):
+        """
+        get config path
+
+        Args:
+            weight_path (str): The path to the weight
+
+        Returns:
+            config_path (str): The path to the config
+
+        """
+        self.uncompress_tar_file()
+        config_path = Path(self.eval_config.weight_path).parent.parent / "config.yaml"
+        return config_path
+
+    def update_config(self):
+        """update evalution config"""
+        self.pdx_config.update_dataset(self.global_config.dataset_dir, "TSDataset")
+
+    def get_eval_kwargs(self) -> dict:
+        """get key-value arguments of model evalution function
+
+        Returns:
+            dict: the arguments of evaluation function.
+        """
+        return {
+            "weight_path": self.eval_config.weight_path,
+            "device": self.get_device(using_device_number=1),
+        }
+
+    def uncompress_tar_file(self):
+        """unpackage the tar file containing training outputs and update weight path"""
+        if tarfile.is_tarfile(self.eval_config.weight_path):
+            dest_path = Path(self.eval_config.weight_path).parent
+            with tarfile.open(self.eval_config.weight_path, "r") as tar:
+                tar.extractall(path=dest_path)
+            self.eval_config.weight_path = dest_path.joinpath(
+                "best_accuracy.pdparams/best_model/model.pdparams"
+            )

paddlex/modules/ts_forecast/exportor.py

@@ -0,0 +1,45 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import tarfile
+from pathlib import Path
+
+from ..base import BaseExportor
+from .model_list import MODELS
+from ...utils import logging
+
+
+class TSFCExportor(BaseExportor):
+    """Image Classification Model Exportor"""
+
+    entities = MODELS
+
+    def get_config_path(self, weight_path):
+        """
+        get config path
+
+        Args:
+            weight_path (str): The path to the weight
+
+        Returns:
+            config_path (str): The path to the config
+
+        """
+        config_path = Path(self.export_config.weight_path).parent.parent / "config.yaml"
+        if not config_path.exists():
+            logging.warning(
+                f"The config file(`{config_path}`) related to weight file(`{weight_path}`) is not exist, use default instead."
+            )
+            config_path = None
+        return config_path

paddlex/modules/ts_forecast/model_list.py

@@ -0,0 +1,24 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+MODELS = [
+    "DLinear",
+    "NLinear",
+    "Nonstationary",
+    "PatchTST",
+    "RLinear",
+    "TiDE",
+    "TimesNet",
+]