paddlex 3.0.0b2__py3-none-any.whl → 3.0.0rc1__py3-none-any.whl

paddlex/modules/ts_classification/dataset_checker/dataset_src/convert_dataset.py

@@ -1,4 +1,4 @@
-# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,12 +14,8 @@
 
 
 import os
-import shutil
-import json
-import random
-import numpy as np
+
 import pandas as pd
-from tqdm import tqdm
 
 from .....utils.errors import ConvertFailedError
 

paddlex/modules/ts_classification/dataset_checker/dataset_src/split_dataset.py

@@ -1,4 +1,4 @@
-# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,13 +13,13 @@
 # limitations under the License.
 
 
+import math
 import os
 import os.path as osp
 import shutil
-import random
-import math
+
 import pandas as pd
-from tqdm import tqdm
+
 from .....utils.logging import info
 
 

paddlex/modules/ts_classification/evaluator.py

@@ -1,4 +1,4 @@
-# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

paddlex/modules/ts_classification/exportor.py

@@ -1,4 +1,4 @@
-# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,12 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import tarfile
 from pathlib import Path
 
+from ...utils import logging
 from ..base import BaseExportor
 from .model_list import MODELS
-from ...utils import logging
 
 
 class TSCLSExportor(BaseExportor):

paddlex/modules/ts_classification/model_list.py

@@ -1,4 +1,4 @@
-# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

paddlex/modules/ts_classification/trainer.py

@@ -1,4 +1,4 @@
-# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,13 +13,11 @@
 # limitations under the License.
 
 import os
-import json
-import time
 import tarfile
 from pathlib import Path
 
+from ...utils.flags import FLAGS_json_format_model
 from ..base import BaseTrainer
-from ...utils.config import AttrDict
 from .model_list import MODELS
 
 
@@ -34,7 +32,21 @@ class TSCLSTrainer(BaseTrainer):
         os.makedirs(self.global_config.output, exist_ok=True)
         self.update_config()
         self.dump_config()
-        train_result = self.pdx_model.train(**self.get_train_kwargs())
+        train_args = self.get_train_kwargs()
+        export_with_pir = (
+            self.global_config.get("export_with_pir", False) or FLAGS_json_format_model
+        )
+        train_args.update(
+            {
+                "uniform_output_enabled": self.train_config.get(
+                    "uniform_output_enabled", True
+                ),
+                "export_with_pir": export_with_pir,
+            }
+        )
+        if self.benchmark_config is not None:
+            train_args.update({"benchmark": self.benchmark_config})
+        train_result = self.pdx_model.train(**train_args)
         assert (
             train_result.returncode == 0
         ), f"Encountered an unexpected error({train_result.returncode}) in \
@@ -77,6 +89,8 @@ training!"
             self.pdx_config.update_epochs(self.train_config.epochs_iters)
         if self.train_config.log_interval is not None:
             self.pdx_config.update_log_interval(self.train_config.log_interval)
+        if self.train_config.get("dy2st", False):
+            self.pdx_config.update_to_static(self.train_config.dy2st)
         if self.global_config.output is not None:
             self.pdx_config.update_save_dir(self.global_config.output)
 
@@ -89,4 +103,6 @@ training!"
         train_args = {"device": self.get_device(using_device_number=1)}
         if self.global_config.output is not None:
             train_args["save_dir"] = self.global_config.output
+        # amp support 'O1', 'O2', 'OFF'
+        train_args["amp"] = self.train_config.get("amp", "OFF")
         return train_args

paddlex/modules/ts_forecast/__init__.py

@@ -1,4 +1,4 @@
-# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,6 +14,6 @@
 
 
 from .dataset_checker import TSFCDatasetChecker
-from .trainer import TSFCTrainer
 from .evaluator import TSFCEvaluator
 from .exportor import TSFCExportor
+from .trainer import TSFCTrainer

paddlex/modules/ts_forecast/dataset_checker/__init__.py

@@ -1,4 +1,4 @@
-# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,17 +13,17 @@
 # limitations under the License.
 
 
+import json
 import os
 import os.path as osp
-from collections import defaultdict, Counter
+from collections import Counter, defaultdict
+from pathlib import Path
 
 from PIL import Image
-import json
 
 from ...base import BaseDatasetChecker
-from .dataset_src import check, convert, split_dataset, deep_analyse
-
 from ..model_list import MODELS
+from .dataset_src import check, convert, deep_analyse, split_dataset
 
 
 class TSFCDatasetChecker(BaseDatasetChecker):
@@ -32,6 +32,20 @@ class TSFCDatasetChecker(BaseDatasetChecker):
     entities = MODELS
     sample_num = 10
 
+    def get_dataset_root(self, dataset_dir: str) -> str:
+        """find the dataset root dir
+
+        Args:
+            dataset_dir (str): the directory that contain dataset.
+
+        Returns:
+            str: the root directory of dataset.
+        """
+        anno_dirs = list(Path(dataset_dir).glob("**/train.csv"))
+        assert len(anno_dirs) == 1
+        dataset_dir = anno_dirs[0].parent.as_posix()
+        return dataset_dir
+
     def convert_dataset(self, src_dataset_dir: str) -> str:
         """convert the dataset from other type to specified type
 

paddlex/modules/ts_forecast/dataset_checker/dataset_src/__init__.py

@@ -1,4 +1,4 @@
-# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 
+from .analyse_dataset import deep_analyse
 from .check_dataset import check
 from .convert_dataset import convert
 from .split_dataset import split_dataset
-from .analyse_dataset import deep_analyse

paddlex/modules/ts_forecast/dataset_checker/dataset_src/analyse_dataset.py

@@ -1,4 +1,4 @@
-# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,14 +13,6 @@
 # limitations under the License.
 
 
-import os
-import platform
-from pathlib import Path
-
-import pandas as pd
-import numpy as np
-
-
 def deep_analyse(dataset_dir, output):
     """class analysis for dataset"""
 

paddlex/modules/ts_forecast/dataset_checker/dataset_src/check_dataset.py

@@ -1,4 +1,4 @@
-# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,10 +13,10 @@
 # limitations under the License.
 
 
-import os
 import os.path as osp
 from collections import defaultdict
 from pathlib import Path
+
 import pandas as pd
 
 from .....utils.errors import DatasetFileNotFoundError

paddlex/modules/ts_forecast/dataset_checker/dataset_src/convert_dataset.py

@@ -1,4 +1,4 @@
-# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,12 +14,8 @@
 
 
 import os
-import shutil
-import json
-import random
-import numpy as np
+
 import pandas as pd
-from tqdm import tqdm
 
 from .....utils.errors import ConvertFailedError
 

paddlex/modules/ts_forecast/dataset_checker/dataset_src/split_dataset.py

@@ -1,4 +1,4 @@
-# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,13 +13,13 @@
 # limitations under the License.
 
 
+import math
 import os
 import os.path as osp
 import shutil
-import random
-import math
+
 import pandas as pd
-from tqdm import tqdm
+
 from .....utils.logging import info
 
 

paddlex/modules/ts_forecast/evaluator.py

@@ -1,4 +1,4 @@
-# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

paddlex/modules/ts_forecast/exportor.py

@@ -1,4 +1,4 @@
-# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,12 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import tarfile
 from pathlib import Path
 
+from ...utils import logging
 from ..base import BaseExportor
 from .model_list import MODELS
-from ...utils import logging
 
 
 class TSFCExportor(BaseExportor):

paddlex/modules/ts_forecast/model_list.py

@@ -1,4 +1,4 @@
-# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

paddlex/modules/ts_forecast/trainer.py

@@ -1,4 +1,4 @@
-# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,13 +13,11 @@
 # limitations under the License.
 
 import os
-import json
-import time
 import tarfile
 from pathlib import Path
 
+from ...utils.flags import FLAGS_json_format_model
 from ..base import BaseTrainer
-from ...utils.config import AttrDict
 from .model_list import MODELS
 
 
@@ -34,7 +32,21 @@ class TSFCTrainer(BaseTrainer):
         os.makedirs(self.global_config.output, exist_ok=True)
         self.update_config()
         self.dump_config()
-        train_result = self.pdx_model.train(**self.get_train_kwargs())
+        train_args = self.get_train_kwargs()
+        export_with_pir = (
+            self.global_config.get("export_with_pir", False) or FLAGS_json_format_model
+        )
+        train_args.update(
+            {
+                "uniform_output_enabled": self.train_config.get(
+                    "uniform_output_enabled", True
+                ),
+                "export_with_pir": export_with_pir,
+            }
+        )
+        if self.benchmark_config is not None:
+            train_args.update({"benchmark": self.benchmark_config})
+        train_result = self.pdx_model.train(**train_args)
         assert (
             train_result.returncode == 0
         ), f"Encountered an unexpected error({train_result.returncode}) in \
@@ -77,6 +89,8 @@ training!"
             self.pdx_config.update_epochs(self.train_config.epochs_iters)
         if self.train_config.log_interval is not None:
             self.pdx_config.update_log_interval(self.train_config.log_interval)
+        if self.train_config.get("dy2st", False):
+            self.pdx_config.update_to_static(self.train_config.dy2st)
         if self.global_config.output is not None:
             self.pdx_config.update_save_dir(self.global_config.output)
 
@@ -89,4 +103,6 @@ training!"
         train_args = {"device": self.get_device(using_device_number=1)}
         if self.global_config.output is not None:
             train_args["save_dir"] = self.global_config.output
+        # amp support 'O1', 'O2', 'OFF'
+        train_args["amp"] = self.train_config.get("amp", "OFF")
         return train_args

paddlex/modules/video_classification/__init__.py

@@ -0,0 +1,18 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .dataset_checker import VideoClsDatasetChecker
+from .evaluator import VideoClsEvaluator
+from .exportor import VideoClsExportor
+from .trainer import VideoClsTrainer

paddlex/modules/video_classification/dataset_checker/__init__.py

@@ -0,0 +1,93 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pathlib import Path
+
+from ...base import BaseDatasetChecker
+from ..model_list import MODELS
+from .dataset_src import check, deep_analyse, split_dataset
+
+
+class VideoClsDatasetChecker(BaseDatasetChecker):
+    """Dataset Checker for Image Classification Model"""
+
+    entities = MODELS
+    sample_num = 10
+
+    def get_dataset_root(self, dataset_dir: str) -> str:
+        """find the dataset root dir
+
+        Args:
+            dataset_dir (str): the directory that contain dataset.
+
+        Returns:
+            str: the root directory of dataset.
+        """
+        anno_dirs = list(Path(dataset_dir).glob("**/videos"))
+        assert len(anno_dirs) == 1
+        dataset_dir = anno_dirs[0].parent.as_posix()
+        return dataset_dir
+
+    def split_dataset(self, src_dataset_dir: str) -> str:
+        """repartition the train and validation dataset
+
+        Args:
+            src_dataset_dir (str): the root directory of dataset.
+
+        Returns:
+            str: the root directory of splited dataset.
+        """
+        return split_dataset(
+            src_dataset_dir,
+            self.check_dataset_config.split.train_percent,
+            self.check_dataset_config.split.val_percent,
+        )
+
+    def check_dataset(self, dataset_dir: str, sample_num: int = sample_num) -> dict:
+        """check if the dataset meets the specifications and get dataset summary
+
+        Args:
+            dataset_dir (str): the root directory of dataset.
+            sample_num (int): the number to be sampled.
+        Returns:
+            dict: dataset summary.
+        """
+        return check(dataset_dir, self.output)
+
+    def analyse(self, dataset_dir: str) -> dict:
+        """deep analyse dataset
+
+        Args:
+            dataset_dir (str): the root directory of dataset.
+
+        Returns:
+            dict: the deep analysis results.
+        """
+        return deep_analyse(dataset_dir, self.output)
+
+    def get_show_type(self) -> str:
+        """get the show type of dataset
+
+        Returns:
+            str: show type
+        """
+        return "video"
+
+    def get_dataset_type(self) -> str:
+        """return the dataset type
+
+        Returns:
+            str: dataset type
+        """
+        return "VideoClsDataset"

paddlex/modules/video_classification/dataset_checker/dataset_src/__init__.py

@@ -0,0 +1,18 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from .analyse_dataset import deep_analyse
+from .check_dataset import check
+from .split_dataset import split_dataset

paddlex/modules/video_classification/dataset_checker/dataset_src/analyse_dataset.py

@@ -0,0 +1,93 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import platform
+from collections import defaultdict
+
+import numpy as np
+
+from .....utils.deps import function_requires_deps, is_dep_available
+
+if is_dep_available("matplotlib"):
+    import matplotlib.pyplot as plt
+    from matplotlib import font_manager
+
+from .....utils.file_interface import custom_open
+from .....utils.fonts import PINGFANG_FONT_FILE_PATH
+
+
+@function_requires_deps("matplotlib")
+def deep_analyse(dataset_path, output):
+    """class analysis for dataset"""
+    tags = ["train", "val"]
+    labels_cnt = defaultdict(str)
+    label_path = os.path.join(dataset_path, "label.txt")
+    with custom_open(label_path, "r") as f:
+        lines = f.readlines()
+    for line in lines:
+        line = line.strip().split()
+        labels_cnt[line[0]] = " ".join(line[1:])
+    for tag in tags:
+        anno_path = os.path.join(dataset_path, f"{tag}.txt")
+        classes_num = defaultdict(int)
+        for i in range(len(labels_cnt)):
+            classes_num[labels_cnt[str(i)]] = 0
+        with custom_open(anno_path, "r") as f:
+            lines = f.readlines()
+        for line in lines:
+            line = line.strip().split()
+            classes_num[labels_cnt[line[1]]] += 1
+        if tag == "train":
+            cnts_train = [cat_ids for cat_name, cat_ids in classes_num.items()]
+        elif tag == "val":
+            cnts_val = [cat_ids for cat_name, cat_ids in classes_num.items()]
+
+    classes = [cat_name for cat_name, cat_ids in classes_num.items()]
+    sorted_id = sorted(
+        range(len(cnts_train)), key=lambda k: cnts_train[k], reverse=True
+    )
+    cnts_train_sorted = [cnts_train[index] for index in sorted_id]
+    cnts_val_sorted = [cnts_val[index] for index in sorted_id]
+    classes_sorted = [classes[index] for index in sorted_id]
+    x = np.arange(len(classes))
+    width = 0.5
+
+    # bar
+    os_system = platform.system().lower()
+    if os_system == "windows":
+        plt.rcParams["font.sans-serif"] = "FangSong"
+    else:
+        font = font_manager.FontProperties(fname=PINGFANG_FONT_FILE_PATH, size=10)
+    fig, ax = plt.subplots(figsize=(max(8, int(len(classes) / 5)), 5), dpi=300)
+    ax.bar(x, cnts_train_sorted, width=0.5, label="train")
+    ax.bar(x + width, cnts_val_sorted, width=0.5, label="val")
+    plt.xticks(
+        x + width / 2,
+        classes_sorted,
+        rotation=90,
+        fontproperties=None if os_system == "windows" else font,
+    )
+    ax.set_xlabel(
+        "类别名称", fontproperties=None if os_system == "windows" else font, fontsize=12
+    )
+    ax.set_ylabel(
+        "视频数量", fontproperties=None if os_system == "windows" else font, fontsize=12
+    )
+    plt.legend(loc=1)
+    fig.tight_layout()
+    file_path = os.path.join(output, "histogram.png")
+    fig.savefig(file_path, dpi=300)
+
+    return {"histogram": os.path.join("check_dataset", "histogram.png")}