paddlex 2.1.0__py3-none-any.whl → 3.0.0__py3-none-any.whl

paddlex/modules/text_detection/dataset_checker/dataset_src/split_dataset.py

@@ -0,0 +1,140 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import glob
+import os.path
+import shutil
+
+import numpy as np
+
+from .....utils.file_interface import custom_open
+from .....utils.logging import info
+
+
+def split_dataset(dataset_root, train_rate, val_rate):
+    """
+    将图像数据集按照比例分成训练集、验证集和测试集，并生成对应的.txt文件。
+
+    Args:
+        dataset_root (str): 数据集根目录路径。
+        train_rate (int): 训练集占总数据集的比例（%）。
+        val_rate (int): 验证集占总数据集的比例（%）。
+
+    Returns:
+        str: 数据划分结果信息。
+    """
+    sum_rate = train_rate + val_rate
+    assert sum_rate == 100, f"训练集、验证集比例之和需要等于100，请修改后重试"
+    assert (
+        train_rate > 0 and val_rate > 0
+    ), f"The train_rate({train_rate}) and val_rate({val_rate}) should be greater than 0!"
+
+    image_dir = os.path.join(dataset_root, "images")
+    tags = ["train.txt", "val.txt"]
+
+    image_files = get_files(image_dir, ["png", "jpg", "jpeg", "PNG", "JPG", "JPEG"])
+    label_files = get_labels_files(dataset_root, ["train.txt", "val.txt"])
+
+    for tag in tags:
+        src_file = os.path.join(dataset_root, tag)
+        dst_file = os.path.join(dataset_root, f"{tag}.bak")
+        info(
+            f"The original annotation file {src_file} has been backed up to {dst_file}."
+        )
+        shutil.move(src_file, dst_file)
+
+    image_num = len(image_files)
+    label_num = len(label_files)
+    assert image_num != 0, f"原始图像数量({image_num})为0, 请检查后重试"
+    assert (
+        image_num == label_num
+    ), f"原始图像数量({image_num})和标注图像数量({label_num})不相等，请检查后重试"
+
+    image_files = np.array(image_files)
+    label_files = np.array(label_files)
+    state = np.random.get_state()
+    np.random.shuffle(image_files)
+    np.random.set_state(state)
+    np.random.shuffle(label_files)
+
+    start = 0
+    rate_list = [train_rate, val_rate]
+    name_list = ["train", "val"]
+    for i, name in enumerate(name_list):
+        info("Creating {}.txt...".format(name))
+
+        rate = rate_list[i]
+        if rate == 0:
+            txt_file = os.path.join(dataset_root, name + ".txt")
+            with custom_open(txt_file, "w") as f:
+                f.write("")
+            continue
+
+        end = start + round(image_num * rate / 100)
+        if sum(rate_list[i + 1 :]) == 0:
+            end = image_num
+
+        txt_file = os.path.join(dataset_root, name + ".txt")
+        with custom_open(txt_file, "w") as f:
+            for id in range(start, end):
+                right = label_files[id]
+                f.write(right)
+        start = end
+
+    return dataset_root
+
+
+def get_files(input_dir, format=["jpg", "png"]):
+    """
+    在给定目录下获取符合指定文件格式的所有文件路径
+
+    Args:
+        input_dir (str): 目标文件夹路径
+        format (Union[str, List[str]]): 需要获取的文件格式, 可以是字符串或者字符串列表
+
+    Returns:
+        List[str]: 符合格式的所有文件路径列表，返回排序后的结果
+    """
+    res = []
+    if not isinstance(format, (list, tuple)):
+        format = [format]
+    for item in format:
+        pattern = os.path.join(input_dir, f"**/*.{item}")
+        files = glob.glob(pattern, recursive=True)
+        res.extend(files)
+    return sorted(res)
+
+
+def get_labels_files(input_dir, format=["train.txt", "val.txt"]):
+    """
+    在给定目录下获取符合指定文件格式的所有文件路径
+
+    Args:
+        input_dir (str): 目标文件夹路径
+        format (Union[str, List[str]]): 需要获取的文件格式, 可以是字符串或者字符串列表
+
+    Returns:
+        List[str]: 符合格式的所有文件路径列表，返回排序后的结果
+    """
+    res = []
+    if not isinstance(format, (list, tuple)):
+        format = [format]
+    for tag in format:
+        file_list = os.path.join(input_dir, f"{tag}")
+        if os.path.exists(file_list):
+            with custom_open(file_list, "r") as f:
+                all_lines = f.readlines()
+                res.extend(all_lines)
+    return sorted(res)

paddlex/modules/text_detection/evaluator.py

@@ -0,0 +1,41 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from ..base import BaseEvaluator
+from .model_list import MODELS
+
+
+class TextDetEvaluator(BaseEvaluator):
+    """Text Detection Model Evaluator"""
+
+    entities = MODELS
+
+    def update_config(self):
+        """update evaluation config"""
+        if self.eval_config.log_interval:
+            self.pdx_config.update_log_interval(self.eval_config.log_interval)
+
+        self.pdx_config.update_dataset(self.global_config.dataset_dir, "TextDetDataset")
+
+    def get_eval_kwargs(self) -> dict:
+        """get key-value arguments of model evaluation function
+
+        Returns:
+            dict: the arguments of evaluation function.
+        """
+        return {
+            "weight_path": self.eval_config.weight_path,
+            "device": self.get_device(),
+        }

paddlex/modules/text_detection/exportor.py

@@ -0,0 +1,22 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ..base import BaseExportor
+from .model_list import MODELS
+
+
+class TextDetExportor(BaseExportor):
+    """Text Detection Model Exportor"""
+
+    entities = MODELS

paddlex/modules/text_detection/model_list.py

@@ -0,0 +1,26 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+MODELS = [
+    "PP-OCRv5_mobile_det",
+    "PP-OCRv5_server_det",
+    "PP-OCRv4_mobile_det",
+    "PP-OCRv4_server_det",
+    "PP-OCRv4_mobile_seal_det",
+    "PP-OCRv4_server_seal_det",
+    "PP-OCRv3_mobile_det",
+    "PP-OCRv3_server_det",
+]
+
+CURVE_MODELS = ["PP-OCRv4_mobile_seal_det", "PP-OCRv4_server_seal_det"]

paddlex/modules/text_detection/trainer.py

@@ -0,0 +1,65 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from ..base import BaseTrainer
+from .model_list import MODELS
+
+
+class TextDetTrainer(BaseTrainer):
+    """Text Detection Model Trainer"""
+
+    entities = MODELS
+
+    def update_config(self):
+        """update training config"""
+        if self.train_config.log_interval:
+            self.pdx_config.update_log_interval(self.train_config.log_interval)
+        if self.train_config.eval_interval:
+            self.pdx_config._update_eval_interval_by_epoch(
+                self.train_config.eval_interval
+            )
+        if self.train_config.save_interval:
+            self.pdx_config.update_save_interval(self.train_config.save_interval)
+
+        self.pdx_config.update_dataset(self.global_config.dataset_dir, "TextDetDataset")
+        if self.train_config.pretrain_weight_path:
+            self.pdx_config.update_pretrained_weights(
+                self.train_config.pretrain_weight_path
+            )
+        if self.train_config.batch_size is not None:
+            self.pdx_config.update_batch_size(self.train_config.batch_size)
+        if self.train_config.learning_rate is not None:
+            self.pdx_config.update_learning_rate(self.train_config.learning_rate)
+        if self.train_config.epochs_iters is not None:
+            self.pdx_config._update_epochs(self.train_config.epochs_iters)
+        if (
+            self.train_config.resume_path is not None
+            and self.train_config.resume_path != ""
+        ):
+            self.pdx_config._update_checkpoints(self.train_config.resume_path)
+        if self.global_config.output is not None:
+            self.pdx_config._update_output_dir(self.global_config.output)
+
+    def get_train_kwargs(self) -> dict:
+        """get key-value arguments of model training function
+
+        Returns:
+            dict: the arguments of training function.
+        """
+        return {
+            "device": self.get_device(),
+            "dy2st": self.train_config.get("dy2st", False),
+            "amp": self.train_config.get("amp", "OFF"),  # amp support 'O1', 'O2', 'OFF'
+        }

paddlex/modules/text_recognition/__init__.py

@@ -0,0 +1,18 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .dataset_checker import TextRecDatasetChecker
+from .evaluator import TextRecEvaluator
+from .exportor import TextRecExportor
+from .trainer import TextRecTrainer

paddlex/modules/text_recognition/dataset_checker/__init__.py

@@ -0,0 +1,125 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import json
+import os
+import os.path as osp
+from collections import Counter, defaultdict
+from pathlib import Path
+
+from PIL import Image
+
+from ...base import BaseDatasetChecker
+from ..model_list import MODELS
+from .dataset_src import check, convert, deep_analyse, split_dataset
+
+
+class TextRecDatasetChecker(BaseDatasetChecker):
+    """Dataset Checker for Text Recognition Model"""
+
+    entities = MODELS
+    sample_num = 10
+
+    def get_dataset_root(self, dataset_dir: str) -> str:
+        """find the dataset root dir
+
+        Args:
+            dataset_dir (str): the directory that contain dataset.
+
+        Returns:
+            str: the root directory of dataset.
+        """
+        anno_dirs = list(Path(dataset_dir).glob("**/train.txt"))
+        assert len(anno_dirs) == 1
+        dataset_dir = anno_dirs[0].parent.as_posix()
+        return dataset_dir
+
+    def convert_dataset(self, src_dataset_dir: str) -> str:
+        """convert the dataset from other type to specified type
+
+        Args:
+            src_dataset_dir (str): the root directory of dataset.
+
+        Returns:
+            str: the root directory of converted dataset.
+        """
+        return convert(
+            self.check_dataset_config.convert.src_dataset_type, src_dataset_dir
+        )
+
+    def split_dataset(self, src_dataset_dir: str) -> str:
+        """repartition the train and validation dataset
+
+        Args:
+            src_dataset_dir (str): the root directory of dataset.
+
+        Returns:
+            str: the root directory of splited dataset.
+        """
+        return split_dataset(
+            src_dataset_dir,
+            self.check_dataset_config.split.train_percent,
+            self.check_dataset_config.split.val_percent,
+        )
+
+    def check_dataset(self, dataset_dir: str, sample_num: int = sample_num) -> dict:
+        """check if the dataset meets the specifications and get dataset summary
+
+        Args:
+            dataset_dir (str): the root directory of dataset.
+            sample_num (int): the number to be sampled.
+        Returns:
+            dict: dataset summary.
+        """
+        return check(
+            dataset_dir,
+            self.output,
+            sample_num=10,
+            dataset_type=self.get_dataset_type(),
+        )
+
+    def analyse(self, dataset_dir: str) -> dict:
+        """deep analyse dataset
+
+        Args:
+            dataset_dir (str): the root directory of dataset.
+
+        Returns:
+            dict: the deep analysis results.
+        """
+        if self.global_config["model"] in ["LaTeX_OCR_rec"]:
+            datatype = "LaTeXOCRDataset"
+        else:
+            datatype = "MSTextRecDataset"
+        return deep_analyse(dataset_dir, self.output, datatype=datatype)
+
+    def get_show_type(self) -> str:
+        """get the show type of dataset
+
+        Returns:
+            str: show type
+        """
+        return "image"
+
+    def get_dataset_type(self) -> str:
+        """return the dataset type
+
+        Returns:
+            str: dataset type
+        """
+        if self.global_config["model"] in ["LaTeX_OCR_rec"]:
+            return "LaTeXOCRDataset"
+        else:
+            return "MSTextRecDataset"

paddlex/modules/text_recognition/dataset_checker/dataset_src/__init__.py

@@ -0,0 +1,19 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from .analyse_dataset import deep_analyse
+from .check_dataset import check
+from .convert_dataset import convert
+from .split_dataset import split_dataset

paddlex/modules/text_recognition/dataset_checker/dataset_src/analyse_dataset.py

@@ -0,0 +1,162 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import math
+import os
+import platform
+from collections import defaultdict
+
+import numpy as np
+
+from .....utils.deps import function_requires_deps, is_dep_available
+from .....utils.file_interface import custom_open
+from .....utils.fonts import PINGFANG_FONT_FILE_PATH
+from .....utils.logging import warning
+
+if is_dep_available("opencv-contrib-python"):
+    import cv2
+if is_dep_available("matplotlib"):
+    import matplotlib.pyplot as plt
+    from matplotlib import font_manager
+    from matplotlib.backends.backend_agg import FigureCanvasAgg
+
+
+def simple_analyse(dataset_path, images_dict):
+    """
+    Analyse the dataset samples by return image path and label path
+
+    Args:
+        dataset_path (str): dataset path
+        ds_meta (dict): dataset meta
+        images_dict (dict): train, val and test image path
+
+    Returns:
+        tuple: tuple of sample number, image path and label path for train, val and text subdataset.
+
+    """
+    tags = ["train", "val", "test"]
+    sample_cnts = defaultdict(int)
+    img_paths = defaultdict(list)
+    res = [None] * 6
+
+    for tag in tags:
+        file_list = os.path.join(dataset_path, f"{tag}.txt")
+        if not os.path.exists(file_list):
+            if tag in ("train", "val"):
+                res.insert(0, "数据集不符合规范，请先通过数据校准")
+                return res
+            else:
+                continue
+        else:
+            with custom_open(file_list, "r") as f:
+                all_lines = f.readlines()
+
+            # Each line corresponds to a sample
+            sample_cnts[tag] = len(all_lines)
+            img_paths[tag] = images_dict[tag]
+
+    return (
+        "完成数据分析",
+        sample_cnts[tags[0]],
+        sample_cnts[tags[1]],
+        sample_cnts[tags[2]],
+        img_paths[tags[0]],
+        img_paths[tags[1]],
+        img_paths[tags[2]],
+    )
+
+
+@function_requires_deps("matplotlib", "opencv-contrib-python")
+def deep_analyse(dataset_path, output, datatype="MSTextRecDataset"):
+    """class analysis for dataset"""
+    tags = ["train", "val"]
+    labels_cnt = {}
+    x_max = []
+    classes_max = []
+    for tag in tags:
+        image_path = os.path.join(dataset_path, f"{tag}.txt")
+        str_nums = []
+        with custom_open(image_path, "r") as f:
+            lines = f.readlines()
+        for line in lines:
+            line = line.strip().split("\t")
+            if len(line) != 2:
+                warning(f"Error in {line}.")
+                continue
+            str_nums.append(len(line[1]))
+        if datatype == "LaTeXOCRDataset":
+            max_length = min(768, max(str_nums))
+            interval = 20
+        else:
+            max_length = min(100, max(str_nums))
+            interval = 5
+        start = 0
+        for i in range(1, math.ceil((max_length / interval))):
+            stop = i * interval
+            num_str = sum(start < i <= stop for i in str_nums)
+            labels_cnt[f"{start}-{stop}"] = num_str
+            start = stop
+        if sum(max_length < i for i in str_nums) != 0:
+            labels_cnt[f"> {max_length}"] = sum(max_length < i for i in str_nums)
+        if tag == "train":
+            cnts_train = [cat_ids for cat_name, cat_ids in labels_cnt.items()]
+            x_train = np.arange(len(cnts_train))
+            if len(x_train) > len(x_max):
+                x_max = x_train
+                classes_max = [cat_name for cat_name, cat_ids in labels_cnt.items()]
+        elif tag == "val":
+            cnts_val = [cat_ids for cat_name, cat_ids in labels_cnt.items()]
+            x_val = np.arange(len(cnts_val))
+            if len(x_val) > len(x_max):
+                x_max = x_val
+                classes_max = [cat_name for cat_name, cat_ids in labels_cnt.items()]
+
+    width = 0.3
+
+    # bar
+    os_system = platform.system().lower()
+    if os_system == "windows":
+        plt.rcParams["font.sans-serif"] = "FangSong"
+    else:
+        font = font_manager.FontProperties(fname=PINGFANG_FONT_FILE_PATH, size=15)
+    if datatype == "LaTeXOCRDataset":
+        fig, ax = plt.subplots(figsize=(15, 9), dpi=120)
+        xlabel_name = "公式长度区间"
+    else:
+        fig, ax = plt.subplots(figsize=(10, 5), dpi=120)
+        xlabel_name = "文本字长度区间"
+    ax.bar(x_train, cnts_train, width=0.3, label="train")
+    ax.bar(x_val + width, cnts_val, width=0.3, label="val")
+    plt.xticks(x_max + width / 2, classes_max, rotation=90)
+    plt.legend(prop={"size": 18})
+    ax.set_xlabel(
+        xlabel_name,
+        fontproperties=None if os_system == "windows" else font,
+        fontsize=12,
+    )
+    ax.set_ylabel(
+        "图片数量", fontproperties=None if os_system == "windows" else font, fontsize=12
+    )
+
+    canvas = FigureCanvasAgg(fig)
+    canvas.draw()
+    width, height = fig.get_size_inches() * fig.get_dpi()
+    pie_array = np.frombuffer(canvas.tostring_rgb(), dtype="uint8").reshape(
+        int(height), int(width), 3
+    )
+    fig1_path = os.path.join(output, "histogram.png")
+    cv2.imwrite(fig1_path, pie_array)
+
+    return {"histogram": os.path.join("check_dataset", "histogram.png")}