paddlex 2.0.0rc4__py3-none-any.whl → 3.0.0b2__py3-none-any.whl

paddlex/modules/table_recognition/dataset_checker/__init__.py

@@ -0,0 +1,83 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import os.path as osp
+from collections import defaultdict, Counter
+from PIL import Image
+import json
+
+from ...base import BaseDatasetChecker
+from .dataset_src import check, split_dataset, deep_analyse
+
+from ..model_list import MODELS
+
+
+class TableRecDatasetChecker(BaseDatasetChecker):
+    """Dataset Checker for Table Recognition Model"""
+
+    entities = MODELS
+
+    def convert_dataset(self, src_dataset_dir: str) -> str:
+        """convert the dataset from other type to specified type
+
+        Args:
+            src_dataset_dir (str): the root directory of dataset.
+
+        Returns:
+            str: the root directory of converted dataset.
+        """
+        return src_dataset_dir
+
+    def split_dataset(self, src_dataset_dir: str) -> str:
+        """repartition the train and validation dataset
+
+        Args:
+            src_dataset_dir (str): the root directory of dataset.
+
+        Returns:
+            str: the root directory of splited dataset.
+        """
+        return split_dataset(
+            src_dataset_dir,
+            self.check_dataset_config.split.train_percent,
+            self.check_dataset_config.split.val_percent,
+        )
+
+    def check_dataset(self, dataset_dir: str) -> dict:
+        """check if the dataset meets the specifications and get dataset summary
+
+        Args:
+            dataset_dir (str): the root directory of dataset.
+        Returns:
+            dict: dataset summary.
+        """
+        return check(dataset_dir, self.global_config.output, sample_num=10)
+
+    def get_show_type(self) -> str:
+        """get the show type of dataset
+
+        Returns:
+            str: show type
+        """
+        return "image"
+
+    def get_dataset_type(self) -> str:
+        """return the dataset type
+
+        Returns:
+            str: dataset type
+        """
+        return "PubTabTableRecDataset"

paddlex/modules/table_recognition/dataset_checker/dataset_src/__init__.py

@@ -0,0 +1,18 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from .check_dataset import check
+from .split_dataset import split_dataset
+from .analyse_dataset import deep_analyse

paddlex/modules/table_recognition/dataset_checker/dataset_src/analyse_dataset.py

@@ -0,0 +1,58 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+from collections import defaultdict
+from .....utils.file_interface import custom_open
+
+
+def simple_analyse(dataset_path):
+    """
+    Analyse the dataset samples by return image path and label path
+
+    Args:
+        dataset_path (str): dataset path
+
+    Returns:
+        tuple: tuple of sample number, image path and label path for train, val and text subdataset.
+
+    """
+    tags = ["train", "val", "test"]
+    sample_cnts = defaultdict(int)
+    img_paths = defaultdict(list)
+    res = [None] * 6
+
+    for tag in tags:
+        file_list = os.path.join(dataset_path, f"{tag}.txt")
+        if not os.path.exists(file_list):
+            if tag in ("train", "val"):
+                res.insert(0, "数据集不符合规范，请先通过数据校准")
+                return res
+            else:
+                continue
+        else:
+            with custom_open(file_list, "r") as f:
+                all_lines = f.readlines()
+
+            # Each line corresponds to a sample
+            sample_cnts[tag] = len(all_lines)
+            # img_paths[tag] = images_dict[tag]
+
+    return f"训练数据样本数: {sample_cnts[tags[0]]}\t评估数据样本数: {sample_cnts[tags[1]]}"
+
+
+def deep_analyse(dataset_path, output=None):
+    """class analysis for dataset"""
+    return simple_analyse(dataset_path)

paddlex/modules/table_recognition/dataset_checker/dataset_src/check_dataset.py

@@ -0,0 +1,87 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import json
+import os.path as osp
+from collections import defaultdict
+from .....utils.errors import DatasetFileNotFoundError, CheckFailedError
+
+
+def check(dataset_dir, output, dataset_type="PubTabTableRecDataset", sample_num=10):
+    """
+    Check whether the dataset is valid.
+    """
+
+    if dataset_type == "PubTabTableRecDataset":
+        # Custom dataset
+        if not osp.exists(dataset_dir) or not osp.isdir(dataset_dir):
+            raise DatasetFileNotFoundError(file_path=dataset_dir)
+
+        tags = ["train", "val"]
+        max_recorded_sample_cnts = 50
+        sample_cnts = dict()
+        sample_paths = defaultdict(list)
+
+        for tag in tags:
+            file_list = osp.join(dataset_dir, f"{tag}.txt")
+            if not osp.exists(file_list):
+                if tag in ("train", "val"):
+                    # train and val file lists must exist
+                    raise DatasetFileNotFoundError(
+                        file_path=file_list,
+                        solution=f"Ensure that both `train.txt` and `val.txt` exist in {dataset_dir}",
+                    )
+                else:
+                    # tag == 'test'
+                    continue
+            else:
+                with open(file_list, "r", encoding="utf-8") as f:
+                    all_lines = f.readlines()
+                    sample_cnts[tag] = len(all_lines)
+                    for line in all_lines:
+                        info = json.loads(line.strip("\n"))
+                        file_name = info["filename"]
+                        cells = info["html"]["cells"].copy()
+                        structure = info["html"]["structure"]["tokens"].copy()
+
+                        img_path = osp.join(dataset_dir, file_name)
+                        if len(sample_paths[tag]) < max_recorded_sample_cnts:
+                            sample_paths[tag].append(os.path.relpath(img_path, output))
+
+                        if not os.path.exists(img_path):
+                            raise DatasetFileNotFoundError(file_path=img_path)
+
+                        boxes_num = len(cells)
+                        tokens_num = sum(
+                            [
+                                structure.count(x)
+                                for x in ["<td>", "<td", "<eb></eb>", "<td></td>"]
+                            ]
+                        )
+                        if boxes_num != tokens_num:
+                            raise CheckFailedError(
+                                f"The number of cells needs to be consistent with the number of tokens "
+                                "but the number of cells is {boxes_num}, and the number of tokens is {tokens_num}."
+                            )
+        meta = {}
+
+        meta["train_samples"] = sample_cnts["train"]
+        meta["train_sample_paths"] = sample_paths["train"][:sample_num]
+
+        meta["val_samples"] = sample_cnts["val"]
+        meta["val_sample_paths"] = sample_paths["val"][:sample_num]
+
+        return meta

paddlex/modules/table_recognition/dataset_checker/dataset_src/split_dataset.py

@@ -0,0 +1,79 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+from random import shuffle
+from .....utils.file_interface import custom_open
+
+
+def split_dataset(dataset_root, train_rate, val_rate):
+    """
+    将图像数据集按照比例分成训练集、验证集和测试集，并生成对应的.txt文件。
+
+    Args:
+        dataset_root (str): 数据集根目录路径。
+        train_rate (int): 训练集占总数据集的比例（%）。
+        val_rate (int): 验证集占总数据集的比例（%）。
+
+    Returns:
+        str: 数据划分结果信息。
+    """
+    sum_rate = train_rate + val_rate
+    if sum_rate != 100:
+        return "训练集、验证集比例之和需要等于100，请修改后重试"
+    tags = ["train", "val"]
+
+    valid_path = False
+    image_files = []
+    for tag in tags:
+        split_image_list = os.path.abspath(os.path.join(dataset_root, f"{tag}.txt"))
+        rename_image_list = os.path.abspath(
+            os.path.join(dataset_root, f"{tag}.txt.bak")
+        )
+        if os.path.exists(split_image_list):
+            with custom_open(split_image_list, "r") as f:
+                lines = f.readlines()
+            image_files = image_files + lines
+            valid_path = True
+            if not os.path.exists(rename_image_list):
+                os.rename(split_image_list, rename_image_list)
+
+    if not valid_path:
+        return f"数据集目录下保存待划分文件{tags[0]}.txt或{tags[1]}.txt不存在，请检查后重试"
+
+    shuffle(image_files)
+    start = 0
+    image_num = len(image_files)
+    rate_list = [train_rate, val_rate]
+    for i, tag in enumerate(tags):
+
+        rate = rate_list[i]
+        if rate == 0:
+            continue
+        if rate > 100 or rate < 0:
+            return f"{tag} 数据集的比例应该在0~100之间."
+
+        end = start + round(image_num * rate / 100)
+        if sum(rate_list[i + 1 :]) == 0:
+            end = image_num
+
+        txt_file = os.path.abspath(os.path.join(dataset_root, tag + ".txt"))
+        with custom_open(txt_file, "w") as f:
+            m = 0
+            for id in range(start, end):
+                m += 1
+                f.write(image_files[id])
+        start = end
+    return dataset_root

paddlex/modules/table_recognition/evaluator.py

@@ -0,0 +1,43 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from ..base import BaseEvaluator
+from .model_list import MODELS
+
+
+class TableRecEvaluator(BaseEvaluator):
+    """Table Recognition Model Evaluator"""
+
+    entities = MODELS
+
+    def update_config(self):
+        """update evalution config"""
+        if self.eval_config.log_interval:
+            self.pdx_config.update_log_interval(self.eval_config.log_interval)
+
+        self.pdx_config.update_dataset(
+            self.global_config.dataset_dir, "PubTabTableRecDataset"
+        )
+
+    def get_eval_kwargs(self) -> dict:
+        """get key-value arguments of model evalution function
+
+        Returns:
+            dict: the arguments of evaluation function.
+        """
+        return {
+            "weight_path": self.eval_config.weight_path,
+            "device": self.get_device(),
+        }

paddlex/modules/table_recognition/exportor.py

@@ -0,0 +1,22 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ..base import BaseExportor
+from .model_list import MODELS
+
+
+class TableRecExportor(BaseExportor):
+    """Table Recognition Model Exportor"""
+
+    entities = MODELS

paddlex/modules/table_recognition/model_list.py

@@ -0,0 +1,19 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+MODELS = [
+    "SLANet",
+    "SLANet_plus",
+]

paddlex/modules/table_recognition/trainer.py

@@ -0,0 +1,70 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+from pathlib import Path
+
+from ..base import BaseTrainer
+from ...utils.config import AttrDict
+from .model_list import MODELS
+
+
+class TableRecTrainer(BaseTrainer):
+    """Table Recognition Model Trainer"""
+
+    entities = MODELS
+
+    def update_config(self):
+        """update training config"""
+        if self.train_config.log_interval:
+            self.pdx_config.update_log_interval(self.train_config.log_interval)
+        if self.train_config.eval_interval:
+            self.pdx_config._update_eval_interval_by_epoch(
+                self.train_config.eval_interval
+            )
+        if self.train_config.save_interval:
+            self.pdx_config.update_save_interval(self.train_config.save_interval)
+
+        self.pdx_config.update_dataset(
+            self.global_config.dataset_dir, "PubTabTableRecDataset"
+        )
+        if self.train_config.pretrain_weight_path:
+            self.pdx_config.update_pretrained_weights(
+                self.train_config.pretrain_weight_path
+            )
+        if self.train_config.batch_size is not None:
+            self.pdx_config.update_batch_size(self.train_config.batch_size)
+        if self.train_config.learning_rate is not None:
+            self.pdx_config.update_learning_rate(self.train_config.learning_rate)
+        if self.train_config.epochs_iters is not None:
+            self.pdx_config._update_epochs(self.train_config.epochs_iters)
+        if (
+            self.train_config.resume_path is not None
+            and self.train_config.resume_path != ""
+        ):
+            self.pdx_config._update_checkpoints(self.train_config.resume_path)
+        if self.global_config.output is not None:
+            self.pdx_config._update_output_dir(self.global_config.output)
+
+    def get_train_kwargs(self) -> dict:
+        """get key-value arguments of model training function
+
+        Returns:
+            dict: the arguments of training function.
+        """
+        return {
+            "device": self.get_device(),
+            "dy2st": self.train_config.get("dy2st", False),
+        }

paddlex/modules/text_detection/__init__.py

@@ -0,0 +1,18 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .dataset_checker import TextDetDatasetChecker
+from .trainer import TextDetTrainer
+from .evaluator import TextDetEvaluator
+from .exportor import TextDetExportor

paddlex/modules/text_detection/dataset_checker/__init__.py

@@ -0,0 +1,94 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import os.path as osp
+from collections import defaultdict, Counter
+
+import json
+
+from ...base import BaseDatasetChecker
+from .dataset_src import check, split_dataset, deep_analyse
+
+from ..model_list import MODELS
+
+
+class TextDetDatasetChecker(BaseDatasetChecker):
+    """Dataset Checker for Text Detection Model"""
+
+    entities = MODELS
+
+    def convert_dataset(self, src_dataset_dir: str) -> str:
+        """convert the dataset from other type to specified type
+
+        Args:
+            src_dataset_dir (str): the root directory of dataset.
+
+        Returns:
+            str: the root directory of converted dataset.
+        """
+        return src_dataset_dir
+
+    def split_dataset(self, src_dataset_dir: str) -> str:
+        """repartition the train and validation dataset
+
+        Args:
+            src_dataset_dir (str): the root directory of dataset.
+
+        Returns:
+            str: the root directory of splited dataset.
+        """
+        return split_dataset(
+            src_dataset_dir,
+            self.check_dataset_config.split.train_percent,
+            self.check_dataset_config.split.val_percent,
+        )
+
+    def check_dataset(self, dataset_dir: str) -> dict:
+        """check if the dataset meets the specifications and get dataset summary
+
+        Args:
+            dataset_dir (str): the root directory of dataset.
+        Returns:
+            dict: dataset summary.
+        """
+        return check(dataset_dir, self.global_config.output, sample_num=10)
+
+    def analyse(self, dataset_dir: str) -> dict:
+        """deep analyse dataset
+
+        Args:
+            dataset_dir (str): the root directory of dataset.
+
+        Returns:
+            dict: the deep analysis results.
+        """
+        return deep_analyse(dataset_dir, self.output)
+
+    def get_show_type(self) -> str:
+        """get the show type of dataset
+
+        Returns:
+            str: show type
+        """
+        return "image"
+
+    def get_dataset_type(self) -> str:
+        """return the dataset type
+
+        Returns:
+            str: dataset type
+        """
+        return "TextDetDataset"

paddlex/modules/text_detection/dataset_checker/dataset_src/__init__.py

@@ -0,0 +1,18 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from .check_dataset import check
+from .split_dataset import split_dataset
+from .analyse_dataset import deep_analyse