deepdoctection 0.32__py3-none-any.whl → 0.34__py3-none-any.whl

deepdoctection/datasets/instances/fintabnet.py

@@ -33,7 +33,7 @@ Module for Fintabnet dataset. Place the dataset as follows
 from __future__ import annotations
 
 from pathlib import Path
-from typing import List, Mapping, Sequence, Union
+from typing import Mapping, Sequence, Union
 
 from ...dataflow import DataFlow, MapData, MultiProcessMapData
 from ...dataflow.common import FlattenData
@@ -43,10 +43,10 @@ from ...mapper.cats import cat_to_sub_cat, filter_cat
 from ...mapper.maputils import curry
 from ...mapper.misc import image_ann_to_image, maybe_ann_to_sub_image
 from ...mapper.pubstruct import pub_to_image
-from ...utils.detection_types import JsonDict
 from ...utils.file_utils import set_mp_spawn
 from ...utils.logger import LoggingRecord, logger
 from ...utils.settings import CellType, DatasetType, LayoutType, ObjectTypes, TableType
+from ...utils.types import PubtabnetDict
 from ...utils.utils import to_bool
 from ..base import _BuiltInDataset
 from ..dataflow_builder import DataFlowBaseBuilder
@@ -83,38 +83,38 @@ _URL = (
     "fintabnet.tar.gz?_ga=2.17492593.994196051.1634564576-1173244232.1625045842"
 )
 _SPLITS: Mapping[str, str] = {"train": "train", "val": "val", "test": "test"}
-_TYPE = DatasetType.object_detection
+_TYPE = DatasetType.OBJECT_DETECTION
 _LOCATION = "fintabnet"
 _ANNOTATION_FILES: Mapping[str, str] = {
     "train": "FinTabNet_1.0.0_table_train.jsonl",
     "test": "FinTabNet_1.0.0_table_test.jsonl",
     "val": "FinTabNet_1.0.0_table_val.jsonl",
 }
-_INIT_CATEGORIES = [LayoutType.table, LayoutType.cell, TableType.item]
+_INIT_CATEGORIES = [LayoutType.TABLE, LayoutType.CELL, TableType.ITEM]
 _SUB_CATEGORIES: Mapping[ObjectTypes, Mapping[ObjectTypes, Sequence[ObjectTypes]]]
 _SUB_CATEGORIES = {
-    LayoutType.cell: {
-        CellType.header: [CellType.header, CellType.body],
-        CellType.row_number: [],
-        CellType.column_number: [],
-        CellType.row_span: [],
-        CellType.column_span: [],
-        CellType.spanning: [CellType.spanning, LayoutType.cell],
+    LayoutType.CELL: {
+        CellType.HEADER: [CellType.HEADER, CellType.BODY],
+        CellType.ROW_NUMBER: [],
+        CellType.COLUMN_NUMBER: [],
+        CellType.ROW_SPAN: [],
+        CellType.COLUMN_SPAN: [],
+        CellType.SPANNING: [CellType.SPANNING, LayoutType.CELL],
     },
-    TableType.item: {TableType.item: [LayoutType.row, LayoutType.column]},
-    CellType.header: {
-        CellType.row_number: [],
-        CellType.column_number: [],
-        CellType.row_span: [],
-        CellType.column_span: [],
-        CellType.spanning: [CellType.spanning, LayoutType.cell],
+    TableType.ITEM: {TableType.ITEM: [LayoutType.ROW, LayoutType.COLUMN]},
+    CellType.HEADER: {
+        CellType.ROW_NUMBER: [],
+        CellType.COLUMN_NUMBER: [],
+        CellType.ROW_SPAN: [],
+        CellType.COLUMN_SPAN: [],
+        CellType.SPANNING: [CellType.SPANNING, LayoutType.CELL],
     },
-    CellType.body: {
-        CellType.row_number: [],
-        CellType.column_number: [],
-        CellType.row_span: [],
-        CellType.column_span: [],
-        CellType.spanning: [CellType.spanning, LayoutType.cell],
+    CellType.BODY: {
+        CellType.ROW_NUMBER: [],
+        CellType.COLUMN_NUMBER: [],
+        CellType.ROW_SPAN: [],
+        CellType.COLUMN_SPAN: [],
+        CellType.SPANNING: [CellType.SPANNING, LayoutType.CELL],
     },
 }
 
@@ -201,18 +201,17 @@ class FintabnetBuilder(DataFlowBaseBuilder):
 
         # Map
         @curry
-        def _map_filename(dp: JsonDict, workdir: Path) -> JsonDict:
+        def _map_filename(dp: PubtabnetDict, workdir: Path) -> PubtabnetDict:
             dp["filename"] = workdir / "pdf" / dp["filename"]
             return dp
 
-        map_filename = _map_filename(self.get_workdir())  # pylint: disable=E1120  # 259
-        df = MapData(df, map_filename)
+        df = MapData(df, _map_filename(self.get_workdir()))
 
         buffer_size = 200 if max_datapoints is None else min(max_datapoints, 200) - 1
 
         pub_mapper = pub_to_image(
-            self.categories.get_categories(name_as_key=True, init=True),
-            load_image,
+            categories_name_as_key=self.categories.get_categories(name_as_key=True, init=True),
+            load_image=load_image,
             fake_score=fake_score,
             rows_and_cols=rows_and_cols,
             dd_pipe_like=False,
@@ -233,39 +232,39 @@ class FintabnetBuilder(DataFlowBaseBuilder):
         if build_mode == "table":
 
             @curry
-            def _crop_and_add_image(dp: Image, category_names: List[str]) -> Image:
+            def _crop_and_add_image(dp: Image, category_names: list[str]) -> Image:
                 return image_ann_to_image(dp, category_names=category_names)
 
             df = MapData(
                 df,
                 _crop_and_add_image(  # pylint: disable=E1120
                     category_names=[
-                        LayoutType.table,
-                        LayoutType.cell,
-                        CellType.header,
-                        CellType.body,
-                        TableType.item,
-                        LayoutType.row,
-                        LayoutType.column,
+                        LayoutType.TABLE,
+                        LayoutType.CELL,
+                        CellType.HEADER,
+                        CellType.BODY,
+                        TableType.ITEM,
+                        LayoutType.ROW,
+                        LayoutType.COLUMN,
                     ]
                 ),
             )
             df = MapData(
                 df,
                 maybe_ann_to_sub_image(  # pylint: disable=E1120  # 259
-                    category_names_sub_image=LayoutType.table,
+                    category_names_sub_image=LayoutType.TABLE,
                     category_names=[
-                        LayoutType.cell,
-                        CellType.header,
-                        CellType.body,
-                        TableType.item,
-                        LayoutType.row,
-                        LayoutType.column,
+                        LayoutType.CELL,
+                        CellType.HEADER,
+                        CellType.BODY,
+                        TableType.ITEM,
+                        LayoutType.ROW,
+                        LayoutType.COLUMN,
                     ],
                     add_summary=True,
                 ),
             )
-            df = MapData(df, lambda dp: [ann.image for ann in dp.get_annotation_iter(category_names=LayoutType.table)])
+            df = MapData(df, lambda dp: [ann.image for ann in dp.get_annotation(category_names=LayoutType.TABLE)])
             df = FlattenData(df)
             df = MapData(df, lambda dp: dp[0])
 

deepdoctection/datasets/instances/funsd.py

@@ -41,16 +41,16 @@ from ...dataflow import DataFlow, MapData, SerializerFiles
 from ...datasets.info import DatasetInfo
 from ...mapper.cats import cat_to_sub_cat, filter_cat
 from ...mapper.xfundstruct import xfund_to_image
-from ...utils.detection_types import JsonDict, Pathlike
 from ...utils.fs import load_json
 from ...utils.settings import BioTag, DatasetType, LayoutType, ObjectTypes, TokenClasses, TokenClassWithTag, WordType
+from ...utils.types import FunsdDict, PathLikeOrStr
 from ..base import _BuiltInDataset
 from ..dataflow_builder import DataFlowBaseBuilder
 from ..info import DatasetCategories
 from ..registry import dataset_registry
 
 
-def load_file(path_ann: Pathlike) -> JsonDict:
+def load_file(path_ann: PathLikeOrStr) -> FunsdDict:
     """
     Loading json file
 
@@ -80,28 +80,28 @@ _LICENSE = (
 
 _URL = "https://guillaumejaume.github.io/FUNSD/download/"
 _SPLITS: Mapping[str, str] = {"train": "training_data", "test": "testing_data"}
-_TYPE = DatasetType.token_classification
+_TYPE = DatasetType.TOKEN_CLASSIFICATION
 _LOCATION = "funsd"
 _ANNOTATION_FILES: Mapping[str, str] = {"train": "annotations", "test": "annotations"}
 
-_INIT_CATEGORIES = [LayoutType.word, LayoutType.text]
+_INIT_CATEGORIES = [LayoutType.WORD, LayoutType.TEXT]
 _SUB_CATEGORIES: Dict[ObjectTypes, Dict[ObjectTypes, List[ObjectTypes]]]
 _SUB_CATEGORIES = {
-    LayoutType.word: {
-        WordType.token_class: [TokenClasses.other, TokenClasses.question, TokenClasses.answer, TokenClasses.header],
-        WordType.tag: [BioTag.inside, BioTag.outside, BioTag.begin],
-        WordType.token_tag: [
-            TokenClassWithTag.b_answer,
-            TokenClassWithTag.b_header,
-            TokenClassWithTag.b_question,
-            TokenClassWithTag.i_answer,
-            TokenClassWithTag.i_header,
-            TokenClassWithTag.i_question,
-            BioTag.outside,
+    LayoutType.WORD: {
+        WordType.TOKEN_CLASS: [TokenClasses.OTHER, TokenClasses.QUESTION, TokenClasses.ANSWER, TokenClasses.HEADER],
+        WordType.TAG: [BioTag.INSIDE, BioTag.OUTSIDE, BioTag.BEGIN],
+        WordType.TOKEN_TAG: [
+            TokenClassWithTag.B_ANSWER,
+            TokenClassWithTag.B_HEADER,
+            TokenClassWithTag.B_QUESTION,
+            TokenClassWithTag.I_ANSWER,
+            TokenClassWithTag.I_HEADER,
+            TokenClassWithTag.I_QUESTION,
+            BioTag.OUTSIDE,
         ],
     },
-    LayoutType.text: {
-        WordType.token_class: [TokenClasses.other, TokenClasses.question, TokenClasses.answer, TokenClasses.header]
+    LayoutType.TEXT: {
+        WordType.TOKEN_CLASS: [TokenClasses.OTHER, TokenClasses.QUESTION, TokenClasses.ANSWER, TokenClasses.HEADER]
     },
 }
 
@@ -159,14 +159,14 @@ class FunsdBuilder(DataFlowBaseBuilder):
         # Map
         categories_name_as_key = self.categories.get_categories(init=True, name_as_key=True)
         category_names_mapping = {
-            "other": TokenClasses.other,
-            "question": TokenClasses.question,
-            "answer": TokenClasses.answer,
-            "header": TokenClasses.header,
+            "other": TokenClasses.OTHER,
+            "question": TokenClasses.QUESTION,
+            "answer": TokenClasses.ANSWER,
+            "header": TokenClasses.HEADER,
         }
         ner_token_to_id_mapping = self.categories.get_sub_categories(
-            categories=LayoutType.word,
-            sub_categories={LayoutType.word: [WordType.token_tag, WordType.tag, WordType.token_class]},
+            categories=LayoutType.WORD,
+            sub_categories={LayoutType.WORD: [WordType.TOKEN_TAG, WordType.TAG, WordType.TOKEN_CLASS]},
             keys=False,
             values_as_dict=True,
             name_as_key=True,

deepdoctection/datasets/instances/iiitar13k.py

@@ -47,10 +47,10 @@ from ...datasets.info import DatasetInfo
 from ...mapper.maputils import curry
 from ...mapper.misc import xml_to_dict
 from ...mapper.pascalstruct import pascal_voc_dict_to_image
-from ...utils.detection_types import JsonDict
 from ...utils.file_utils import lxml_available
 from ...utils.fs import get_package_path
 from ...utils.settings import DatasetType, LayoutType
+from ...utils.types import JsonDict
 from ..base import _BuiltInDataset
 from ..dataflow_builder import DataFlowBaseBuilder
 from ..info import DatasetCategories
@@ -76,7 +76,7 @@ _LICENSE = "NN"
 _URL = "http://cvit.iiit.ac.in/usodi/iiitar13k.php"
 
 _SPLITS: Mapping[str, str] = {"train": "training_images", "val": "validation_images", "test": "test_images"}
-_TYPE = DatasetType.object_detection
+_TYPE = DatasetType.OBJECT_DETECTION
 _LOCATION = "iiitar13k"
 _ANNOTATION_FILES: Mapping[str, str] = {
     "train": "training_xml",
@@ -84,7 +84,7 @@ _ANNOTATION_FILES: Mapping[str, str] = {
     "test": "test_xml",
 }
 
-_INIT_CATEGORIES = [LayoutType.table, LayoutType.logo, LayoutType.figure, LayoutType.signature]
+_INIT_CATEGORIES = [LayoutType.TABLE, LayoutType.LOGO, LayoutType.FIGURE, LayoutType.SIGNATURE]
 
 
 @dataset_registry.register("iiitar13k")
@@ -176,11 +176,11 @@ class IIITar13KBuilder(DataFlowBaseBuilder):
                 filter_empty_image=True,
                 fake_score=fake_score,
                 category_name_mapping={
-                    "natural_image": LayoutType.figure,
-                    "figure": LayoutType.figure,
-                    "logo": LayoutType.logo,
-                    "signature": LayoutType.signature,
-                    "table": LayoutType.table,
+                    "natural_image": LayoutType.FIGURE,
+                    "figure": LayoutType.FIGURE,
+                    "logo": LayoutType.LOGO,
+                    "signature": LayoutType.SIGNATURE,
+                    "table": LayoutType.FIGURE,
                 },
             ),
         )

deepdoctection/datasets/instances/layouttest.py

@@ -52,7 +52,7 @@ _LICENSE = (
 )
 
 _SPLITS: Mapping[str, str] = {"test": "test", "predict": "predict"}
-_TYPE = DatasetType.object_detection
+_TYPE = DatasetType.OBJECT_DETECTION
 _LOCATION = "testlayout"
 
 _ANNOTATION_FILES: Mapping[str, str] = {
@@ -60,7 +60,7 @@ _ANNOTATION_FILES: Mapping[str, str] = {
     "predict": "xrf_layout_test_predict.jsonl",
 }
 
-_INIT_CATEGORIES = [LayoutType.text, LayoutType.title, LayoutType.list, LayoutType.table, LayoutType.figure]
+_INIT_CATEGORIES = [LayoutType.TEXT, LayoutType.TITLE, LayoutType.LIST, LayoutType.TABLE, LayoutType.FIGURE]
 
 
 @dataset_registry.register("testlayout")

deepdoctection/datasets/instances/publaynet.py

@@ -61,12 +61,12 @@ _URL = (
     "publaynet.tar.gz?_ga=2.23017467.1796315263.1628754613-1173244232.1625045842"
 )
 _SPLITS: Mapping[str, str] = {"train": "train", "val": "val"}
-_TYPE = DatasetType.object_detection
+_TYPE = DatasetType.OBJECT_DETECTION
 
 _LOCATION = "publaynet"
 
 _ANNOTATION_FILES: Mapping[str, str] = {"train": "train.json", "val": "val.json"}
-_INIT_CATEGORIES = [LayoutType.text, LayoutType.title, LayoutType.list, LayoutType.table, LayoutType.figure]
+_INIT_CATEGORIES = [LayoutType.TEXT, LayoutType.TITLE, LayoutType.LIST, LayoutType.TABLE, LayoutType.FIGURE]
 
 
 @dataset_registry.register("publaynet")
@@ -120,7 +120,7 @@ class PublaynetBuilder(DataFlowBaseBuilder):
         df = SerializerCoco.load(path, max_datapoints=max_datapoints)
 
         # Map
-        df = MapDataComponent(df, lambda dp: self.get_workdir() / self.get_split(split) / dp, "file_name")
+        df = MapDataComponent(df, lambda dp: (self.get_workdir() / self.get_split(split) / dp).as_posix(), "file_name")
         coco_mapper = coco_to_image(  # pylint: disable=E1120  # 259
             self.categories.get_categories(init=True),
             load_image,

deepdoctection/datasets/instances/pubtables1m.py

@@ -50,10 +50,10 @@ from ...mapper.cats import filter_cat
 from ...mapper.maputils import curry
 from ...mapper.misc import xml_to_dict
 from ...mapper.pascalstruct import pascal_voc_dict_to_image
-from ...utils.detection_types import JsonDict
 from ...utils.file_utils import lxml_available
 from ...utils.fs import get_package_path
 from ...utils.settings import CellType, DatasetType, LayoutType
+from ...utils.types import JsonDict
 from ..base import _BuiltInDataset
 from ..dataflow_builder import DataFlowBaseBuilder
 from ..info import DatasetCategories
@@ -80,14 +80,14 @@ _LICENSE = "Community Data License Agreement – Permissive, Version 1.0"
 _URL = "https://msropendata.com/datasets/505fcbe3-1383-42b1-913a-f651b8b712d3"
 
 _SPLITS: Mapping[str, str] = {"train": "train", "val": "val", "test": "test"}
-_TYPE = DatasetType.object_detection
+_TYPE = DatasetType.OBJECT_DETECTION
 _LOCATION = "PubTables1M"
 _ANNOTATION_FILES: Mapping[str, str] = {
     "train": "PubTables1M-Detection-PASCAL-VOC/train",
     "val": "PubTables1M-Detection-PASCAL-VOC/val",
     "test": "PubTables1M-Detection-PASCAL-VOC/test",
 }
-_INIT_CATEGORIES_DET = [LayoutType.table, LayoutType.table_rotated]
+_INIT_CATEGORIES_DET = [LayoutType.TABLE, LayoutType.TABLE_ROTATED]
 
 
 @dataset_registry.register("pubtables1m_det")
@@ -180,7 +180,7 @@ class Pubtables1MBuilder(DataFlowBaseBuilder):
                 load_image,
                 filter_empty_image=True,
                 fake_score=fake_score,
-                category_name_mapping={"table": LayoutType.table, "table rotated": LayoutType.table_rotated},
+                category_name_mapping={"table": LayoutType.TABLE, "table rotated": LayoutType.TABLE_ROTATED},
             ),
         )
 
@@ -195,13 +195,13 @@ _ANNOTATION_FILES_STRUCT: Mapping[str, str] = {
 }
 
 _INIT_CATEGORIES_STRUCT = [
-    LayoutType.table,
-    LayoutType.row,
-    LayoutType.column,
-    CellType.spanning,
-    CellType.row_header,
-    CellType.column_header,
-    CellType.projected_row_header,
+    LayoutType.TABLE,
+    LayoutType.ROW,
+    LayoutType.COLUMN,
+    CellType.SPANNING,
+    CellType.ROW_HEADER,
+    CellType.COLUMN_HEADER,
+    CellType.PROJECTED_ROW_HEADER,
 ]
 
 _IMAGES: Mapping[str, str] = {
@@ -302,13 +302,13 @@ class Pubtables1MBuilderStruct(DataFlowBaseBuilder):
                 filter_empty_image=True,
                 fake_score=fake_score,
                 category_name_mapping={
-                    "table": LayoutType.table,
-                    "table spanning cell": CellType.spanning,
-                    "table row": LayoutType.row,
-                    "table row header": CellType.row_header,
-                    "table projected row header": CellType.projected_row_header,
-                    "table column": LayoutType.column,
-                    "table column header": CellType.column_header,
+                    "table": LayoutType.TABLE,
+                    "table spanning cell": CellType.SPANNING,
+                    "table row": LayoutType.ROW,
+                    "table row header": CellType.ROW_HEADER,
+                    "table projected row header": CellType.PROJECTED_ROW_HEADER,
+                    "table column": LayoutType.COLUMN,
+                    "table column header": CellType.COLUMN_HEADER,
                 },
             ),
         )

deepdoctection/datasets/instances/pubtabnet.py

@@ -29,16 +29,16 @@ Module for Pubtabnet dataset. Place the dataset as follows
 """
 from __future__ import annotations
 
-from typing import Dict, List, Mapping, Union
+from typing import Mapping, Union
 
 from ...dataflow import DataFlow, MapData
 from ...dataflow.custom_serialize import SerializerJsonlines
 from ...datasets.info import DatasetInfo
 from ...mapper.cats import cat_to_sub_cat, filter_cat
 from ...mapper.pubstruct import pub_to_image
-from ...utils.detection_types import JsonDict
 from ...utils.logger import LoggingRecord, logger
 from ...utils.settings import CellType, DatasetType, LayoutType, ObjectTypes, TableType, WordType
+from ...utils.types import PubtabnetDict
 from ..base import _BuiltInDataset
 from ..dataflow_builder import DataFlowBaseBuilder
 from ..info import DatasetCategories
@@ -70,38 +70,38 @@ _URL = (
     "pubtabnet.tar.gz?_ga=2.267291150.146828643.1629125962-1173244232.1625045842"
 )
 _SPLITS: Mapping[str, str] = {"train": "train", "val": "val", "test": "test"}
-_TYPE = DatasetType.object_detection
+_TYPE = DatasetType.OBJECT_DETECTION
 _LOCATION = "pubtabnet"
 _ANNOTATION_FILES: Mapping[str, str] = {"all": "PubTabNet_2.0.0.jsonl"}
 
-_INIT_CATEGORIES = [LayoutType.cell, TableType.item, LayoutType.table, LayoutType.word]
-_SUB_CATEGORIES: Dict[ObjectTypes, Dict[ObjectTypes, List[ObjectTypes]]]
+_INIT_CATEGORIES = [LayoutType.CELL, TableType.ITEM, LayoutType.TABLE, LayoutType.WORD]
+_SUB_CATEGORIES: dict[ObjectTypes, dict[ObjectTypes, list[ObjectTypes]]]
 _SUB_CATEGORIES = {
-    TableType.item: {TableType.item: [LayoutType.row, LayoutType.column]},
-    LayoutType.cell: {
-        CellType.header: [CellType.header, CellType.body],
-        CellType.row_number: [],
-        CellType.column_number: [],
-        CellType.row_span: [],
-        CellType.column_span: [],
-        CellType.spanning: [CellType.spanning],
+    TableType.ITEM: {TableType.ITEM: [LayoutType.ROW, LayoutType.COLUMN]},
+    LayoutType.CELL: {
+        CellType.HEADER: [CellType.HEADER, CellType.BODY],
+        CellType.ROW_NUMBER: [],
+        CellType.COLUMN_NUMBER: [],
+        CellType.ROW_SPAN: [],
+        CellType.COLUMN_SPAN: [],
+        CellType.SPANNING: [CellType.SPANNING],
     },
-    CellType.header: {
-        CellType.row_number: [],
-        CellType.column_number: [],
-        CellType.row_span: [],
-        CellType.column_span: [],
-        CellType.spanning: [CellType.spanning],
+    CellType.HEADER: {
+        CellType.ROW_NUMBER: [],
+        CellType.COLUMN_NUMBER: [],
+        CellType.ROW_SPAN: [],
+        CellType.COLUMN_SPAN: [],
+        CellType.SPANNING: [CellType.SPANNING],
     },
-    CellType.body: {
-        CellType.row_number: [],
-        CellType.column_number: [],
-        CellType.row_span: [],
-        CellType.column_span: [],
-        CellType.spanning: [CellType.spanning],
+    CellType.BODY: {
+        CellType.ROW_NUMBER: [],
+        CellType.COLUMN_NUMBER: [],
+        CellType.ROW_SPAN: [],
+        CellType.COLUMN_SPAN: [],
+        CellType.SPANNING: [CellType.SPANNING],
     },
-    LayoutType.table: {TableType.html: [TableType.html]},
-    LayoutType.word: {WordType.characters: [WordType.characters]},
+    LayoutType.TABLE: {TableType.HTML: [TableType.HTML]},
+    LayoutType.WORD: {WordType.CHARACTERS: [WordType.CHARACTERS]},
 }
 
 
@@ -170,7 +170,7 @@ class PubtabnetBuilder(DataFlowBaseBuilder):
         df = SerializerJsonlines.load(path, max_datapoints=max_datapoints)
 
         # Map
-        def replace_filename(dp: JsonDict) -> JsonDict:
+        def replace_filename(dp: PubtabnetDict) -> PubtabnetDict:
             dp["filename"] = self.get_workdir() / dp["split"] / dp["filename"]
             return dp
 
@@ -178,7 +178,7 @@ class PubtabnetBuilder(DataFlowBaseBuilder):
         df = MapData(df, lambda dp: dp if dp["split"] == split else None)
         pub_mapper = pub_to_image(
             self.categories.get_categories(name_as_key=True, init=True),
-            load_image,
+            load_image=load_image,
             fake_score=fake_score,
             rows_and_cols=rows_and_cols,
             dd_pipe_like=dd_pipe_like,
@@ -187,6 +187,7 @@ class PubtabnetBuilder(DataFlowBaseBuilder):
         )
 
         df = MapData(df, pub_mapper)
+
         if self.categories.is_cat_to_sub_cat():
             df = MapData(
                 df,

deepdoctection/datasets/instances/rvlcdip.py

@@ -36,12 +36,12 @@ from typing import Mapping, Union
 
 from ...dataflow import DataFlow, MapData
 from ...dataflow.custom_serialize import SerializerTabsepFiles
-from ...datapoint.annotation import CategoryAnnotation, SummaryAnnotation
+from ...datapoint.annotation import CategoryAnnotation
 from ...datapoint.image import Image
 from ...mapper.cats import filter_summary
 from ...mapper.maputils import curry
 from ...utils.fs import load_image_from_file
-from ...utils.settings import DatasetType, DocumentType, PageType, TypeOrStr
+from ...utils.settings import DatasetType, DocumentType, PageType, SummaryType, TypeOrStr
 from ..base import _BuiltInDataset
 from ..dataflow_builder import DataFlowBaseBuilder
 from ..info import DatasetCategories, DatasetInfo
@@ -64,27 +64,27 @@ _LICENSE = (
 _URL = "https://www.cs.cmu.edu/~aharley/rvl-cdip/"
 
 _SPLITS: Mapping[str, str] = {"train": "train", "val": "val", "test": "test"}
-_TYPE = DatasetType.sequence_classification
+_TYPE = DatasetType.SEQUENCE_CLASSIFICATION
 _LOCATION = "rvl-cdip"
 
 _ANNOTATION_FILES: Mapping[str, str] = {"train": "labels/train.txt", "val": "labels/val.txt", "test": "labels/test.txt"}
 _INIT_CATEGORIES = [
-    DocumentType.letter,
-    DocumentType.form,
-    DocumentType.email,
-    DocumentType.handwritten,
-    DocumentType.advertisement,
-    DocumentType.scientific_report,
-    DocumentType.scientific_publication,
-    DocumentType.specification,
-    DocumentType.file_folder,
-    DocumentType.news_article,
-    DocumentType.budget,
-    DocumentType.invoice,
-    DocumentType.presentation,
-    DocumentType.questionnaire,
-    DocumentType.resume,
-    DocumentType.memo,
+    DocumentType.LETTER,
+    DocumentType.FORM,
+    DocumentType.EMAIL,
+    DocumentType.HANDWRITTEN,
+    DocumentType.ADVERTISEMENT,
+    DocumentType.SCIENTIFIC_REPORT,
+    DocumentType.SCIENTIFIC_PUBLICATION,
+    DocumentType.SPECIFICATION,
+    DocumentType.FILE_FOLDER,
+    DocumentType.NEWS_ARTICLE,
+    DocumentType.BUDGET,
+    DocumentType.INVOICE,
+    DocumentType.PRESENTATION,
+    DocumentType.QUESTIONNAIRE,
+    DocumentType.RESUME,
+    DocumentType.MEMO,
 ]
 
 
@@ -139,15 +139,15 @@ class RvlcdipBuilder(DataFlowBaseBuilder):
 
         @curry
         def _map_str_to_image(dp: str, load_img: bool) -> Image:
-            location, label = dp.split()[0], dp.split()[1]
-            label = str(int(label) + 1)
+            location, label_str = dp.split()[0], dp.split()[1]
+            label = int(label_str) + 1
             file_name = os.path.split(location)[1]
             image = Image(location=(self.get_workdir() / "images" / location).as_posix(), file_name=file_name)
             image.image = load_image_from_file(image.location)
-            summary = SummaryAnnotation()
+            summary = CategoryAnnotation(category_name=SummaryType.SUMMARY)
             categories_dict = self.categories.get_categories(init=True)
             summary.dump_sub_category(
-                PageType.document_type, CategoryAnnotation(category_name=categories_dict[label], category_id=str(label))
+                PageType.DOCUMENT_TYPE, CategoryAnnotation(category_name=categories_dict[label], category_id=label)
             )
             image.summary = summary
             if not load_img:
@@ -159,15 +159,14 @@ class RvlcdipBuilder(DataFlowBaseBuilder):
         if self.categories.is_filtered():
             df = MapData(
                 df,
-                filter_summary({PageType.document_type: self.categories.get_categories(as_dict=False, filtered=True)}),
+                filter_summary({PageType.DOCUMENT_TYPE: self.categories.get_categories(as_dict=False, filtered=True)}),
             )
 
             @curry
-            def _re_map_cat_ids(dp: Image, filtered_categories_name_as_key: Mapping[TypeOrStr, str]) -> Image:
-                if dp.summary:
-                    if PageType.document_type in dp.summary.sub_categories:
-                        summary_cat = dp.summary.get_sub_category(PageType.document_type)
-                        summary_cat.category_id = filtered_categories_name_as_key[summary_cat.category_name]
+            def _re_map_cat_ids(dp: Image, filtered_categories_name_as_key: Mapping[TypeOrStr, int]) -> Image:
+                if PageType.DOCUMENT_TYPE in dp.summary.sub_categories:
+                    summary_cat = dp.summary.get_sub_category(PageType.DOCUMENT_TYPE)
+                    summary_cat.category_id = filtered_categories_name_as_key[summary_cat.category_name]
                 return dp
 
             df = MapData(df, _re_map_cat_ids(self.categories.get_categories(filtered=True, name_as_key=True)))