deepdoctection 0.31__py3-none-any.whl → 0.33__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of deepdoctection might be problematic. Click here for more details.

Files changed (131) hide show
  1. deepdoctection/__init__.py +16 -29
  2. deepdoctection/analyzer/dd.py +70 -59
  3. deepdoctection/configs/conf_dd_one.yaml +34 -31
  4. deepdoctection/dataflow/common.py +9 -5
  5. deepdoctection/dataflow/custom.py +5 -5
  6. deepdoctection/dataflow/custom_serialize.py +75 -18
  7. deepdoctection/dataflow/parallel_map.py +3 -3
  8. deepdoctection/dataflow/serialize.py +4 -4
  9. deepdoctection/dataflow/stats.py +3 -3
  10. deepdoctection/datapoint/annotation.py +41 -56
  11. deepdoctection/datapoint/box.py +9 -8
  12. deepdoctection/datapoint/convert.py +6 -6
  13. deepdoctection/datapoint/image.py +56 -44
  14. deepdoctection/datapoint/view.py +245 -150
  15. deepdoctection/datasets/__init__.py +1 -4
  16. deepdoctection/datasets/adapter.py +35 -26
  17. deepdoctection/datasets/base.py +14 -12
  18. deepdoctection/datasets/dataflow_builder.py +3 -3
  19. deepdoctection/datasets/info.py +24 -26
  20. deepdoctection/datasets/instances/doclaynet.py +51 -51
  21. deepdoctection/datasets/instances/fintabnet.py +46 -46
  22. deepdoctection/datasets/instances/funsd.py +25 -24
  23. deepdoctection/datasets/instances/iiitar13k.py +13 -10
  24. deepdoctection/datasets/instances/layouttest.py +4 -3
  25. deepdoctection/datasets/instances/publaynet.py +5 -5
  26. deepdoctection/datasets/instances/pubtables1m.py +24 -21
  27. deepdoctection/datasets/instances/pubtabnet.py +32 -30
  28. deepdoctection/datasets/instances/rvlcdip.py +30 -30
  29. deepdoctection/datasets/instances/xfund.py +26 -26
  30. deepdoctection/datasets/save.py +6 -6
  31. deepdoctection/eval/__init__.py +1 -4
  32. deepdoctection/eval/accmetric.py +32 -33
  33. deepdoctection/eval/base.py +8 -9
  34. deepdoctection/eval/cocometric.py +15 -13
  35. deepdoctection/eval/eval.py +41 -37
  36. deepdoctection/eval/tedsmetric.py +30 -23
  37. deepdoctection/eval/tp_eval_callback.py +16 -19
  38. deepdoctection/extern/__init__.py +2 -7
  39. deepdoctection/extern/base.py +339 -134
  40. deepdoctection/extern/d2detect.py +85 -113
  41. deepdoctection/extern/deskew.py +14 -11
  42. deepdoctection/extern/doctrocr.py +141 -130
  43. deepdoctection/extern/fastlang.py +27 -18
  44. deepdoctection/extern/hfdetr.py +71 -62
  45. deepdoctection/extern/hflayoutlm.py +504 -211
  46. deepdoctection/extern/hflm.py +230 -0
  47. deepdoctection/extern/model.py +488 -302
  48. deepdoctection/extern/pdftext.py +23 -19
  49. deepdoctection/extern/pt/__init__.py +1 -3
  50. deepdoctection/extern/pt/nms.py +6 -2
  51. deepdoctection/extern/pt/ptutils.py +29 -19
  52. deepdoctection/extern/tessocr.py +39 -38
  53. deepdoctection/extern/texocr.py +18 -18
  54. deepdoctection/extern/tp/tfutils.py +57 -9
  55. deepdoctection/extern/tp/tpcompat.py +21 -14
  56. deepdoctection/extern/tp/tpfrcnn/__init__.py +20 -0
  57. deepdoctection/extern/tp/tpfrcnn/common.py +7 -3
  58. deepdoctection/extern/tp/tpfrcnn/config/__init__.py +20 -0
  59. deepdoctection/extern/tp/tpfrcnn/config/config.py +13 -10
  60. deepdoctection/extern/tp/tpfrcnn/modeling/__init__.py +20 -0
  61. deepdoctection/extern/tp/tpfrcnn/modeling/backbone.py +18 -8
  62. deepdoctection/extern/tp/tpfrcnn/modeling/generalized_rcnn.py +12 -6
  63. deepdoctection/extern/tp/tpfrcnn/modeling/model_box.py +14 -9
  64. deepdoctection/extern/tp/tpfrcnn/modeling/model_cascade.py +8 -5
  65. deepdoctection/extern/tp/tpfrcnn/modeling/model_fpn.py +22 -17
  66. deepdoctection/extern/tp/tpfrcnn/modeling/model_frcnn.py +21 -14
  67. deepdoctection/extern/tp/tpfrcnn/modeling/model_mrcnn.py +19 -11
  68. deepdoctection/extern/tp/tpfrcnn/modeling/model_rpn.py +15 -10
  69. deepdoctection/extern/tp/tpfrcnn/predict.py +9 -4
  70. deepdoctection/extern/tp/tpfrcnn/preproc.py +12 -8
  71. deepdoctection/extern/tp/tpfrcnn/utils/__init__.py +20 -0
  72. deepdoctection/extern/tp/tpfrcnn/utils/box_ops.py +10 -2
  73. deepdoctection/extern/tpdetect.py +45 -53
  74. deepdoctection/mapper/__init__.py +3 -8
  75. deepdoctection/mapper/cats.py +27 -29
  76. deepdoctection/mapper/cocostruct.py +10 -10
  77. deepdoctection/mapper/d2struct.py +27 -26
  78. deepdoctection/mapper/hfstruct.py +13 -8
  79. deepdoctection/mapper/laylmstruct.py +178 -37
  80. deepdoctection/mapper/maputils.py +12 -11
  81. deepdoctection/mapper/match.py +2 -2
  82. deepdoctection/mapper/misc.py +11 -9
  83. deepdoctection/mapper/pascalstruct.py +4 -4
  84. deepdoctection/mapper/prodigystruct.py +5 -5
  85. deepdoctection/mapper/pubstruct.py +84 -92
  86. deepdoctection/mapper/tpstruct.py +5 -5
  87. deepdoctection/mapper/xfundstruct.py +33 -33
  88. deepdoctection/pipe/__init__.py +1 -1
  89. deepdoctection/pipe/anngen.py +12 -14
  90. deepdoctection/pipe/base.py +52 -106
  91. deepdoctection/pipe/common.py +72 -59
  92. deepdoctection/pipe/concurrency.py +16 -11
  93. deepdoctection/pipe/doctectionpipe.py +24 -21
  94. deepdoctection/pipe/language.py +20 -25
  95. deepdoctection/pipe/layout.py +20 -16
  96. deepdoctection/pipe/lm.py +75 -105
  97. deepdoctection/pipe/order.py +194 -89
  98. deepdoctection/pipe/refine.py +111 -124
  99. deepdoctection/pipe/segment.py +156 -161
  100. deepdoctection/pipe/{cell.py → sub_layout.py} +50 -40
  101. deepdoctection/pipe/text.py +37 -36
  102. deepdoctection/pipe/transform.py +19 -16
  103. deepdoctection/train/__init__.py +6 -12
  104. deepdoctection/train/d2_frcnn_train.py +48 -41
  105. deepdoctection/train/hf_detr_train.py +41 -30
  106. deepdoctection/train/hf_layoutlm_train.py +153 -135
  107. deepdoctection/train/tp_frcnn_train.py +32 -31
  108. deepdoctection/utils/concurrency.py +1 -1
  109. deepdoctection/utils/context.py +13 -6
  110. deepdoctection/utils/develop.py +4 -4
  111. deepdoctection/utils/env_info.py +87 -125
  112. deepdoctection/utils/file_utils.py +6 -11
  113. deepdoctection/utils/fs.py +22 -18
  114. deepdoctection/utils/identifier.py +2 -2
  115. deepdoctection/utils/logger.py +16 -15
  116. deepdoctection/utils/metacfg.py +7 -7
  117. deepdoctection/utils/mocks.py +93 -0
  118. deepdoctection/utils/pdf_utils.py +11 -11
  119. deepdoctection/utils/settings.py +185 -181
  120. deepdoctection/utils/tqdm.py +1 -1
  121. deepdoctection/utils/transform.py +14 -9
  122. deepdoctection/utils/types.py +104 -0
  123. deepdoctection/utils/utils.py +7 -7
  124. deepdoctection/utils/viz.py +74 -72
  125. {deepdoctection-0.31.dist-info → deepdoctection-0.33.dist-info}/METADATA +30 -21
  126. deepdoctection-0.33.dist-info/RECORD +146 -0
  127. {deepdoctection-0.31.dist-info → deepdoctection-0.33.dist-info}/WHEEL +1 -1
  128. deepdoctection/utils/detection_types.py +0 -68
  129. deepdoctection-0.31.dist-info/RECORD +0 -144
  130. {deepdoctection-0.31.dist-info → deepdoctection-0.33.dist-info}/LICENSE +0 -0
  131. {deepdoctection-0.31.dist-info → deepdoctection-0.33.dist-info}/top_level.txt +0 -0
@@ -20,11 +20,11 @@ Class AttrDict for maintaining configs and some functions for generating and sav
20
20
  """
21
21
 
22
22
  import pprint
23
- from typing import Any, Dict, List
23
+ from typing import Any
24
24
 
25
25
  import yaml
26
26
 
27
- from .detection_types import Pathlike
27
+ from .types import PathLikeOrStr
28
28
 
29
29
 
30
30
  # Copyright (c) Tensorpack Contributors
@@ -67,13 +67,13 @@ class AttrDict:
67
67
 
68
68
  __repr__ = __str__
69
69
 
70
- def to_dict(self) -> Dict[str, Any]:
70
+ def to_dict(self) -> dict[str, Any]:
71
71
  """Convert to a nested dict."""
72
72
  return {
73
73
  k: v.to_dict() if isinstance(v, AttrDict) else v for k, v in self.__dict__.items() if not k.startswith("_")
74
74
  }
75
75
 
76
- def from_dict(self, d: Dict[str, Any]) -> None: # pylint: disable=C0103
76
+ def from_dict(self, d: dict[str, Any]) -> None: # pylint: disable=C0103
77
77
  """
78
78
  Generate an instance from a dict
79
79
  """
@@ -86,7 +86,7 @@ class AttrDict:
86
86
  else:
87
87
  setattr(self, k, v)
88
88
 
89
- def update_args(self, args: List[str]) -> None:
89
+ def update_args(self, args: list[str]) -> None:
90
90
  """
91
91
  Update from command line args.
92
92
  """
@@ -122,7 +122,7 @@ class AttrDict:
122
122
  raise NotImplementedError()
123
123
 
124
124
 
125
- def set_config_by_yaml(path_yaml: Pathlike) -> AttrDict:
125
+ def set_config_by_yaml(path_yaml: PathLikeOrStr) -> AttrDict:
126
126
  """
127
127
  Use to initialize the config class for tensorpack faster rcnn
128
128
 
@@ -139,7 +139,7 @@ def set_config_by_yaml(path_yaml: Pathlike) -> AttrDict:
139
139
  return config
140
140
 
141
141
 
142
- def save_config_to_yaml(config: AttrDict, path_yaml: Pathlike) -> None:
142
+ def save_config_to_yaml(config: AttrDict, path_yaml: PathLikeOrStr) -> None:
143
143
  """
144
144
  :param config: The configuration instance as an AttrDict
145
145
  :param path_yaml: Save the config class for tensorpack faster rcnn
@@ -0,0 +1,93 @@
1
+ # -*- coding: utf-8 -*-
2
+ # File: mocks.py
3
+
4
+ # Copyright 2024 Dr. Janis Meyer. All rights reserved.
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+
18
+ """
19
+ Some classes with the purpose to mock the original classes from the Tensorpack library, if Tensorpack is not installed
20
+ """
21
+
22
+ from deepdoctection.utils.error import DependencyError
23
+
24
+
25
+ def layer_register(log_shape): # pylint: disable=W0613
26
+ """Mock layer_register function from tensorpack."""
27
+
28
+ def inner(inputs): # pylint: disable=W0613
29
+ pass
30
+
31
+ return inner
32
+
33
+
34
+ def under_name_scope():
35
+ """Mock under_name_scope function from tensorpack."""
36
+
37
+ def inner(inputs): # pylint: disable=W0613
38
+ pass
39
+
40
+ return inner
41
+
42
+
43
+ def memoized(func):
44
+ """Mock memoized function from tensorpack."""
45
+ return func
46
+
47
+
48
+ def memoized_method(func):
49
+ """Mock memoized_method function from tensorpack."""
50
+ return func
51
+
52
+
53
+ def auto_reuse_variable_scope(inputs): # pylint: disable=W0613
54
+ """Mock auto_reuse_variable_scope function from tensorpack."""
55
+
56
+
57
+ class ModelDesc: # pylint: disable=R0903
58
+ """Mock ModelDesc class from tensorpack."""
59
+
60
+ def __init__(self) -> None:
61
+ raise DependencyError("Tensorpack not found.")
62
+
63
+
64
+ class ImageAugmentor: # pylint: disable=R0903
65
+ """Mock ImageAugmentor class from tensorpack."""
66
+
67
+ def __init__(self) -> None:
68
+ raise DependencyError("Tensorpack not found.")
69
+
70
+
71
+ class Callback: # pylint: disable=R0903
72
+ """Mock Callback class from tensor"""
73
+
74
+ def __init__(self) -> None:
75
+ raise DependencyError("Tensorpack not found.")
76
+
77
+
78
+ class Config: # pylint: disable=R0903
79
+ """Mock class for Config"""
80
+
81
+ pass # pylint: disable=W0107
82
+
83
+
84
+ class Tree: # pylint: disable=R0903
85
+ """Mock class for Tree"""
86
+
87
+ pass # pylint: disable=W0107
88
+
89
+
90
+ class IterableDataset: # pylint: disable=R0903
91
+ """Mock class for IterableDataset"""
92
+
93
+ pass # pylint: disable=W0107
@@ -25,23 +25,23 @@ import sys
25
25
  from errno import ENOENT
26
26
  from io import BytesIO
27
27
  from shutil import copyfile
28
- from typing import Generator, List, Optional, Tuple
28
+ from typing import Generator, Optional
29
29
 
30
30
  from numpy import uint8
31
31
  from pypdf import PdfReader, PdfWriter, errors
32
32
 
33
33
  from .context import save_tmp_file, timeout_manager
34
- from .detection_types import ImageType, Pathlike
35
34
  from .error import DependencyError, FileExtensionError
36
35
  from .file_utils import pdf_to_cairo_available, pdf_to_ppm_available, qpdf_available
37
36
  from .logger import LoggingRecord, logger
37
+ from .types import PathLikeOrStr, PixelValues
38
38
  from .utils import is_file_extension
39
39
  from .viz import viz_handler
40
40
 
41
41
  __all__ = ["decrypt_pdf_document", "get_pdf_file_reader", "get_pdf_file_writer", "PDFStreamer", "pdf_to_np_array"]
42
42
 
43
43
 
44
- def decrypt_pdf_document(path: Pathlike) -> bool:
44
+ def decrypt_pdf_document(path: PathLikeOrStr) -> bool:
45
45
  """
46
46
  Decrypting a pdf. As copying a pdf document removes the password that protects pdf, this method
47
47
  generates a copy and decrypts the copy using qpdf. The result is saved as the original
@@ -73,7 +73,7 @@ def decrypt_pdf_document(path: Pathlike) -> bool:
73
73
  return False
74
74
 
75
75
 
76
- def get_pdf_file_reader(path: Pathlike) -> PdfReader:
76
+ def get_pdf_file_reader(path: PathLikeOrStr) -> PdfReader:
77
77
  """
78
78
  Creates a file reader object from a pdf document. Will try to decrypt the document if it is
79
79
  encrypted. (See `decrypt_pdf_document` to understand what is meant with "decrypt").
@@ -133,7 +133,7 @@ class PDFStreamer:
133
133
 
134
134
  """
135
135
 
136
- def __init__(self, path: Pathlike) -> None:
136
+ def __init__(self, path: PathLikeOrStr) -> None:
137
137
  """
138
138
  :param path: to a pdf.
139
139
  """
@@ -143,7 +143,7 @@ class PDFStreamer:
143
143
  def __len__(self) -> int:
144
144
  return len(self.file_reader.pages)
145
145
 
146
- def __iter__(self) -> Generator[Tuple[bytes, int], None, None]:
146
+ def __iter__(self) -> Generator[tuple[bytes, int], None, None]:
147
147
  for k in range(len(self)):
148
148
  buffer = BytesIO()
149
149
  writer = get_pdf_file_writer()
@@ -157,9 +157,9 @@ class PDFStreamer:
157
157
 
158
158
 
159
159
  def _input_to_cli_str(
160
- input_file_name: Pathlike, output_file_name: Pathlike, dpi: int, size: Optional[Tuple[int, int]] = None
161
- ) -> List[str]:
162
- cmd_args: List[str] = []
160
+ input_file_name: PathLikeOrStr, output_file_name: PathLikeOrStr, dpi: int, size: Optional[tuple[int, int]] = None
161
+ ) -> list[str]:
162
+ cmd_args: list[str] = []
163
163
 
164
164
  if pdf_to_ppm_available():
165
165
  command = "pdftoppm"
@@ -196,7 +196,7 @@ class PopplerError(RuntimeError):
196
196
  self.args = (status, message)
197
197
 
198
198
 
199
- def _run_poppler(poppler_args: List[str]) -> None:
199
+ def _run_poppler(poppler_args: list[str]) -> None:
200
200
  try:
201
201
  proc = subprocess.Popen(poppler_args) # pylint: disable=R1732
202
202
  except OSError as error:
@@ -209,7 +209,7 @@ def _run_poppler(poppler_args: List[str]) -> None:
209
209
  raise PopplerError(status=proc.returncode, message="Syntax Error: PDF cannot be read with Poppler")
210
210
 
211
211
 
212
- def pdf_to_np_array(pdf_bytes: bytes, size: Optional[Tuple[int, int]] = None, dpi: int = 200) -> ImageType:
212
+ def pdf_to_np_array(pdf_bytes: bytes, size: Optional[tuple[int, int]] = None, dpi: int = 200) -> PixelValues:
213
213
  """
214
214
  Convert a single pdf page from its byte representation to a numpy array. This function will save the pdf as to a tmp
215
215
  file and then call poppler via `pdftoppm` resp. `pdftocairo` if the former is not available.