deepdoctection 0.31__py3-none-any.whl → 0.33__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of deepdoctection might be problematic. Click here for more details.
- deepdoctection/__init__.py +16 -29
- deepdoctection/analyzer/dd.py +70 -59
- deepdoctection/configs/conf_dd_one.yaml +34 -31
- deepdoctection/dataflow/common.py +9 -5
- deepdoctection/dataflow/custom.py +5 -5
- deepdoctection/dataflow/custom_serialize.py +75 -18
- deepdoctection/dataflow/parallel_map.py +3 -3
- deepdoctection/dataflow/serialize.py +4 -4
- deepdoctection/dataflow/stats.py +3 -3
- deepdoctection/datapoint/annotation.py +41 -56
- deepdoctection/datapoint/box.py +9 -8
- deepdoctection/datapoint/convert.py +6 -6
- deepdoctection/datapoint/image.py +56 -44
- deepdoctection/datapoint/view.py +245 -150
- deepdoctection/datasets/__init__.py +1 -4
- deepdoctection/datasets/adapter.py +35 -26
- deepdoctection/datasets/base.py +14 -12
- deepdoctection/datasets/dataflow_builder.py +3 -3
- deepdoctection/datasets/info.py +24 -26
- deepdoctection/datasets/instances/doclaynet.py +51 -51
- deepdoctection/datasets/instances/fintabnet.py +46 -46
- deepdoctection/datasets/instances/funsd.py +25 -24
- deepdoctection/datasets/instances/iiitar13k.py +13 -10
- deepdoctection/datasets/instances/layouttest.py +4 -3
- deepdoctection/datasets/instances/publaynet.py +5 -5
- deepdoctection/datasets/instances/pubtables1m.py +24 -21
- deepdoctection/datasets/instances/pubtabnet.py +32 -30
- deepdoctection/datasets/instances/rvlcdip.py +30 -30
- deepdoctection/datasets/instances/xfund.py +26 -26
- deepdoctection/datasets/save.py +6 -6
- deepdoctection/eval/__init__.py +1 -4
- deepdoctection/eval/accmetric.py +32 -33
- deepdoctection/eval/base.py +8 -9
- deepdoctection/eval/cocometric.py +15 -13
- deepdoctection/eval/eval.py +41 -37
- deepdoctection/eval/tedsmetric.py +30 -23
- deepdoctection/eval/tp_eval_callback.py +16 -19
- deepdoctection/extern/__init__.py +2 -7
- deepdoctection/extern/base.py +339 -134
- deepdoctection/extern/d2detect.py +85 -113
- deepdoctection/extern/deskew.py +14 -11
- deepdoctection/extern/doctrocr.py +141 -130
- deepdoctection/extern/fastlang.py +27 -18
- deepdoctection/extern/hfdetr.py +71 -62
- deepdoctection/extern/hflayoutlm.py +504 -211
- deepdoctection/extern/hflm.py +230 -0
- deepdoctection/extern/model.py +488 -302
- deepdoctection/extern/pdftext.py +23 -19
- deepdoctection/extern/pt/__init__.py +1 -3
- deepdoctection/extern/pt/nms.py +6 -2
- deepdoctection/extern/pt/ptutils.py +29 -19
- deepdoctection/extern/tessocr.py +39 -38
- deepdoctection/extern/texocr.py +18 -18
- deepdoctection/extern/tp/tfutils.py +57 -9
- deepdoctection/extern/tp/tpcompat.py +21 -14
- deepdoctection/extern/tp/tpfrcnn/__init__.py +20 -0
- deepdoctection/extern/tp/tpfrcnn/common.py +7 -3
- deepdoctection/extern/tp/tpfrcnn/config/__init__.py +20 -0
- deepdoctection/extern/tp/tpfrcnn/config/config.py +13 -10
- deepdoctection/extern/tp/tpfrcnn/modeling/__init__.py +20 -0
- deepdoctection/extern/tp/tpfrcnn/modeling/backbone.py +18 -8
- deepdoctection/extern/tp/tpfrcnn/modeling/generalized_rcnn.py +12 -6
- deepdoctection/extern/tp/tpfrcnn/modeling/model_box.py +14 -9
- deepdoctection/extern/tp/tpfrcnn/modeling/model_cascade.py +8 -5
- deepdoctection/extern/tp/tpfrcnn/modeling/model_fpn.py +22 -17
- deepdoctection/extern/tp/tpfrcnn/modeling/model_frcnn.py +21 -14
- deepdoctection/extern/tp/tpfrcnn/modeling/model_mrcnn.py +19 -11
- deepdoctection/extern/tp/tpfrcnn/modeling/model_rpn.py +15 -10
- deepdoctection/extern/tp/tpfrcnn/predict.py +9 -4
- deepdoctection/extern/tp/tpfrcnn/preproc.py +12 -8
- deepdoctection/extern/tp/tpfrcnn/utils/__init__.py +20 -0
- deepdoctection/extern/tp/tpfrcnn/utils/box_ops.py +10 -2
- deepdoctection/extern/tpdetect.py +45 -53
- deepdoctection/mapper/__init__.py +3 -8
- deepdoctection/mapper/cats.py +27 -29
- deepdoctection/mapper/cocostruct.py +10 -10
- deepdoctection/mapper/d2struct.py +27 -26
- deepdoctection/mapper/hfstruct.py +13 -8
- deepdoctection/mapper/laylmstruct.py +178 -37
- deepdoctection/mapper/maputils.py +12 -11
- deepdoctection/mapper/match.py +2 -2
- deepdoctection/mapper/misc.py +11 -9
- deepdoctection/mapper/pascalstruct.py +4 -4
- deepdoctection/mapper/prodigystruct.py +5 -5
- deepdoctection/mapper/pubstruct.py +84 -92
- deepdoctection/mapper/tpstruct.py +5 -5
- deepdoctection/mapper/xfundstruct.py +33 -33
- deepdoctection/pipe/__init__.py +1 -1
- deepdoctection/pipe/anngen.py +12 -14
- deepdoctection/pipe/base.py +52 -106
- deepdoctection/pipe/common.py +72 -59
- deepdoctection/pipe/concurrency.py +16 -11
- deepdoctection/pipe/doctectionpipe.py +24 -21
- deepdoctection/pipe/language.py +20 -25
- deepdoctection/pipe/layout.py +20 -16
- deepdoctection/pipe/lm.py +75 -105
- deepdoctection/pipe/order.py +194 -89
- deepdoctection/pipe/refine.py +111 -124
- deepdoctection/pipe/segment.py +156 -161
- deepdoctection/pipe/{cell.py → sub_layout.py} +50 -40
- deepdoctection/pipe/text.py +37 -36
- deepdoctection/pipe/transform.py +19 -16
- deepdoctection/train/__init__.py +6 -12
- deepdoctection/train/d2_frcnn_train.py +48 -41
- deepdoctection/train/hf_detr_train.py +41 -30
- deepdoctection/train/hf_layoutlm_train.py +153 -135
- deepdoctection/train/tp_frcnn_train.py +32 -31
- deepdoctection/utils/concurrency.py +1 -1
- deepdoctection/utils/context.py +13 -6
- deepdoctection/utils/develop.py +4 -4
- deepdoctection/utils/env_info.py +87 -125
- deepdoctection/utils/file_utils.py +6 -11
- deepdoctection/utils/fs.py +22 -18
- deepdoctection/utils/identifier.py +2 -2
- deepdoctection/utils/logger.py +16 -15
- deepdoctection/utils/metacfg.py +7 -7
- deepdoctection/utils/mocks.py +93 -0
- deepdoctection/utils/pdf_utils.py +11 -11
- deepdoctection/utils/settings.py +185 -181
- deepdoctection/utils/tqdm.py +1 -1
- deepdoctection/utils/transform.py +14 -9
- deepdoctection/utils/types.py +104 -0
- deepdoctection/utils/utils.py +7 -7
- deepdoctection/utils/viz.py +74 -72
- {deepdoctection-0.31.dist-info → deepdoctection-0.33.dist-info}/METADATA +30 -21
- deepdoctection-0.33.dist-info/RECORD +146 -0
- {deepdoctection-0.31.dist-info → deepdoctection-0.33.dist-info}/WHEEL +1 -1
- deepdoctection/utils/detection_types.py +0 -68
- deepdoctection-0.31.dist-info/RECORD +0 -144
- {deepdoctection-0.31.dist-info → deepdoctection-0.33.dist-info}/LICENSE +0 -0
- {deepdoctection-0.31.dist-info → deepdoctection-0.33.dist-info}/top_level.txt +0 -0
deepdoctection/utils/metacfg.py
CHANGED
|
@@ -20,11 +20,11 @@ Class AttrDict for maintaining configs and some functions for generating and sav
|
|
|
20
20
|
"""
|
|
21
21
|
|
|
22
22
|
import pprint
|
|
23
|
-
from typing import Any
|
|
23
|
+
from typing import Any
|
|
24
24
|
|
|
25
25
|
import yaml
|
|
26
26
|
|
|
27
|
-
from .
|
|
27
|
+
from .types import PathLikeOrStr
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
# Copyright (c) Tensorpack Contributors
|
|
@@ -67,13 +67,13 @@ class AttrDict:
|
|
|
67
67
|
|
|
68
68
|
__repr__ = __str__
|
|
69
69
|
|
|
70
|
-
def to_dict(self) ->
|
|
70
|
+
def to_dict(self) -> dict[str, Any]:
|
|
71
71
|
"""Convert to a nested dict."""
|
|
72
72
|
return {
|
|
73
73
|
k: v.to_dict() if isinstance(v, AttrDict) else v for k, v in self.__dict__.items() if not k.startswith("_")
|
|
74
74
|
}
|
|
75
75
|
|
|
76
|
-
def from_dict(self, d:
|
|
76
|
+
def from_dict(self, d: dict[str, Any]) -> None: # pylint: disable=C0103
|
|
77
77
|
"""
|
|
78
78
|
Generate an instance from a dict
|
|
79
79
|
"""
|
|
@@ -86,7 +86,7 @@ class AttrDict:
|
|
|
86
86
|
else:
|
|
87
87
|
setattr(self, k, v)
|
|
88
88
|
|
|
89
|
-
def update_args(self, args:
|
|
89
|
+
def update_args(self, args: list[str]) -> None:
|
|
90
90
|
"""
|
|
91
91
|
Update from command line args.
|
|
92
92
|
"""
|
|
@@ -122,7 +122,7 @@ class AttrDict:
|
|
|
122
122
|
raise NotImplementedError()
|
|
123
123
|
|
|
124
124
|
|
|
125
|
-
def set_config_by_yaml(path_yaml:
|
|
125
|
+
def set_config_by_yaml(path_yaml: PathLikeOrStr) -> AttrDict:
|
|
126
126
|
"""
|
|
127
127
|
Use to initialize the config class for tensorpack faster rcnn
|
|
128
128
|
|
|
@@ -139,7 +139,7 @@ def set_config_by_yaml(path_yaml: Pathlike) -> AttrDict:
|
|
|
139
139
|
return config
|
|
140
140
|
|
|
141
141
|
|
|
142
|
-
def save_config_to_yaml(config: AttrDict, path_yaml:
|
|
142
|
+
def save_config_to_yaml(config: AttrDict, path_yaml: PathLikeOrStr) -> None:
|
|
143
143
|
"""
|
|
144
144
|
:param config: The configuration instance as an AttrDict
|
|
145
145
|
:param path_yaml: Save the config class for tensorpack faster rcnn
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# File: mocks.py
|
|
3
|
+
|
|
4
|
+
# Copyright 2024 Dr. Janis Meyer. All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
7
|
+
# you may not use this file except in compliance with the License.
|
|
8
|
+
# You may obtain a copy of the License at
|
|
9
|
+
#
|
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
+
#
|
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
15
|
+
# See the License for the specific language governing permissions and
|
|
16
|
+
# limitations under the License.
|
|
17
|
+
|
|
18
|
+
"""
|
|
19
|
+
Some classes with the purpose to mock the original classes from the Tensorpack library, if Tensorpack is not installed
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from deepdoctection.utils.error import DependencyError
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def layer_register(log_shape): # pylint: disable=W0613
|
|
26
|
+
"""Mock layer_register function from tensorpack."""
|
|
27
|
+
|
|
28
|
+
def inner(inputs): # pylint: disable=W0613
|
|
29
|
+
pass
|
|
30
|
+
|
|
31
|
+
return inner
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def under_name_scope():
|
|
35
|
+
"""Mock under_name_scope function from tensorpack."""
|
|
36
|
+
|
|
37
|
+
def inner(inputs): # pylint: disable=W0613
|
|
38
|
+
pass
|
|
39
|
+
|
|
40
|
+
return inner
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def memoized(func):
|
|
44
|
+
"""Mock memoized function from tensorpack."""
|
|
45
|
+
return func
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def memoized_method(func):
|
|
49
|
+
"""Mock memoized_method function from tensorpack."""
|
|
50
|
+
return func
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def auto_reuse_variable_scope(inputs): # pylint: disable=W0613
|
|
54
|
+
"""Mock auto_reuse_variable_scope function from tensorpack."""
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class ModelDesc: # pylint: disable=R0903
|
|
58
|
+
"""Mock ModelDesc class from tensorpack."""
|
|
59
|
+
|
|
60
|
+
def __init__(self) -> None:
|
|
61
|
+
raise DependencyError("Tensorpack not found.")
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class ImageAugmentor: # pylint: disable=R0903
|
|
65
|
+
"""Mock ImageAugmentor class from tensorpack."""
|
|
66
|
+
|
|
67
|
+
def __init__(self) -> None:
|
|
68
|
+
raise DependencyError("Tensorpack not found.")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class Callback: # pylint: disable=R0903
|
|
72
|
+
"""Mock Callback class from tensor"""
|
|
73
|
+
|
|
74
|
+
def __init__(self) -> None:
|
|
75
|
+
raise DependencyError("Tensorpack not found.")
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class Config: # pylint: disable=R0903
|
|
79
|
+
"""Mock class for Config"""
|
|
80
|
+
|
|
81
|
+
pass # pylint: disable=W0107
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class Tree: # pylint: disable=R0903
|
|
85
|
+
"""Mock class for Tree"""
|
|
86
|
+
|
|
87
|
+
pass # pylint: disable=W0107
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class IterableDataset: # pylint: disable=R0903
|
|
91
|
+
"""Mock class for IterableDataset"""
|
|
92
|
+
|
|
93
|
+
pass # pylint: disable=W0107
|
|
@@ -25,23 +25,23 @@ import sys
|
|
|
25
25
|
from errno import ENOENT
|
|
26
26
|
from io import BytesIO
|
|
27
27
|
from shutil import copyfile
|
|
28
|
-
from typing import Generator,
|
|
28
|
+
from typing import Generator, Optional
|
|
29
29
|
|
|
30
30
|
from numpy import uint8
|
|
31
31
|
from pypdf import PdfReader, PdfWriter, errors
|
|
32
32
|
|
|
33
33
|
from .context import save_tmp_file, timeout_manager
|
|
34
|
-
from .detection_types import ImageType, Pathlike
|
|
35
34
|
from .error import DependencyError, FileExtensionError
|
|
36
35
|
from .file_utils import pdf_to_cairo_available, pdf_to_ppm_available, qpdf_available
|
|
37
36
|
from .logger import LoggingRecord, logger
|
|
37
|
+
from .types import PathLikeOrStr, PixelValues
|
|
38
38
|
from .utils import is_file_extension
|
|
39
39
|
from .viz import viz_handler
|
|
40
40
|
|
|
41
41
|
__all__ = ["decrypt_pdf_document", "get_pdf_file_reader", "get_pdf_file_writer", "PDFStreamer", "pdf_to_np_array"]
|
|
42
42
|
|
|
43
43
|
|
|
44
|
-
def decrypt_pdf_document(path:
|
|
44
|
+
def decrypt_pdf_document(path: PathLikeOrStr) -> bool:
|
|
45
45
|
"""
|
|
46
46
|
Decrypting a pdf. As copying a pdf document removes the password that protects pdf, this method
|
|
47
47
|
generates a copy and decrypts the copy using qpdf. The result is saved as the original
|
|
@@ -73,7 +73,7 @@ def decrypt_pdf_document(path: Pathlike) -> bool:
|
|
|
73
73
|
return False
|
|
74
74
|
|
|
75
75
|
|
|
76
|
-
def get_pdf_file_reader(path:
|
|
76
|
+
def get_pdf_file_reader(path: PathLikeOrStr) -> PdfReader:
|
|
77
77
|
"""
|
|
78
78
|
Creates a file reader object from a pdf document. Will try to decrypt the document if it is
|
|
79
79
|
encrypted. (See `decrypt_pdf_document` to understand what is meant with "decrypt").
|
|
@@ -133,7 +133,7 @@ class PDFStreamer:
|
|
|
133
133
|
|
|
134
134
|
"""
|
|
135
135
|
|
|
136
|
-
def __init__(self, path:
|
|
136
|
+
def __init__(self, path: PathLikeOrStr) -> None:
|
|
137
137
|
"""
|
|
138
138
|
:param path: to a pdf.
|
|
139
139
|
"""
|
|
@@ -143,7 +143,7 @@ class PDFStreamer:
|
|
|
143
143
|
def __len__(self) -> int:
|
|
144
144
|
return len(self.file_reader.pages)
|
|
145
145
|
|
|
146
|
-
def __iter__(self) -> Generator[
|
|
146
|
+
def __iter__(self) -> Generator[tuple[bytes, int], None, None]:
|
|
147
147
|
for k in range(len(self)):
|
|
148
148
|
buffer = BytesIO()
|
|
149
149
|
writer = get_pdf_file_writer()
|
|
@@ -157,9 +157,9 @@ class PDFStreamer:
|
|
|
157
157
|
|
|
158
158
|
|
|
159
159
|
def _input_to_cli_str(
|
|
160
|
-
input_file_name:
|
|
161
|
-
) ->
|
|
162
|
-
cmd_args:
|
|
160
|
+
input_file_name: PathLikeOrStr, output_file_name: PathLikeOrStr, dpi: int, size: Optional[tuple[int, int]] = None
|
|
161
|
+
) -> list[str]:
|
|
162
|
+
cmd_args: list[str] = []
|
|
163
163
|
|
|
164
164
|
if pdf_to_ppm_available():
|
|
165
165
|
command = "pdftoppm"
|
|
@@ -196,7 +196,7 @@ class PopplerError(RuntimeError):
|
|
|
196
196
|
self.args = (status, message)
|
|
197
197
|
|
|
198
198
|
|
|
199
|
-
def _run_poppler(poppler_args:
|
|
199
|
+
def _run_poppler(poppler_args: list[str]) -> None:
|
|
200
200
|
try:
|
|
201
201
|
proc = subprocess.Popen(poppler_args) # pylint: disable=R1732
|
|
202
202
|
except OSError as error:
|
|
@@ -209,7 +209,7 @@ def _run_poppler(poppler_args: List[str]) -> None:
|
|
|
209
209
|
raise PopplerError(status=proc.returncode, message="Syntax Error: PDF cannot be read with Poppler")
|
|
210
210
|
|
|
211
211
|
|
|
212
|
-
def pdf_to_np_array(pdf_bytes: bytes, size: Optional[
|
|
212
|
+
def pdf_to_np_array(pdf_bytes: bytes, size: Optional[tuple[int, int]] = None, dpi: int = 200) -> PixelValues:
|
|
213
213
|
"""
|
|
214
214
|
Convert a single pdf page from its byte representation to a numpy array. This function will save the pdf as to a tmp
|
|
215
215
|
file and then call poppler via `pdftoppm` resp. `pdftocairo` if the former is not available.
|