deepdoctection 0.42.1__py3-none-any.whl → 0.43.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of deepdoctection might be problematic. Click here for more details.
- deepdoctection/__init__.py +4 -2
- deepdoctection/analyzer/__init__.py +2 -1
- deepdoctection/analyzer/config.py +919 -0
- deepdoctection/analyzer/dd.py +36 -62
- deepdoctection/analyzer/factory.py +311 -141
- deepdoctection/configs/conf_dd_one.yaml +100 -44
- deepdoctection/configs/profiles.jsonl +32 -0
- deepdoctection/dataflow/__init__.py +9 -6
- deepdoctection/dataflow/base.py +33 -15
- deepdoctection/dataflow/common.py +96 -75
- deepdoctection/dataflow/custom.py +36 -29
- deepdoctection/dataflow/custom_serialize.py +135 -91
- deepdoctection/dataflow/parallel_map.py +33 -31
- deepdoctection/dataflow/serialize.py +15 -10
- deepdoctection/dataflow/stats.py +41 -28
- deepdoctection/datapoint/__init__.py +4 -6
- deepdoctection/datapoint/annotation.py +104 -66
- deepdoctection/datapoint/box.py +190 -130
- deepdoctection/datapoint/convert.py +66 -39
- deepdoctection/datapoint/image.py +151 -95
- deepdoctection/datapoint/view.py +383 -236
- deepdoctection/datasets/__init__.py +2 -6
- deepdoctection/datasets/adapter.py +11 -11
- deepdoctection/datasets/base.py +118 -81
- deepdoctection/datasets/dataflow_builder.py +18 -12
- deepdoctection/datasets/info.py +76 -57
- deepdoctection/datasets/instances/__init__.py +6 -2
- deepdoctection/datasets/instances/doclaynet.py +17 -14
- deepdoctection/datasets/instances/fintabnet.py +16 -22
- deepdoctection/datasets/instances/funsd.py +11 -6
- deepdoctection/datasets/instances/iiitar13k.py +9 -9
- deepdoctection/datasets/instances/layouttest.py +9 -9
- deepdoctection/datasets/instances/publaynet.py +9 -9
- deepdoctection/datasets/instances/pubtables1m.py +13 -13
- deepdoctection/datasets/instances/pubtabnet.py +13 -15
- deepdoctection/datasets/instances/rvlcdip.py +8 -8
- deepdoctection/datasets/instances/xfund.py +11 -9
- deepdoctection/datasets/registry.py +18 -11
- deepdoctection/datasets/save.py +12 -11
- deepdoctection/eval/__init__.py +3 -2
- deepdoctection/eval/accmetric.py +72 -52
- deepdoctection/eval/base.py +29 -10
- deepdoctection/eval/cocometric.py +14 -12
- deepdoctection/eval/eval.py +56 -41
- deepdoctection/eval/registry.py +6 -3
- deepdoctection/eval/tedsmetric.py +24 -9
- deepdoctection/eval/tp_eval_callback.py +13 -12
- deepdoctection/extern/__init__.py +1 -1
- deepdoctection/extern/base.py +176 -97
- deepdoctection/extern/d2detect.py +127 -92
- deepdoctection/extern/deskew.py +19 -10
- deepdoctection/extern/doctrocr.py +162 -108
- deepdoctection/extern/fastlang.py +25 -17
- deepdoctection/extern/hfdetr.py +137 -60
- deepdoctection/extern/hflayoutlm.py +329 -248
- deepdoctection/extern/hflm.py +67 -33
- deepdoctection/extern/model.py +108 -762
- deepdoctection/extern/pdftext.py +37 -12
- deepdoctection/extern/pt/nms.py +15 -1
- deepdoctection/extern/pt/ptutils.py +13 -9
- deepdoctection/extern/tessocr.py +87 -54
- deepdoctection/extern/texocr.py +29 -14
- deepdoctection/extern/tp/tfutils.py +36 -8
- deepdoctection/extern/tp/tpcompat.py +54 -16
- deepdoctection/extern/tp/tpfrcnn/config/config.py +20 -4
- deepdoctection/extern/tpdetect.py +4 -2
- deepdoctection/mapper/__init__.py +1 -1
- deepdoctection/mapper/cats.py +117 -76
- deepdoctection/mapper/cocostruct.py +35 -17
- deepdoctection/mapper/d2struct.py +56 -29
- deepdoctection/mapper/hfstruct.py +32 -19
- deepdoctection/mapper/laylmstruct.py +221 -185
- deepdoctection/mapper/maputils.py +71 -35
- deepdoctection/mapper/match.py +76 -62
- deepdoctection/mapper/misc.py +68 -44
- deepdoctection/mapper/pascalstruct.py +13 -12
- deepdoctection/mapper/prodigystruct.py +33 -19
- deepdoctection/mapper/pubstruct.py +42 -32
- deepdoctection/mapper/tpstruct.py +39 -19
- deepdoctection/mapper/xfundstruct.py +20 -13
- deepdoctection/pipe/__init__.py +1 -2
- deepdoctection/pipe/anngen.py +104 -62
- deepdoctection/pipe/base.py +226 -107
- deepdoctection/pipe/common.py +206 -123
- deepdoctection/pipe/concurrency.py +74 -47
- deepdoctection/pipe/doctectionpipe.py +108 -47
- deepdoctection/pipe/language.py +41 -24
- deepdoctection/pipe/layout.py +45 -18
- deepdoctection/pipe/lm.py +146 -78
- deepdoctection/pipe/order.py +205 -119
- deepdoctection/pipe/refine.py +111 -63
- deepdoctection/pipe/registry.py +1 -1
- deepdoctection/pipe/segment.py +213 -142
- deepdoctection/pipe/sub_layout.py +76 -46
- deepdoctection/pipe/text.py +52 -33
- deepdoctection/pipe/transform.py +8 -6
- deepdoctection/train/d2_frcnn_train.py +87 -69
- deepdoctection/train/hf_detr_train.py +72 -40
- deepdoctection/train/hf_layoutlm_train.py +85 -46
- deepdoctection/train/tp_frcnn_train.py +56 -28
- deepdoctection/utils/concurrency.py +59 -16
- deepdoctection/utils/context.py +40 -19
- deepdoctection/utils/develop.py +26 -17
- deepdoctection/utils/env_info.py +86 -37
- deepdoctection/utils/error.py +16 -10
- deepdoctection/utils/file_utils.py +246 -71
- deepdoctection/utils/fs.py +162 -43
- deepdoctection/utils/identifier.py +29 -16
- deepdoctection/utils/logger.py +49 -32
- deepdoctection/utils/metacfg.py +83 -21
- deepdoctection/utils/pdf_utils.py +119 -62
- deepdoctection/utils/settings.py +24 -10
- deepdoctection/utils/tqdm.py +10 -5
- deepdoctection/utils/transform.py +182 -46
- deepdoctection/utils/utils.py +61 -28
- deepdoctection/utils/viz.py +150 -104
- deepdoctection-0.43.1.dist-info/METADATA +376 -0
- deepdoctection-0.43.1.dist-info/RECORD +149 -0
- deepdoctection/analyzer/_config.py +0 -146
- deepdoctection-0.42.1.dist-info/METADATA +0 -431
- deepdoctection-0.42.1.dist-info/RECORD +0 -148
- {deepdoctection-0.42.1.dist-info → deepdoctection-0.43.1.dist-info}/WHEEL +0 -0
- {deepdoctection-0.42.1.dist-info → deepdoctection-0.43.1.dist-info}/licenses/LICENSE +0 -0
- {deepdoctection-0.42.1.dist-info → deepdoctection-0.43.1.dist-info}/top_level.txt +0 -0
|
@@ -16,8 +16,9 @@
|
|
|
16
16
|
# limitations under the License.
|
|
17
17
|
|
|
18
18
|
"""
|
|
19
|
-
|
|
19
|
+
`DatasetRegistry` for registering built-in and custom datasets
|
|
20
20
|
"""
|
|
21
|
+
|
|
21
22
|
import inspect
|
|
22
23
|
|
|
23
24
|
import catalogue # type: ignore
|
|
@@ -36,17 +37,22 @@ def get_dataset(name: str) -> DatasetBase:
|
|
|
36
37
|
"""
|
|
37
38
|
Returns an instance of a dataset with a given name. This instance can be used to customize the dataflow output
|
|
38
39
|
|
|
39
|
-
|
|
40
|
+
Example:
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
dataset = get_dataset("some_name")
|
|
44
|
+
dataset.dataflow.categories.filter_categories(["cat1","cat2"])
|
|
45
|
+
df = dataset.dataflow.build(split="train")
|
|
40
46
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
47
|
+
for dp in df:
|
|
48
|
+
# do something
|
|
49
|
+
```
|
|
44
50
|
|
|
45
|
-
|
|
46
|
-
|
|
51
|
+
Args:
|
|
52
|
+
name: A dataset name
|
|
47
53
|
|
|
48
|
-
:
|
|
49
|
-
|
|
54
|
+
Returns:
|
|
55
|
+
An instance of a dataset
|
|
50
56
|
"""
|
|
51
57
|
ds = dataset_registry.get(name)
|
|
52
58
|
if inspect.isclass(ds):
|
|
@@ -58,8 +64,9 @@ def print_dataset_infos(add_license: bool = True, add_info: bool = True) -> None
|
|
|
58
64
|
"""
|
|
59
65
|
Prints a table with all registered datasets and some basic information (name, license and optionally description)
|
|
60
66
|
|
|
61
|
-
:
|
|
62
|
-
|
|
67
|
+
Args:
|
|
68
|
+
add_license: Whether to add the license type of the dataset
|
|
69
|
+
add_info: Whether to add a description of the dataset
|
|
63
70
|
"""
|
|
64
71
|
|
|
65
72
|
data = dataset_registry.get_all()
|
deepdoctection/datasets/save.py
CHANGED
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
# limitations under the License.
|
|
17
17
|
|
|
18
18
|
"""
|
|
19
|
-
|
|
19
|
+
Saving samples from a DataFlow
|
|
20
20
|
"""
|
|
21
21
|
|
|
22
22
|
import json
|
|
@@ -42,17 +42,18 @@ def dataflow_to_json(
|
|
|
42
42
|
highest_hierarchy_only: bool = False,
|
|
43
43
|
) -> None:
|
|
44
44
|
"""
|
|
45
|
-
Save a dataflow consisting of `datapoint.Image` to a jsonl file. Each image will be dumped into a separate
|
|
46
|
-
JSON object.
|
|
45
|
+
Save a dataflow consisting of `datapoint.Image` to a `jsonl` file. Each image will be dumped into a separate
|
|
46
|
+
`JSON` object.
|
|
47
47
|
|
|
48
|
-
:
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
48
|
+
Args:
|
|
49
|
+
df: Input dataflow
|
|
50
|
+
path: Path to save the file(s) to
|
|
51
|
+
single_files: Will save image results to a single `JSON` file. If False all images of the dataflow will be
|
|
52
|
+
dumped into a single `.jsonl` file.
|
|
53
|
+
file_name: file name, only needed for `jsonl` files
|
|
54
|
+
max_datapoints: Will stop saving after dumping max_datapoint images.
|
|
55
|
+
save_image_in_json: Will save the image to the `JSON` object
|
|
56
|
+
highest_hierarchy_only: If `True` it will remove all image attributes of `ImageAnnotation`s
|
|
56
57
|
"""
|
|
57
58
|
path = Path(path)
|
|
58
59
|
if single_files:
|
deepdoctection/eval/__init__.py
CHANGED
|
@@ -16,8 +16,9 @@
|
|
|
16
16
|
# limitations under the License.
|
|
17
17
|
|
|
18
18
|
"""
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
## Evalulation metrics and Evaluators
|
|
20
|
+
|
|
21
|
+
Contains metrics (customized for special tasks), evaluators and Tensorpack callback
|
|
21
22
|
"""
|
|
22
23
|
|
|
23
24
|
from .accmetric import *
|
deepdoctection/eval/accmetric.py
CHANGED
|
@@ -16,8 +16,9 @@
|
|
|
16
16
|
# limitations under the License.
|
|
17
17
|
|
|
18
18
|
"""
|
|
19
|
-
|
|
19
|
+
Accuracy metrics (micro, macro, F1 and per label) for classification tasks.
|
|
20
20
|
"""
|
|
21
|
+
|
|
21
22
|
from collections import Counter
|
|
22
23
|
from typing import Counter as TypeCounter
|
|
23
24
|
from typing import Mapping, Optional, Sequence, Union
|
|
@@ -73,14 +74,19 @@ def _confusion(np_label_gt: NDArray[int32], np_label_pr: NDArray[int32]) -> NDAr
|
|
|
73
74
|
|
|
74
75
|
def accuracy(label_gt: Sequence[int], label_predictions: Sequence[int], masks: Optional[Sequence[int]] = None) -> float:
|
|
75
76
|
"""
|
|
76
|
-
Calculates the accuracy given predictions and labels. Ignores masked indices.
|
|
77
|
-
`sklearn.metrics.accuracy_score`
|
|
77
|
+
Calculates the accuracy given predictions and labels. Ignores masked indices.
|
|
78
|
+
Uses `sklearn.metrics.accuracy_score`
|
|
79
|
+
|
|
80
|
+
Args:
|
|
81
|
+
label_gt: List of ground truth labels
|
|
82
|
+
label_predictions: List of predictions. Must have the same length as label_gt
|
|
83
|
+
masks: An optional list with masks to ignore some samples
|
|
78
84
|
|
|
79
|
-
:
|
|
80
|
-
|
|
81
|
-
:param masks: An optional list with masks to ignore some samples.
|
|
85
|
+
Returns:
|
|
86
|
+
Accuracy score with only unmasked values considered
|
|
82
87
|
|
|
83
|
-
:
|
|
88
|
+
Raises:
|
|
89
|
+
ValueError: If lengths of label_gt and label_predictions are not equal
|
|
84
90
|
"""
|
|
85
91
|
|
|
86
92
|
np_label_gt, np_label_pr = np.asarray(label_gt), np.asarray(label_predictions)
|
|
@@ -99,13 +105,15 @@ def confusion(
|
|
|
99
105
|
label_gt: Sequence[int], label_predictions: Sequence[int], masks: Optional[Sequence[int]] = None
|
|
100
106
|
) -> NDArray[int32]:
|
|
101
107
|
"""
|
|
102
|
-
Calculates the
|
|
108
|
+
Calculates the confusion matrix given the predictions and labels.
|
|
103
109
|
|
|
104
|
-
:
|
|
105
|
-
|
|
106
|
-
|
|
110
|
+
Args:
|
|
111
|
+
label_gt: List of ground truth labels
|
|
112
|
+
label_predictions: List of predictions. Must have the same length as label_gt
|
|
113
|
+
masks: List with masks of same length as label_gt
|
|
107
114
|
|
|
108
|
-
:
|
|
115
|
+
Returns:
|
|
116
|
+
Confusion matrix as numpy array
|
|
109
117
|
"""
|
|
110
118
|
|
|
111
119
|
np_label_gt, np_label_pr = np.asarray(label_gt), np.asarray(label_predictions)
|
|
@@ -123,14 +131,16 @@ def precision(
|
|
|
123
131
|
micro: bool = False,
|
|
124
132
|
) -> NDArray[float32]:
|
|
125
133
|
"""
|
|
126
|
-
Calculates the precision for a multi
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
+
Calculates the precision for a multi-classification problem using a confusion matrix.
|
|
135
|
+
|
|
136
|
+
Args:
|
|
137
|
+
label_gt: List of ground truth labels
|
|
138
|
+
label_predictions: List of predictions. Must have the same length as label_gt
|
|
139
|
+
masks: List with masks of same length as label_gt
|
|
140
|
+
micro: If True, calculates the micro average precision
|
|
141
|
+
|
|
142
|
+
Returns:
|
|
143
|
+
Precision scores by category or micro average
|
|
134
144
|
"""
|
|
135
145
|
np_label_gt, np_label_pr = np.asarray(label_gt), np.asarray(label_predictions)
|
|
136
146
|
|
|
@@ -153,14 +163,16 @@ def recall(
|
|
|
153
163
|
micro: bool = False,
|
|
154
164
|
) -> NDArray[float32]:
|
|
155
165
|
"""
|
|
156
|
-
Calculates the recall for a multi
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
166
|
+
Calculates the recall for a multi-classification problem using a confusion matrix.
|
|
167
|
+
|
|
168
|
+
Args:
|
|
169
|
+
label_gt: List of ground truth labels
|
|
170
|
+
label_predictions: List of predictions. Must have the same length as label_gt
|
|
171
|
+
masks: List with masks of same length as label_gt
|
|
172
|
+
micro: If True, calculates the micro average recall
|
|
173
|
+
|
|
174
|
+
Returns:
|
|
175
|
+
Recall scores by category or micro average
|
|
164
176
|
"""
|
|
165
177
|
np_label_gt, np_label_pr = np.asarray(label_gt), np.asarray(label_predictions)
|
|
166
178
|
|
|
@@ -184,15 +196,17 @@ def f1_score(
|
|
|
184
196
|
per_label: bool = True,
|
|
185
197
|
) -> NDArray[float32]:
|
|
186
198
|
"""
|
|
187
|
-
Calculates the
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
199
|
+
Calculates the `F1` score for a multi-classification problem.
|
|
200
|
+
|
|
201
|
+
Args:
|
|
202
|
+
label_gt: List of ground truth labels
|
|
203
|
+
label_predictions: List of predictions. Must have the same length as label_gt
|
|
204
|
+
masks: List with masks of same length as label_gt
|
|
205
|
+
micro: If True, calculates the micro average f1 score
|
|
206
|
+
per_label: If True, returns the f1 score per label, otherwise returns the mean of all f1's
|
|
207
|
+
|
|
208
|
+
Returns:
|
|
209
|
+
`F1` scores by category, micro average, or mean value
|
|
196
210
|
"""
|
|
197
211
|
|
|
198
212
|
np_precision = precision(label_gt, label_predictions, masks, micro)
|
|
@@ -205,7 +219,13 @@ def f1_score(
|
|
|
205
219
|
|
|
206
220
|
class ClassificationMetric(MetricBase):
|
|
207
221
|
"""
|
|
208
|
-
|
|
222
|
+
Base metric class for classification metrics.
|
|
223
|
+
|
|
224
|
+
Attributes:
|
|
225
|
+
mapper: Function to map images to `category_id`
|
|
226
|
+
_cats: Optional sequence of `ObjectTypes`
|
|
227
|
+
_sub_cats: Optional mapping of object types to object types or sequences of `ObjectTypes`
|
|
228
|
+
_summary_sub_cats: Optional sequence of `ObjectTypes` for summary
|
|
209
229
|
"""
|
|
210
230
|
|
|
211
231
|
mapper = image_to_cat_id
|
|
@@ -279,20 +299,20 @@ class ClassificationMetric(MetricBase):
|
|
|
279
299
|
summary_sub_category_names: Optional[Union[TypeOrStr, Sequence[TypeOrStr]]] = None,
|
|
280
300
|
) -> None:
|
|
281
301
|
"""
|
|
282
|
-
Set categories that are supposed to be evaluated.
|
|
283
|
-
passed explicitly.
|
|
284
|
-
|
|
285
|
-
**Example:**
|
|
286
|
-
|
|
287
|
-
You want to evaluate sub_cat1, sub_cat2 of cat1 and sub_cat3 of cat2. Set
|
|
302
|
+
Set categories that are supposed to be evaluated.
|
|
288
303
|
|
|
289
|
-
|
|
304
|
+
If `sub_categories` have to be considered, they need to be passed explicitly.
|
|
290
305
|
|
|
306
|
+
Example:
|
|
307
|
+
```python
|
|
308
|
+
# Evaluate sub_cat1, sub_cat2 of cat1 and sub_cat3 of cat2
|
|
309
|
+
set_categories(sub_category_names={cat1: [sub_cat1, sub_cat2], cat2: sub_cat3})
|
|
310
|
+
```
|
|
291
311
|
|
|
292
|
-
:
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
312
|
+
Args:
|
|
313
|
+
category_names: List of category names
|
|
314
|
+
sub_category_names: Dict of categories and their sub categories to be evaluated
|
|
315
|
+
summary_sub_category_names: String or list of summary sub categories
|
|
296
316
|
"""
|
|
297
317
|
|
|
298
318
|
if category_names is not None:
|
|
@@ -352,7 +372,7 @@ class ClassificationMetric(MetricBase):
|
|
|
352
372
|
def sub_cats(
|
|
353
373
|
self,
|
|
354
374
|
) -> Optional[Union[Mapping[ObjectTypes, ObjectTypes], Mapping[ObjectTypes, Sequence[ObjectTypes]]]]:
|
|
355
|
-
"""
|
|
375
|
+
"""`sub_cats`"""
|
|
356
376
|
return self._sub_cats
|
|
357
377
|
|
|
358
378
|
@property
|
|
@@ -474,7 +494,7 @@ class RecallMetric(PrecisionMetric):
|
|
|
474
494
|
@metric_registry.register("f1")
|
|
475
495
|
class F1Metric(PrecisionMetric):
|
|
476
496
|
"""
|
|
477
|
-
Metric induced by `f1_score`. Will calculate the
|
|
497
|
+
Metric induced by `f1_score`. Will calculate the F1 per category
|
|
478
498
|
"""
|
|
479
499
|
|
|
480
500
|
name = "F1"
|
|
@@ -523,7 +543,7 @@ class RecallMetricMicro(PrecisionMetricMicro):
|
|
|
523
543
|
@metric_registry.register("f1_micro")
|
|
524
544
|
class F1MetricMicro(PrecisionMetricMicro):
|
|
525
545
|
"""
|
|
526
|
-
Metric induced by `f1_score`. Will calculate the micro average
|
|
546
|
+
Metric induced by `f1_score`. Will calculate the micro average F1
|
|
527
547
|
"""
|
|
528
548
|
|
|
529
549
|
name = "Micro F1"
|
deepdoctection/eval/base.py
CHANGED
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
# limitations under the License.
|
|
17
17
|
|
|
18
18
|
"""
|
|
19
|
-
|
|
19
|
+
Base classes for evaluations and metrics
|
|
20
20
|
"""
|
|
21
21
|
|
|
22
22
|
from abc import ABC, abstractmethod
|
|
@@ -42,6 +42,11 @@ class MetricBase(ABC):
|
|
|
42
42
|
Using `get_distance`, ground truth and prediction dataflow can be read in and evaluated.
|
|
43
43
|
`dump` is a helper method that is often called via `get_distance`. Here, the dataflows should be
|
|
44
44
|
executed and the results should be saved in separate lists.
|
|
45
|
+
|
|
46
|
+
Attributes:
|
|
47
|
+
name (str): Name of the metric, usually the class name.
|
|
48
|
+
metric (Callable[[Any, Any], Optional[Any]]): The metric function that computes the distance.
|
|
49
|
+
_results (list[MetricResults]): Internal storage for results of the metric computation.
|
|
45
50
|
"""
|
|
46
51
|
|
|
47
52
|
name: str
|
|
@@ -65,6 +70,9 @@ class MetricBase(ABC):
|
|
|
65
70
|
def get_requirements(cls) -> list[Requirement]:
|
|
66
71
|
"""
|
|
67
72
|
Get a list of requirements for running the detector
|
|
73
|
+
|
|
74
|
+
Returns:
|
|
75
|
+
List of requirements
|
|
68
76
|
"""
|
|
69
77
|
raise NotImplementedError()
|
|
70
78
|
|
|
@@ -74,11 +82,15 @@ class MetricBase(ABC):
|
|
|
74
82
|
cls, dataflow_gt: DataFlow, dataflow_predictions: DataFlow, categories: DatasetCategories
|
|
75
83
|
) -> list[MetricResults]:
|
|
76
84
|
"""
|
|
77
|
-
Takes of the ground truth
|
|
85
|
+
Takes of the ground truth dataflow as well as the dataflow and generates the metric results.
|
|
86
|
+
|
|
87
|
+
Args:
|
|
88
|
+
dataflow_gt: Dataflow with ground truth annotations.
|
|
89
|
+
dataflow_predictions: Dataflow with predictions.
|
|
90
|
+
categories: DatasetCategories with respect to the underlying dataset.
|
|
78
91
|
|
|
79
|
-
:
|
|
80
|
-
|
|
81
|
-
:param categories: DatasetCategories with respect to the underlying dataset.
|
|
92
|
+
Returns:
|
|
93
|
+
List of metric results
|
|
82
94
|
"""
|
|
83
95
|
raise NotImplementedError()
|
|
84
96
|
|
|
@@ -91,9 +103,13 @@ class MetricBase(ABC):
|
|
|
91
103
|
Dump the dataflow with ground truth annotations and predictions. Use it as auxiliary method and call it from
|
|
92
104
|
`get_distance`.
|
|
93
105
|
|
|
94
|
-
:
|
|
95
|
-
|
|
96
|
-
|
|
106
|
+
Args:
|
|
107
|
+
dataflow_gt: Dataflow with ground truth annotations.
|
|
108
|
+
dataflow_predictions: Dataflow with predictions.
|
|
109
|
+
categories: DatasetCategories with respect to the underlying dataset.
|
|
110
|
+
|
|
111
|
+
Returns:
|
|
112
|
+
Tuple containing ground truth and predictions
|
|
97
113
|
"""
|
|
98
114
|
raise NotImplementedError()
|
|
99
115
|
|
|
@@ -103,8 +119,11 @@ class MetricBase(ABC):
|
|
|
103
119
|
Converts the result from `get_distance` to a dict. It concatenates all keys of the inner dict and uses
|
|
104
120
|
the metric result 'val' as value.
|
|
105
121
|
|
|
106
|
-
:
|
|
107
|
-
|
|
122
|
+
Args:
|
|
123
|
+
results: List of dict as input
|
|
124
|
+
|
|
125
|
+
Returns:
|
|
126
|
+
MetricResults: Dict with metric results.
|
|
108
127
|
"""
|
|
109
128
|
output: MetricResults = {}
|
|
110
129
|
for res in results:
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
# limitations under the License.
|
|
17
17
|
|
|
18
18
|
"""
|
|
19
|
-
|
|
19
|
+
Metrics that require the `COCOeval` class.
|
|
20
20
|
"""
|
|
21
21
|
from __future__ import annotations
|
|
22
22
|
|
|
@@ -65,8 +65,7 @@ _F1_DEFAULTS = [
|
|
|
65
65
|
_MAX_DET_INDEX = [2, 2, 2, 2, 2, 2, 0, 1, 2, 2, 2, 2]
|
|
66
66
|
|
|
67
67
|
"""
|
|
68
|
-
|
|
69
|
-
https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocotools/cocoeval.py
|
|
68
|
+
Taken from <https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocotools/cocoeval.py>
|
|
70
69
|
"""
|
|
71
70
|
|
|
72
71
|
|
|
@@ -236,10 +235,11 @@ class CocoMetric(MetricBase):
|
|
|
236
235
|
@classmethod
|
|
237
236
|
def get_summary_default_parameters(cls) -> list[JsonDict]:
|
|
238
237
|
"""
|
|
239
|
-
|
|
238
|
+
Get default parameters of evaluation results. May differ from other `CocoMetric` classes.
|
|
240
239
|
|
|
241
|
-
:
|
|
242
|
-
|
|
240
|
+
Returns:
|
|
241
|
+
List of dict with default configuration, e.g. setting of average precision, iou threshold,
|
|
242
|
+
area range and maximum detections.
|
|
243
243
|
"""
|
|
244
244
|
if cls._f1_score:
|
|
245
245
|
for el, idx in zip(_F1_DEFAULTS, [2, 2]):
|
|
@@ -267,12 +267,14 @@ class CocoMetric(MetricBase):
|
|
|
267
267
|
"""
|
|
268
268
|
Setting params for different coco metric modes.
|
|
269
269
|
|
|
270
|
-
:
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
270
|
+
Args:
|
|
271
|
+
max_detections: The maximum number of detections to consider
|
|
272
|
+
area_range: The area range to classify objects as `all`, `small`, `medium` and `large`
|
|
273
|
+
f1_score: Will use F1-score setting with default `iouThr=0.9`. To be more precise it does not calculate
|
|
274
|
+
the F1-score but the precision and recall for a given `iou` threshold. Use the harmonic mean to
|
|
275
|
+
get the ultimate F1-score.
|
|
276
|
+
f1_iou: Use with `f1_score=True` and reset the f1 iou threshold
|
|
277
|
+
per_category: Whether to calculate metrics per category
|
|
276
278
|
"""
|
|
277
279
|
if max_detections is not None:
|
|
278
280
|
assert len(max_detections) == 3, max_detections
|
deepdoctection/eval/eval.py
CHANGED
|
@@ -65,26 +65,28 @@ class Evaluator:
|
|
|
65
65
|
erasing process and after that passing the predictor. Predicted and gt datapoints will be converted into the
|
|
66
66
|
required metric input format and dumped into lists. Both lists will be passed to `MetricBase.get_distance`.
|
|
67
67
|
|
|
68
|
-
|
|
69
|
-
|
|
68
|
+
Note:
|
|
70
69
|
You can evaluate the predictor on a subset of categories by filtering the ground truth dataset. When using
|
|
71
70
|
the coco metric all predicted objects that are not in the set of filtered objects will be not taken into
|
|
72
71
|
account.
|
|
73
72
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
73
|
+
Example:
|
|
74
|
+
```python
|
|
75
|
+
publaynet = get_dataset("publaynet")
|
|
76
|
+
publaynet.dataflow.categories.filter_categories(categories=["TEXT","TITLE"])
|
|
77
|
+
coco_metric = metric_registry.get("coco")
|
|
78
|
+
profile = ModelCatalog.get_profile("layout/d2_model_0829999_layout_inf_only.pt")
|
|
79
|
+
path_weights = ModelCatalog.get_full_path_weights("layout/d2_model_0829999_layout_inf_only.pt")
|
|
80
|
+
path_config_yaml= ModelCatalog.get_full_path_configs("layout/d2_model_0829999_layout_inf_only.pt")
|
|
80
81
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
82
|
+
layout_detector = D2FrcnnDetector(path_config_yaml, path_weights, profile.categories)
|
|
83
|
+
layout_service = ImageLayoutService(layout_detector)
|
|
84
|
+
evaluator = Evaluator(publaynet, layout_service, coco_metric)
|
|
84
85
|
|
|
85
|
-
|
|
86
|
+
output = evaluator.run(max_datapoints=10)
|
|
87
|
+
```
|
|
86
88
|
|
|
87
|
-
For another example check the script in
|
|
89
|
+
For another example check the script in `Evaluation` of table recognition`
|
|
88
90
|
"""
|
|
89
91
|
|
|
90
92
|
def __init__(
|
|
@@ -98,9 +100,10 @@ class Evaluator:
|
|
|
98
100
|
"""
|
|
99
101
|
Evaluating a pipeline component on a dataset with a given metric.
|
|
100
102
|
|
|
101
|
-
:
|
|
102
|
-
|
|
103
|
-
|
|
103
|
+
Args:
|
|
104
|
+
dataset: dataset
|
|
105
|
+
component_or_pipeline: A pipeline component with predictor and annotation factory.
|
|
106
|
+
metric: metric
|
|
104
107
|
"""
|
|
105
108
|
|
|
106
109
|
self.dataset = dataset
|
|
@@ -191,11 +194,13 @@ class Evaluator:
|
|
|
191
194
|
"""
|
|
192
195
|
Start evaluation process and return the results.
|
|
193
196
|
|
|
194
|
-
:
|
|
195
|
-
|
|
196
|
-
|
|
197
|
+
Args:
|
|
198
|
+
output_as_dict: Return result in a list or dict.
|
|
199
|
+
dataflow_build_kwargs: Pass the necessary arguments in order to build the dataflow, e.g. `split`,
|
|
200
|
+
`build_mode`, `max_datapoints` etc.
|
|
197
201
|
|
|
198
|
-
:
|
|
202
|
+
Returns:
|
|
203
|
+
dict with metric results.
|
|
199
204
|
"""
|
|
200
205
|
|
|
201
206
|
df_gt = self.dataset.dataflow.build(**dataflow_build_kwargs)
|
|
@@ -279,12 +284,15 @@ class Evaluator:
|
|
|
279
284
|
Visualize ground truth and prediction datapoint. Given a dataflow config it will run predictions per sample
|
|
280
285
|
and concat the prediction image (with predicted bounding boxes) with ground truth image.
|
|
281
286
|
|
|
282
|
-
:
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
287
|
+
Args:
|
|
288
|
+
interactive: If set to True will open an interactive image, otherwise it will return a `np.array` that
|
|
289
|
+
can be displayed differently (e.g. `matplotlib`). Note that, if the interactive mode is being
|
|
290
|
+
used, more than one sample can be iteratively be displayed.
|
|
291
|
+
kwargs: Dataflow configs for displaying specific image splits and visualisation configs:
|
|
292
|
+
`show_tables`, `show_layouts`, `show_table_structure`, `show_words`
|
|
293
|
+
|
|
294
|
+
Returns:
|
|
295
|
+
Image as `np.array`
|
|
288
296
|
"""
|
|
289
297
|
|
|
290
298
|
show_tables = kwargs.pop("show_tables", True)
|
|
@@ -349,13 +357,15 @@ class WandbTableAgent:
|
|
|
349
357
|
"""
|
|
350
358
|
A class that creates a W&B table of sample predictions and sends them to the W&B server.
|
|
351
359
|
|
|
360
|
+
Example:
|
|
361
|
+
```python
|
|
352
362
|
df ... # some dataflow
|
|
353
363
|
agent = WandbTableAgent(myrun,"MY_DATASET",50,{"1":"FOO"})
|
|
354
364
|
for dp in df:
|
|
355
365
|
agent.dump(dp)
|
|
356
366
|
|
|
357
367
|
agent.log()
|
|
358
|
-
|
|
368
|
+
```
|
|
359
369
|
"""
|
|
360
370
|
|
|
361
371
|
def __init__(
|
|
@@ -368,16 +378,17 @@ class WandbTableAgent:
|
|
|
368
378
|
cat_to_sub_cat: Optional[Mapping[TypeOrStr, TypeOrStr]] = None,
|
|
369
379
|
):
|
|
370
380
|
"""
|
|
371
|
-
:
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
+
Args:
|
|
382
|
+
wandb_run: An `wandb.run` instance for tracking. Use `run=wandb.init(project=project, config=config,
|
|
383
|
+
**kwargs)` to generate a `run`.
|
|
384
|
+
dataset_name: name for tracking
|
|
385
|
+
num_samples: When dumping images to a table it will stop adding samples after `num_samples` instances
|
|
386
|
+
categories: dict of all possible categories
|
|
387
|
+
sub_categories: dict of sub categories. If provided, these categories will define the classes for the
|
|
388
|
+
table
|
|
389
|
+
cat_to_sub_cat: dict of category to sub category keys. Suppose your category `foo` has a sub category
|
|
390
|
+
defined by the key `sub_foo`. The range sub category values must then be given by
|
|
391
|
+
`sub_categories` and to extract the sub category values one must pass `{"foo": "sub_foo"}`
|
|
381
392
|
"""
|
|
382
393
|
|
|
383
394
|
self.dataset_name = dataset_name
|
|
@@ -400,8 +411,11 @@ class WandbTableAgent:
|
|
|
400
411
|
Dump image to a table. Add this while iterating over samples. After `num_samples` it will stop appending samples
|
|
401
412
|
to the table
|
|
402
413
|
|
|
403
|
-
:
|
|
404
|
-
|
|
414
|
+
Args:
|
|
415
|
+
dp: `Image` instance
|
|
416
|
+
|
|
417
|
+
Returns:
|
|
418
|
+
`Image` instance
|
|
405
419
|
"""
|
|
406
420
|
if self.num_samples > self._counter:
|
|
407
421
|
dp = maybe_load_image(dp)
|
|
@@ -421,9 +435,10 @@ class WandbTableAgent:
|
|
|
421
435
|
|
|
422
436
|
def _build_table(self) -> Table:
|
|
423
437
|
"""
|
|
424
|
-
Builds wandb.Table
|
|
438
|
+
Builds `wandb.Table` instance for logging evaluation
|
|
425
439
|
|
|
426
|
-
|
|
440
|
+
Returns:
|
|
441
|
+
Table object to log evaluation
|
|
427
442
|
"""
|
|
428
443
|
return Table(columns=self._table_cols, data=self._table_rows)
|
|
429
444
|
|
deepdoctection/eval/registry.py
CHANGED
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
# limitations under the License.
|
|
17
17
|
|
|
18
18
|
"""
|
|
19
|
-
|
|
19
|
+
MetricRegistry for registering and retrieving evaluation metrics
|
|
20
20
|
"""
|
|
21
21
|
|
|
22
22
|
import catalogue # type: ignore
|
|
@@ -30,7 +30,10 @@ def get_metric(name: str) -> MetricBase:
|
|
|
30
30
|
"""
|
|
31
31
|
Returns an instance of a metric with a given name.
|
|
32
32
|
|
|
33
|
-
:
|
|
34
|
-
|
|
33
|
+
Args:
|
|
34
|
+
name: A metric name
|
|
35
|
+
|
|
36
|
+
Returns:
|
|
37
|
+
A metric instance
|
|
35
38
|
"""
|
|
36
39
|
return metric_registry.get(name)()
|