deepdoctection 0.31__py3-none-any.whl → 0.33__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of deepdoctection might be problematic. Click here for more details.

Files changed (131) hide show
  1. deepdoctection/__init__.py +16 -29
  2. deepdoctection/analyzer/dd.py +70 -59
  3. deepdoctection/configs/conf_dd_one.yaml +34 -31
  4. deepdoctection/dataflow/common.py +9 -5
  5. deepdoctection/dataflow/custom.py +5 -5
  6. deepdoctection/dataflow/custom_serialize.py +75 -18
  7. deepdoctection/dataflow/parallel_map.py +3 -3
  8. deepdoctection/dataflow/serialize.py +4 -4
  9. deepdoctection/dataflow/stats.py +3 -3
  10. deepdoctection/datapoint/annotation.py +41 -56
  11. deepdoctection/datapoint/box.py +9 -8
  12. deepdoctection/datapoint/convert.py +6 -6
  13. deepdoctection/datapoint/image.py +56 -44
  14. deepdoctection/datapoint/view.py +245 -150
  15. deepdoctection/datasets/__init__.py +1 -4
  16. deepdoctection/datasets/adapter.py +35 -26
  17. deepdoctection/datasets/base.py +14 -12
  18. deepdoctection/datasets/dataflow_builder.py +3 -3
  19. deepdoctection/datasets/info.py +24 -26
  20. deepdoctection/datasets/instances/doclaynet.py +51 -51
  21. deepdoctection/datasets/instances/fintabnet.py +46 -46
  22. deepdoctection/datasets/instances/funsd.py +25 -24
  23. deepdoctection/datasets/instances/iiitar13k.py +13 -10
  24. deepdoctection/datasets/instances/layouttest.py +4 -3
  25. deepdoctection/datasets/instances/publaynet.py +5 -5
  26. deepdoctection/datasets/instances/pubtables1m.py +24 -21
  27. deepdoctection/datasets/instances/pubtabnet.py +32 -30
  28. deepdoctection/datasets/instances/rvlcdip.py +30 -30
  29. deepdoctection/datasets/instances/xfund.py +26 -26
  30. deepdoctection/datasets/save.py +6 -6
  31. deepdoctection/eval/__init__.py +1 -4
  32. deepdoctection/eval/accmetric.py +32 -33
  33. deepdoctection/eval/base.py +8 -9
  34. deepdoctection/eval/cocometric.py +15 -13
  35. deepdoctection/eval/eval.py +41 -37
  36. deepdoctection/eval/tedsmetric.py +30 -23
  37. deepdoctection/eval/tp_eval_callback.py +16 -19
  38. deepdoctection/extern/__init__.py +2 -7
  39. deepdoctection/extern/base.py +339 -134
  40. deepdoctection/extern/d2detect.py +85 -113
  41. deepdoctection/extern/deskew.py +14 -11
  42. deepdoctection/extern/doctrocr.py +141 -130
  43. deepdoctection/extern/fastlang.py +27 -18
  44. deepdoctection/extern/hfdetr.py +71 -62
  45. deepdoctection/extern/hflayoutlm.py +504 -211
  46. deepdoctection/extern/hflm.py +230 -0
  47. deepdoctection/extern/model.py +488 -302
  48. deepdoctection/extern/pdftext.py +23 -19
  49. deepdoctection/extern/pt/__init__.py +1 -3
  50. deepdoctection/extern/pt/nms.py +6 -2
  51. deepdoctection/extern/pt/ptutils.py +29 -19
  52. deepdoctection/extern/tessocr.py +39 -38
  53. deepdoctection/extern/texocr.py +18 -18
  54. deepdoctection/extern/tp/tfutils.py +57 -9
  55. deepdoctection/extern/tp/tpcompat.py +21 -14
  56. deepdoctection/extern/tp/tpfrcnn/__init__.py +20 -0
  57. deepdoctection/extern/tp/tpfrcnn/common.py +7 -3
  58. deepdoctection/extern/tp/tpfrcnn/config/__init__.py +20 -0
  59. deepdoctection/extern/tp/tpfrcnn/config/config.py +13 -10
  60. deepdoctection/extern/tp/tpfrcnn/modeling/__init__.py +20 -0
  61. deepdoctection/extern/tp/tpfrcnn/modeling/backbone.py +18 -8
  62. deepdoctection/extern/tp/tpfrcnn/modeling/generalized_rcnn.py +12 -6
  63. deepdoctection/extern/tp/tpfrcnn/modeling/model_box.py +14 -9
  64. deepdoctection/extern/tp/tpfrcnn/modeling/model_cascade.py +8 -5
  65. deepdoctection/extern/tp/tpfrcnn/modeling/model_fpn.py +22 -17
  66. deepdoctection/extern/tp/tpfrcnn/modeling/model_frcnn.py +21 -14
  67. deepdoctection/extern/tp/tpfrcnn/modeling/model_mrcnn.py +19 -11
  68. deepdoctection/extern/tp/tpfrcnn/modeling/model_rpn.py +15 -10
  69. deepdoctection/extern/tp/tpfrcnn/predict.py +9 -4
  70. deepdoctection/extern/tp/tpfrcnn/preproc.py +12 -8
  71. deepdoctection/extern/tp/tpfrcnn/utils/__init__.py +20 -0
  72. deepdoctection/extern/tp/tpfrcnn/utils/box_ops.py +10 -2
  73. deepdoctection/extern/tpdetect.py +45 -53
  74. deepdoctection/mapper/__init__.py +3 -8
  75. deepdoctection/mapper/cats.py +27 -29
  76. deepdoctection/mapper/cocostruct.py +10 -10
  77. deepdoctection/mapper/d2struct.py +27 -26
  78. deepdoctection/mapper/hfstruct.py +13 -8
  79. deepdoctection/mapper/laylmstruct.py +178 -37
  80. deepdoctection/mapper/maputils.py +12 -11
  81. deepdoctection/mapper/match.py +2 -2
  82. deepdoctection/mapper/misc.py +11 -9
  83. deepdoctection/mapper/pascalstruct.py +4 -4
  84. deepdoctection/mapper/prodigystruct.py +5 -5
  85. deepdoctection/mapper/pubstruct.py +84 -92
  86. deepdoctection/mapper/tpstruct.py +5 -5
  87. deepdoctection/mapper/xfundstruct.py +33 -33
  88. deepdoctection/pipe/__init__.py +1 -1
  89. deepdoctection/pipe/anngen.py +12 -14
  90. deepdoctection/pipe/base.py +52 -106
  91. deepdoctection/pipe/common.py +72 -59
  92. deepdoctection/pipe/concurrency.py +16 -11
  93. deepdoctection/pipe/doctectionpipe.py +24 -21
  94. deepdoctection/pipe/language.py +20 -25
  95. deepdoctection/pipe/layout.py +20 -16
  96. deepdoctection/pipe/lm.py +75 -105
  97. deepdoctection/pipe/order.py +194 -89
  98. deepdoctection/pipe/refine.py +111 -124
  99. deepdoctection/pipe/segment.py +156 -161
  100. deepdoctection/pipe/{cell.py → sub_layout.py} +50 -40
  101. deepdoctection/pipe/text.py +37 -36
  102. deepdoctection/pipe/transform.py +19 -16
  103. deepdoctection/train/__init__.py +6 -12
  104. deepdoctection/train/d2_frcnn_train.py +48 -41
  105. deepdoctection/train/hf_detr_train.py +41 -30
  106. deepdoctection/train/hf_layoutlm_train.py +153 -135
  107. deepdoctection/train/tp_frcnn_train.py +32 -31
  108. deepdoctection/utils/concurrency.py +1 -1
  109. deepdoctection/utils/context.py +13 -6
  110. deepdoctection/utils/develop.py +4 -4
  111. deepdoctection/utils/env_info.py +87 -125
  112. deepdoctection/utils/file_utils.py +6 -11
  113. deepdoctection/utils/fs.py +22 -18
  114. deepdoctection/utils/identifier.py +2 -2
  115. deepdoctection/utils/logger.py +16 -15
  116. deepdoctection/utils/metacfg.py +7 -7
  117. deepdoctection/utils/mocks.py +93 -0
  118. deepdoctection/utils/pdf_utils.py +11 -11
  119. deepdoctection/utils/settings.py +185 -181
  120. deepdoctection/utils/tqdm.py +1 -1
  121. deepdoctection/utils/transform.py +14 -9
  122. deepdoctection/utils/types.py +104 -0
  123. deepdoctection/utils/utils.py +7 -7
  124. deepdoctection/utils/viz.py +74 -72
  125. {deepdoctection-0.31.dist-info → deepdoctection-0.33.dist-info}/METADATA +30 -21
  126. deepdoctection-0.33.dist-info/RECORD +146 -0
  127. {deepdoctection-0.31.dist-info → deepdoctection-0.33.dist-info}/WHEEL +1 -1
  128. deepdoctection/utils/detection_types.py +0 -68
  129. deepdoctection-0.31.dist-info/RECORD +0 -144
  130. {deepdoctection-0.31.dist-info → deepdoctection-0.33.dist-info}/LICENSE +0 -0
  131. {deepdoctection-0.31.dist-info → deepdoctection-0.33.dist-info}/top_level.txt +0 -0
@@ -8,23 +8,27 @@
8
8
  This file is modified from
9
9
  <https://github.com/tensorpack/tensorpack/blob/1a79d595f7eda9dc9dc8428f4461680ed2222ab6/examples/FasterRCNN/data.py>
10
10
  """
11
+ from __future__ import annotations
11
12
 
12
13
  from typing import Any, List, Optional, Tuple
13
14
 
14
15
  import numpy as np
15
-
16
- # pylint: disable=import-error
17
- from tensorpack.dataflow.imgaug import AugmentorList, ImageAugmentor
16
+ from lazy_imports import try_import
18
17
 
19
18
  from ....datapoint.convert import box_to_point4, point4_to_box
20
- from ....utils.detection_types import ImageType, JsonDict
21
19
  from ....utils.error import MalformedData
22
20
  from ....utils.logger import log_once
21
+ from ....utils.types import JsonDict, PixelValues
23
22
  from .common import filter_boxes_inside_shape, np_iou
24
23
  from .modeling.model_fpn import get_all_anchors_fpn
25
24
  from .utils.np_box_ops import area as np_area
26
25
  from .utils.np_box_ops import ioa as np_ioa
27
26
 
27
+ # pylint: disable=import-error
28
+
29
+
30
+ with try_import() as import_guard:
31
+ from tensorpack.dataflow.imgaug import AugmentorList, ImageAugmentor
28
32
  # pylint: enable=import-error
29
33
 
30
34
 
@@ -210,15 +214,15 @@ def get_multilevel_rpn_anchor_input(
210
214
 
211
215
 
212
216
  def get_anchor_labels(
213
- anchors: ImageType,
214
- gt_boxes: ImageType,
215
- crowd_boxes: ImageType,
217
+ anchors: PixelValues,
218
+ gt_boxes: PixelValues,
219
+ crowd_boxes: PixelValues,
216
220
  batch_per_image: int,
217
221
  front_ground_ratio: float,
218
222
  positive_anchor_threshold: float,
219
223
  negative_anchor_threshold: float,
220
224
  crowd_overlap_threshold: float,
221
- ) -> (ImageType, ImageType):
225
+ ) -> (PixelValues, PixelValues):
222
226
  """
223
227
  Label each anchor as fg/bg/ignore.
224
228
 
@@ -0,0 +1,20 @@
1
+ # -*- coding: utf-8 -*-
2
+ # File: __init__.py
3
+
4
+ # Copyright 2021 Dr. Janis Meyer. All rights reserved.
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+
18
+ """
19
+ Init file for code for Tensorpack utils
20
+ """
@@ -15,8 +15,16 @@ and
15
15
  """
16
16
 
17
17
  # pylint: disable=import-error
18
- import tensorflow as tf
19
- from tensorpack.tfutils.scope_utils import under_name_scope
18
+
19
+ from lazy_imports import try_import
20
+
21
+ with try_import() as tf_import_guard:
22
+ import tensorflow as tf
23
+ from tensorpack.tfutils.scope_utils import under_name_scope
24
+
25
+ if not tf_import_guard.is_successful():
26
+ from .....utils.mocks import under_name_scope
27
+
20
28
 
21
29
  # pylint: enable=import-error
22
30
 
@@ -18,63 +18,57 @@
18
18
  """
19
19
  TP Faster RCNN model as predictor for deepdoctection pipeline
20
20
  """
21
+ from __future__ import annotations
21
22
 
22
23
  from abc import ABC
23
- from copy import copy
24
24
  from pathlib import Path
25
- from typing import Dict, List, Mapping, Optional, Sequence, Union
25
+ from typing import Mapping, Optional, Sequence, Union
26
26
 
27
- from ..utils.detection_types import ImageType, Requirement
28
- from ..utils.file_utils import get_tensorflow_requirement, get_tensorpack_requirement, tensorpack_available
27
+ from ..utils.file_utils import get_tensorflow_requirement, get_tensorpack_requirement
29
28
  from ..utils.metacfg import set_config_by_yaml
30
- from ..utils.settings import ObjectTypes, TypeOrStr, get_type
31
- from .base import DetectionResult, ObjectDetector, PredictorBase
32
-
33
- if tensorpack_available():
34
- from .tp.tpcompat import TensorpackPredictor
35
- from .tp.tpfrcnn.config.config import model_frcnn_config
36
- from .tp.tpfrcnn.modeling.generalized_rcnn import ResNetFPNModel
37
- from .tp.tpfrcnn.predict import tp_predict_image
29
+ from ..utils.settings import DefaultType, ObjectTypes, TypeOrStr, get_type
30
+ from ..utils.types import PathLikeOrStr, PixelValues, Requirement
31
+ from .base import DetectionResult, ModelCategories, ObjectDetector
32
+ from .tp.tpcompat import TensorpackPredictor
33
+ from .tp.tpfrcnn.config.config import model_frcnn_config
34
+ from .tp.tpfrcnn.modeling.generalized_rcnn import ResNetFPNModel
35
+ from .tp.tpfrcnn.predict import tp_predict_image
38
36
 
39
37
 
40
38
  class TPFrcnnDetectorMixin(ObjectDetector, ABC):
41
39
  """Base class for TP FRCNN detector. This class only implements the basic wrapper functions"""
42
40
 
43
- def __init__(self, categories: Mapping[str, TypeOrStr], filter_categories: Optional[Sequence[TypeOrStr]] = None):
44
- self.categories = copy(categories) # type: ignore
41
+ def __init__(self, categories: Mapping[int, TypeOrStr], filter_categories: Optional[Sequence[TypeOrStr]] = None):
42
+ categories = {k: get_type(v) for k, v in categories.items()}
43
+ categories.update({0: get_type("background")})
44
+ self.categories = ModelCategories(categories)
45
45
  if filter_categories:
46
- filter_categories = [get_type(cat) for cat in filter_categories]
47
- self.filter_categories = filter_categories
48
- self._tp_categories = self._map_to_tp_categories(categories)
46
+ self.categories.filter_categories = tuple(get_type(cat) for cat in filter_categories)
49
47
 
50
- def _map_category_names(self, detection_results: List[DetectionResult]) -> List[DetectionResult]:
48
+ def _map_category_names(self, detection_results: list[DetectionResult]) -> list[DetectionResult]:
51
49
  """
52
50
  Populating category names to detection results
53
51
 
54
52
  :param detection_results: list of detection results
55
53
  :return: List of detection results with attribute class_name populated
56
54
  """
57
- filtered_detection_result: List[DetectionResult] = []
55
+ filtered_detection_result: list[DetectionResult] = []
58
56
  for result in detection_results:
59
- result.class_name = self._tp_categories[str(result.class_id)]
60
- if self.filter_categories:
61
- if result.class_name not in self.filter_categories:
62
- filtered_detection_result.append(result)
63
- else:
57
+ result.class_name = self.categories.categories.get(
58
+ result.class_id if result.class_id else -1, DefaultType.DEFAULT_TYPE
59
+ )
60
+ if result.class_name != DefaultType.DEFAULT_TYPE:
64
61
  filtered_detection_result.append(result)
65
62
  return filtered_detection_result
66
63
 
67
64
  @staticmethod
68
- def _map_to_tp_categories(categories: Mapping[str, TypeOrStr]) -> Dict[str, ObjectTypes]:
69
- categories = {str(key): get_type(categories[val]) for key, val in enumerate(categories, 1)}
70
- categories["0"] = get_type("background")
71
- return categories # type: ignore
72
-
73
- @staticmethod
74
- def get_name(path_weights: str, architecture: str) -> str:
65
+ def get_name(path_weights: PathLikeOrStr, architecture: str) -> str:
75
66
  """Returns the name of the model"""
76
67
  return f"Tensorpack_{architecture}" + "_".join(Path(path_weights).parts[-2:])
77
68
 
69
+ def get_category_names(self) -> tuple[ObjectTypes, ...]:
70
+ return self.categories.get_categories(as_dict=False)
71
+
78
72
 
79
73
  class TPFrcnnDetector(TensorpackPredictor, TPFrcnnDetectorMixin):
80
74
  """
@@ -99,10 +93,10 @@ class TPFrcnnDetector(TensorpackPredictor, TPFrcnnDetectorMixin):
99
93
 
100
94
  def __init__(
101
95
  self,
102
- path_yaml: str,
103
- path_weights: str,
104
- categories: Mapping[str, TypeOrStr],
105
- config_overwrite: Optional[List[str]] = None,
96
+ path_yaml: PathLikeOrStr,
97
+ path_weights: PathLikeOrStr,
98
+ categories: Mapping[int, TypeOrStr],
99
+ config_overwrite: Optional[list[str]] = None,
106
100
  ignore_mismatch: bool = False,
107
101
  filter_categories: Optional[Sequence[TypeOrStr]] = None,
108
102
  ):
@@ -127,24 +121,19 @@ class TPFrcnnDetector(TensorpackPredictor, TPFrcnnDetectorMixin):
127
121
  :param filter_categories: The model might return objects that are not supposed to be predicted and that should
128
122
  be filtered. Pass a list of category names that must not be returned
129
123
  """
130
- self.path_yaml = path_yaml
131
-
132
- self.categories = copy(categories) # type: ignore
124
+ self.path_yaml = Path(path_yaml)
133
125
  self.config_overwrite = config_overwrite
134
- if filter_categories:
135
- filter_categories = [get_type(cat) for cat in filter_categories]
136
- self.filter_categories = filter_categories
137
- model = TPFrcnnDetector.get_wrapped_model(path_yaml, self.categories, config_overwrite)
126
+
127
+ model = TPFrcnnDetector.get_wrapped_model(path_yaml, categories, config_overwrite)
138
128
  TensorpackPredictor.__init__(self, model, path_weights, ignore_mismatch)
139
129
  TPFrcnnDetectorMixin.__init__(self, categories, filter_categories)
140
130
 
141
131
  self.name = self.get_name(path_weights, self._model.cfg.TAG)
142
132
  self.model_id = self.get_model_id()
143
- assert self._number_gpus > 0, "Model only support inference with GPU"
144
133
 
145
134
  @staticmethod
146
135
  def get_wrapped_model(
147
- path_yaml: str, categories: Mapping[str, ObjectTypes], config_overwrite: Union[List[str], None]
136
+ path_yaml: PathLikeOrStr, categories: Mapping[int, TypeOrStr], config_overwrite: Union[list[str], None]
148
137
  ) -> ResNetFPNModel:
149
138
  """
150
139
  Calls all necessary methods to build TP ResNetFPNModel
@@ -166,7 +155,7 @@ class TPFrcnnDetector(TensorpackPredictor, TPFrcnnDetectorMixin):
166
155
  model_frcnn_config(config=hyper_param_config, categories=categories, print_summary=False)
167
156
  return ResNetFPNModel(config=hyper_param_config)
168
157
 
169
- def predict(self, np_img: ImageType) -> List[DetectionResult]:
158
+ def predict(self, np_img: PixelValues) -> list[DetectionResult]:
170
159
  """
171
160
  Prediction per image.
172
161
 
@@ -183,15 +172,18 @@ class TPFrcnnDetector(TensorpackPredictor, TPFrcnnDetectorMixin):
183
172
  return self._map_category_names(detection_results)
184
173
 
185
174
  @classmethod
186
- def get_requirements(cls) -> List[Requirement]:
175
+ def get_requirements(cls) -> list[Requirement]:
187
176
  return [get_tensorflow_requirement(), get_tensorpack_requirement()]
188
177
 
189
- def clone(self) -> PredictorBase:
178
+ def clone(self) -> TPFrcnnDetector:
190
179
  return self.__class__(
191
- self.path_yaml,
192
- self.path_weights,
193
- self.categories,
194
- self.config_overwrite,
195
- self.ignore_mismatch,
196
- self.filter_categories,
180
+ path_yaml=self.path_yaml,
181
+ path_weights=self.path_weights,
182
+ categories=dict(self.categories.get_categories()),
183
+ config_overwrite=self.config_overwrite,
184
+ ignore_mismatch=self.ignore_mismatch,
185
+ filter_categories=self.categories.filter_categories,
197
186
  )
187
+
188
+ def clear_model(self) -> None:
189
+ self.tp_predictor = None
@@ -20,9 +20,11 @@ Contains everything that is related to transformation between datapoints
20
20
  """
21
21
  from typing import Callable
22
22
 
23
- from ..utils.file_utils import pytorch_available, transformers_available
24
23
  from .cats import *
25
24
  from .cocostruct import *
25
+ from .d2struct import *
26
+ from .hfstruct import *
27
+ from .laylmstruct import *
26
28
  from .maputils import *
27
29
  from .match import *
28
30
  from .misc import *
@@ -32,12 +34,5 @@ from .pubstruct import *
32
34
  from .tpstruct import *
33
35
  from .xfundstruct import *
34
36
 
35
- if pytorch_available() and transformers_available():
36
- from .hfstruct import *
37
- from .laylmstruct import *
38
-
39
- if pytorch_available():
40
- from .d2struct import *
41
-
42
37
  # Mapper
43
38
  Mapper = Callable[[Image], Optional[Image]]
@@ -21,19 +21,19 @@ builder method of a dataset.
21
21
  """
22
22
 
23
23
  from collections import defaultdict
24
- from typing import Any, Dict, List, Literal, Mapping, Optional, Sequence, Tuple, Union
24
+ from typing import Any, Literal, Mapping, Optional, Sequence, Union
25
25
 
26
- from ..datapoint.annotation import CategoryAnnotation, ContainerAnnotation, ImageAnnotation, SummaryAnnotation
26
+ from ..datapoint.annotation import DEFAULT_CATEGORY_ID, CategoryAnnotation, ContainerAnnotation, ImageAnnotation
27
27
  from ..datapoint.image import Image
28
- from ..utils.settings import ObjectTypes, TypeOrStr, get_type
28
+ from ..utils.settings import ObjectTypes, SummaryType, TypeOrStr, get_type
29
29
  from .maputils import LabelSummarizer, curry
30
30
 
31
31
 
32
32
  @curry
33
33
  def cat_to_sub_cat(
34
34
  dp: Image,
35
- categories_dict_names_as_key: Dict[TypeOrStr, str],
36
- cat_to_sub_cat_dict: Optional[Dict[TypeOrStr, TypeOrStr]] = None,
35
+ categories_dict_names_as_key: dict[TypeOrStr, int],
36
+ cat_to_sub_cat_dict: Optional[dict[TypeOrStr, TypeOrStr]] = None,
37
37
  ) -> Image:
38
38
  """
39
39
  Replace some category with its affiliated sub category of CategoryAnnotations. Suppose your category name is `foo`
@@ -49,13 +49,12 @@ def cat_to_sub_cat(
49
49
  if cat_to_sub_cat_dict is None:
50
50
  return dp
51
51
  cat_to_sub_cat_dict_obj_type = {get_type(key): get_type(value) for key, value in cat_to_sub_cat_dict.items()}
52
- categories_dict = categories_dict_names_as_key
53
52
  for ann in dp.get_annotation_iter(category_names=list(cat_to_sub_cat_dict_obj_type.keys())):
54
53
  sub_cat_type = cat_to_sub_cat_dict_obj_type[get_type(ann.category_name)]
55
54
  sub_cat = ann.get_sub_category(sub_cat_type)
56
55
  if sub_cat:
57
56
  ann.category_name = sub_cat.category_name
58
- ann.category_id = categories_dict[ann.category_name]
57
+ ann.category_id = categories_dict_names_as_key[ann.category_name]
59
58
 
60
59
  return dp
61
60
 
@@ -63,7 +62,7 @@ def cat_to_sub_cat(
63
62
  @curry
64
63
  def re_assign_cat_ids(
65
64
  dp: Image,
66
- categories_dict_name_as_key: Optional[Dict[TypeOrStr, str]] = None,
65
+ categories_dict_name_as_key: Optional[dict[TypeOrStr, int]] = None,
67
66
  cat_to_sub_cat_mapping: Optional[Mapping[ObjectTypes, Any]] = None,
68
67
  ) -> Image:
69
68
  """
@@ -89,7 +88,7 @@ def re_assign_cat_ids(
89
88
  :return: Image
90
89
  """
91
90
 
92
- anns_to_remove: List[ImageAnnotation] = []
91
+ anns_to_remove: list[ImageAnnotation] = []
93
92
  for ann in dp.get_annotation_iter():
94
93
  if categories_dict_name_as_key is not None:
95
94
  if ann.category_name in categories_dict_name_as_key:
@@ -103,7 +102,7 @@ def re_assign_cat_ids(
103
102
  for key in sub_cat_keys_to_sub_cat_values:
104
103
  sub_cat_values_dict = sub_cat_keys_to_sub_cat_values[key]
105
104
  sub_category = ann.get_sub_category(key)
106
- sub_category.category_id = sub_cat_values_dict.get(sub_category.category_name, "")
105
+ sub_category.category_id = sub_cat_values_dict.get(sub_category.category_name, DEFAULT_CATEGORY_ID)
107
106
 
108
107
  for ann in anns_to_remove:
109
108
  dp.remove(ann)
@@ -113,7 +112,7 @@ def re_assign_cat_ids(
113
112
 
114
113
  @curry
115
114
  def filter_cat(
116
- dp: Image, categories_as_list_filtered: List[TypeOrStr], categories_as_list_unfiltered: List[TypeOrStr]
115
+ dp: Image, categories_as_list_filtered: list[TypeOrStr], categories_as_list_unfiltered: list[TypeOrStr]
117
116
  ) -> Image:
118
117
  """
119
118
  Filters category annotations based on the on a list of categories to be kept and a list of all possible
@@ -132,7 +131,7 @@ def filter_cat(
132
131
  remove_cats_mapper = remove_cats(category_names=cats_to_remove_list) # pylint: disable=E1120 # 259
133
132
  dp = remove_cats_mapper(dp)
134
133
 
135
- categories_dict_name_as_key = {v: str(k) for k, v in enumerate(categories_as_list_filtered, 1)}
134
+ categories_dict_name_as_key = {v: k for k, v in enumerate(categories_as_list_filtered, 1)}
136
135
  re_assign_cat_ids_mapper = re_assign_cat_ids( # pylint: disable=E1120
137
136
  categories_dict_name_as_key=categories_dict_name_as_key
138
137
  )
@@ -160,13 +159,13 @@ def filter_summary(
160
159
  :return: Image or None
161
160
  """
162
161
  for key, values in sub_cat_to_sub_cat_names_or_ids.items():
163
- if mode == "name" and dp.summary:
162
+ if mode == "name":
164
163
  if dp.summary.get_sub_category(get_type(key)).category_name in values:
165
164
  return dp
166
- elif mode == "value" and dp.summary:
165
+ elif mode == "value":
167
166
  if dp.summary.get_sub_category(get_type(key)).value in values: # type: ignore
168
167
  return dp
169
- elif dp.summary:
168
+ else:
170
169
  if dp.summary.get_sub_category(get_type(key)).category_id in values:
171
170
  return dp
172
171
  return None
@@ -179,7 +178,7 @@ def image_to_cat_id(
179
178
  sub_categories: Optional[Union[Mapping[TypeOrStr, TypeOrStr], Mapping[TypeOrStr, Sequence[TypeOrStr]]]] = None,
180
179
  summary_sub_category_names: Optional[Union[TypeOrStr, Sequence[TypeOrStr]]] = None,
181
180
  id_name_or_value: Literal["id", "name", "value"] = "id",
182
- ) -> Tuple[Dict[TypeOrStr, Union[List[int], List[int]]], str]:
181
+ ) -> tuple[dict[TypeOrStr, list[int]], str]:
183
182
  """
184
183
  Extracts all category_ids, sub category information or summary sub category information with given names into a
185
184
  defaultdict. This mapping is useful when running evaluation with e.g. an accuracy metric.
@@ -199,7 +198,7 @@ def image_to_cat_id(
199
198
 
200
199
  will return
201
200
 
202
- ({'foo':['1', '1'], 'bak':[ '2'], 'baz':['3']}, image_id)
201
+ ({'foo':[1,1], 'bak':[2], 'baz':[3]}, image_id)
203
202
 
204
203
 
205
204
  **Example 2:**
@@ -213,7 +212,7 @@ def image_to_cat_id(
213
212
 
214
213
  will return
215
214
 
216
- ({'foo_sub_1':['5', '6']}, image_id)
215
+ ({'foo_sub_1':[5,6]}, image_id)
217
216
 
218
217
 
219
218
 
@@ -238,7 +237,7 @@ def image_to_cat_id(
238
237
  if not summary_sub_category_names:
239
238
  summary_sub_category_names = []
240
239
 
241
- tmp_sub_category_names: Dict[str, Sequence[str]] = {}
240
+ tmp_sub_category_names: dict[str, Sequence[str]] = {}
242
241
 
243
242
  if sub_categories is not None:
244
243
  for key, val in sub_categories.items():
@@ -252,13 +251,13 @@ def image_to_cat_id(
252
251
  if category_names or sub_categories:
253
252
  for ann in dp.get_annotation_iter():
254
253
  if ann.category_name in category_names:
255
- cat_container[ann.category_name].append(int(ann.category_id))
254
+ cat_container[ann.category_name].append(ann.category_id)
256
255
  if ann.category_name in tmp_sub_category_names:
257
256
  for sub_cat_name in tmp_sub_category_names[ann.category_name]:
258
257
  sub_cat = ann.get_sub_category(get_type(sub_cat_name))
259
258
  if sub_cat is not None:
260
259
  if id_name_or_value == "id":
261
- cat_container[sub_cat_name].append(int(sub_cat.category_id))
260
+ cat_container[sub_cat_name].append(sub_cat.category_id)
262
261
  if id_name_or_value == "name":
263
262
  cat_container[sub_cat_name].append(sub_cat.category_name) # type: ignore
264
263
  if id_name_or_value == "value":
@@ -269,11 +268,11 @@ def image_to_cat_id(
269
268
  )
270
269
  cat_container[sub_cat_name].append(sub_cat.value) # type: ignore
271
270
 
272
- if dp.summary is not None and summary_sub_category_names:
271
+ if summary_sub_category_names:
273
272
  for sub_cat_name in summary_sub_category_names:
274
273
  sub_cat = dp.summary.get_sub_category(get_type(sub_cat_name))
275
274
  if id_name_or_value == "id":
276
- cat_container[sub_cat_name].append(int(sub_cat.category_id))
275
+ cat_container[sub_cat_name].append(sub_cat.category_id)
277
276
  if id_name_or_value == "name":
278
277
  cat_container[sub_cat_name].append(sub_cat.category_name) # type: ignore
279
278
  if id_name_or_value == "value":
@@ -344,15 +343,14 @@ def remove_cats(
344
343
  dp.remove(ann)
345
344
 
346
345
  if summary_sub_categories is not None:
347
- if dp.summary is not None:
348
- for sub_cat in summary_sub_categories:
349
- dp.summary.remove_sub_category(get_type(sub_cat))
346
+ for sub_cat in summary_sub_categories:
347
+ dp.summary.remove_sub_category(get_type(sub_cat))
350
348
 
351
349
  return dp
352
350
 
353
351
 
354
352
  @curry
355
- def add_summary(dp: Image, categories: Mapping[str, ObjectTypes]) -> Image:
353
+ def add_summary(dp: Image, categories: Mapping[int, ObjectTypes]) -> Image:
356
354
  """
357
355
  Adding a summary with the number of categories in an image.
358
356
 
@@ -366,10 +364,10 @@ def add_summary(dp: Image, categories: Mapping[str, ObjectTypes]) -> Image:
366
364
  for ann in anns:
367
365
  summarizer.dump(ann.category_id)
368
366
  summary_dict = summarizer.get_summary()
369
- summary = SummaryAnnotation()
367
+ summary = CategoryAnnotation(category_name=SummaryType.SUMMARY)
370
368
  for cat_id, val in summary_dict.items():
371
369
  summary.dump_sub_category(
372
- categories[cat_id], CategoryAnnotation(category_name=categories[cat_id], category_id=str(val))
370
+ categories[cat_id], CategoryAnnotation(category_name=categories[cat_id], category_id=val)
373
371
  )
374
372
  dp.summary = summary
375
373
  return dp
@@ -20,21 +20,21 @@ Module for mapping annotations in coco style structure
20
20
  """
21
21
 
22
22
  import os
23
- from typing import Dict, List, Mapping, Optional, Tuple
23
+ from typing import Mapping, Optional
24
24
 
25
25
  from ..datapoint.annotation import CategoryAnnotation, ImageAnnotation
26
26
  from ..datapoint.box import BoundingBox
27
27
  from ..datapoint.image import Image
28
- from ..utils.detection_types import JsonDict
29
28
  from ..utils.fs import load_image_from_file
30
29
  from ..utils.settings import ObjectTypes
30
+ from ..utils.types import CocoDatapointDict, JsonDict
31
31
  from .maputils import MappingContextManager, curry, maybe_get_fake_score
32
32
 
33
33
 
34
34
  @curry
35
35
  def coco_to_image(
36
- dp: JsonDict,
37
- categories: Dict[str, str],
36
+ dp: CocoDatapointDict,
37
+ categories: dict[int, ObjectTypes],
38
38
  load_image: bool,
39
39
  filter_empty_image: bool,
40
40
  fake_score: bool,
@@ -88,7 +88,7 @@ def coco_to_image(
88
88
  bbox = BoundingBox(absolute_coords=True, ulx=x_1, uly=y_1, height=h, width=w)
89
89
 
90
90
  annotation = ImageAnnotation(
91
- category_name=categories[str(ann["category_id"])],
91
+ category_name=categories[ann["category_id"]],
92
92
  bounding_box=bbox,
93
93
  category_id=ann["category_id"],
94
94
  score=maybe_get_fake_score(fake_score),
@@ -98,8 +98,8 @@ def coco_to_image(
98
98
 
99
99
  if coarse_sub_cat_name and coarse_mapping:
100
100
  sub_cat = CategoryAnnotation(
101
- category_name=categories[str(coarse_mapping[ann["category_id"]])],
102
- category_id=str(coarse_mapping[ann["category_id"]]),
101
+ category_name=categories[coarse_mapping[ann["category_id"]]],
102
+ category_id=coarse_mapping[ann["category_id"]],
103
103
  )
104
104
  annotation.dump_sub_category(coarse_sub_cat_name, sub_cat)
105
105
 
@@ -109,7 +109,7 @@ def coco_to_image(
109
109
  return image
110
110
 
111
111
 
112
- def image_to_coco(dp: Image) -> Tuple[JsonDict, List[JsonDict]]:
112
+ def image_to_coco(dp: Image) -> tuple[JsonDict, list[JsonDict]]:
113
113
  """
114
114
  Converting an image back into the coco format. As images and anns are separated it will return a dict with the
115
115
  image information and one for its annotations.
@@ -122,7 +122,7 @@ def image_to_coco(dp: Image) -> Tuple[JsonDict, List[JsonDict]]:
122
122
  raise TypeError(f"datapoints must be of type Image, is of type {type(dp)}")
123
123
 
124
124
  img: JsonDict = {}
125
- anns: List[JsonDict] = []
125
+ anns: list[JsonDict] = []
126
126
 
127
127
  img["id"] = int("".join([s for s in dp.image_id if s.isdigit()]))
128
128
  img["width"] = dp.width
@@ -133,7 +133,7 @@ def image_to_coco(dp: Image) -> Tuple[JsonDict, List[JsonDict]]:
133
133
  ann: JsonDict = {
134
134
  "id": int("".join([s for s in img_ann.annotation_id if s.isdigit()])),
135
135
  "image_id": img["id"],
136
- "category_id": int(img_ann.category_id),
136
+ "category_id": img_ann.category_id,
137
137
  }
138
138
  if img_ann.score:
139
139
  ann["score"] = img_ann.score