deepdoctection 0.31__py3-none-any.whl → 0.33__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of deepdoctection might be problematic. Click here for more details.

Files changed (131) hide show
  1. deepdoctection/__init__.py +16 -29
  2. deepdoctection/analyzer/dd.py +70 -59
  3. deepdoctection/configs/conf_dd_one.yaml +34 -31
  4. deepdoctection/dataflow/common.py +9 -5
  5. deepdoctection/dataflow/custom.py +5 -5
  6. deepdoctection/dataflow/custom_serialize.py +75 -18
  7. deepdoctection/dataflow/parallel_map.py +3 -3
  8. deepdoctection/dataflow/serialize.py +4 -4
  9. deepdoctection/dataflow/stats.py +3 -3
  10. deepdoctection/datapoint/annotation.py +41 -56
  11. deepdoctection/datapoint/box.py +9 -8
  12. deepdoctection/datapoint/convert.py +6 -6
  13. deepdoctection/datapoint/image.py +56 -44
  14. deepdoctection/datapoint/view.py +245 -150
  15. deepdoctection/datasets/__init__.py +1 -4
  16. deepdoctection/datasets/adapter.py +35 -26
  17. deepdoctection/datasets/base.py +14 -12
  18. deepdoctection/datasets/dataflow_builder.py +3 -3
  19. deepdoctection/datasets/info.py +24 -26
  20. deepdoctection/datasets/instances/doclaynet.py +51 -51
  21. deepdoctection/datasets/instances/fintabnet.py +46 -46
  22. deepdoctection/datasets/instances/funsd.py +25 -24
  23. deepdoctection/datasets/instances/iiitar13k.py +13 -10
  24. deepdoctection/datasets/instances/layouttest.py +4 -3
  25. deepdoctection/datasets/instances/publaynet.py +5 -5
  26. deepdoctection/datasets/instances/pubtables1m.py +24 -21
  27. deepdoctection/datasets/instances/pubtabnet.py +32 -30
  28. deepdoctection/datasets/instances/rvlcdip.py +30 -30
  29. deepdoctection/datasets/instances/xfund.py +26 -26
  30. deepdoctection/datasets/save.py +6 -6
  31. deepdoctection/eval/__init__.py +1 -4
  32. deepdoctection/eval/accmetric.py +32 -33
  33. deepdoctection/eval/base.py +8 -9
  34. deepdoctection/eval/cocometric.py +15 -13
  35. deepdoctection/eval/eval.py +41 -37
  36. deepdoctection/eval/tedsmetric.py +30 -23
  37. deepdoctection/eval/tp_eval_callback.py +16 -19
  38. deepdoctection/extern/__init__.py +2 -7
  39. deepdoctection/extern/base.py +339 -134
  40. deepdoctection/extern/d2detect.py +85 -113
  41. deepdoctection/extern/deskew.py +14 -11
  42. deepdoctection/extern/doctrocr.py +141 -130
  43. deepdoctection/extern/fastlang.py +27 -18
  44. deepdoctection/extern/hfdetr.py +71 -62
  45. deepdoctection/extern/hflayoutlm.py +504 -211
  46. deepdoctection/extern/hflm.py +230 -0
  47. deepdoctection/extern/model.py +488 -302
  48. deepdoctection/extern/pdftext.py +23 -19
  49. deepdoctection/extern/pt/__init__.py +1 -3
  50. deepdoctection/extern/pt/nms.py +6 -2
  51. deepdoctection/extern/pt/ptutils.py +29 -19
  52. deepdoctection/extern/tessocr.py +39 -38
  53. deepdoctection/extern/texocr.py +18 -18
  54. deepdoctection/extern/tp/tfutils.py +57 -9
  55. deepdoctection/extern/tp/tpcompat.py +21 -14
  56. deepdoctection/extern/tp/tpfrcnn/__init__.py +20 -0
  57. deepdoctection/extern/tp/tpfrcnn/common.py +7 -3
  58. deepdoctection/extern/tp/tpfrcnn/config/__init__.py +20 -0
  59. deepdoctection/extern/tp/tpfrcnn/config/config.py +13 -10
  60. deepdoctection/extern/tp/tpfrcnn/modeling/__init__.py +20 -0
  61. deepdoctection/extern/tp/tpfrcnn/modeling/backbone.py +18 -8
  62. deepdoctection/extern/tp/tpfrcnn/modeling/generalized_rcnn.py +12 -6
  63. deepdoctection/extern/tp/tpfrcnn/modeling/model_box.py +14 -9
  64. deepdoctection/extern/tp/tpfrcnn/modeling/model_cascade.py +8 -5
  65. deepdoctection/extern/tp/tpfrcnn/modeling/model_fpn.py +22 -17
  66. deepdoctection/extern/tp/tpfrcnn/modeling/model_frcnn.py +21 -14
  67. deepdoctection/extern/tp/tpfrcnn/modeling/model_mrcnn.py +19 -11
  68. deepdoctection/extern/tp/tpfrcnn/modeling/model_rpn.py +15 -10
  69. deepdoctection/extern/tp/tpfrcnn/predict.py +9 -4
  70. deepdoctection/extern/tp/tpfrcnn/preproc.py +12 -8
  71. deepdoctection/extern/tp/tpfrcnn/utils/__init__.py +20 -0
  72. deepdoctection/extern/tp/tpfrcnn/utils/box_ops.py +10 -2
  73. deepdoctection/extern/tpdetect.py +45 -53
  74. deepdoctection/mapper/__init__.py +3 -8
  75. deepdoctection/mapper/cats.py +27 -29
  76. deepdoctection/mapper/cocostruct.py +10 -10
  77. deepdoctection/mapper/d2struct.py +27 -26
  78. deepdoctection/mapper/hfstruct.py +13 -8
  79. deepdoctection/mapper/laylmstruct.py +178 -37
  80. deepdoctection/mapper/maputils.py +12 -11
  81. deepdoctection/mapper/match.py +2 -2
  82. deepdoctection/mapper/misc.py +11 -9
  83. deepdoctection/mapper/pascalstruct.py +4 -4
  84. deepdoctection/mapper/prodigystruct.py +5 -5
  85. deepdoctection/mapper/pubstruct.py +84 -92
  86. deepdoctection/mapper/tpstruct.py +5 -5
  87. deepdoctection/mapper/xfundstruct.py +33 -33
  88. deepdoctection/pipe/__init__.py +1 -1
  89. deepdoctection/pipe/anngen.py +12 -14
  90. deepdoctection/pipe/base.py +52 -106
  91. deepdoctection/pipe/common.py +72 -59
  92. deepdoctection/pipe/concurrency.py +16 -11
  93. deepdoctection/pipe/doctectionpipe.py +24 -21
  94. deepdoctection/pipe/language.py +20 -25
  95. deepdoctection/pipe/layout.py +20 -16
  96. deepdoctection/pipe/lm.py +75 -105
  97. deepdoctection/pipe/order.py +194 -89
  98. deepdoctection/pipe/refine.py +111 -124
  99. deepdoctection/pipe/segment.py +156 -161
  100. deepdoctection/pipe/{cell.py → sub_layout.py} +50 -40
  101. deepdoctection/pipe/text.py +37 -36
  102. deepdoctection/pipe/transform.py +19 -16
  103. deepdoctection/train/__init__.py +6 -12
  104. deepdoctection/train/d2_frcnn_train.py +48 -41
  105. deepdoctection/train/hf_detr_train.py +41 -30
  106. deepdoctection/train/hf_layoutlm_train.py +153 -135
  107. deepdoctection/train/tp_frcnn_train.py +32 -31
  108. deepdoctection/utils/concurrency.py +1 -1
  109. deepdoctection/utils/context.py +13 -6
  110. deepdoctection/utils/develop.py +4 -4
  111. deepdoctection/utils/env_info.py +87 -125
  112. deepdoctection/utils/file_utils.py +6 -11
  113. deepdoctection/utils/fs.py +22 -18
  114. deepdoctection/utils/identifier.py +2 -2
  115. deepdoctection/utils/logger.py +16 -15
  116. deepdoctection/utils/metacfg.py +7 -7
  117. deepdoctection/utils/mocks.py +93 -0
  118. deepdoctection/utils/pdf_utils.py +11 -11
  119. deepdoctection/utils/settings.py +185 -181
  120. deepdoctection/utils/tqdm.py +1 -1
  121. deepdoctection/utils/transform.py +14 -9
  122. deepdoctection/utils/types.py +104 -0
  123. deepdoctection/utils/utils.py +7 -7
  124. deepdoctection/utils/viz.py +74 -72
  125. {deepdoctection-0.31.dist-info → deepdoctection-0.33.dist-info}/METADATA +30 -21
  126. deepdoctection-0.33.dist-info/RECORD +146 -0
  127. {deepdoctection-0.31.dist-info → deepdoctection-0.33.dist-info}/WHEEL +1 -1
  128. deepdoctection/utils/detection_types.py +0 -68
  129. deepdoctection-0.31.dist-info/RECORD +0 -144
  130. {deepdoctection-0.31.dist-info → deepdoctection-0.33.dist-info}/LICENSE +0 -0
  131. {deepdoctection-0.31.dist-info → deepdoctection-0.33.dist-info}/top_level.txt +0 -0
@@ -19,23 +19,25 @@
19
19
  Methods that convert incoming data to dataflows.
20
20
  """
21
21
 
22
+ from __future__ import annotations
23
+
22
24
  import itertools
23
25
  import json
24
26
  import os
25
27
  from collections import defaultdict
26
28
  from pathlib import Path
27
- from typing import DefaultDict, Dict, List, Optional, Sequence, Union
29
+ from typing import Any, DefaultDict, Dict, Iterator, List, Optional, Sequence, TextIO, Union
28
30
 
29
31
  from jsonlines import Reader, Writer
30
32
  from tabulate import tabulate
31
33
  from termcolor import colored
32
34
 
33
35
  from ..utils.context import timed_operation
34
- from ..utils.detection_types import JsonDict, Pathlike
35
36
  from ..utils.error import FileExtensionError
36
37
  from ..utils.identifier import get_uuid_from_str
37
38
  from ..utils.pdf_utils import PDFStreamer
38
39
  from ..utils.tqdm import get_tqdm
40
+ from ..utils.types import JsonDict, PathLikeOrStr
39
41
  from ..utils.utils import is_file_extension
40
42
  from .base import DataFlow
41
43
  from .common import FlattenData, JoinData, MapData
@@ -53,6 +55,59 @@ def _reset_df_and_get_length(df: DataFlow) -> int:
53
55
  return length
54
56
 
55
57
 
58
+ class FileClosingIterator:
59
+ """
60
+ A custom iterator that closes the file object once the iteration is complete.
61
+
62
+ This iterator is used to ensure that the file object is properly closed after
63
+ reading the data from it. It is used in the context of reading data from a file
64
+ in a streaming manner, where the data is not loaded into memory all at once.
65
+
66
+ **Example:**
67
+
68
+ file = open(path, "r")
69
+ iterator = Reader(file)
70
+ closing_iterator = FileClosingIterator(file, iter(iterator))
71
+
72
+ df = CustomDataFromIterable(closing_iterator, max_datapoints=max_datapoints) # set up a dataflow
73
+
74
+ """
75
+
76
+ def __init__(self, file_obj: TextIO, iterator: Iterator[Any]):
77
+ """
78
+ Initializes the FileClosingIterator with a file object and its iterator.
79
+
80
+ :param file_obj (TextIO): The file object to read data from.
81
+ :param iterator (Iterator): The actual iterator of the file object.
82
+ """
83
+ self.file_obj = file_obj
84
+ self.iterator = iterator
85
+
86
+ def __iter__(self) -> FileClosingIterator:
87
+ """
88
+ Returns the iterator object itself.
89
+
90
+ :return: FileClosingIterator: The instance of the class itself.
91
+ """
92
+ return self
93
+
94
+ def __next__(self) -> Any:
95
+ """
96
+ Returns the next item from the file object's iterator.
97
+ Closes the file object if the iteration is finished.
98
+
99
+ :return: The next item from the file object's iterator.
100
+
101
+ Raises:
102
+ StopIteration: If there are no more items to return.
103
+ """
104
+ try:
105
+ return next(self.iterator)
106
+ except StopIteration as exc:
107
+ self.file_obj.close()
108
+ raise StopIteration from exc
109
+
110
+
56
111
  class SerializerJsonlines:
57
112
  """
58
113
  Serialize a dataflow from a jsonlines file. Alternatively, save a dataflow of JSON objects to a .jsonl file.
@@ -66,7 +121,7 @@ class SerializerJsonlines:
66
121
  """
67
122
 
68
123
  @staticmethod
69
- def load(path: Pathlike, max_datapoints: Optional[int] = None) -> CustomDataFromIterable:
124
+ def load(path: PathLikeOrStr, max_datapoints: Optional[int] = None) -> CustomDataFromIterable:
70
125
  """
71
126
  :param path: a path to a .jsonl file.
72
127
  :param max_datapoints: Will stop the iteration once max_datapoints have been streamed
@@ -75,10 +130,11 @@ class SerializerJsonlines:
75
130
  """
76
131
  file = open(path, "r") # pylint: disable=W1514,R1732
77
132
  iterator = Reader(file)
78
- return CustomDataFromIterable(iterator, max_datapoints=max_datapoints)
133
+ closing_iterator = FileClosingIterator(file, iter(iterator))
134
+ return CustomDataFromIterable(closing_iterator, max_datapoints=max_datapoints)
79
135
 
80
136
  @staticmethod
81
- def save(df: DataFlow, path: Pathlike, file_name: str, max_datapoints: Optional[int] = None) -> None:
137
+ def save(df: DataFlow, path: PathLikeOrStr, file_name: str, max_datapoints: Optional[int] = None) -> None:
82
138
  """
83
139
  Writes a dataflow iteratively to a .jsonl file. Every datapoint must be a dict where all items are serializable.
84
140
  As the length of the dataflow cannot be determined in every case max_datapoint prevents generating an
@@ -120,7 +176,7 @@ class SerializerTabsepFiles:
120
176
  """
121
177
 
122
178
  @staticmethod
123
- def load(path: Pathlike, max_datapoins: Optional[int] = None) -> CustomDataFromList:
179
+ def load(path: PathLikeOrStr, max_datapoins: Optional[int] = None) -> CustomDataFromList:
124
180
  """
125
181
  :param path: a path to a .txt file.
126
182
  :param max_datapoins: Will stop the iteration once max_datapoints have been streamed
@@ -133,7 +189,7 @@ class SerializerTabsepFiles:
133
189
  return CustomDataFromList(file_list, max_datapoints=max_datapoins)
134
190
 
135
191
  @staticmethod
136
- def save(df: DataFlow, path: Pathlike, file_name: str, max_datapoints: Optional[int] = None) -> None:
192
+ def save(df: DataFlow, path: PathLikeOrStr, file_name: str, max_datapoints: Optional[int] = None) -> None:
137
193
  """
138
194
  Writes a dataflow iteratively to a .txt file. Every datapoint must be a string.
139
195
  As the length of the dataflow cannot be determined in every case max_datapoint prevents generating an
@@ -168,7 +224,7 @@ class SerializerFiles:
168
224
 
169
225
  @staticmethod
170
226
  def load(
171
- path: Pathlike,
227
+ path: PathLikeOrStr,
172
228
  file_type: Union[str, Sequence[str]],
173
229
  max_datapoints: Optional[int] = None,
174
230
  shuffle: Optional[bool] = False,
@@ -190,15 +246,14 @@ class SerializerFiles:
190
246
  df2: DataFlow
191
247
  df3: DataFlow
192
248
 
193
- if isinstance(path, str):
194
- path = Path(path)
249
+ path = Path(path)
195
250
  if not path.exists():
196
251
  raise NotADirectoryError(f"The path {path} to the directory or file does not exist")
197
252
 
198
253
  if shuffle:
199
254
  sort = False
200
- it1 = os.walk(path, topdown=False)
201
- it2 = os.walk(path, topdown=False)
255
+ it1 = os.walk(os.fspath(path), topdown=False)
256
+ it2 = os.walk(os.fspath(path), topdown=False)
202
257
  df1 = CustomDataFromIterable(it1)
203
258
  df2 = CustomDataFromIterable(it2)
204
259
  df1 = MapData(df1, lambda dp: None if len(dp[2]) == 0 else dp)
@@ -237,7 +292,7 @@ class CocoParser:
237
292
  :param annotation_file: location of annotation file
238
293
  """
239
294
 
240
- def __init__(self, annotation_file: Optional[Pathlike] = None) -> None:
295
+ def __init__(self, annotation_file: Optional[PathLikeOrStr] = None) -> None:
241
296
  self.dataset: JsonDict = {}
242
297
  self.anns: Dict[int, JsonDict] = {}
243
298
  self.cats: Dict[int, JsonDict] = {}
@@ -465,7 +520,7 @@ class SerializerCoco:
465
520
  """
466
521
 
467
522
  @staticmethod
468
- def load(path: Pathlike, max_datapoints: Optional[int] = None) -> DataFlow:
523
+ def load(path: PathLikeOrStr, max_datapoints: Optional[int] = None) -> DataFlow:
469
524
  """
470
525
  Loads a .json file and generates a dataflow.
471
526
 
@@ -478,7 +533,7 @@ class SerializerCoco:
478
533
 
479
534
  {'image':{'id',...},'annotations':[{'id':…,'bbox':...}]}
480
535
 
481
- for each single image id.
536
+ for each image id. We use the type hint CocoDatapointDict to describe this dictionary
482
537
 
483
538
  :param max_datapoints: Will stop the iteration once max_datapoints have been streamed.
484
539
  :param path: a path to a .json file.
@@ -525,7 +580,7 @@ class SerializerPdfDoc:
525
580
  """
526
581
 
527
582
  @staticmethod
528
- def load(path: Pathlike, max_datapoints: Optional[int] = None) -> DataFlow:
583
+ def load(path: PathLikeOrStr, max_datapoints: Optional[int] = None) -> DataFlow:
529
584
  """
530
585
  Loads the document page wise and returns a dataflow accordingly.
531
586
 
@@ -552,14 +607,16 @@ class SerializerPdfDoc:
552
607
  return df
553
608
 
554
609
  @staticmethod
555
- def save(path: Pathlike) -> None:
610
+ def save(path: PathLikeOrStr) -> None:
556
611
  """
557
612
  Not implemented
558
613
  """
559
614
  raise NotImplementedError()
560
615
 
561
616
  @staticmethod
562
- def split(path: Pathlike, path_target: Optional[Pathlike] = None, max_datapoint: Optional[int] = None) -> None:
617
+ def split(
618
+ path: PathLikeOrStr, path_target: Optional[PathLikeOrStr] = None, max_datapoint: Optional[int] = None
619
+ ) -> None:
563
620
  """
564
621
  Split a document into single pages.
565
622
  """
@@ -23,7 +23,7 @@ import uuid
23
23
  import weakref
24
24
  from abc import ABC, abstractmethod
25
25
  from contextlib import contextmanager
26
- from typing import Any, Callable, Iterator, List, no_type_check
26
+ from typing import Any, Callable, Iterator, no_type_check
27
27
 
28
28
  import zmq
29
29
 
@@ -236,7 +236,7 @@ class MultiThreadMapData(_ParallelMapData):
236
236
  self._strict = strict
237
237
  self.num_thread = num_thread
238
238
  self.map_func = map_func
239
- self._threads: List[Any] = []
239
+ self._threads: list[Any] = []
240
240
  self._evt = None
241
241
 
242
242
  def reset_state(self) -> None:
@@ -284,7 +284,7 @@ class _MultiProcessZMQDataFlow(DataFlow, ABC):
284
284
  if os.name == "nt":
285
285
  raise EnvironmentError("ZMQ IPC doesn't support windows")
286
286
  self._reset_done = False
287
- self._procs: List[Any] = []
287
+ self._procs: list[Any] = []
288
288
  self.context = None
289
289
  self.socket = None
290
290
 
@@ -12,7 +12,7 @@ Some DataFlow classes for serialization. Many classes have been taken from
12
12
 
13
13
  import pickle
14
14
  from copy import copy
15
- from typing import Any, Iterable, Iterator, List, Optional, Tuple, Union
15
+ from typing import Any, Iterable, Iterator, Optional, Union
16
16
 
17
17
  import numpy as np
18
18
 
@@ -23,7 +23,7 @@ from .base import DataFlow, RNGDataFlow
23
23
  class DataFromList(RNGDataFlow):
24
24
  """Wrap a list of datapoints to a DataFlow"""
25
25
 
26
- def __init__(self, lst: List[Any], shuffle: bool = True) -> None:
26
+ def __init__(self, lst: list[Any], shuffle: bool = True) -> None:
27
27
  """
28
28
  :param lst: input list. Each element is a datapoint.
29
29
  :param shuffle: shuffle data.
@@ -79,11 +79,11 @@ class FakeData(RNGDataFlow):
79
79
 
80
80
  def __init__(
81
81
  self,
82
- shapes: List[Union[List[Any], Tuple[Any]]],
82
+ shapes: list[Union[list[Any], tuple[Any]]],
83
83
  size: int = 1000,
84
84
  random: bool = True,
85
85
  dtype: str = "float32",
86
- domain: Tuple[Union[float, int], Union[float, int]] = (0, 1),
86
+ domain: tuple[Union[float, int], Union[float, int]] = (0, 1),
87
87
  ):
88
88
  """
89
89
  :param shapes: a list of lists/tuples. Shapes of each component.
@@ -18,7 +18,7 @@
18
18
  """
19
19
  Dataflows for calculating statistical values of the underlying dataset
20
20
  """
21
- from typing import Any, Optional, Tuple, Union
21
+ from typing import Any, Optional, Union
22
22
 
23
23
  import numpy as np
24
24
  import numpy.typing as npt
@@ -45,7 +45,7 @@ class MeanFromDataFlow(ProxyDataFlow):
45
45
  def __init__(
46
46
  self,
47
47
  df: DataFlow,
48
- axis: Optional[Union[int, Tuple[int], Tuple[int, int], Tuple[int, int, int]]] = None,
48
+ axis: Optional[Union[int, tuple[int], tuple[int, int], tuple[int, int, int]]] = None,
49
49
  key: Optional[str] = None,
50
50
  max_datapoints: Optional[int] = None,
51
51
  ):
@@ -165,7 +165,7 @@ class StdFromDataFlow(ProxyDataFlow):
165
165
  def __init__(
166
166
  self,
167
167
  df: DataFlow,
168
- axis: Optional[Union[int, Tuple[int], Tuple[int, int], Tuple[int, int, int]]] = None,
168
+ axis: Optional[Union[int, tuple[int], tuple[int, int], tuple[int, int, int]]] = None,
169
169
  key: Optional[str] = None,
170
170
  max_datapoints: Optional[int] = None,
171
171
  ):
@@ -18,34 +18,38 @@
18
18
  """
19
19
  Dataclass for annotations and their derived classes.
20
20
  """
21
+ from __future__ import annotations
21
22
 
22
23
  from abc import ABC, abstractmethod
23
24
  from dataclasses import dataclass, field
24
- from typing import Any, Dict, List, Optional, Union, no_type_check
25
+ from typing import Optional, Union, no_type_check
25
26
 
26
- from ..utils.detection_types import JsonDict
27
27
  from ..utils.error import AnnotationError, UUIDError
28
28
  from ..utils.identifier import get_uuid, is_uuid_like
29
29
  from ..utils.logger import LoggingRecord, logger
30
- from ..utils.settings import DefaultType, ObjectTypes, SummaryType, TypeOrStr, get_type
30
+ from ..utils.settings import DefaultType, ObjectTypes, TypeOrStr, get_type
31
+ from ..utils.types import AnnotationDict
31
32
  from .box import BoundingBox
32
33
  from .convert import as_dict
33
34
 
34
35
 
35
36
  @no_type_check
36
- def ann_from_dict(cls, **kwargs):
37
+ def ann_from_dict(cls, **kwargs: AnnotationDict):
37
38
  """
38
39
  A factory function to create subclasses of annotations from a given dict
39
40
  """
40
41
  _init_kwargs = {
41
42
  "external_id": kwargs.get("external_id"),
42
43
  "category_name": kwargs.get("category_name"),
43
- "category_id": kwargs.get("category_id"),
44
+ "category_id": kwargs.get("category_id", DEFAULT_CATEGORY_ID),
44
45
  "score": kwargs.get("score"),
45
46
  "service_id": kwargs.get("service_id"),
46
47
  "model_id": kwargs.get("model_id"),
47
48
  "session_id": kwargs.get("session_id"),
48
49
  }
50
+ _init_kwargs["category_id"] = (
51
+ int(_init_kwargs["category_id"]) if (_init_kwargs)["category_id"] not in ("None", "") else DEFAULT_CATEGORY_ID
52
+ )
49
53
  ann = cls(**_init_kwargs)
50
54
  ann.active = kwargs.get("active")
51
55
  ann._annotation_id = kwargs.get("_annotation_id") # pylint: disable=W0212
@@ -134,7 +138,7 @@ class Annotation(ABC):
134
138
  raise AnnotationError("Annotation_id must be uuid3 string")
135
139
 
136
140
  @abstractmethod
137
- def get_defining_attributes(self) -> List[str]:
141
+ def get_defining_attributes(self) -> list[str]:
138
142
  """
139
143
  Defining attributes of an annotation instance are attributes, of which you think that they uniquely
140
144
  describe the annotation object. If you do not provide an external id, only the defining attributes will be used
@@ -151,7 +155,7 @@ class Annotation(ABC):
151
155
  raise AnnotationError(f"Attribute {attr} must have __str__ method")
152
156
 
153
157
  @staticmethod
154
- def set_annotation_id(annotation: "CategoryAnnotation", *container_id_context: Optional[str]) -> str:
158
+ def set_annotation_id(annotation: CategoryAnnotation, *container_id_context: Optional[str]) -> str:
155
159
  """
156
160
  Defines the `annotation_id` by attributes of the annotation class as well as by external parameters given by a
157
161
  tuple or list of container id contexts.
@@ -167,7 +171,7 @@ class Annotation(ABC):
167
171
  attributes_values = [str(getattr(annotation, attribute)) for attribute in attributes]
168
172
  return get_uuid(*attributes_values, *container_id_context) # type: ignore
169
173
 
170
- def as_dict(self) -> Dict[str, Any]:
174
+ def as_dict(self) -> AnnotationDict:
171
175
  """
172
176
  Returning the full dataclass as dict. Uses the custom `convert.as_dict` to disregard attributes defined by
173
177
  `remove_keys`.
@@ -187,7 +191,7 @@ class Annotation(ABC):
187
191
 
188
192
  @classmethod
189
193
  @abstractmethod
190
- def from_dict(cls, **kwargs: JsonDict) -> "Annotation":
194
+ def from_dict(cls, **kwargs: AnnotationDict) -> Annotation:
191
195
  """
192
196
  Method to initialize a derived class from dict.
193
197
 
@@ -199,7 +203,7 @@ class Annotation(ABC):
199
203
 
200
204
  @staticmethod
201
205
  @abstractmethod
202
- def get_state_attributes() -> List[str]:
206
+ def get_state_attributes() -> list[str]:
203
207
  """
204
208
  Similar to `get_defining_attributes` but for `state_id`
205
209
 
@@ -242,6 +246,9 @@ class Annotation(ABC):
242
246
  return get_uuid(self.annotation_id, *container_ids)
243
247
 
244
248
 
249
+ DEFAULT_CATEGORY_ID = -1
250
+
251
+
245
252
  @dataclass
246
253
  class CategoryAnnotation(Annotation):
247
254
  """
@@ -268,12 +275,12 @@ class CategoryAnnotation(Annotation):
268
275
  `dump_relationship` instead.
269
276
  """
270
277
 
271
- category_name: TypeOrStr = field(default=DefaultType.default_type)
272
- _category_name: ObjectTypes = field(default=DefaultType.default_type, init=False)
273
- category_id: str = field(default="")
278
+ category_name: TypeOrStr = field(default=DefaultType.DEFAULT_TYPE)
279
+ _category_name: ObjectTypes = field(default=DefaultType.DEFAULT_TYPE, init=False)
280
+ category_id: int = field(default=DEFAULT_CATEGORY_ID)
274
281
  score: Optional[float] = field(default=None)
275
- sub_categories: Dict[ObjectTypes, "CategoryAnnotation"] = field(default_factory=dict, init=False, repr=True)
276
- relationships: Dict[ObjectTypes, List[str]] = field(default_factory=dict, init=False, repr=True)
282
+ sub_categories: dict[ObjectTypes, CategoryAnnotation] = field(default_factory=dict, init=False, repr=True)
283
+ relationships: dict[ObjectTypes, list[str]] = field(default_factory=dict, init=False, repr=True)
277
284
 
278
285
  @property # type: ignore
279
286
  def category_name(self) -> ObjectTypes:
@@ -287,13 +294,11 @@ class CategoryAnnotation(Annotation):
287
294
  self._category_name = get_type(category_name)
288
295
 
289
296
  def __post_init__(self) -> None:
290
- self.category_id = str(self.category_id)
291
- assert self.category_name
292
297
  self._assert_attributes_have_str(state_id=True)
293
298
  super().__post_init__()
294
299
 
295
300
  def dump_sub_category(
296
- self, sub_category_name: TypeOrStr, annotation: "CategoryAnnotation", *container_id_context: Optional[str]
301
+ self, sub_category_name: TypeOrStr, annotation: CategoryAnnotation, *container_id_context: Optional[str]
297
302
  ) -> None:
298
303
  """
299
304
  Storage of sub-categories. As sub-categories usually only depend on very few attributes and the parent
@@ -324,7 +329,7 @@ class CategoryAnnotation(Annotation):
324
329
  )
325
330
  self.sub_categories[get_type(sub_category_name)] = annotation
326
331
 
327
- def get_sub_category(self, sub_category_name: ObjectTypes) -> "CategoryAnnotation":
332
+ def get_sub_category(self, sub_category_name: ObjectTypes) -> CategoryAnnotation:
328
333
  """
329
334
  Return a sub category by its key.
330
335
 
@@ -362,7 +367,7 @@ class CategoryAnnotation(Annotation):
362
367
  if annotation_id not in self.relationships[key_type]:
363
368
  self.relationships[key_type].append(annotation_id)
364
369
 
365
- def get_relationship(self, key: ObjectTypes) -> List[str]:
370
+ def get_relationship(self, key: ObjectTypes) -> list[str]:
366
371
  """
367
372
  Returns a list of annotation ids stored with a given relationship key.
368
373
 
@@ -373,7 +378,7 @@ class CategoryAnnotation(Annotation):
373
378
  return self.relationships[key]
374
379
  return []
375
380
 
376
- def remove_relationship(self, key: ObjectTypes, annotation_ids: Optional[Union[List[str], str]] = None) -> None:
381
+ def remove_relationship(self, key: ObjectTypes, annotation_ids: Optional[Union[list[str], str]] = None) -> None:
377
382
  """
378
383
  Remove relationship by some given keys and ids. If no annotation ids are provided all relationship according
379
384
  to the key will be removed.
@@ -394,25 +399,25 @@ class CategoryAnnotation(Annotation):
394
399
  else:
395
400
  self.relationships[key].clear()
396
401
 
397
- def get_defining_attributes(self) -> List[str]:
402
+ def get_defining_attributes(self) -> list[str]:
398
403
  return ["category_name", "category_id"]
399
404
 
400
405
  @staticmethod
401
- def remove_keys() -> List[str]:
406
+ def remove_keys() -> list[str]:
402
407
  """
403
408
  A list of attributes to suspend from as_dict creation.
404
409
 
405
- :return: List of attributes.
410
+ :return: list of attributes.
406
411
  """
407
412
  return []
408
413
 
409
414
  @classmethod
410
- def from_dict(cls, **kwargs: JsonDict) -> "CategoryAnnotation":
415
+ def from_dict(cls, **kwargs: AnnotationDict) -> CategoryAnnotation:
411
416
  category_ann = ann_from_dict(cls, **kwargs)
412
417
  return category_ann
413
418
 
414
419
  @staticmethod
415
- def get_state_attributes() -> List[str]:
420
+ def get_state_attributes() -> list[str]:
416
421
  return ["active", "sub_categories", "relationships"]
417
422
 
418
423
 
@@ -432,20 +437,20 @@ class ImageAnnotation(CategoryAnnotation):
432
437
  """
433
438
 
434
439
  bounding_box: Optional[BoundingBox] = field(default=None)
435
- image: Optional["Image"] = field(default=None, init=False, repr=False) # type: ignore
440
+ image: Optional[Image] = field(default=None, init=False, repr=False) # type: ignore # pylint: disable=E0602
436
441
 
437
- def get_defining_attributes(self) -> List[str]:
442
+ def get_defining_attributes(self) -> list[str]:
438
443
  return ["category_name", "bounding_box"]
439
444
 
440
445
  @classmethod
441
- def from_dict(cls, **kwargs: JsonDict) -> "ImageAnnotation":
446
+ def from_dict(cls, **kwargs: AnnotationDict) -> ImageAnnotation:
442
447
  image_ann = ann_from_dict(cls, **kwargs)
443
448
  if box_kwargs := kwargs.get("bounding_box"):
444
449
  image_ann.bounding_box = BoundingBox.from_dict(**box_kwargs)
445
450
  return image_ann
446
451
 
447
452
  @staticmethod
448
- def get_state_attributes() -> List[str]:
453
+ def get_state_attributes() -> list[str]:
449
454
  return ["active", "sub_categories", "relationships", "image"]
450
455
 
451
456
  def get_bounding_box(self, image_id: Optional[str] = None) -> BoundingBox:
@@ -462,31 +467,10 @@ class ImageAnnotation(CategoryAnnotation):
462
467
  def get_summary(self, key: ObjectTypes) -> CategoryAnnotation:
463
468
  """Get summary sub categories from `image`. Raises `ValueError` if `key` is not available"""
464
469
  if self.image:
465
- if self.image.summary:
466
- return self.image.summary.get_sub_category(key)
470
+ return self.image.summary.get_sub_category(key)
467
471
  raise AnnotationError(f"Summary does not exist for {self.annotation_id} and key: {key}")
468
472
 
469
473
 
470
- @dataclass
471
- class SummaryAnnotation(CategoryAnnotation):
472
- """
473
- A dataclass for adding summaries. The various summaries can be stored as sub categories.
474
-
475
- Summary annotations should be stored in the attribute provided: `image.Image.summary` and should not be
476
- dumped as a category.
477
- """
478
-
479
- def __post_init__(self) -> None:
480
- self._category_name = SummaryType.summary
481
- super().__post_init__()
482
-
483
- @classmethod
484
- def from_dict(cls, **kwargs: JsonDict) -> "SummaryAnnotation":
485
- summary_ann = ann_from_dict(cls, **kwargs)
486
- summary_ann.category_name = SummaryType.summary
487
- return summary_ann
488
-
489
-
490
474
  @dataclass
491
475
  class ContainerAnnotation(CategoryAnnotation):
492
476
  """
@@ -496,13 +480,14 @@ class ContainerAnnotation(CategoryAnnotation):
496
480
  value: Attribute to store the value. Use strings.
497
481
  """
498
482
 
499
- value: Optional[Union[List[str], str]] = field(default=None)
483
+ value: Optional[Union[list[str], str]] = field(default=None)
500
484
 
501
- def get_defining_attributes(self) -> List[str]:
485
+ def get_defining_attributes(self) -> list[str]:
502
486
  return ["category_name", "value"]
503
487
 
504
488
  @classmethod
505
- def from_dict(cls, **kwargs: JsonDict) -> "SummaryAnnotation":
489
+ def from_dict(cls, **kwargs: AnnotationDict) -> ContainerAnnotation:
506
490
  container_ann = ann_from_dict(cls, **kwargs)
507
- container_ann.value = kwargs.get("value")
491
+ value = kwargs.get("value", "")
492
+ container_ann.value = value if isinstance(value, str) else list(value)
508
493
  return container_ann
@@ -21,18 +21,19 @@ Implementation of BoundingBox class and related methods
21
21
 
22
22
  from dataclasses import dataclass
23
23
  from math import ceil, floor
24
- from typing import List, Optional, Sequence, no_type_check
24
+ from typing import Optional, Sequence, no_type_check
25
25
 
26
26
  import numpy as np
27
27
  import numpy.typing as npt
28
+ from lazy_imports import try_import
28
29
  from numpy import float32
29
30
 
30
- from ..utils.detection_types import ImageType
31
31
  from ..utils.error import BoundingBoxError
32
32
  from ..utils.file_utils import cocotools_available
33
33
  from ..utils.logger import LoggingRecord, logger
34
+ from ..utils.types import PixelValues
34
35
 
35
- if cocotools_available():
36
+ with try_import() as import_guard:
36
37
  import pycocotools.mask as coco_mask
37
38
 
38
39
 
@@ -220,7 +221,7 @@ class BoundingBox:
220
221
  return self.uly + 0.5 * self.height
221
222
 
222
223
  @property
223
- def center(self) -> List[float]:
224
+ def center(self) -> list[float]:
224
225
  """
225
226
  Bounding box center [x,y]
226
227
  """
@@ -263,7 +264,7 @@ class BoundingBox:
263
264
  * np_poly_scale
264
265
  )
265
266
 
266
- def to_list(self, mode: str, scale_x: float = 1.0, scale_y: float = 1.0) -> List[float]:
267
+ def to_list(self, mode: str, scale_x: float = 1.0, scale_y: float = 1.0) -> list[float]:
267
268
  """
268
269
  Returns the coordinates as list
269
270
 
@@ -344,7 +345,7 @@ class BoundingBox:
344
345
  return f"Bounding Box ulx: {self.ulx}, uly: {self.uly}, lrx: {self.lrx}, lry: {self.lry}"
345
346
 
346
347
  @staticmethod
347
- def remove_keys() -> List[str]:
348
+ def remove_keys() -> list[str]:
348
349
  """
349
350
  A list of attributes to suspend from as_dict creation.
350
351
  """
@@ -397,8 +398,8 @@ def intersection_box(
397
398
 
398
399
 
399
400
  def crop_box_from_image(
400
- np_image: ImageType, crop_box: BoundingBox, width: Optional[float] = None, height: Optional[float] = None
401
- ) -> ImageType:
401
+ np_image: PixelValues, crop_box: BoundingBox, width: Optional[float] = None, height: Optional[float] = None
402
+ ) -> PixelValues:
402
403
  """
403
404
  Crop a box (the crop_box) from a np_image. Will floor the left and ceil the right coordinate point.
404
405
 
@@ -30,10 +30,10 @@ from numpy import uint8
30
30
  from numpy.typing import NDArray
31
31
  from pypdf import PdfReader
32
32
 
33
- from ..utils.detection_types import ImageType
34
33
  from ..utils.develop import deprecated
35
34
  from ..utils.error import DependencyError
36
35
  from ..utils.pdf_utils import pdf_to_np_array
36
+ from ..utils.types import PixelValues
37
37
  from ..utils.viz import viz_handler
38
38
 
39
39
  __all__ = [
@@ -75,7 +75,7 @@ def as_dict(obj: Any, dict_factory) -> Union[Any]: # type: ignore
75
75
  return copy.deepcopy(obj)
76
76
 
77
77
 
78
- def convert_b64_to_np_array(image: str) -> ImageType:
78
+ def convert_b64_to_np_array(image: str) -> PixelValues:
79
79
  """
80
80
  Converts an image in base4 string encoding representation to a numpy array of shape (width,height,channel).
81
81
 
@@ -86,7 +86,7 @@ def convert_b64_to_np_array(image: str) -> ImageType:
86
86
  return viz_handler.convert_b64_to_np(image).astype(uint8)
87
87
 
88
88
 
89
- def convert_np_array_to_b64(np_image: ImageType) -> str:
89
+ def convert_np_array_to_b64(np_image: PixelValues) -> str:
90
90
  """
91
91
  Converts an image from numpy array into a base64 string encoding representation
92
92
 
@@ -97,7 +97,7 @@ def convert_np_array_to_b64(np_image: ImageType) -> str:
97
97
 
98
98
 
99
99
  @no_type_check
100
- def convert_np_array_to_b64_b(np_image: ImageType) -> bytes:
100
+ def convert_np_array_to_b64_b(np_image: PixelValues) -> bytes:
101
101
  """
102
102
  Converts an image from numpy array into a base64 bytes encoding representation
103
103
 
@@ -108,7 +108,7 @@ def convert_np_array_to_b64_b(np_image: ImageType) -> bytes:
108
108
 
109
109
 
110
110
  @deprecated("Use convert_pdf_bytes_to_np_array_v2", "2022-02-23")
111
- def convert_pdf_bytes_to_np_array(pdf_bytes: bytes, dpi: Optional[int] = None) -> ImageType:
111
+ def convert_pdf_bytes_to_np_array(pdf_bytes: bytes, dpi: Optional[int] = None) -> PixelValues:
112
112
  """
113
113
  Converts a pdf passed as bytes into a numpy array. Note, that this method expects poppler to be installed.
114
114
  Please check the installation guides at https://poppler.freedesktop.org/ . If no value for dpi is provided
@@ -143,7 +143,7 @@ def convert_pdf_bytes_to_np_array(pdf_bytes: bytes, dpi: Optional[int] = None) -
143
143
  return np_array.astype(uint8)
144
144
 
145
145
 
146
- def convert_pdf_bytes_to_np_array_v2(pdf_bytes: bytes, dpi: Optional[int] = None) -> ImageType:
146
+ def convert_pdf_bytes_to_np_array_v2(pdf_bytes: bytes, dpi: Optional[int] = None) -> PixelValues:
147
147
  """
148
148
  Converts a pdf passed as bytes into a numpy array. Note, that this method expects poppler to be installed. This
149
149
  function, however does not rely on the wrapper pdf2image but uses a function of this lib which calls poppler