deepdoctection 0.43.6__tar.gz → 0.44.1__tar.gz

{deepdoctection-0.43.6 → deepdoctection-0.44.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deepdoctection
-Version: 0.43.6
+Version: 0.44.1
 Summary: Repository for Document AI
 Home-page: https://github.com/deepdoctection/deepdoctection
 Author: Dr. Janis Meyer
@@ -27,7 +27,7 @@ Requires-Dist: networkx>=2.7.1
 Requires-Dist: numpy<2.0,>=1.21
 Requires-Dist: packaging>=20.0
 Requires-Dist: Pillow>=10.0.0
-Requires-Dist: pypdf>=3.16.0
+Requires-Dist: pypdf>=6.0.0
 Requires-Dist: pypdfium2>=4.30.0
 Requires-Dist: pyyaml>=6.0.1
 Requires-Dist: pyzmq>=16
@@ -46,7 +46,7 @@ Requires-Dist: networkx>=2.7.1; extra == "tf"
 Requires-Dist: numpy<2.0,>=1.21; extra == "tf"
 Requires-Dist: packaging>=20.0; extra == "tf"
 Requires-Dist: Pillow>=10.0.0; extra == "tf"
-Requires-Dist: pypdf>=3.16.0; extra == "tf"
+Requires-Dist: pypdf>=6.0.0; extra == "tf"
 Requires-Dist: pypdfium2>=4.30.0; extra == "tf"
 Requires-Dist: pyyaml>=6.0.1; extra == "tf"
 Requires-Dist: pyzmq>=16; extra == "tf"
@@ -78,7 +78,7 @@ Requires-Dist: networkx>=2.7.1; extra == "pt"
 Requires-Dist: numpy<2.0,>=1.21; extra == "pt"
 Requires-Dist: packaging>=20.0; extra == "pt"
 Requires-Dist: Pillow>=10.0.0; extra == "pt"
-Requires-Dist: pypdf>=3.16.0; extra == "pt"
+Requires-Dist: pypdf>=6.0.0; extra == "pt"
 Requires-Dist: pypdfium2>=4.30.0; extra == "pt"
 Requires-Dist: pyyaml>=6.0.1; extra == "pt"
 Requires-Dist: pyzmq>=16; extra == "pt"

{deepdoctection-0.43.6 → deepdoctection-0.44.1}/deepdoctection/__init__.py

@@ -25,7 +25,7 @@ from .utils.logger import LoggingRecord, logger
 
 # pylint: enable=wrong-import-position
 
-__version__ = "0.43.6"
+__version__ = "0.44.1"
 
 _IMPORT_STRUCTURE = {
     "analyzer": ["config_sanity_checks", "get_dd_analyzer", "ServiceFactory"],
@@ -92,6 +92,7 @@ _IMPORT_STRUCTURE = {
         "convert_pdf_bytes_to_np_array_v2",
         "as_dict",
         "ImageAnnotationBaseView",
+        "MetaAnnotation",
         "Image",
         "Word",
         "Layout",
@@ -105,6 +106,7 @@ _IMPORT_STRUCTURE = {
         "DatasetAdapter",
         "DatasetBase",
         "MergeDataset",
+        "DatasetCard",
         "CustomDataset",
         "DataFlowBaseBuilder",
         "DatasetInfo",
@@ -313,6 +315,8 @@ _IMPORT_STRUCTURE = {
         "get_apted_requirement",
         "distance_available",
         "get_distance_requirement",
+        "numpy_v1_available",
+        "get_numpy_v1_requirement",
         "transformers_available",
         "get_transformers_requirement",
         "detectron2_available",

{deepdoctection-0.43.6 → deepdoctection-0.44.1}/deepdoctection/datapoint/__init__.py

@@ -34,5 +34,5 @@ After all, the point here is not to provide an optimal processing environment.
 from .annotation import *
 from .box import *
 from .convert import *
-from .image import Image
+from .image import Image, MetaAnnotation
 from .view import *

{deepdoctection-0.43.6 → deepdoctection-0.44.1}/deepdoctection/datapoint/image.py

@@ -25,7 +25,7 @@ from collections import defaultdict
 from dataclasses import dataclass, field
 from os import environ, fspath
 from pathlib import Path
-from typing import Any, Optional, Sequence, Union, no_type_check
+from typing import Any, Optional, Sequence, TypedDict, Union, no_type_check
 
 import numpy as np
 from numpy import uint8
@@ -40,6 +40,55 @@ from .box import crop_box_from_image, global_to_local_coords, intersection_box
 from .convert import as_dict, convert_b64_to_np_array, convert_np_array_to_b64, convert_pdf_bytes_to_np_array_v2
 
 
+class MetaAnnotationDict(TypedDict):
+    """MetaAnnotationDict"""
+
+    image_annotations: list[str]
+    sub_categories: dict[str, dict[str, list[str]]]
+    relationships: dict[str, list[str]]
+    summaries: list[str]
+
+
+@dataclass(frozen=True)
+class MetaAnnotation:
+    """
+    An immutable dataclass that stores information about what `Image` are being
+    modified through a pipeline component.
+
+    Attributes:
+        image_annotations: Tuple of `ObjectTypes` representing image annotations.
+        sub_categories: Dictionary mapping `ObjectTypes` to dicts of `ObjectTypes` to sets of `ObjectTypes`
+        for sub-categories.
+        relationships: Dictionary mapping `ObjectTypes` to sets of `ObjectTypes` for relationships.
+        summaries: Tuple of `ObjectTypes` representing summaries.
+    """
+
+    image_annotations: tuple[ObjectTypes, ...] = field(default=())
+    sub_categories: dict[ObjectTypes, dict[ObjectTypes, set[ObjectTypes]]] = field(default_factory=dict)
+    relationships: dict[ObjectTypes, set[ObjectTypes]] = field(default_factory=dict)
+    summaries: tuple[ObjectTypes, ...] = field(default=())
+
+    def as_dict(self) -> MetaAnnotationDict:
+        """
+        Returns the MetaAnnotation as a dictionary, with all `ObjectTypes` converted to strings.
+
+        Returns:
+            A dictionary representation of the MetaAnnotation where all `ObjectTypes` are converted to strings.
+        """
+        return {
+            "image_annotations": [obj.value for obj in self.image_annotations],
+            "sub_categories": {
+                outer_key.value: {
+                    inner_key.value: [val.value for val in inner_values]
+                    for inner_key, inner_values in outer_value.items()
+                }
+                for outer_key, outer_value in self.sub_categories.items()
+            },
+            "relationships": {key.value: [val.value for val in values] for key, values in self.relationships.items()},
+            "summaries": [obj.value for obj in self.summaries],
+        }
+
+
 @dataclass
 class Image:
     """

{deepdoctection-0.43.6 → deepdoctection-0.44.1}/deepdoctection/datapoint/view.py

@@ -42,13 +42,60 @@ from ..utils.settings import (
     get_type,
 )
 from ..utils.transform import ResizeTransform, box_to_point4, point4_to_box
-from ..utils.types import HTML, AnnotationDict, Chunks, ImageDict, PathLikeOrStr, PixelValues, Text_, csv
+from ..utils.types import HTML, AnnotationDict, Chunks, ImageDict, PathLikeOrStr, PixelValues, csv
 from ..utils.viz import draw_boxes, interactive_imshow, viz_handler
 from .annotation import CategoryAnnotation, ContainerAnnotation, ImageAnnotation, ann_from_dict
 from .box import BoundingBox, crop_box_from_image
 from .image import Image
 
 
+@dataclass(frozen=True)
+class Text_:
+    """
+    Immutable dataclass for storing structured text extraction results.
+
+    Attributes:
+        text: The concatenated text string.
+        words: List of word strings.
+        ann_ids: List of annotation IDs for each word.
+        token_classes: List of token class names for each word.
+        token_class_ann_ids: List of annotation IDs for each token class.
+        token_tags: List of token tag names for each word.
+        token_tag_ann_ids: List of annotation IDs for each token tag.
+        token_class_ids: List of token class IDs.
+        token_tag_ids: List of token tag IDs.
+    """
+
+    text: str = ""
+    words: list[str] = field(default_factory=list)
+    ann_ids: list[str] = field(default_factory=list)
+    token_classes: list[str] = field(default_factory=list)
+    token_class_ann_ids: list[str] = field(default_factory=list)
+    token_tags: list[str] = field(default_factory=list)
+    token_tag_ann_ids: list[str] = field(default_factory=list)
+    token_class_ids: list[str] = field(default_factory=list)
+    token_tag_ids: list[str] = field(default_factory=list)
+
+    def as_dict(self) -> dict[str, Union[list[str], str]]:
+        """
+        Returns the Text_ as a dictionary.
+
+        Returns:
+            A dictionary representation of the Text_ dataclass.
+        """
+        return {
+            "text": self.text,
+            "words": self.words,
+            "ann_ids": self.ann_ids,
+            "token_classes": self.token_classes,
+            "token_class_ann_ids": self.token_class_ann_ids,
+            "token_tags": self.token_tags,
+            "token_tag_ann_ids": self.token_tag_ann_ids,
+            "token_class_ids": self.token_class_ids,
+            "token_tag_ids": self.token_tag_ids,
+        }
+
+
 class ImageAnnotationBaseView(ImageAnnotation):
     """
     Consumption class for having easier access to categories added to an `ImageAnnotation`.
@@ -263,13 +310,28 @@ class Layout(ImageAnnotationBaseView):
         """
         words = self.get_ordered_words()
         if words:
-            characters, ann_ids, token_classes, token_tags, token_classes_ids, token_tag_ids = zip(
+            (
+                characters,
+                ann_ids,
+                token_classes,
+                token_class_ann_ids,
+                token_tags,
+                token_tag_ann_ids,
+                token_classes_ids,
+                token_tag_ids,
+            ) = map(list, zip(
                 *[
                     (
                         word.characters,
                         word.annotation_id,
                         word.token_class,
+                        word.get_sub_category(WordType.TOKEN_CLASS).annotation_id
+                        if WordType.TOKEN_CLASS in word.sub_categories
+                        else None,
                         word.token_tag,
+                        word.get_sub_category(WordType.TOKEN_TAG).annotation_id
+                        if WordType.TOKEN_TAG in word.sub_categories
+                        else None,
                         word.get_sub_category(WordType.TOKEN_CLASS).category_id
                         if WordType.TOKEN_CLASS in word.sub_categories
                         else None,
@@ -279,25 +341,40 @@ class Layout(ImageAnnotationBaseView):
                     )
                     for word in words
                 ]
-            )
+            ))
         else:
-            characters, ann_ids, token_classes, token_tags, token_classes_ids, token_tag_ids = (
-                [],  # type: ignore
-                [],  # type: ignore
-                [],  # type: ignore
-                [],  # type: ignore
-                [],  # type: ignore
-                [],  # type: ignore
+            (
+                characters,
+                ann_ids,
+                token_classes,
+                token_class_ann_ids,
+                token_tags,
+                token_tag_ann_ids,
+                token_classes_ids,
+                token_tag_ids,
+            ) = (
+                [],
+                [],
+                [],
+                [],
+                [],
+                [],
+                [],
+                [],
             )
-        return {
-            "text": " ".join(characters),
-            "words": characters,
-            "ann_ids": ann_ids,
-            "token_classes": token_classes,
-            "token_tags": token_tags,
-            "token_class_ids": token_classes_ids,
-            "token_tag_ids": token_tag_ids,
-        }
+
+        return Text_(
+            text=" ".join(characters), # type: ignore
+            words=characters, # type: ignore
+            ann_ids=ann_ids, # type: ignore
+            token_classes=token_classes, # type: ignore
+            token_class_ann_ids=token_class_ann_ids, # type: ignore
+            token_tags=token_tags, # type: ignore
+            token_tag_ann_ids=token_tag_ann_ids, # type: ignore
+            token_class_ids=token_classes_ids, # type: ignore
+            token_tag_ids=token_tag_ids, # type: ignore
+        )
+
 
     def get_attribute_names(self) -> set[str]:
         attr_names = (
@@ -590,14 +667,16 @@ class Table(Layout):
 
     @property
     def csv_(self) -> list[list[list[Text_]]]:
+        """
+        Returns:
+            A csv-style representation of a table as list of lists of cell.text_.
+        """
         cells = self.cells
         table_list = [[[] for _ in range(self.number_of_columns)] for _ in range(self.number_of_rows)]  # type: ignore
         for cell in cells:
             table_list[cell.row_number - 1][cell.column_number - 1].append(cell.text_)  # type: ignore
         return table_list
 
-
-
     def __str__(self) -> str:
         out = " ".join([" ".join(row + ["\n"]) for row in self.csv])
         return out
@@ -624,26 +703,34 @@ class Table(Layout):
         words: list[str] = []
         ann_ids: list[str] = []
         token_classes: list[str] = []
+        token_class_ann_ids: list[str] = []
         token_tags: list[str] = []
+        token_tag_ann_ids: list[str] = []
         token_class_ids: list[str] = []
         token_tag_ids: list[str] = []
         for cell in cells:
-            text.append(cell.text_["text"])
-            words.extend(cell.text_["words"])
-            ann_ids.extend(cell.text_["ann_ids"])
-            token_classes.extend(cell.text_["token_classes"])
-            token_tags.extend(cell.text_["token_tags"])
-            token_class_ids.extend(cell.text_["token_class_ids"])
-            token_tag_ids.extend(cell.text_["token_tag_ids"])
-        return {
-            "text": " ".join(text),
-            "words": words,
-            "ann_ids": ann_ids,
-            "token_classes": token_classes,
-            "token_tags": token_tags,
-            "token_class_ids": token_class_ids,
-            "token_tag_ids": token_tag_ids,
-        }
+            text_ = cell.text_
+            text.append(text_.text)
+            words.extend(text_.words)
+            ann_ids.extend(text_.ann_ids)
+            token_classes.extend(text_.token_classes)
+            token_class_ann_ids.extend(text_.token_class_ann_ids)
+            token_tags.extend(text_.token_tags)
+            token_tag_ann_ids.extend(text_.token_tag_ann_ids)
+            token_class_ids.extend(text_.token_class_ids)
+            token_tag_ids.extend(text_.token_tag_ids)
+        return Text_(
+            text=" ".join(text),
+            words=words,
+            ann_ids=ann_ids,
+            token_classes=token_classes,
+            token_class_ann_ids=token_class_ann_ids,
+            token_tags=token_tags,
+            token_tag_ann_ids=token_tag_ann_ids,
+            token_class_ids=token_class_ids,
+            token_tag_ids=token_tag_ids,
+        )
+
 
     @property
     def words(self) -> list[ImageAnnotationBaseView]:
@@ -1051,7 +1138,7 @@ class Page(Image):
 
             ```python
                 {"text": text string,
-                 "text_list": list of single words,
+                 "words": list of single words,
                  "annotation_ids": word annotation ids}
          ```
         """
@@ -1060,26 +1147,34 @@ class Page(Image):
         words: list[str] = []
         ann_ids: list[str] = []
         token_classes: list[str] = []
+        token_class_ann_ids: list[str] = []
         token_tags: list[str] = []
+        token_tag_ann_ids: list[str] = []
         token_class_ids: list[str] = []
         token_tag_ids: list[str] = []
         for block in block_with_order:
-            text.append(block.text_["text"])  # type: ignore
-            words.extend(block.text_["words"])  # type: ignore
-            ann_ids.extend(block.text_["ann_ids"])  # type: ignore
-            token_classes.extend(block.text_["token_classes"])  # type: ignore
-            token_tags.extend(block.text_["token_tags"])  # type: ignore
-            token_class_ids.extend(block.text_["token_class_ids"])  # type: ignore
-            token_tag_ids.extend(block.text_["token_tag_ids"])  # type: ignore
-        return {
-            "text": " ".join(text),
-            "words": words,
-            "ann_ids": ann_ids,
-            "token_classes": token_classes,
-            "token_tags": token_tags,
-            "token_class_ids": token_class_ids,
-            "token_tag_ids": token_tag_ids,
-        }
+            text_ = block.text_
+            text.append(text_.text)  # type: ignore
+            words.extend(text_.words)  # type: ignore
+            ann_ids.extend(text_.ann_ids)  # type: ignore
+            token_classes.extend(text_.token_classes)  # type: ignore
+            token_class_ann_ids.extend(text_.token_class_ann_ids)  # type: ignore
+            token_tags.extend(text_.token_tags)  # type: ignore
+            token_tag_ann_ids.extend(text_.token_tag_ann_ids)  # type: ignore
+            token_class_ids.extend(text_.token_class_ids)  # type: ignore
+            token_tag_ids.extend(text_.token_tag_ids)  # type: ignore
+        return Text_(
+            text=" ".join(text),
+            words=words,
+            ann_ids=ann_ids,
+            token_classes=token_classes,
+            token_class_ann_ids=token_class_ann_ids,
+            token_tags=token_tags,
+            token_tag_ann_ids=token_tag_ann_ids,
+            token_class_ids=token_class_ids,
+            token_tag_ids=token_tag_ann_ids,
+        )
+
 
     def get_layout_context(self, annotation_id: str, context_size: int = 3) -> list[ImageAnnotationBaseView]:
         """