deepdoctection 0.30__py3-none-any.whl → 0.32__py3-none-any.whl

deepdoctection/mapper/laylmstruct.py

@@ -20,32 +20,30 @@ Module for mapping annotations from image to layout lm input structure. Heavily
 <https://github.com/NielsRogge/Transformers-Tutorials>
 """
 
+from __future__ import annotations
+
 import random
 from dataclasses import dataclass, field
 from typing import Any, Callable, Dict, List, Literal, NewType, Optional, Sequence, Union
 
 import numpy as np
 import numpy.typing as npt
+from lazy_imports import try_import
 
 from ..datapoint.annotation import ContainerAnnotation
 from ..datapoint.convert import box_to_point4, point4_to_box
 from ..datapoint.image import Image
+from ..datapoint.view import Page
 from ..utils.detection_types import JsonDict
-from ..utils.file_utils import pytorch_available, transformers_available
 from ..utils.settings import DatasetType, LayoutType, PageType, Relationships, WordType
 from ..utils.transform import ResizeTransform, normalize_image
 from .maputils import curry
 
-if pytorch_available():
+with try_import() as import_guard:
     import torch
 
-if transformers_available():
-    from transformers import (  # pylint: disable=W0611
-        BatchEncoding,
-        PreTrainedTokenizerFast,
-        RobertaTokenizerFast,
-        XLMRobertaTokenizerFast,
-    )
+with try_import() as tr_import_guard:
+    from transformers import BatchEncoding, PreTrainedTokenizerFast  # pylint: disable=W0611
 
 __all__ = [
     "image_to_raw_layoutlm_features",
@@ -54,12 +52,17 @@ __all__ = [
     "image_to_layoutlm_features",
     "DataCollator",
     "LayoutLMFeatures",
+    "image_to_raw_lm_features",
+    "image_to_lm_features",
 ]
 
 RawLayoutLMFeatures = NewType("RawLayoutLMFeatures", JsonDict)
+RawLMFeatures = NewType("RawLMFeatures", JsonDict)
 LayoutLMFeatures = NewType("LayoutLMFeatures", JsonDict)
+LMFeatures = NewType("LMFeatures", JsonDict)
 InputDataClass = NewType("InputDataClass", JsonDict)
 
+
 """
 <https://github.com/huggingface/transformers/src/transformers/data/data_collator.py>
 A DataCollator is a function that takes a list of samples from a Dataset and collate them into a batch, as a dictionary
@@ -146,7 +149,7 @@ def image_to_raw_layoutlm_features(
             raise TypeError(f"char_cat must be of type ContainerAnnotation but is of type {type(char_cat)}")
         word = char_cat.value
         if not isinstance(word, str):
-            raise ValueError(f"word must be of type str but is of type {type(word)}")
+            raise TypeError(f"word must be of type str but is of type {type(word)}")
         all_words.append(word)
 
         box = ann.get_bounding_box(dp.image_id)
@@ -208,7 +211,7 @@ def image_to_raw_layoutlm_features(
     return raw_features
 
 
-def features_to_pt_tensors(features: LayoutLMFeatures) -> LayoutLMFeatures:
+def layoutlm_features_to_pt_tensors(features: LayoutLMFeatures) -> LayoutLMFeatures:
     """
     Converting list of floats to pytorch tensors
     :param features: LayoutLMFeatures
@@ -216,7 +219,8 @@ def features_to_pt_tensors(features: LayoutLMFeatures) -> LayoutLMFeatures:
     """
 
     _image_key = "pixel_values" if "pixel_values" in features else "image"
-    features["bbox"] = torch.tensor(features["bbox"], dtype=torch.long)
+    if "bbox" in features:
+        features["bbox"] = torch.tensor(features["bbox"], dtype=torch.long)
     if "labels" in features:
         features["labels"] = torch.tensor(features["labels"], dtype=torch.long)
     if _image_key in features:
@@ -230,12 +234,12 @@ def features_to_pt_tensors(features: LayoutLMFeatures) -> LayoutLMFeatures:
 
 
 def _tokenize_with_sliding_window(
-    raw_features: List[RawLayoutLMFeatures],
-    tokenizer: "PreTrainedTokenizerFast",
+    raw_features: List[Union[RawLayoutLMFeatures, RawLMFeatures]],
+    tokenizer: PreTrainedTokenizerFast,
     sliding_window_stride: int,
     max_batch_size: int,
     return_tensors: Optional[Literal["pt"]] = None,
-) -> Union[JsonDict, "BatchEncoding"]:
+) -> Union[JsonDict, BatchEncoding]:
     """
     Runs a tokenizer: If there are no overflowing tokens, the tokenizer output will be returned as it is.
     If there are overflowing tokens, sliding windows have to be built. As it is easier to prepare the sliding windows
@@ -398,8 +402,8 @@ def _tokenize_with_sliding_window(
 
 
 def raw_features_to_layoutlm_features(
-    raw_features: Union[RawLayoutLMFeatures, List[RawLayoutLMFeatures]],
-    tokenizer: "PreTrainedTokenizerFast",
+    raw_features: Union[RawLayoutLMFeatures, RawLMFeatures, List[Union[RawLayoutLMFeatures, RawLMFeatures]]],
+    tokenizer: PreTrainedTokenizerFast,
     padding: Literal["max_length", "do_not_pad", "longest"] = "max_length",
     truncation: bool = True,
     return_overflowing_tokens: bool = False,
@@ -407,6 +411,7 @@ def raw_features_to_layoutlm_features(
     remove_columns_for_training: bool = False,
     sliding_window_stride: int = 0,
     max_batch_size: int = 0,
+    remove_bounding_boxes: bool = False,
 ) -> LayoutLMFeatures:
     """
     Mapping raw features to tokenized input sequences for LayoutLM models.
@@ -563,8 +568,11 @@ def raw_features_to_layoutlm_features(
         input_dict.pop("ann_ids")
         input_dict.pop("tokens")
 
+    if remove_bounding_boxes:
+        input_dict.pop("bbox")
+
     if return_tensors == "pt":
-        return features_to_pt_tensors(LayoutLMFeatures(input_dict))
+        return layoutlm_features_to_pt_tensors(LayoutLMFeatures(input_dict))
     return LayoutLMFeatures(input_dict)
 
 
@@ -595,13 +603,14 @@ class LayoutLMDataCollator:
                            with windows shifted `sliding_window_stride` to the right.
     """
 
-    tokenizer: "PreTrainedTokenizerFast"
+    tokenizer: PreTrainedTokenizerFast
     padding: Literal["max_length", "do_not_pad", "longest"] = field(default="max_length")
     truncation: bool = field(default=True)
     return_overflowing_tokens: bool = field(default=False)
     return_tensors: Optional[Literal["pt"]] = field(default=None)
     sliding_window_stride: int = field(default=0)
     max_batch_size: int = field(default=0)
+    remove_bounding_box_features: bool = field(default=False)
 
     def __post_init__(self) -> None:
         assert isinstance(self.tokenizer, PreTrainedTokenizerFast), "Tokenizer must be a fast tokenizer"
@@ -620,7 +629,7 @@ class LayoutLMDataCollator:
                  token_type_ids, attention_masks, boxes, labels`.
         """
         return raw_features_to_layoutlm_features(
-            raw_features,
+            raw_features,  # type: ignore
             self.tokenizer,
             self.padding,
             self.truncation,
@@ -629,13 +638,14 @@ class LayoutLMDataCollator:
             True,
             self.sliding_window_stride,
             self.max_batch_size,
+            self.remove_bounding_box_features,
         )
 
 
 @curry
 def image_to_layoutlm_features(
     dp: Image,
-    tokenizer: "PreTrainedTokenizerFast",
+    tokenizer: PreTrainedTokenizerFast,
     padding: Literal["max_length", "do_not_pad", "longest"] = "max_length",
     truncation: bool = True,
     return_overflowing_tokens: bool = False,
@@ -724,3 +734,136 @@ def image_to_layoutlm_features(
         sliding_window_stride=sliding_window_stride,
     )
     return features
+
+
+@curry
+def image_to_raw_lm_features(
+    dp: Image,
+    dataset_type: Optional[Literal["sequence_classification", "token_classification"]] = None,
+    use_token_tag: bool = True,
+    text_container: Optional[LayoutType] = LayoutType.word,
+    floating_text_block_categories: Optional[Sequence[LayoutType]] = None,
+    include_residual_text_container: bool = False,
+) -> Optional[RawLMFeatures]:
+    """
+    Mapping a datapoint into an intermediate format for bert-like models. Features will be provided into a dict and
+    this mapping can be used for sequence or token classification as well as for inference. To generate input features
+    for the model please `use raw_features_to_layoutlm_features`.
+
+
+    :param dp: Image
+    :param dataset_type: Either SEQUENCE_CLASSIFICATION or TOKEN_CLASSIFICATION. When using a built-in dataset use
+    :param use_token_tag: Will only be used for dataset_type="token_classification". If use_token_tag=True, will use
+                          labels from sub category `WordType.token_tag` (with `B,I,O` suffix), otherwise
+                          `WordType.token_class`.
+    :param text_container: A LayoutType to get the text from. It will steer the output of `Layout.words`.
+    :param floating_text_block_categories: A list of top level layout objects
+    :param include_residual_text_container: This will regard synthetic text line annotations as floating text
+                                            blocks and therefore incorporate all image annotations of category
+                                            `word` when building text strings.
+    :return: dictionary with the following arguments:
+            'image_id', 'width', 'height', 'ann_ids', 'words', 'bbox' and 'dataset_type'.
+    """
+
+    raw_features: RawLMFeatures = RawLMFeatures({})
+
+    page = Page.from_image(dp, text_container, floating_text_block_categories, include_residual_text_container)
+
+    text_ = page.text_
+
+    # pylint: disable=E1137  #3162
+    raw_features["image_id"] = page.image_id
+    raw_features["width"] = page.width
+    raw_features["height"] = page.height
+    raw_features["ann_ids"] = text_["ann_ids"]
+    raw_features["words"] = text_["words"]
+    # We use a dummy bounding box for all bounding boxes so that we can pass the raw features to
+    # raw_features_to_layoutlm_features
+    raw_features["bbox"] = [_CLS_BOX] * len(text_["words"])
+    raw_features["dataset_type"] = dataset_type
+
+    if use_token_tag and text_["token_tags"]:
+        raw_features["labels"] = text_["token_tags"]
+    elif text_["token_classes"]:
+        raw_features["labels"] = text_["token_classes"]
+    elif page.document_type is not None:
+        document_type_id = (
+            int(page.image_orig.summary.get_sub_category(PageType.document_type).category_id) - 1  # type: ignore
+        )
+        raw_features["labels"] = [document_type_id]
+
+    raw_features["dataset_type"] = dataset_type
+    # pylint: enable=E1137
+    return raw_features
+
+
+@curry
+def image_to_lm_features(
+    dp: Image,
+    tokenizer: PreTrainedTokenizerFast,
+    padding: Literal["max_length", "do_not_pad", "longest"] = "max_length",
+    truncation: bool = True,
+    return_overflowing_tokens: bool = False,
+    return_tensors: Optional[Literal["pt"]] = "pt",
+    sliding_window_stride: int = 0,
+    text_container: Optional[LayoutType] = LayoutType.word,
+    floating_text_block_categories: Optional[Sequence[LayoutType]] = None,
+    include_residual_text_container: bool = False,
+) -> Optional[LayoutLMFeatures]:
+    """
+    Mapping function to generate layoutlm features from `Image` to be used for inference in a pipeline component.
+    `LanguageModelPipelineComponent` has a positional argument `mapping_to_lm_input_func` that must be chosen
+    with respect to the language model chosen. This mapper is devoted to generating features for LayoutLM. It will be
+    used internally in `LMTokenClassifierService`.
+
+            tokenizer = LayoutLMTokenizer.from_pretrained("mrm8488/layoutlm-finetuned-funsd")
+            layoutlm = HFLayoutLmTokenClassifier("path/to/config.json","path/to/model.bin",
+                                                  categories_explicit=['B-ANSWER', 'B-QUESTION', 'O'])
+
+            layoutlm_service = LMTokenClassifierService(tokenizer,layoutlm)
+
+    :param dp: Image datapoint
+    :param tokenizer: Tokenizer compatible with the language model
+    :param padding: A padding strategy to be passed to the tokenizer. Must bei either `max_length, longest` or
+                    `do_not_pad`.
+    :param truncation: If "True" will truncate to a maximum length specified with the argument max_length or to the
+                       maximum acceptable input length for the model if that argument is not provided. This will
+                       truncate token by token, removing a token from the longest sequence in the pair if a pair of
+                       sequences (or a batch of pairs) is provided.
+                       If `False` then no truncation (i.e., can output batch with sequence lengths greater than the
+                       model maximum admissible input size).
+    :param return_overflowing_tokens: If a sequence (due to a truncation strategy) overflows the overflowing tokens
+                                      can be returned as an additional batch element. Not that in this case, the number
+                                      of input batch samples will be smaller than the output batch samples.
+    :param return_tensors: Output tensor features. Either 'pt' for PyTorch models or None, if features should be
+                           returned in list objects.
+    :param sliding_window_stride: If the output of the tokenizer exceeds the max_length sequence length a sliding
+                                  windows will be created with each window having max_length sequence input. When using
+                                  `sliding_window_stride=0` no strides will be created, otherwise it will create slides
+                                  with windows shifted `sliding_window_stride` to the right.
+    :param text_container: A LayoutType to get the text from. It will steer the output of `Layout.words`.
+    :param floating_text_block_categories: A list of top level layout objects
+    :param include_residual_text_container: This will regard synthetic text line annotations as floating text
+                                            blocks and therefore incorporate all image annotations of category
+                                            `word` when building text strings.
+    :return: A dict of lm features
+    """
+    raw_features = image_to_raw_lm_features(  # pylint: disable=E1102
+        dataset_type=None,
+        use_token_tag=True,
+        text_container=text_container,
+        floating_text_block_categories=floating_text_block_categories,
+        include_residual_text_container=include_residual_text_container,
+    )(dp)
+    if raw_features is None:
+        return None
+    features = raw_features_to_layoutlm_features(
+        raw_features,
+        tokenizer,
+        padding,
+        truncation,
+        return_overflowing_tokens,
+        return_tensors=return_tensors,
+        sliding_window_stride=sliding_window_stride,
+    )
+    return features

deepdoctection/mapper/maputils.py

@@ -18,6 +18,8 @@
 """
 Utility functions related to mapping tasks
 """
+from __future__ import annotations
+
 import functools
 import itertools
 import traceback
@@ -28,8 +30,8 @@ import numpy as np
 from tabulate import tabulate
 from termcolor import colored
 
-from ..datapoint.box import BoundingBoxError
 from ..utils.detection_types import DP, BaseExceptionType, S, T
+from ..utils.error import AnnotationError, BoundingBoxError, ImageError, UUIDError
 from ..utils.logger import LoggingRecord, logger
 from ..utils.settings import ObjectTypes
 
@@ -55,7 +57,7 @@ class MappingContextManager:
         self.context_error = True
         self.kwargs = kwargs
 
-    def __enter__(self) -> "MappingContextManager":
+    def __enter__(self) -> MappingContextManager:
         """
         context enter
         """
@@ -72,7 +74,18 @@ class MappingContextManager:
         """
         if (
             exc_type
-            in (KeyError, ValueError, IndexError, AssertionError, TypeError, BoundingBoxError, FileNotFoundError)
+            in (
+                KeyError,
+                ValueError,
+                IndexError,
+                AssertionError,
+                TypeError,
+                FileNotFoundError,
+                BoundingBoxError,
+                AnnotationError,
+                ImageError,
+                UUIDError,
+            )
             and exc_tb is not None
         ):
             frame_summary = traceback.extract_tb(exc_tb)[0]

deepdoctection/mapper/misc.py

@@ -19,19 +19,22 @@
 Module for small mapping functions
 """
 
+from __future__ import annotations
+
 import ast
 import os
 from typing import List, Mapping, Optional, Sequence, Union
 
+from lazy_imports import try_import
+
 from ..datapoint.convert import convert_pdf_bytes_to_np_array_v2
 from ..datapoint.image import Image
 from ..utils.detection_types import JsonDict
-from ..utils.file_utils import lxml_available
 from ..utils.fs import get_load_image_func, load_image_from_file
 from ..utils.utils import is_file_extension
 from .maputils import MappingContextManager, curry
 
-if lxml_available():
+with try_import() as import_guard:
     from lxml import etree  # pylint: disable=W0611
 
 
@@ -175,7 +178,7 @@ def maybe_ann_to_sub_image(
 
 
 @curry
-def xml_to_dict(dp: JsonDict, xslt_obj: "etree.XSLT") -> JsonDict:
+def xml_to_dict(dp: JsonDict, xslt_obj: etree.XSLT) -> JsonDict:
     """
     Convert a xml object into a dict using a xsl style sheet.
 

deepdoctection/mapper/prodigystruct.py

@@ -128,7 +128,7 @@ def prodigy_to_image(
             else:
                 label = span["label"]
             if not isinstance(label, str):
-                raise ValueError("label could not assigned to be a string")
+                raise TypeError("label must be a string")
 
             annotation = ImageAnnotation(
                 category_name=label,

deepdoctection/mapper/pubstruct.py

@@ -75,12 +75,14 @@ def _cell_token(html: Sequence[str]) -> List[List[int]]:
 def _item_spans(html: Sequence[str], index_cells: Sequence[Sequence[int]], item: str) -> List[List[int]]:
     item_spans = [
         [
-            int(html[index_cell - 1].replace(item + "=", "").replace('"', ""))
-            if (item in html[index_cell - 1] and html[index_cell] == ">")
-            else (
-                int(html[index_cell - 2].replace(item + "=", "").replace('"', ""))
-                if (item in html[index_cell - 2] and html[index_cell] == ">")
-                else 1
+            (
+                int(html[index_cell - 1].replace(item + "=", "").replace('"', ""))
+                if (item in html[index_cell - 1] and html[index_cell] == ">")
+                else (
+                    int(html[index_cell - 2].replace(item + "=", "").replace('"', ""))
+                    if (item in html[index_cell - 2] and html[index_cell] == ">")
+                    else 1
+                )
             )
             for index_cell in index_cell_per_row
         ]
@@ -210,9 +212,7 @@ def _add_items(image: Image, item_type: str, categories_name_as_key: Dict[str, s
         items = image.get_annotation(category_names=TableType.item)
         item_type_anns = [ann for ann in items if ann.get_sub_category(TableType.item).category_name == item_type]
         item_type_anns.sort(
-            key=lambda x: x.bounding_box.cx  # type: ignore
-            if item_type == LayoutType.column
-            else x.bounding_box.cy  # type: ignore
+            key=lambda x: (x.bounding_box.cx if item_type == LayoutType.column else x.bounding_box.cy)  # type: ignore
         )
         if table.bounding_box:
             tmp_item_xy = table.bounding_box.uly + 1.0 if item_type == LayoutType.row else table.bounding_box.ulx + 1.0
@@ -389,7 +389,7 @@ def pub_to_image_uncur(  # pylint: disable=R0914
     with MappingContextManager(str(idx)) as mapping_context:
         max_rs, max_cs = 0, 0
         if idx is None:
-            raise ValueError("No valid datapoint external id")
+            raise TypeError("imgid is None but must be a string")
 
         image = Image(file_name=os.path.split(dp["filename"])[1], location=dp["filename"], external_id=idx)
 

deepdoctection/mapper/tpstruct.py

@@ -22,15 +22,15 @@ import os.path
 from typing import Optional, Sequence, Union
 
 import numpy as np
+from lazy_imports import try_import
 
 from ..datapoint.annotation import ImageAnnotation
 from ..datapoint.image import Image
 from ..utils.detection_types import JsonDict
-from ..utils.file_utils import tf_available
 from ..utils.settings import ObjectTypes
 from .maputils import curry
 
-if tf_available():
+with try_import() as import_guard:
     from tensorflow import convert_to_tensor, uint8  # type: ignore # pylint: disable=E0401
     from tensorflow.image import non_max_suppression  # type: ignore # pylint: disable=E0401
 
@@ -67,7 +67,7 @@ def image_to_tp_frcnn_training(
         all_categories.append(ann.category_id)
 
         if add_mask:
-            raise NotImplementedError
+            raise NotImplementedError()
 
     output["gt_boxes"] = np.asarray(all_boxes, dtype="float32")
     output["gt_labels"] = np.asarray(all_categories, dtype="int32")

deepdoctection/pipe/__init__.py

@@ -22,7 +22,6 @@ Contains pipeline components that can be plugged into each other and predictors
 
 from .anngen import *
 from .base import *
-from .cell import *
 from .common import *
 from .concurrency import *
 from .doctectionpipe import *
@@ -33,5 +32,6 @@ from .order import *
 from .refine import *
 from .registry import *
 from .segment import *
+from .sub_layout import *
 from .text import *
 from .transform import *

deepdoctection/pipe/anngen.py

@@ -42,11 +42,14 @@ class DatapointManager:
     The manager is part of each `PipelineComponent`.
     """
 
-    def __init__(self) -> None:
+    def __init__(self, service_id: str, model_id: Optional[str] = None) -> None:
         self._datapoint: Optional[Image] = None
         self._cache_anns: Dict[str, ImageAnnotation] = {}
         self.datapoint_is_passed: bool = False
         self.category_id_mapping: Optional[Mapping[int, int]] = None
+        self.service_id = service_id
+        self.model_id = model_id
+        self.session_id: Optional[str] = None
 
     @property
     def datapoint(self) -> Image:
@@ -55,7 +58,7 @@ class DatapointManager:
         """
         if self._datapoint is not None:
             return self._datapoint
-        raise ValueError("no datapoint passed")
+        raise ValueError("No datapoint passed")
 
     @datapoint.setter
     def datapoint(self, dp: Image) -> None:
@@ -154,6 +157,9 @@ class DatapointManager:
                 bounding_box=box,
                 category_id=str(detect_result.class_id),
                 score=detect_result.score,
+                service_id=self.service_id,
+                model_id=self.model_id,
+                session_id=self.session_id,
             )
             if to_annotation_id is not None:
                 parent_ann = self._cache_anns[to_annotation_id]
@@ -208,7 +214,14 @@ class DatapointManager:
                 "annotation_id": annotation_id,
             },
         ) as annotation_context:
-            cat_ann = CategoryAnnotation(category_name=category_name, category_id=str(category_id), score=score)
+            cat_ann = CategoryAnnotation(
+                category_name=category_name,
+                category_id=str(category_id),
+                score=score,
+                service_id=self.service_id,
+                model_id=self.model_id,
+                session_id=self.session_id,
+            )
             self._cache_anns[annotation_id].dump_sub_category(sub_cat_key, cat_ann)
         if annotation_context.context_error:
             return None
@@ -246,7 +259,13 @@ class DatapointManager:
             },
         ) as annotation_context:
             cont_ann = ContainerAnnotation(
-                category_name=category_name, category_id=str(category_id), value=value, score=score
+                category_name=category_name,
+                category_id=str(category_id),
+                value=value,
+                score=score,
+                service_id=self.service_id,
+                model_id=self.model_id,
+                session_id=self.session_id,
             )
             self._cache_anns[annotation_id].dump_sub_category(sub_cat_key, cont_ann)
         if annotation_context.context_error:
@@ -257,7 +276,7 @@ class DatapointManager:
         self,
         summary_key: ObjectTypes,
         summary_name: ObjectTypes,
-        summary_number: int,
+        summary_number: Optional[int] = None,
         summary_value: Optional[str] = None,
         summary_score: Optional[float] = None,
         annotation_id: Optional[str] = None,
@@ -294,16 +313,24 @@ class DatapointManager:
                 "annotation_id": annotation_id,
             },
         ) as annotation_context:
-            if summary_value:
+            if summary_value is not None:
                 ann = ContainerAnnotation(
                     category_name=summary_name,
-                    category_id=str(summary_number),
+                    category_id=str(summary_number) if summary_number is not None else "",
                     value=summary_value,
                     score=summary_score,
+                    service_id=self.service_id,
+                    model_id=self.model_id,
+                    session_id=self.session_id,
                 )
             else:
                 ann = CategoryAnnotation(
-                    category_name=summary_name, category_id=str(summary_number), score=summary_score
+                    category_name=summary_name,
+                    category_id=str(summary_number) if summary_number is not None else "",
+                    score=summary_score,
+                    service_id=self.service_id,
+                    model_id=self.model_id,
+                    session_id=self.session_id,
                 )
             image.summary.dump_sub_category(summary_key, ann, image.image_id)