deepdoctection 0.44.0__py3-none-any.whl → 0.45.0__py3-none-any.whl

deepdoctection/__init__.py

@@ -25,11 +25,10 @@ from .utils.logger import LoggingRecord, logger
 
 # pylint: enable=wrong-import-position
 
-__version__ = "0.44.0"
+__version__ = "0.45.0"
 
 _IMPORT_STRUCTURE = {
-    "analyzer": ["config_sanity_checks", "get_dd_analyzer", "ServiceFactory"],
-    "configs": ["update_cfg_from_defaults"],
+    "analyzer": ["config_sanity_checks", "get_dd_analyzer", "ServiceFactory", "update_cfg_from_defaults"],
     "dataflow": [
         "DataFlowTerminated",
         "DataFlowResetStateNotCalled",
@@ -186,7 +185,9 @@ _IMPORT_STRUCTURE = {
         "HFLayoutLmv3SequenceClassifier",
         "HFLiltTokenClassifier",
         "HFLiltSequenceClassifier",
+        "HFLmTokenClassifier",
         "HFLmSequenceClassifier",
+        "HFLmLanguageDetector",
         "ModelProfile",
         "ModelCatalog",
         "print_model_infos",
@@ -315,6 +316,8 @@ _IMPORT_STRUCTURE = {
         "get_apted_requirement",
         "distance_available",
         "get_distance_requirement",
+        "numpy_v1_available",
+        "get_numpy_v1_requirement",
         "transformers_available",
         "get_transformers_requirement",
         "detectron2_available",

deepdoctection/analyzer/config.py

@@ -520,6 +520,13 @@ cfg.USE_LAYOUT_LINK = False
 # (e.g., by grouping orphan text containers). Only applicable if list items were previously grouped.
 cfg.USE_LINE_MATCHER = False
 
+# Enables a sequence classification pipeline component, e.g. a LayoutLM or a Bert-like model.
+cfg.USE_LM_SEQUENCE_CLASS = False
+
+# Enables a token classification pipeline component, e.g. a LayoutLM or Bert-like model
+cfg.USE_LM_TOKEN_CLASS = False
+
+
 # Relevant when LIB = TF. Specifies the layout detection model.
 # This model should detect multiple or single objects across an entire page.
 # Currently, only one default model is supported.
@@ -899,6 +906,40 @@ cfg.LAYOUT_LINK.PARENTAL_CATEGORIES = [LayoutType.FIGURE, LayoutType.TABLE]
 # These are typically smaller or subordinate elements (e.g., captions).
 cfg.LAYOUT_LINK.CHILD_CATEGORIES = [LayoutType.CAPTION]
 
+
+# Weights configuration for sequence classifier. This will be a fine-tuned version of a LayoutLM, LayoutLMv2,
+# LayoutXLM, LayoutLMv3, LiLT or Roberta base model for sequence classification.
+cfg.LM_SEQUENCE_CLASS.WEIGHTS = None
+
+# When predicting document classes, it might be possible that some pages are empty or do not contain any text, in
+# which case the model will be unable to predict anything. If set to `True` it will
+# assign images with no features the category `TokenClasses.OTHER`.
+cfg.LM_SEQUENCE_CLASS.USE_OTHER_AS_DEFAULT_CATEGORY = False
+
+# Weights configuration for sequence classifier. This will be a fine-tuned version of a LayoutLM, LayoutLMv2,
+# LayoutXLM, LayoutLMv3, LiLT or Roberta base model for token classification.
+cfg.LM_TOKEN_CLASS.WEIGHTS = None
+
+# When predicting token classes, it might be possible that some words might not get sent to the model because they are
+# categorized as not eligible token (e.g. empty string). If set to `True` it will assign all words without token
+# as `TokenClasses.OTHER`.
+cfg.LM_TOKEN_CLASS.USE_OTHER_AS_DEFAULT_CATEGORY = False
+
+# Using bounding boxes of segments instead of words might improve model accuracy
+# for models that have been trained on segments rather than words (e.g. LiLT, LayoutLMv3).
+# Choose a single or a sequence of layout segments to use their bounding boxes. Note,
+# that the layout segments need to have a child-relationship with words. If a word
+# does not appear as child, it will use the word bounding box.
+cfg.LM_TOKEN_CLASS.SEGMENT_POSITIONS = None
+
+# If the output of the `tokenizer` exceeds the `max_length` sequence length, a
+# sliding window will be created with each window having `max_length` sequence
+# input. When using `SLIDING_WINDOW_STRIDE=0` no strides will be created,
+# otherwise it will create slides with windows shifted `SLIDING_WINDOW_STRIDE` to
+# the right.
+cfg.LM_TOKEN_CLASS.SLIDING_WINDOW_STRIDE = 0
+
+
 # Freezes the configuration to make it immutable.
 # This prevents accidental modification at runtime.
 cfg.freeze()

deepdoctection/analyzer/factory.py

@@ -19,9 +19,10 @@
 `ServiceFactory` for building analyzers
 """
 
+from __future__ import annotations
 
 from os import environ
-from typing import Union
+from typing import TYPE_CHECKING, Union
 
 from lazy_imports import try_import
 
@@ -29,6 +30,18 @@ from ..extern.base import ImageTransformer, ObjectDetector, PdfMiner
 from ..extern.d2detect import D2FrcnnDetector, D2FrcnnTracingDetector
 from ..extern.doctrocr import DoctrTextlineDetector, DoctrTextRecognizer
 from ..extern.hfdetr import HFDetrDerivedDetector
+from ..extern.hflayoutlm import (
+    HFLayoutLmSequenceClassifier,
+    HFLayoutLmTokenClassifier,
+    HFLayoutLmv2SequenceClassifier,
+    HFLayoutLmv2TokenClassifier,
+    HFLayoutLmv3SequenceClassifier,
+    HFLayoutLmv3TokenClassifier,
+    HFLiltSequenceClassifier,
+    HFLiltTokenClassifier,
+    get_tokenizer_from_model_class,
+)
+from ..extern.hflm import HFLmSequenceClassifier, HFLmTokenClassifier
 from ..extern.model import ModelCatalog, ModelDownloadManager
 from ..extern.pdftext import PdfPlumberTextDetector
 from ..extern.tessocr import TesseractOcrDetector, TesseractRotationTransformer
@@ -45,6 +58,7 @@ from ..pipe.common import (
 )
 from ..pipe.doctectionpipe import DoctectionPipe
 from ..pipe.layout import ImageLayoutService, skip_if_category_or_service_extracted
+from ..pipe.lm import LMSequenceClassifierService, LMTokenClassifierService
 from ..pipe.order import TextOrderService
 from ..pipe.refine import TableSegmentationRefinementService
 from ..pipe.segment import PubtablesSegmentationService, TableSegmentationService
@@ -60,6 +74,10 @@ from ..utils.transform import PadTransform
 with try_import() as image_guard:
     from botocore.config import Config  # type: ignore
 
+if TYPE_CHECKING:
+    from ..extern.hflayoutlm import LayoutSequenceModels, LayoutTokenModels
+    from ..extern.hflm import LmSequenceModels, LmTokenModels
+
 
 __all__ = [
     "ServiceFactory",
@@ -841,6 +859,226 @@ class ServiceFactory:
         """
         return ServiceFactory._build_text_order_service(config)
 
+    @staticmethod
+    def _build_sequence_classifier(config: AttrDict) -> Union[LayoutSequenceModels, LmSequenceModels]:
+        """
+        Builds and returns a sequence classifier instance.
+
+        Args:
+            config: Configuration object that determines the type of sequence classifier to construct.
+
+        Returns:
+            A sequence classifier instance constructed according to the specified configuration.
+        """
+        config_path = ModelCatalog.get_full_path_configs(config.LM_SEQUENCE_CLASS.WEIGHTS)
+        weights_path = ModelDownloadManager.maybe_download_weights_and_configs(config.LM_SEQUENCE_CLASS.WEIGHTS)
+        profile = ModelCatalog.get_profile(config.LM_SEQUENCE_CLASS.WEIGHTS)
+        categories = profile.categories if profile.categories is not None else {}
+        use_xlm_tokenizer = "xlm_tokenizer" == profile.architecture
+
+        if profile.model_wrapper in ("HFLayoutLmSequenceClassifier",):
+            return HFLayoutLmSequenceClassifier(
+                path_config_json=config_path,
+                path_weights=weights_path,
+                categories=categories,
+                device=config.DEVICE,
+                use_xlm_tokenizer=use_xlm_tokenizer,
+            )
+        if profile.model_wrapper in ("HFLayoutLmv2SequenceClassifier",):
+            return HFLayoutLmv2SequenceClassifier(
+                path_config_json=config_path,
+                path_weights=weights_path,
+                categories=categories,
+                device=config.DEVICE,
+                use_xlm_tokenizer=use_xlm_tokenizer,
+            )
+        if profile.model_wrapper in ("HFLayoutLmv3SequenceClassifier",):
+            return HFLayoutLmv3SequenceClassifier(
+                path_config_json=config_path,
+                path_weights=weights_path,
+                categories=categories,
+                device=config.DEVICE,
+                use_xlm_tokenizer=use_xlm_tokenizer,
+            )
+        if profile.model_wrapper in ("HFLiltSequenceClassifier",):
+            return HFLiltSequenceClassifier(
+                path_config_json=config_path,
+                path_weights=weights_path,
+                categories=categories,
+                device=config.DEVICE,
+                use_xlm_tokenizer=use_xlm_tokenizer,
+            )
+        if profile.model_wrapper in ("HFLmSequenceClassifier",):
+            return HFLmSequenceClassifier(
+                path_config_json=config_path,
+                path_weights=weights_path,
+                categories=categories,
+                device=config.DEVICE,
+                use_xlm_tokenizer=use_xlm_tokenizer,
+            )
+        raise ValueError(f"Unsupported model wrapper: {profile.model_wrapper}")
+
+    @staticmethod
+    def build_sequence_classifier(config: AttrDict) -> Union[LayoutSequenceModels, LmSequenceModels]:
+        """
+        Builds and returns a sequence classifier instance.
+
+        Args:
+            config: Configuration object that determines the type of sequence classifier to construct.
+
+        Returns:
+            A sequence classifier instance constructed according to the specified configuration.
+        """
+        return ServiceFactory._build_sequence_classifier(config)
+
+    @staticmethod
+    def _build_sequence_classifier_service(
+        config: AttrDict, sequence_classifier: Union[LayoutSequenceModels, LmSequenceModels]
+    ) -> LMSequenceClassifierService:
+        """
+        Building a sequence classifier service.
+
+        Args:
+            config: Configuration object.
+            sequence_classifier: Sequence classifier instance.
+
+        Returns:
+            LMSequenceClassifierService: Text order service instance.
+        """
+        tokenizer_fast = get_tokenizer_from_model_class(
+            sequence_classifier.model.__class__.__name__, sequence_classifier.use_xlm_tokenizer
+        )
+
+        return LMSequenceClassifierService(
+            tokenizer=tokenizer_fast,
+            language_model=sequence_classifier,
+            use_other_as_default_category=config.LM_SEQUENCE_CLASS.USE_OTHER_AS_DEFAULT_CATEGORY,
+        )
+
+    @staticmethod
+    def build_sequence_classifier_service(
+        config: AttrDict, sequence_classifier: Union[LayoutSequenceModels, LmSequenceModels]
+    ) -> LMSequenceClassifierService:
+        """
+        Building a sequence classifier service.
+
+        Args:
+            config: Configuration object.
+            sequence_classifier: Sequence classifier instance.
+
+        Returns:
+            LMSequenceClassifierService: Text order service instance.
+        """
+        return ServiceFactory._build_sequence_classifier_service(config, sequence_classifier)
+
+    @staticmethod
+    def _build_token_classifier(config: AttrDict) -> Union[LayoutTokenModels, LmTokenModels]:
+        """
+        Builds and returns a token classifier model.
+
+        Args:
+            config: Configuration object.
+
+        Returns:
+            The instantiated token classifier model.
+        """
+        config_path = ModelCatalog.get_full_path_configs(config.LM_TOKEN_CLASS.WEIGHTS)
+        weights_path = ModelDownloadManager.maybe_download_weights_and_configs(config.LM_TOKEN_CLASS.WEIGHTS)
+        profile = ModelCatalog.get_profile(config.LM_TOKEN_CLASS.WEIGHTS)
+        categories = profile.categories if profile.categories is not None else {}
+        use_xlm_tokenizer = "xlm_tokenizer" == profile.architecture
+        if profile.model_wrapper in ("HFLayoutLmTokenClassifier",):
+            return HFLayoutLmTokenClassifier(
+                path_config_json=config_path,
+                path_weights=weights_path,
+                categories=categories,
+                device=config.DEVICE,
+                use_xlm_tokenizer=use_xlm_tokenizer,
+            )
+        if profile.model_wrapper in ("HFLayoutLmv2TokenClassifier",):
+            return HFLayoutLmv2TokenClassifier(
+                path_config_json=config_path,
+                path_weights=weights_path,
+                categories=categories,
+                device=config.DEVICE,
+            )
+        if profile.model_wrapper in ("HFLayoutLmv3TokenClassifier",):
+            return HFLayoutLmv3TokenClassifier(
+                path_config_json=config_path,
+                path_weights=weights_path,
+                categories=categories,
+                device=config.DEVICE,
+            )
+        if profile.model_wrapper in ("HFLiltTokenClassifier",):
+            return HFLiltTokenClassifier(
+                path_config_json=config_path,
+                path_weights=weights_path,
+                categories=categories,
+                device=config.DEVICE,
+            )
+        if profile.model_wrapper in ("HFLmTokenClassifier",):
+            return HFLmTokenClassifier(
+                path_config_json=config_path,
+                path_weights=weights_path,
+                categories=categories,
+            )
+        raise ValueError(f"Unsupported model wrapper: {profile.model_wrapper}")
+
+    @staticmethod
+    def build_token_classifier(config: AttrDict) -> Union[LayoutTokenModels, LmTokenModels]:
+        """
+        Builds and returns a token classifier model.
+
+        Args:
+            config: Configuration object.
+
+        Returns:
+            The instantiated token classifier model.
+        """
+        return ServiceFactory._build_token_classifier(config)
+
+    @staticmethod
+    def _build_token_classifier_service(
+        config: AttrDict, token_classifier: Union[LayoutTokenModels, LmTokenModels]
+    ) -> LMTokenClassifierService:
+        """
+        Building a token classifier service.
+
+        Args:
+            config: Configuration object.
+            token_classifier: Token classifier instance.
+
+        Returns:
+             A LMTokenClassifierService instance.
+        """
+        tokenizer_fast = get_tokenizer_from_model_class(
+            token_classifier.model.__class__.__name__, token_classifier.use_xlm_tokenizer
+        )
+
+        return LMTokenClassifierService(
+            tokenizer=tokenizer_fast,
+            language_model=token_classifier,
+            use_other_as_default_category=config.LM_TOKEN_CLASS.USE_OTHER_AS_DEFAULT_CATEGORY,
+            segment_positions=config.LM_TOKEN_CLASS.SEGMENT_POSITIONS,
+            sliding_window_stride=config.LM_TOKEN_CLASS.SLIDING_WINDOW_STRIDE,
+        )
+
+    @staticmethod
+    def build_token_classifier_service(
+        config: AttrDict, token_classifier: Union[LayoutTokenModels, LmTokenModels]
+    ) -> LMTokenClassifierService:
+        """
+        Building a token classifier service.
+
+        Args:
+            config: Configuration object.
+            token_classifier: Token classifier instance.
+
+        Returns:
+             A LMTokenClassifierService instance.
+        """
+        return ServiceFactory._build_token_classifier_service(config, token_classifier)
+
     @staticmethod
     def _build_page_parsing_service(config: AttrDict) -> PageParsingService:
         """
@@ -955,6 +1193,16 @@ class ServiceFactory:
             line_list_matching_service = ServiceFactory.build_line_matching_service(config)
             pipe_component_list.append(line_list_matching_service)
 
+        if config.USE_LM_SEQUENCE_CLASS:
+            sequence_classifier = ServiceFactory.build_sequence_classifier(config)
+            sequence_classifier_service = ServiceFactory.build_sequence_classifier_service(config, sequence_classifier)
+            pipe_component_list.append(sequence_classifier_service)
+
+        if config.USE_LM_TOKEN_CLASS:
+            token_classifier = ServiceFactory.build_token_classifier(config)
+            token_classifier_service = ServiceFactory.build_token_classifier_service(config, token_classifier)
+            pipe_component_list.append(token_classifier_service)
+
         page_parsing_service = ServiceFactory.build_page_parsing_service(config)
 
         return DoctectionPipe(pipeline_component_list=pipe_component_list, page_parsing_service=page_parsing_service)

deepdoctection/configs/profiles.jsonl

@@ -30,4 +30,5 @@
 {"name": "Felix92/doctr-torch-parseq-multilingual-v1/pytorch_model.bin", "description": "", "size": [63286381], "tp_model": false, "config": "Felix92/doctr-torch-parseq-multilingual-v1/config.json", "preprocessor_config": null, "hf_repo_id": "Felix92/doctr-torch-parseq-multilingual-v1", "hf_model_name": "pytorch_model.bin", "hf_config_file": ["config.json"], "urls": null, "categories": {}, "categories_orig": null, "dl_library": "PT", "model_wrapper": "DoctrTextRecognizer", "architecture": "parseq", "padding": null}
 {"name": "doctr/crnn_vgg16_bn/pt/master-fde31e4a.pt", "description": "MASTER", "size": [63286381], "tp_model": false, "config": null, "preprocessor_config": null, "hf_repo_id": null, "hf_model_name": null, "hf_config_file": null, "urls": ["https://doctr-static.mindee.com/models?id=v0.7.0/master-fde31e4a.pt&src=0"], "categories": {}, "categories_orig": null, "dl_library": "PT", "model_wrapper": "DoctrTextRecognizer", "architecture": "master", "padding": null}
 {"name": "Aryn/deformable-detr-DocLayNet/model.safetensors", "description": "Deformable DEtection TRansformer (DETR), trained on DocLayNet (including 80k annotated pages in 11 classes).", "size": [115511753], "tp_model": false, "config": "Aryn/deformable-detr-DocLayNet/config.json", "preprocessor_config": "Aryn/deformable-detr-DocLayNet/preprocessor_config.json", "hf_repo_id": "Aryn/deformable-detr-DocLayNet", "hf_model_name": "model.safetensors", "hf_config_file": ["config.json", "preprocessor_config.json"], "urls": null, "categories": {"1": "default_type", "2": "caption", "11": "text", "12": "title", "3": "footnote", "4": "formula", "5": "list_item", "6": "page_footer", "7": "page_header", "8": "figure", "9": "section_header", "10": "table"}, "categories_orig": null, "dl_library": "PT", "model_wrapper": "HFDetrDerivedDetector", "architecture": null, "padding": null}
-{"name": "deepdoctection/tatr_tab_struct_v2/model.safetensors", "description": "Table Transformer (DETR) model trained on PubTables1M. It was introduced in the paper Aligning benchmark datasets for table structure recognition by Smock et al. This model is devoted to table structure recognition and assumes to receive a slightly croppedtable as input. It will predict rows, column and spanning cells. Use a padding of around 5 pixels. This artefact has been converted from deepdoctection/tatr_tab_struct_v2/pytorch_model.bin and should be used to reduce security issues", "size": [115511753], "tp_model": false, "config": "deepdoctection/tatr_tab_struct_v2/config.json", "preprocessor_config": "deepdoctection/tatr_tab_struct_v2/preprocessor_config.json", "hf_repo_id": "deepdoctection/tatr_tab_struct_v2", "hf_model_name": "model.safetensors", "hf_config_file": ["config.json", "preprocessor_config.json"], "urls": null, "categories": {"1": "table", "2": "column", "3": "row", "4": "column_header", "5": "projected_row_header", "6": "spanning"}, "categories_orig": null, "dl_library": "PT", "model_wrapper": "HFDetrDerivedDetector", "architecture": null, "padding": null}
+{"name": "deepdoctection/tatr_tab_struct_v2/model.safetensors", "description": "Table Transformer (DETR) model trained on PubTables1M. It was introduced in the paper Aligning benchmark datasets for table structure recognition by Smock et al. This model is devoted to table structure recognition and assumes to receive a slightly croppedtable as input. It will predict rows, column and spanning cells. Use a padding of around 5 pixels. This artefact has been converted from deepdoctection/tatr_tab_struct_v2/pytorch_model.bin and should be used to reduce security issues", "size": [115511753], "tp_model": false, "config": "deepdoctection/tatr_tab_struct_v2/config.json", "preprocessor_config": "deepdoctection/tatr_tab_struct_v2/preprocessor_config.json", "hf_repo_id": "deepdoctection/tatr_tab_struct_v2", "hf_model_name": "model.safetensors", "hf_config_file": ["config.json", "preprocessor_config.json"], "urls": null, "categories": {"1": "table", "2": "column", "3": "row", "4": "column_header", "5": "projected_row_header", "6": "spanning"}, "categories_orig": null, "dl_library": "PT", "model_wrapper": "HFDetrDerivedDetector", "architecture": null, "padding": null}
+{"name": "papluca/xlm-roberta-base-language-detection/model.safetensors", "description": "This model is an XLM-RoBERTa transformer model with a classification head on top (i.e. a linear layer on top of the pooled output). For additional information please refer to the xlm-roberta-base model card or to the paper Unsupervised Cross-lingual Representation Learning at Scale by Conneau et al.", "size": [101971449], "tp_model": false, "config": "papluca/xlm-roberta-base-language-detection/config.json", "preprocessor_config": null, "hf_repo_id": "papluca/xlm-roberta-base-language-detection", "hf_model_name": "model.safetensors", "hf_config_file": ["config.json"], "urls": null, "categories": {"1": "jpn", "2": "dut", "3": "ara", "4": "pol", "5": "deu", "6": "ita", "7": "por", "8": "tur", "9": "spa", "10": "hin", "11": "gre", "12": "urd", "13": "bul", "14": "eng", "15": "fre", "16": "chi", "17": "rus", "18": "tha", "19": "swa", "20": "vie"}, "categories_orig": null, "dl_library": "PT", "model_wrapper": "HFLmLanguageDetector", "architecture": null, "padding": null}

deepdoctection/datapoint/image.py

@@ -42,6 +42,7 @@ from .convert import as_dict, convert_b64_to_np_array, convert_np_array_to_b64,
 
 class MetaAnnotationDict(TypedDict):
     """MetaAnnotationDict"""
+
     image_annotations: list[str]
     sub_categories: dict[str, dict[str, list[str]]]
     relationships: dict[str, list[str]]