deepdoctection 0.39__py3-none-any.whl → 0.39.1__py3-none-any.whl

deepdoctection/__init__.py

@@ -25,7 +25,7 @@ from .utils.logger import LoggingRecord, logger
 
 # pylint: enable=wrong-import-position
 
-__version__ = "0.39"
+__version__ = "0.39.1"
 
 _IMPORT_STRUCTURE = {
     "analyzer": ["config_sanity_checks", "get_dd_analyzer", "ServiceFactory"],

deepdoctection/pipe/base.py

@@ -24,7 +24,7 @@ from __future__ import annotations
 from abc import ABC, abstractmethod
 from collections import defaultdict
 from dataclasses import dataclass, field
-from typing import Any, Mapping, Optional, Union
+from typing import Any, Mapping, Optional, Union, Callable
 from uuid import uuid1
 
 from ..dataflow import DataFlow, MapData
@@ -33,6 +33,7 @@ from ..mapper.misc import curry
 from ..utils.context import timed_operation
 from ..utils.identifier import get_uuid_from_str
 from ..utils.settings import ObjectTypes
+from ..utils.types import DP
 from .anngen import DatapointManager
 
 
@@ -76,6 +77,30 @@ class PipelineComponent(ABC):
         self.service_id = self.get_service_id()
         self.dp_manager = DatapointManager(self.service_id, model_id)
         self.timer_on = False
+        self.filter_func: Callable[[DP], bool] = lambda dp: False
+
+    def set_inbound_filter(self, filter_func: Callable[[DP], bool]) -> None:
+        """
+        Set a filter function to decide, if an image of the inbound dataflow should be passed to self.serve.
+        The filter function should return a boolean value. If the function returns True, the image will not be processed
+        by this pipeline component.
+
+        **Example:**
+
+            ```python
+            def do_not_process_tables(dp: Image) -> bool:
+                 if "table" not in dp.get_categories_from_current_state():
+                    return True
+                 return False
+
+            layout_component = ImageLayoutService(...)
+            layout_component.set_inbound_filter(do_not_process_tables)
+            ```
+
+
+        :param filter_func: A function that takes an image datapoint and returns a boolean value
+        """
+        self.filter_func = filter_func # type: ignore
 
     @abstractmethod
     def serve(self, dp: Image) -> None:
@@ -92,6 +117,12 @@ class PipelineComponent(ABC):
         """
         raise NotImplementedError()
 
+    def _pass_datapoint(self, dp: Image) -> None:
+        self.dp_manager.datapoint = dp
+        if not self.filter_func(dp):
+            self.serve(dp)
+
+
     def pass_datapoint(self, dp: Image) -> Image:
         """
         Acceptance, handover to dp_manager, transformation and forwarding of dp. To measure the time, use
@@ -103,11 +134,9 @@ class PipelineComponent(ABC):
         """
         if self.timer_on:
             with timed_operation(self.__class__.__name__):
-                self.dp_manager.datapoint = dp
-                self.serve(dp)
+                self._pass_datapoint(dp)
         else:
-            self.dp_manager.datapoint = dp
-            self.serve(dp)
+            self._pass_datapoint(dp)
         return self.dp_manager.datapoint
 
     def predict_dataflow(self, df: DataFlow) -> DataFlow:
@@ -205,6 +234,7 @@ class Pipeline(ABC):
 
     **Example:**
 
+            ```python
             layout = LayoutPipeComponent(layout_detector ...)
             text = TextExtractPipeComponent(text_detector ...)
             simple_pipe = MyPipeline(pipeline_component = [layout, text])
@@ -212,6 +242,7 @@ class Pipeline(ABC):
 
             for page in doc_dataflow:
                 print(page)
+            ```
 
     In doing so, page contains all document structures determined via the pipeline (either directly from the Image core
     model or already processed further).
@@ -225,10 +256,12 @@ class Pipeline(ABC):
 
     **Example:**
 
+           ```python
            pipe = MyPipeline(pipeline_component = [layout, text])
            pipe.set_session_id = True
 
            df = pipe.analyze(input = "path/to/dir") # session_id is generated automatically
+           ```
     """
 
     def __init__(self, pipeline_component_list: list[PipelineComponent]) -> None:

deepdoctection/pipe/common.py

@@ -349,8 +349,8 @@ class AnnotationNmsService(PipelineComponent):
     def __init__(
         self,
         nms_pairs: Sequence[Sequence[TypeOrStr]],
-        thresholds: Union[float, list[float]],
-        priority: Optional[list[Union[Optional[TypeOrStr]]]] = None,
+        thresholds: Union[float, Sequence[float]],
+        priority: Optional[Sequence[Union[Optional[TypeOrStr]]]] = None,
     ):
         """
         :param nms_pairs: Groups of categories, either as string or by `ObjectType`.
@@ -362,7 +362,7 @@ class AnnotationNmsService(PipelineComponent):
             self.threshold = [thresholds for _ in self.nms_pairs]
         else:
             assert len(self.nms_pairs) == len(thresholds), "Sequences of nms_pairs and thresholds must have same length"
-            self.threshold = thresholds
+            self.threshold = thresholds # type: ignore
         if priority:
             assert len(self.nms_pairs) == len(priority), "Sequences of nms_pairs and priority must have same length"
 

deepdoctection/train/hf_detr_train.py

@@ -73,7 +73,7 @@ class DetrDerivedTrainer(Trainer):
         model: Union[PreTrainedModel, nn.Module],
         args: TrainingArguments,
         data_collator: DetrDataCollator,
-        train_dataset: Dataset[Any],
+        train_dataset: DatasetAdapter,
     ):
         self.evaluator: Optional[Evaluator] = None
         self.build_eval_kwargs: Optional[dict[str, Any]] = None

deepdoctection/utils/settings.py

@@ -101,7 +101,6 @@ class DocumentType(ObjectTypes):
     GOVERNMENT_TENDERS = "government_tenders"
     MANUALS = "manuals"
     PATENTS = "patents"
-    MARK = "mark"
 
 
 @object_types_registry.register("LayoutType")
@@ -132,6 +131,7 @@ class LayoutType(ObjectTypes):
     PAGE_NUMBER = "page_number"
     KEY_VALUE_AREA = "key_value_area"
     LIST_ITEM = "list_item"
+    MARK = "mark"
 
 
 @object_types_registry.register("TableType")

{deepdoctection-0.39.dist-info → deepdoctection-0.39.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: deepdoctection
-Version: 0.39
+Version: 0.39.1
 Summary: Repository for Document AI
 Home-page: https://github.com/deepdoctection/deepdoctection
 Author: Dr. Janis Meyer
@@ -61,7 +61,7 @@ Requires-Dist: python-doctr==0.8.1; extra == "tf"
 Requires-Dist: pycocotools>=2.0.2; extra == "tf"
 Requires-Dist: boto3==1.34.102; extra == "tf"
 Requires-Dist: pdfplumber>=0.11.0; extra == "tf"
-Requires-Dist: fasttext==0.9.2; extra == "tf"
+Requires-Dist: fasttext-wheel; extra == "tf"
 Requires-Dist: jdeskew>=0.2.2; extra == "tf"
 Requires-Dist: apted==1.0.3; extra == "tf"
 Requires-Dist: distance==0.1.3; extra == "tf"
@@ -86,12 +86,12 @@ Requires-Dist: termcolor>=1.1; extra == "pt"
 Requires-Dist: tabulate>=0.7.7; extra == "pt"
 Requires-Dist: tqdm==4.64.0; extra == "pt"
 Requires-Dist: timm>=0.9.16; extra == "pt"
-Requires-Dist: transformers>=4.36.0; extra == "pt"
+Requires-Dist: transformers>=4.48.0; extra == "pt"
 Requires-Dist: accelerate>=0.29.1; extra == "pt"
 Requires-Dist: python-doctr==0.8.1; extra == "pt"
 Requires-Dist: boto3==1.34.102; extra == "pt"
 Requires-Dist: pdfplumber>=0.11.0; extra == "pt"
-Requires-Dist: fasttext==0.9.2; extra == "pt"
+Requires-Dist: fasttext-wheel; extra == "pt"
 Requires-Dist: jdeskew>=0.2.2; extra == "pt"
 Requires-Dist: apted==1.0.3; extra == "pt"
 Requires-Dist: distance==0.1.3; extra == "pt"
@@ -99,7 +99,7 @@ Requires-Dist: lxml>=4.9.1; extra == "pt"
 Provides-Extra: docs
 Requires-Dist: tensorpack==0.11; extra == "docs"
 Requires-Dist: boto3==1.34.102; extra == "docs"
-Requires-Dist: transformers>=4.36.0; extra == "docs"
+Requires-Dist: transformers>=4.48.0; extra == "docs"
 Requires-Dist: accelerate>=0.29.1; extra == "docs"
 Requires-Dist: pdfplumber>=0.11.0; extra == "docs"
 Requires-Dist: lxml>=4.9.1; extra == "docs"

{deepdoctection-0.39.dist-info → deepdoctection-0.39.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-deepdoctection/__init__.py,sha256=0nxfBTu-aeg3DYu9g2kEAnt3Y-lCnHSgP31qvCnsLOs,12752
+deepdoctection/__init__.py,sha256=uDowNayqaYZGYaqnGzPSz6pVuHQhtDVRAN_bvPq85Ko,12754
 deepdoctection/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deepdoctection/analyzer/__init__.py,sha256=icClxrd20XutD6LxLgEPIWceSs4j_QfI3szCE-9BL2w,729
 deepdoctection/analyzer/_config.py,sha256=OZMOPlyFv4gcyabPG6KO08EYx-0tUH82Ehs9YDv2B1Q,5027
@@ -103,8 +103,8 @@ deepdoctection/mapper/tpstruct.py,sha256=YNABRibvcISD5Lavg3jouoE4FMdqXEJoM-hNoB_
 deepdoctection/mapper/xfundstruct.py,sha256=_3r3c0K82fnF2h1HxA85h-9ETYrHwcERa6MNc6Ko6Z8,8807
 deepdoctection/pipe/__init__.py,sha256=ywTVoetftdL6plXg2YlBzMfmqBZupq7yXblSVyvvkcQ,1127
 deepdoctection/pipe/anngen.py,sha256=3319l4aaXzcY4w6ItVBNPX8LGS5fHFDVtyVY9KMefac,16393
-deepdoctection/pipe/base.py,sha256=ynNg5SSRuUVxN69VWOO3Oi7WSeGrYwn3A56NQMBJDvw,14222
-deepdoctection/pipe/common.py,sha256=haOb4v0jLX3r41BSC8cVseX2E320_HkSrGlZsQiKE2g,17728
+deepdoctection/pipe/base.py,sha256=F4NusbZ-xYc6wuO-XAngmC8uzahT2ubsu2g9NO8PpVw,15390
+deepdoctection/pipe/common.py,sha256=vlWzvwn8wl7baPbK-917HUWujEGJEkHur_-ilkweKjk,17751
 deepdoctection/pipe/concurrency.py,sha256=AAKRsVgaBEYNluntbDa46SBF1JZ_XqnWLDSWrNvAzEo,9657
 deepdoctection/pipe/doctectionpipe.py,sha256=bGW3ugky-fb-nEe-3bvO6Oc_4_6w82cQboGM_6p2eIo,12530
 deepdoctection/pipe/language.py,sha256=5zI0UQC6Fh12_r2pfVL42HoCGz2hpHrOhpXAn5m-rYw,5451
@@ -119,7 +119,7 @@ deepdoctection/pipe/text.py,sha256=h9q6d3HFOs7LOg-iwdLUPiQxrPqgunBVNmtYMBrfRQE,1
 deepdoctection/pipe/transform.py,sha256=9Om7X7hJeL4jgUwHM1CHa4sb5v7Qo1PtVG0ls_3nI7w,3798
 deepdoctection/train/__init__.py,sha256=YFTRAZF1F7cEAKTdAIi1BLyYb6rSRcwq09Ui5Lu8d6E,1071
 deepdoctection/train/d2_frcnn_train.py,sha256=sFc_G-mEpaM8d1CCE0_6Gl4nBh11X2RYRBA3p_ylFJQ,16000
-deepdoctection/train/hf_detr_train.py,sha256=8ydysxzOPE_IPoNFGaHb7PbKr9Nbl41rcY4lbylQavU,10783
+deepdoctection/train/hf_detr_train.py,sha256=eHSdI11U8oGy93noZxAISfukhRBElj4dBerJ4Xcercw,10785
 deepdoctection/train/hf_layoutlm_train.py,sha256=irSg-IpbVoSlaw1-vZCej2mCZcctONtXr5Z2NQAc_a4,22680
 deepdoctection/train/tp_frcnn_train.py,sha256=pEpXokSVGveqo82pRnhnAmHPmjQ_8wQWpqM4ZyNHJgs,13049
 deepdoctection/utils/__init__.py,sha256=brBceRWeov9WXMiJTjyJOF2rHMP8trGGRRjhMdZ61nI,2371
@@ -135,14 +135,14 @@ deepdoctection/utils/logger.py,sha256=J0OVKiXP_2A82MWbbJoOeMEJ-75aZu5npgaS_yI6mV
 deepdoctection/utils/metacfg.py,sha256=hD76KQ_RnD_5B02qLI2Zxf3WfnsnXhEI_KUTKpw91RI,5711
 deepdoctection/utils/mocks.py,sha256=IkN3-IzAl4eX0ibgKIHg8IY7ykVw6BnpF6XnxKnKaZI,2389
 deepdoctection/utils/pdf_utils.py,sha256=Fi0eZ2GbnO7N61Rd8b8YRKRff4dalHAzkcn3zpGPoic,13119
-deepdoctection/utils/settings.py,sha256=k6OyuWbj-IPeaO9zT9RZ-5Yad1wNhWGYqGLZdtgXAZY,12464
+deepdoctection/utils/settings.py,sha256=hDD6yDX_4pQXwR5ILVwJIj6hb7NXA0-ifnC25ldcUjA,12464
 deepdoctection/utils/tqdm.py,sha256=cBUtR0L1x0KMeYrLP2rrzyzCamCjpQAKroHXLv81_pk,1820
 deepdoctection/utils/transform.py,sha256=3kCgsEeRkG1efCdkfvj7tUFMs-e2jbjbflq826F2GPU,8502
 deepdoctection/utils/types.py,sha256=_3dmPdCIZNLbgU5QP5k_c5phDf18xLe1kYL6t2nM45s,2953
 deepdoctection/utils/utils.py,sha256=csVs_VvCq4QBETPoE2JdTTL4MFYnD4xh-Js5vRb612g,6492
 deepdoctection/utils/viz.py,sha256=Jf8ePNYWlpuyaS6SeTYQ4OyA3eNhtgjvAQZnGNdgHC0,27051
-deepdoctection-0.39.dist-info/LICENSE,sha256=GQ0rUvuGdrMNEI3iHK5UQx6dIMU1QwAuyXsxUHn5MEQ,11351
-deepdoctection-0.39.dist-info/METADATA,sha256=0OcWmWb8bssiwC2_Xnb6hUKyY0ISv6Bc5qVGosbrn3c,19741
-deepdoctection-0.39.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-deepdoctection-0.39.dist-info/top_level.txt,sha256=hs2DdoOL9h4mnHhmO82BT4pz4QATIoOZ20PZmlnxFI8,15
-deepdoctection-0.39.dist-info/RECORD,,
+deepdoctection-0.39.1.dist-info/LICENSE,sha256=GQ0rUvuGdrMNEI3iHK5UQx6dIMU1QwAuyXsxUHn5MEQ,11351
+deepdoctection-0.39.1.dist-info/METADATA,sha256=NBN2dqFMUiXkcJ28xJDwyN6eNP-MmFw64F7dm3kUWTA,19741
+deepdoctection-0.39.1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+deepdoctection-0.39.1.dist-info/top_level.txt,sha256=hs2DdoOL9h4mnHhmO82BT4pz4QATIoOZ20PZmlnxFI8,15
+deepdoctection-0.39.1.dist-info/RECORD,,