deepdoctection 0.37__py3-none-any.whl → 0.37.2__py3-none-any.whl

deepdoctection/__init__.py

@@ -24,7 +24,7 @@ from .utils.logger import LoggingRecord, logger
 
 # pylint: enable=wrong-import-position
 
-__version__ = 0.37
+__version__ = "0.37.2"
 
 _IMPORT_STRUCTURE = {
     "analyzer": ["config_sanity_checks", "get_dd_analyzer", "ServiceFactory"],

deepdoctection/analyzer/factory.py

@@ -327,9 +327,9 @@ class ServiceFactory:
             )
         if config.OCR.USE_TEXTRACT:
             credentials_kwargs = {
-                "aws_access_key_id": environ.get("ACCESS_KEY", None),
-                "aws_secret_access_key": environ.get("SECRET_KEY", None),
-                "config": Config(region_name=environ.get("REGION", None)),
+                "aws_access_key_id": environ.get("AWS_ACCESS_KEY", None),
+                "aws_secret_access_key": environ.get("AWS_SECRET_KEY", None),
+                "config": Config(region_name=environ.get("AWS_REGION", None)),
             }
             return TextractOcrDetector(**credentials_kwargs)
         raise ValueError("You have set USE_OCR=True but any of USE_TESSERACT, USE_DOCTR, USE_TEXTRACT is set to False")

deepdoctection/datapoint/image.py

@@ -32,6 +32,7 @@ from numpy import uint8
 
 from ..utils.error import AnnotationError, BoundingBoxError, ImageError, UUIDError
 from ..utils.identifier import get_uuid, is_uuid_like
+from ..utils.logger import LoggingRecord, logger
 from ..utils.settings import ObjectTypes, SummaryType, get_type
 from ..utils.types import ImageDict, PathLikeOrStr, PixelValues
 from .annotation import Annotation, AnnotationMap, BoundingBox, CategoryAnnotation, ImageAnnotation
@@ -474,8 +475,13 @@ class Image:
 
             for service_id in service_ids:
                 if service_id not in service_id_to_annotation_id:
-                    raise ImageError(f"Service id {service_id} not found")
-                annotation_ids = service_id_to_annotation_id[service_id]
+                    logger.info(
+                        LoggingRecord(
+                            f"Service_id {service_id} for image_id: {self.image_id} not found. Skipping removal."
+                        )
+                    )
+
+                annotation_ids = service_id_to_annotation_id.get(service_id, [])
 
                 for ann_id in annotation_ids:
                     if ann_id not in ann_id_to_annotation_maps:
@@ -747,7 +753,7 @@ class Image:
                 if sub_cat.service_id:
                     service_id_dict[sub_cat.service_id].append(sub_cat.annotation_id)
             if ann.image is not None:
-                for summary_cat_key in ann.image.summary:
+                for summary_cat_key in ann.image.summary.sub_categories:
                     summary_cat = ann.get_summary(summary_cat_key)
                     if summary_cat.service_id:
                         service_id_dict[summary_cat.service_id].append(summary_cat.annotation_id)

deepdoctection/extern/hflayoutlm.py

@@ -48,7 +48,7 @@ with try_import() as pt_import_guard:
     import torch.nn.functional as F
 
 with try_import() as tr_import_guard:
-    from timm.data.constants import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD  # type: ignore
+    from timm.data.constants import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD  # type:ignore
     from transformers import (
         LayoutLMForSequenceClassification,
         LayoutLMForTokenClassification,

deepdoctection/pipe/base.py

@@ -29,6 +29,7 @@ from uuid import uuid1
 
 from ..dataflow import DataFlow, MapData
 from ..datapoint.image import Image
+from ..mapper.misc import curry
 from ..utils.context import timed_operation
 from ..utils.identifier import get_uuid_from_str
 from ..utils.settings import ObjectTypes
@@ -247,17 +248,24 @@ class Pipeline(ABC):
         """
         raise NotImplementedError()
 
-    def _build_pipe(self, df: DataFlow, session_id: Optional[str] = None) -> DataFlow:
+    @staticmethod
+    @curry
+    def _undo(dp: Image, service_ids: Optional[list[str]] = None) -> Image:
         """
-        Composition of the backbone
+        Remove annotations from a datapoint
         """
-        if session_id is None and self.set_session_id:
-            session_id = self.get_session_id()
-        for component in self.pipe_component_list:
-            component.timer_on = True
-            component.dp_manager.session_id = session_id
-            df = component.predict_dataflow(df)
-        return df
+        dp.remove(service_ids=service_ids)
+        return dp
+
+    def undo(self, df: DataFlow, service_ids: Optional[set[str]] = None) -> DataFlow:
+        """
+        Mapping a datapoint via `_undo` within a dataflow pipeline
+
+        :param df: An input dataflow of Images
+        :param service_ids: A set of service ids to remove
+        :return: A output dataflow of Images
+        """
+        return MapData(df, self._undo(service_ids=service_ids))
 
     @abstractmethod
     def analyze(self, **kwargs: Any) -> DataFlow:
@@ -273,6 +281,18 @@ class Pipeline(ABC):
         """
         raise NotImplementedError()
 
+    def _build_pipe(self, df: DataFlow, session_id: Optional[str] = None) -> DataFlow:
+        """
+        Composition of the backbone
+        """
+        if session_id is None and self.set_session_id:
+            session_id = self.get_session_id()
+        for component in self.pipe_component_list:
+            component.timer_on = True
+            component.dp_manager.session_id = session_id
+            df = component.predict_dataflow(df)
+        return df
+
     def get_meta_annotation(self) -> MetaAnnotation:
         """
         Collects meta annotations from all pipeline components and summarizes the returned results

deepdoctection/pipe/doctectionpipe.py

@@ -161,16 +161,17 @@ class DoctectionPipe(Pipeline):
 
         super().__init__(pipeline_component_list)
 
-    def _entry(self, **kwargs: Union[str, bytes, DataFlow, bool, int, PathLikeOrStr, Union[str, List[str]]]) \
-            -> DataFlow:
+    def _entry(
+        self, **kwargs: Union[str, bytes, DataFlow, bool, int, PathLikeOrStr, Union[str, List[str]]]
+    ) -> DataFlow:
         path, file_type, shuffle, max_datapoints, doc_path, dataset_dataflow, b_bytes = _collect_from_kwargs(**kwargs)
 
         df: DataFlow
 
         if isinstance(b_bytes, bytes):
-            df = DoctectionPipe.bytes_to_dataflow(path=doc_path if path is None else path,
-                                                  b_bytes=b_bytes,
-                                                  file_type=file_type)
+            df = DoctectionPipe.bytes_to_dataflow(
+                path=doc_path if path is None else path, b_bytes=b_bytes, file_type=file_type
+            )
 
         elif isinstance(path, (str, Path)):
             if not isinstance(file_type, (str, list)):

deepdoctection/train/hf_layoutlm_train.py

@@ -161,11 +161,12 @@ class LayoutLMTrainer(Trainer):
         model: Union[PreTrainedModel, nn.Module],
         args: TrainingArguments,
         data_collator: LayoutLMDataCollator,
-        train_dataset: Dataset[Any],
+        train_dataset: DatasetAdapter,
+        eval_dataset: Optional[DatasetBase] = None,
     ):
         self.evaluator: Optional[Evaluator] = None
         self.build_eval_kwargs: Optional[dict[str, Any]] = None
-        super().__init__(model, args, data_collator, train_dataset)
+        super().__init__(model, args, data_collator, train_dataset, eval_dataset=eval_dataset)
 
     def setup_evaluator(
         self,
@@ -472,7 +473,7 @@ def train_hf_layoutlm(
         max_batch_size=max_batch_size,  # type: ignore
         remove_bounding_box_features=remove_box_features,
     )
-    trainer = LayoutLMTrainer(model, arguments, data_collator, dataset)
+    trainer = LayoutLMTrainer(model, arguments, data_collator, dataset, eval_dataset=dataset_val)
 
     if arguments.evaluation_strategy in (IntervalStrategy.STEPS,):
         assert metric is not None  # silence mypy

{deepdoctection-0.37.dist-info → deepdoctection-0.37.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepdoctection
-Version: 0.37
+Version: 0.37.2
 Summary: Repository for Document AI
 Home-page: https://github.com/deepdoctection/deepdoctection
 Author: Dr. Janis Meyer

{deepdoctection-0.37.dist-info → deepdoctection-0.37.2.dist-info}/RECORD

@@ -1,9 +1,9 @@
-deepdoctection/__init__.py,sha256=7VELexCFRaBTCXHQpBoKhVi4hqUUgpcsLTqvHXHjufQ,12651
+deepdoctection/__init__.py,sha256=T2sHOc6ZPpx44hWbarp0i_QlAqm0dEmzs7HVg2mL_nM,12655
 deepdoctection/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deepdoctection/analyzer/__init__.py,sha256=icClxrd20XutD6LxLgEPIWceSs4j_QfI3szCE-9BL2w,729
 deepdoctection/analyzer/_config.py,sha256=NZl_REM8Ge2xfxvHN-mZR5KURcHfZii3xfMlKQwckbA,4864
 deepdoctection/analyzer/dd.py,sha256=DUOhOtwipHw5nabYqn3WGR9aZcgP0ma_bi_tjf9xscw,5973
-deepdoctection/analyzer/factory.py,sha256=xmo5F9X7I6lp0ZWJv8QavpMyG8UWYLvMi4qogsZV1_s,31507
+deepdoctection/analyzer/factory.py,sha256=dEUOtdBS3yQGLqMqLR_kq5EYCR3IE30DjHNzE0spoQE,31519
 deepdoctection/configs/__init__.py,sha256=TX_P6tqDOF1LK1mi9ruAl7x0mtv1Asm8cYWCz3Pe2dk,646
 deepdoctection/configs/conf_dd_one.yaml,sha256=td7XsyVhdXkhh5Pie7sT_WNjGTaxBOWgpxhkobHd1H0,2325
 deepdoctection/configs/conf_tesseract.yaml,sha256=oF6szDyoi15FHvq7yFUNIEjfA_jNLhGxoowiRsz_zY4,35
@@ -19,7 +19,7 @@ deepdoctection/datapoint/__init__.py,sha256=3K406GbOPhoEp8koVaSbMocmSsmWifnSZ1SP
 deepdoctection/datapoint/annotation.py,sha256=FEgz4COxVDfjic0gG7kS6iHnWLBIgFnquQ63Cbj2a4Y,22531
 deepdoctection/datapoint/box.py,sha256=tkFuVM6xfx2jL7W4UED4qHXV572LSRdIsVJbrEiyIxI,23524
 deepdoctection/datapoint/convert.py,sha256=O7920pIomyEkzXwxpFsrzfhn7Pl6UzVGhNzv90VcuKU,7099
-deepdoctection/datapoint/image.py,sha256=DIXXXD2yKsacg47Wt_GEYEIe1MQkrd06Yr5xAWv_n64,33047
+deepdoctection/datapoint/image.py,sha256=UDBKXJJpuKAUx0J-DGjvLGqrMV4N3kLpksJYxoVkong,33279
 deepdoctection/datapoint/view.py,sha256=1rVMuqucCrI5zlwyXMADJQBV38V_zSNFqFyBi3cMA1E,44914
 deepdoctection/datasets/__init__.py,sha256=-A3aR90aDsHPmVM35JavfnQ2itYSCn3ujl4krRni1QU,1076
 deepdoctection/datasets/adapter.py,sha256=Ly_vbOAgVI73V41FUccnSX1ECTOyesW_qsuvQuvOZbw,7796
@@ -56,7 +56,7 @@ deepdoctection/extern/deskew.py,sha256=sPoixu8S9he-0wbs-jgxtPE2V9BiP4-3uZlb6F5Y1
 deepdoctection/extern/doctrocr.py,sha256=T3_tvlih22_dVCBZypS1Y8tjQQB1fkAxIbGdUGHIapQ,24473
 deepdoctection/extern/fastlang.py,sha256=F4gK-SEwcCujjxH327ZDzMGWToJ49xS_dCKcePQ9IlY,4780
 deepdoctection/extern/hfdetr.py,sha256=1NPW_u5eH2tP3ixZ91l4WR-O-wLVcrFsLWA7BqID0oM,12055
-deepdoctection/extern/hflayoutlm.py,sha256=KfoWx9_Rpa1Y2L51HLrYvenfWaTB4SVTmVJH00Cqb-s,56510
+deepdoctection/extern/hflayoutlm.py,sha256=T1IBm3C8CtG97-tauo03YqhUac6xdFc2y345BWVMajQ,56509
 deepdoctection/extern/hflm.py,sha256=kwS6kcSlY_2m9u0RzBLTRq-UMM7c1PhyUaDTvSdejus,9217
 deepdoctection/extern/model.py,sha256=ViHHKPvbGmLCPw7ZESv_rmjlkA90UiBU6oZiHOMqNSw,59869
 deepdoctection/extern/pdftext.py,sha256=KS_t27SUiYn_IOS_J2lF9lSSo22vLagxmxvYCY3CqXA,7228
@@ -103,10 +103,10 @@ deepdoctection/mapper/tpstruct.py,sha256=YNABRibvcISD5Lavg3jouoE4FMdqXEJoM-hNoB_
 deepdoctection/mapper/xfundstruct.py,sha256=_3r3c0K82fnF2h1HxA85h-9ETYrHwcERa6MNc6Ko6Z8,8807
 deepdoctection/pipe/__init__.py,sha256=ywTVoetftdL6plXg2YlBzMfmqBZupq7yXblSVyvvkcQ,1127
 deepdoctection/pipe/anngen.py,sha256=3319l4aaXzcY4w6ItVBNPX8LGS5fHFDVtyVY9KMefac,16393
-deepdoctection/pipe/base.py,sha256=Davjkf3D837y9AIITcx7yXdebmVaz6Moyw_5Wi3nfmg,13561
+deepdoctection/pipe/base.py,sha256=ynNg5SSRuUVxN69VWOO3Oi7WSeGrYwn3A56NQMBJDvw,14222
 deepdoctection/pipe/common.py,sha256=haOb4v0jLX3r41BSC8cVseX2E320_HkSrGlZsQiKE2g,17728
 deepdoctection/pipe/concurrency.py,sha256=AAKRsVgaBEYNluntbDa46SBF1JZ_XqnWLDSWrNvAzEo,9657
-deepdoctection/pipe/doctectionpipe.py,sha256=uhsrSuwaHcOMj8b8i6wCpPaZlSxCTaeHVhMokJ8vRSI,11835
+deepdoctection/pipe/doctectionpipe.py,sha256=wCg96P9Pb54i5AVgG02b4FljobM64_qEML_GxiULy-4,11765
 deepdoctection/pipe/language.py,sha256=5zI0UQC6Fh12_r2pfVL42HoCGz2hpHrOhpXAn5m-rYw,5451
 deepdoctection/pipe/layout.py,sha256=xIhnJpyUSbvLbhTXyAKXY1hmG9352jihGYFSclTH_1g,5567
 deepdoctection/pipe/lm.py,sha256=Sp-b7smeslNDyioEfNjuNBUxAuFKn3-OKpCZkGXri_c,16643
@@ -120,7 +120,7 @@ deepdoctection/pipe/transform.py,sha256=9Om7X7hJeL4jgUwHM1CHa4sb5v7Qo1PtVG0ls_3n
 deepdoctection/train/__init__.py,sha256=YFTRAZF1F7cEAKTdAIi1BLyYb6rSRcwq09Ui5Lu8d6E,1071
 deepdoctection/train/d2_frcnn_train.py,sha256=sFc_G-mEpaM8d1CCE0_6Gl4nBh11X2RYRBA3p_ylFJQ,16000
 deepdoctection/train/hf_detr_train.py,sha256=8ydysxzOPE_IPoNFGaHb7PbKr9Nbl41rcY4lbylQavU,10783
-deepdoctection/train/hf_layoutlm_train.py,sha256=e3pekLfe2KeYAI04COiTTL3KKiLDaXxTj0A2vwTvYZo,22425
+deepdoctection/train/hf_layoutlm_train.py,sha256=BNjPgPAvxm4beHULqzo58u-gW7GcTGiZAk2rF6TootM,22532
 deepdoctection/train/tp_frcnn_train.py,sha256=pEpXokSVGveqo82pRnhnAmHPmjQ_8wQWpqM4ZyNHJgs,13049
 deepdoctection/utils/__init__.py,sha256=brBceRWeov9WXMiJTjyJOF2rHMP8trGGRRjhMdZ61nI,2371
 deepdoctection/utils/concurrency.py,sha256=nIhpkSncmv0LBB8PtcOLY-BsRGlfcDpz7foVdgzZd20,4598
@@ -141,8 +141,8 @@ deepdoctection/utils/transform.py,sha256=3kCgsEeRkG1efCdkfvj7tUFMs-e2jbjbflq826F
 deepdoctection/utils/types.py,sha256=_3dmPdCIZNLbgU5QP5k_c5phDf18xLe1kYL6t2nM45s,2953
 deepdoctection/utils/utils.py,sha256=csVs_VvCq4QBETPoE2JdTTL4MFYnD4xh-Js5vRb612g,6492
 deepdoctection/utils/viz.py,sha256=Jf8ePNYWlpuyaS6SeTYQ4OyA3eNhtgjvAQZnGNdgHC0,27051
-deepdoctection-0.37.dist-info/LICENSE,sha256=GQ0rUvuGdrMNEI3iHK5UQx6dIMU1QwAuyXsxUHn5MEQ,11351
-deepdoctection-0.37.dist-info/METADATA,sha256=0qGgmf07xmNRJx55yfMagHcfAoQG6GO9KTw6b0tv0uA,19543
-deepdoctection-0.37.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-deepdoctection-0.37.dist-info/top_level.txt,sha256=hs2DdoOL9h4mnHhmO82BT4pz4QATIoOZ20PZmlnxFI8,15
-deepdoctection-0.37.dist-info/RECORD,,
+deepdoctection-0.37.2.dist-info/LICENSE,sha256=GQ0rUvuGdrMNEI3iHK5UQx6dIMU1QwAuyXsxUHn5MEQ,11351
+deepdoctection-0.37.2.dist-info/METADATA,sha256=XLOCkFBWynZhyZmKpDDRaomDIxPnVpy07WdUkodRF3Y,19545
+deepdoctection-0.37.2.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+deepdoctection-0.37.2.dist-info/top_level.txt,sha256=hs2DdoOL9h4mnHhmO82BT4pz4QATIoOZ20PZmlnxFI8,15
+deepdoctection-0.37.2.dist-info/RECORD,,