deepdoctection 0.37__tar.gz → 0.37.2__tar.gz

{deepdoctection-0.37 → deepdoctection-0.37.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepdoctection
-Version: 0.37
+Version: 0.37.2
 Summary: Repository for Document AI
 Home-page: https://github.com/deepdoctection/deepdoctection
 Author: Dr. Janis Meyer

{deepdoctection-0.37 → deepdoctection-0.37.2}/deepdoctection/__init__.py

@@ -24,7 +24,7 @@ from .utils.logger import LoggingRecord, logger
 
 # pylint: enable=wrong-import-position
 
-__version__ = 0.37
+__version__ = "0.37.2"
 
 _IMPORT_STRUCTURE = {
     "analyzer": ["config_sanity_checks", "get_dd_analyzer", "ServiceFactory"],

{deepdoctection-0.37 → deepdoctection-0.37.2}/deepdoctection/analyzer/factory.py

@@ -327,9 +327,9 @@ class ServiceFactory:
             )
         if config.OCR.USE_TEXTRACT:
             credentials_kwargs = {
-                "aws_access_key_id": environ.get("ACCESS_KEY", None),
-                "aws_secret_access_key": environ.get("SECRET_KEY", None),
-                "config": Config(region_name=environ.get("REGION", None)),
+                "aws_access_key_id": environ.get("AWS_ACCESS_KEY", None),
+                "aws_secret_access_key": environ.get("AWS_SECRET_KEY", None),
+                "config": Config(region_name=environ.get("AWS_REGION", None)),
             }
             return TextractOcrDetector(**credentials_kwargs)
         raise ValueError("You have set USE_OCR=True but any of USE_TESSERACT, USE_DOCTR, USE_TEXTRACT is set to False")

{deepdoctection-0.37 → deepdoctection-0.37.2}/deepdoctection/datapoint/image.py

@@ -32,6 +32,7 @@ from numpy import uint8
 
 from ..utils.error import AnnotationError, BoundingBoxError, ImageError, UUIDError
 from ..utils.identifier import get_uuid, is_uuid_like
+from ..utils.logger import LoggingRecord, logger
 from ..utils.settings import ObjectTypes, SummaryType, get_type
 from ..utils.types import ImageDict, PathLikeOrStr, PixelValues
 from .annotation import Annotation, AnnotationMap, BoundingBox, CategoryAnnotation, ImageAnnotation
@@ -474,8 +475,13 @@ class Image:
 
             for service_id in service_ids:
                 if service_id not in service_id_to_annotation_id:
-                    raise ImageError(f"Service id {service_id} not found")
-                annotation_ids = service_id_to_annotation_id[service_id]
+                    logger.info(
+                        LoggingRecord(
+                            f"Service_id {service_id} for image_id: {self.image_id} not found. Skipping removal."
+                        )
+                    )
+
+                annotation_ids = service_id_to_annotation_id.get(service_id, [])
 
                 for ann_id in annotation_ids:
                     if ann_id not in ann_id_to_annotation_maps:
@@ -747,7 +753,7 @@ class Image:
                 if sub_cat.service_id:
                     service_id_dict[sub_cat.service_id].append(sub_cat.annotation_id)
             if ann.image is not None:
-                for summary_cat_key in ann.image.summary:
+                for summary_cat_key in ann.image.summary.sub_categories:
                     summary_cat = ann.get_summary(summary_cat_key)
                     if summary_cat.service_id:
                         service_id_dict[summary_cat.service_id].append(summary_cat.annotation_id)

{deepdoctection-0.37 → deepdoctection-0.37.2}/deepdoctection/extern/hflayoutlm.py

@@ -48,7 +48,7 @@ with try_import() as pt_import_guard:
     import torch.nn.functional as F
 
 with try_import() as tr_import_guard:
-    from timm.data.constants import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD  # type: ignore
+    from timm.data.constants import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD  # type:ignore
     from transformers import (
         LayoutLMForSequenceClassification,
         LayoutLMForTokenClassification,

{deepdoctection-0.37 → deepdoctection-0.37.2}/deepdoctection/pipe/base.py

@@ -29,6 +29,7 @@ from uuid import uuid1
 
 from ..dataflow import DataFlow, MapData
 from ..datapoint.image import Image
+from ..mapper.misc import curry
 from ..utils.context import timed_operation
 from ..utils.identifier import get_uuid_from_str
 from ..utils.settings import ObjectTypes
@@ -247,17 +248,24 @@ class Pipeline(ABC):
         """
         raise NotImplementedError()
 
-    def _build_pipe(self, df: DataFlow, session_id: Optional[str] = None) -> DataFlow:
+    @staticmethod
+    @curry
+    def _undo(dp: Image, service_ids: Optional[list[str]] = None) -> Image:
         """
-        Composition of the backbone
+        Remove annotations from a datapoint
         """
-        if session_id is None and self.set_session_id:
-            session_id = self.get_session_id()
-        for component in self.pipe_component_list:
-            component.timer_on = True
-            component.dp_manager.session_id = session_id
-            df = component.predict_dataflow(df)
-        return df
+        dp.remove(service_ids=service_ids)
+        return dp
+
+    def undo(self, df: DataFlow, service_ids: Optional[set[str]] = None) -> DataFlow:
+        """
+        Mapping a datapoint via `_undo` within a dataflow pipeline
+
+        :param df: An input dataflow of Images
+        :param service_ids: A set of service ids to remove
+        :return: A output dataflow of Images
+        """
+        return MapData(df, self._undo(service_ids=service_ids))
 
     @abstractmethod
     def analyze(self, **kwargs: Any) -> DataFlow:
@@ -273,6 +281,18 @@ class Pipeline(ABC):
         """
         raise NotImplementedError()
 
+    def _build_pipe(self, df: DataFlow, session_id: Optional[str] = None) -> DataFlow:
+        """
+        Composition of the backbone
+        """
+        if session_id is None and self.set_session_id:
+            session_id = self.get_session_id()
+        for component in self.pipe_component_list:
+            component.timer_on = True
+            component.dp_manager.session_id = session_id
+            df = component.predict_dataflow(df)
+        return df
+
     def get_meta_annotation(self) -> MetaAnnotation:
         """
         Collects meta annotations from all pipeline components and summarizes the returned results

{deepdoctection-0.37 → deepdoctection-0.37.2}/deepdoctection/pipe/doctectionpipe.py

@@ -161,16 +161,17 @@ class DoctectionPipe(Pipeline):
 
         super().__init__(pipeline_component_list)
 
-    def _entry(self, **kwargs: Union[str, bytes, DataFlow, bool, int, PathLikeOrStr, Union[str, List[str]]]) \
-            -> DataFlow:
+    def _entry(
+        self, **kwargs: Union[str, bytes, DataFlow, bool, int, PathLikeOrStr, Union[str, List[str]]]
+    ) -> DataFlow:
         path, file_type, shuffle, max_datapoints, doc_path, dataset_dataflow, b_bytes = _collect_from_kwargs(**kwargs)
 
         df: DataFlow
 
         if isinstance(b_bytes, bytes):
-            df = DoctectionPipe.bytes_to_dataflow(path=doc_path if path is None else path,
-                                                  b_bytes=b_bytes,
-                                                  file_type=file_type)
+            df = DoctectionPipe.bytes_to_dataflow(
+                path=doc_path if path is None else path, b_bytes=b_bytes, file_type=file_type
+            )
 
         elif isinstance(path, (str, Path)):
             if not isinstance(file_type, (str, list)):

{deepdoctection-0.37 → deepdoctection-0.37.2}/deepdoctection/train/hf_layoutlm_train.py

@@ -161,11 +161,12 @@ class LayoutLMTrainer(Trainer):
         model: Union[PreTrainedModel, nn.Module],
         args: TrainingArguments,
         data_collator: LayoutLMDataCollator,
-        train_dataset: Dataset[Any],
+        train_dataset: DatasetAdapter,
+        eval_dataset: Optional[DatasetBase] = None,
     ):
         self.evaluator: Optional[Evaluator] = None
         self.build_eval_kwargs: Optional[dict[str, Any]] = None
-        super().__init__(model, args, data_collator, train_dataset)
+        super().__init__(model, args, data_collator, train_dataset, eval_dataset=eval_dataset)
 
     def setup_evaluator(
         self,
@@ -472,7 +473,7 @@ def train_hf_layoutlm(
         max_batch_size=max_batch_size,  # type: ignore
         remove_bounding_box_features=remove_box_features,
     )
-    trainer = LayoutLMTrainer(model, arguments, data_collator, dataset)
+    trainer = LayoutLMTrainer(model, arguments, data_collator, dataset, eval_dataset=dataset_val)
 
     if arguments.evaluation_strategy in (IntervalStrategy.STEPS,):
         assert metric is not None  # silence mypy

{deepdoctection-0.37 → deepdoctection-0.37.2}/deepdoctection.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepdoctection
-Version: 0.37
+Version: 0.37.2
 Summary: Repository for Document AI
 Home-page: https://github.com/deepdoctection/deepdoctection
 Author: Dr. Janis Meyer