deepdoctection 0.39__tar.gz → 0.39.2__tar.gz

{deepdoctection-0.39 → deepdoctection-0.39.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: deepdoctection
-Version: 0.39
+Version: 0.39.2
 Summary: Repository for Document AI
 Home-page: https://github.com/deepdoctection/deepdoctection
 Author: Dr. Janis Meyer
@@ -61,7 +61,7 @@ Requires-Dist: python-doctr==0.8.1; extra == "tf"
 Requires-Dist: pycocotools>=2.0.2; extra == "tf"
 Requires-Dist: boto3==1.34.102; extra == "tf"
 Requires-Dist: pdfplumber>=0.11.0; extra == "tf"
-Requires-Dist: fasttext==0.9.2; extra == "tf"
+Requires-Dist: fasttext-wheel; extra == "tf"
 Requires-Dist: jdeskew>=0.2.2; extra == "tf"
 Requires-Dist: apted==1.0.3; extra == "tf"
 Requires-Dist: distance==0.1.3; extra == "tf"
@@ -86,12 +86,12 @@ Requires-Dist: termcolor>=1.1; extra == "pt"
 Requires-Dist: tabulate>=0.7.7; extra == "pt"
 Requires-Dist: tqdm==4.64.0; extra == "pt"
 Requires-Dist: timm>=0.9.16; extra == "pt"
-Requires-Dist: transformers>=4.36.0; extra == "pt"
+Requires-Dist: transformers>=4.48.0; extra == "pt"
 Requires-Dist: accelerate>=0.29.1; extra == "pt"
 Requires-Dist: python-doctr==0.8.1; extra == "pt"
 Requires-Dist: boto3==1.34.102; extra == "pt"
 Requires-Dist: pdfplumber>=0.11.0; extra == "pt"
-Requires-Dist: fasttext==0.9.2; extra == "pt"
+Requires-Dist: fasttext-wheel; extra == "pt"
 Requires-Dist: jdeskew>=0.2.2; extra == "pt"
 Requires-Dist: apted==1.0.3; extra == "pt"
 Requires-Dist: distance==0.1.3; extra == "pt"
@@ -99,7 +99,7 @@ Requires-Dist: lxml>=4.9.1; extra == "pt"
 Provides-Extra: docs
 Requires-Dist: tensorpack==0.11; extra == "docs"
 Requires-Dist: boto3==1.34.102; extra == "docs"
-Requires-Dist: transformers>=4.36.0; extra == "docs"
+Requires-Dist: transformers>=4.48.0; extra == "docs"
 Requires-Dist: accelerate>=0.29.1; extra == "docs"
 Requires-Dist: pdfplumber>=0.11.0; extra == "docs"
 Requires-Dist: lxml>=4.9.1; extra == "docs"

{deepdoctection-0.39 → deepdoctection-0.39.2}/deepdoctection/__init__.py

@@ -25,7 +25,7 @@ from .utils.logger import LoggingRecord, logger
 
 # pylint: enable=wrong-import-position
 
-__version__ = "0.39"
+__version__ = "0.39.2"
 
 _IMPORT_STRUCTURE = {
     "analyzer": ["config_sanity_checks", "get_dd_analyzer", "ServiceFactory"],

{deepdoctection-0.39 → deepdoctection-0.39.2}/deepdoctection/analyzer/_config.py

@@ -40,6 +40,7 @@ cfg.TF.CELL.FILTER = None
 cfg.TF.ITEM.WEIGHTS = "item/model-1620000_inf_only.data-00000-of-00001"
 cfg.TF.ITEM.FILTER = None
 
+cfg.PT.ENFORCE_WEIGHTS = False
 cfg.PT.LAYOUT.WEIGHTS = "layout/d2_model_0829999_layout_inf_only.pt"
 cfg.PT.LAYOUT.WEIGHTS_TS = "layout/d2_model_0829999_layout_inf_only.ts"
 cfg.PT.LAYOUT.FILTER = None

{deepdoctection-0.39 → deepdoctection-0.39.2}/deepdoctection/analyzer/factory.py

@@ -98,7 +98,11 @@ class ServiceFactory:
         weights = (
             getattr(config.TF, mode).WEIGHTS
             if config.LIB == "TF"
-            else (getattr(config.PT, mode).WEIGHTS if detectron2_available() else getattr(config.PT, mode).WEIGHTS_TS)
+            else (
+                getattr(config.PT, mode).WEIGHTS
+                if detectron2_available() or config.PT.ENFORCE_WEIGHTS
+                else getattr(config.PT, mode).WEIGHTS_TS
+            )
         )
         filter_categories = (
             getattr(getattr(config.TF, mode), "FILTER")

{deepdoctection-0.39 → deepdoctection-0.39.2}/deepdoctection/datapoint/view.py

@@ -407,6 +407,35 @@ class Table(Layout):
         col_anns = self.base_page.get_annotation(annotation_ids=all_relation_ids, category_names=[LayoutType.COLUMN])
         return col_anns
 
+    def row(self, row_number: int) -> list[ImageAnnotationBaseView]:
+        """
+        Get a list of cells in a row.
+        """
+        all_relation_ids = self.get_relationship(Relationships.CHILD)
+        all_cells = self.base_page.get_annotation(
+            category_names=[LayoutType.CELL, CellType.SPANNING], annotation_ids=all_relation_ids
+        )
+        row_cells = list(
+            filter(lambda c: row_number in (c.row_number, c.row_number + c.row_span - 1), all_cells)  # type: ignore
+        )
+        row_cells.sort(key=lambda c: c.column_number)  # type: ignore
+        return row_cells  # type: ignore
+
+    def column(self, column_number: int) -> list[ImageAnnotationBaseView]:
+        """
+        Get a list of cells in a column.
+        """
+        all_relation_ids = self.get_relationship(Relationships.CHILD)
+        all_cells = self.base_page.get_annotation(
+            category_names=[LayoutType.CELL, CellType.SPANNING], annotation_ids=all_relation_ids
+        )
+        column_cells = list(
+            filter(lambda c: column_number in  # type: ignore
+                             (c.column_number, c.column_number + c.column_span - 1), all_cells)  # type: ignore
+        )
+        column_cells.sort(key=lambda c: c.row_number)  # type: ignore
+        return column_cells  # type: ignore
+
     @property
     def html(self) -> HTML:
         """

{deepdoctection-0.39 → deepdoctection-0.39.2}/deepdoctection/pipe/base.py

@@ -24,7 +24,7 @@ from __future__ import annotations
 from abc import ABC, abstractmethod
 from collections import defaultdict
 from dataclasses import dataclass, field
-from typing import Any, Mapping, Optional, Union
+from typing import Any, Callable, Mapping, Optional, Union
 from uuid import uuid1
 
 from ..dataflow import DataFlow, MapData
@@ -33,6 +33,7 @@ from ..mapper.misc import curry
 from ..utils.context import timed_operation
 from ..utils.identifier import get_uuid_from_str
 from ..utils.settings import ObjectTypes
+from ..utils.types import DP
 from .anngen import DatapointManager
 
 
@@ -76,6 +77,30 @@ class PipelineComponent(ABC):
         self.service_id = self.get_service_id()
         self.dp_manager = DatapointManager(self.service_id, model_id)
         self.timer_on = False
+        self.filter_func: Callable[[DP], bool] = lambda dp: False
+
+    def set_inbound_filter(self, filter_func: Callable[[DP], bool]) -> None:
+        """
+        Set a filter function to decide, if an image of the inbound dataflow should be passed to self.serve.
+        The filter function should return a boolean value. If the function returns True, the image will not be processed
+        by this pipeline component.
+
+        **Example:**
+
+            ```python
+            def do_not_process_tables(dp: Image) -> bool:
+                 if "table" not in dp.get_categories_from_current_state():
+                    return True
+                 return False
+
+            layout_component = ImageLayoutService(...)
+            layout_component.set_inbound_filter(do_not_process_tables)
+            ```
+
+
+        :param filter_func: A function that takes an image datapoint and returns a boolean value
+        """
+        self.filter_func = filter_func  # type: ignore
 
     @abstractmethod
     def serve(self, dp: Image) -> None:
@@ -92,6 +117,11 @@ class PipelineComponent(ABC):
         """
         raise NotImplementedError()
 
+    def _pass_datapoint(self, dp: Image) -> None:
+        self.dp_manager.datapoint = dp
+        if not self.filter_func(dp):
+            self.serve(dp)
+
     def pass_datapoint(self, dp: Image) -> Image:
         """
         Acceptance, handover to dp_manager, transformation and forwarding of dp. To measure the time, use
@@ -103,11 +133,9 @@ class PipelineComponent(ABC):
         """
         if self.timer_on:
             with timed_operation(self.__class__.__name__):
-                self.dp_manager.datapoint = dp
-                self.serve(dp)
+                self._pass_datapoint(dp)
         else:
-            self.dp_manager.datapoint = dp
-            self.serve(dp)
+            self._pass_datapoint(dp)
         return self.dp_manager.datapoint
 
     def predict_dataflow(self, df: DataFlow) -> DataFlow:
@@ -205,6 +233,7 @@ class Pipeline(ABC):
 
     **Example:**
 
+            ```python
             layout = LayoutPipeComponent(layout_detector ...)
             text = TextExtractPipeComponent(text_detector ...)
             simple_pipe = MyPipeline(pipeline_component = [layout, text])
@@ -212,6 +241,7 @@ class Pipeline(ABC):
 
             for page in doc_dataflow:
                 print(page)
+            ```
 
     In doing so, page contains all document structures determined via the pipeline (either directly from the Image core
     model or already processed further).
@@ -225,10 +255,12 @@ class Pipeline(ABC):
 
     **Example:**
 
+           ```python
            pipe = MyPipeline(pipeline_component = [layout, text])
            pipe.set_session_id = True
 
            df = pipe.analyze(input = "path/to/dir") # session_id is generated automatically
+           ```
     """
 
     def __init__(self, pipeline_component_list: list[PipelineComponent]) -> None:

{deepdoctection-0.39 → deepdoctection-0.39.2}/deepdoctection/pipe/common.py

@@ -349,8 +349,8 @@ class AnnotationNmsService(PipelineComponent):
     def __init__(
         self,
         nms_pairs: Sequence[Sequence[TypeOrStr]],
-        thresholds: Union[float, list[float]],
-        priority: Optional[list[Union[Optional[TypeOrStr]]]] = None,
+        thresholds: Union[float, Sequence[float]],
+        priority: Optional[Sequence[Union[Optional[TypeOrStr]]]] = None,
     ):
         """
         :param nms_pairs: Groups of categories, either as string or by `ObjectType`.
@@ -362,7 +362,7 @@ class AnnotationNmsService(PipelineComponent):
             self.threshold = [thresholds for _ in self.nms_pairs]
         else:
             assert len(self.nms_pairs) == len(thresholds), "Sequences of nms_pairs and thresholds must have same length"
-            self.threshold = thresholds
+            self.threshold = thresholds  # type: ignore
         if priority:
             assert len(self.nms_pairs) == len(priority), "Sequences of nms_pairs and priority must have same length"
 

{deepdoctection-0.39 → deepdoctection-0.39.2}/deepdoctection/pipe/lm.py

@@ -265,7 +265,7 @@ class LMSequenceClassifierService(PipelineComponent):
         padding: Literal["max_length", "do_not_pad", "longest"] = "max_length",
         truncation: bool = True,
         return_overflowing_tokens: bool = False,
-        use_other_as_default_category: bool = False
+        use_other_as_default_category: bool = False,
     ) -> None:
         """
         :param tokenizer: Tokenizer, typing allows currently anything. This will be changed in the future
@@ -309,11 +309,10 @@ class LMSequenceClassifierService(PipelineComponent):
         lm_output = None
         if lm_input is None:
             if self.use_other_as_default_category:
-                class_id = self.language_model.categories.get_categories(as_dict=True,
-                                                                         name_as_key=True).get(TokenClasses.OTHER, 1)
-                lm_output = SequenceClassResult(class_name=TokenClasses.OTHER,
-                                                class_id = class_id,
-                                                score=-1.)
+                class_id = self.language_model.categories.get_categories(as_dict=True, name_as_key=True).get(
+                    TokenClasses.OTHER, 1
+                )
+                lm_output = SequenceClassResult(class_name=TokenClasses.OTHER, class_id=class_id, score=-1.0)
         else:
             lm_output = self.language_model.predict(**lm_input)
         if lm_output:

{deepdoctection-0.39 → deepdoctection-0.39.2}/deepdoctection/train/hf_detr_train.py

@@ -73,7 +73,7 @@ class DetrDerivedTrainer(Trainer):
         model: Union[PreTrainedModel, nn.Module],
         args: TrainingArguments,
         data_collator: DetrDataCollator,
-        train_dataset: Dataset[Any],
+        train_dataset: DatasetAdapter,
     ):
         self.evaluator: Optional[Evaluator] = None
         self.build_eval_kwargs: Optional[dict[str, Any]] = None

{deepdoctection-0.39 → deepdoctection-0.39.2}/deepdoctection/train/hf_layoutlm_train.py

@@ -499,9 +499,7 @@ def train_hf_layoutlm(
         )
         pipeline_component_cls = pipeline_component_registry.get(pipeline_component_name)
         if dataset_type == DatasetType.SEQUENCE_CLASSIFICATION:
-            pipeline_component = pipeline_component_cls(tokenizer_fast,
-                                                        dd_model,
-                                                        use_other_as_default_category=True)
+            pipeline_component = pipeline_component_cls(tokenizer_fast, dd_model, use_other_as_default_category=True)
         else:
             pipeline_component = pipeline_component_cls(
                 tokenizer_fast,

{deepdoctection-0.39 → deepdoctection-0.39.2}/deepdoctection.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: deepdoctection
-Version: 0.39
+Version: 0.39.2
 Summary: Repository for Document AI
 Home-page: https://github.com/deepdoctection/deepdoctection
 Author: Dr. Janis Meyer
@@ -61,7 +61,7 @@ Requires-Dist: python-doctr==0.8.1; extra == "tf"
 Requires-Dist: pycocotools>=2.0.2; extra == "tf"
 Requires-Dist: boto3==1.34.102; extra == "tf"
 Requires-Dist: pdfplumber>=0.11.0; extra == "tf"
-Requires-Dist: fasttext==0.9.2; extra == "tf"
+Requires-Dist: fasttext-wheel; extra == "tf"
 Requires-Dist: jdeskew>=0.2.2; extra == "tf"
 Requires-Dist: apted==1.0.3; extra == "tf"
 Requires-Dist: distance==0.1.3; extra == "tf"
@@ -86,12 +86,12 @@ Requires-Dist: termcolor>=1.1; extra == "pt"
 Requires-Dist: tabulate>=0.7.7; extra == "pt"
 Requires-Dist: tqdm==4.64.0; extra == "pt"
 Requires-Dist: timm>=0.9.16; extra == "pt"
-Requires-Dist: transformers>=4.36.0; extra == "pt"
+Requires-Dist: transformers>=4.48.0; extra == "pt"
 Requires-Dist: accelerate>=0.29.1; extra == "pt"
 Requires-Dist: python-doctr==0.8.1; extra == "pt"
 Requires-Dist: boto3==1.34.102; extra == "pt"
 Requires-Dist: pdfplumber>=0.11.0; extra == "pt"
-Requires-Dist: fasttext==0.9.2; extra == "pt"
+Requires-Dist: fasttext-wheel; extra == "pt"
 Requires-Dist: jdeskew>=0.2.2; extra == "pt"
 Requires-Dist: apted==1.0.3; extra == "pt"
 Requires-Dist: distance==0.1.3; extra == "pt"
@@ -99,7 +99,7 @@ Requires-Dist: lxml>=4.9.1; extra == "pt"
 Provides-Extra: docs
 Requires-Dist: tensorpack==0.11; extra == "docs"
 Requires-Dist: boto3==1.34.102; extra == "docs"
-Requires-Dist: transformers>=4.36.0; extra == "docs"
+Requires-Dist: transformers>=4.48.0; extra == "docs"
 Requires-Dist: accelerate>=0.29.1; extra == "docs"
 Requires-Dist: pdfplumber>=0.11.0; extra == "docs"
 Requires-Dist: lxml>=4.9.1; extra == "docs"

{deepdoctection-0.39 → deepdoctection-0.39.2}/deepdoctection.egg-info/requires.txt

@@ -36,7 +36,7 @@ types-urllib3>=1.26.25.14
 [docs]
 tensorpack==0.11
 boto3==1.34.102
-transformers>=4.36.0
+transformers>=4.48.0
 accelerate>=0.29.1
 pdfplumber>=0.11.0
 lxml>=4.9.1
@@ -67,12 +67,12 @@ termcolor>=1.1
 tabulate>=0.7.7
 tqdm==4.64.0
 timm>=0.9.16
-transformers>=4.36.0
+transformers>=4.48.0
 accelerate>=0.29.1
 python-doctr==0.8.1
 boto3==1.34.102
 pdfplumber>=0.11.0
-fasttext==0.9.2
+fasttext-wheel
 jdeskew>=0.2.2
 apted==1.0.3
 distance==0.1.3
@@ -109,7 +109,7 @@ python-doctr==0.8.1
 pycocotools>=2.0.2
 boto3==1.34.102
 pdfplumber>=0.11.0
-fasttext==0.9.2
+fasttext-wheel
 jdeskew>=0.2.2
 apted==1.0.3
 distance==0.1.3

{deepdoctection-0.39 → deepdoctection-0.39.2}/setup.py

@@ -78,7 +78,7 @@ _DEPS = [
     "tensorpack==0.11",
     # PyTorch related dependencies
     "timm>=0.9.16",
-    "transformers>=4.36.0",
+    "transformers>=4.48.0",
     "accelerate>=0.29.1",
     # As maintenance of Detectron2 decreases, we will now use our own Fork the keep updating after rigorous testing.
     # This will hopefully prevent from issues like 233
@@ -90,7 +90,7 @@ _DEPS = [
     "tensorflow-addons>=0.17.1",
     "tf2onnx>=1.9.2",
     "python-doctr==0.8.1",
-    "fasttext==0.9.2",
+    "fasttext-wheel",
     # dev dependencies
     "python-dotenv==1.0.0",
     "click",  # version will not break black
@@ -147,7 +147,7 @@ dist_deps = deps_list(
 additional_deps = deps_list(
     "boto3",
     "pdfplumber",
-    "fasttext",
+    "fasttext-wheel",
     "jdeskew",
     "apted",
     "distance",

{deepdoctection-0.39 → deepdoctection-0.39.2}/deepdoctection/utils/settings.py

@@ -101,7 +101,6 @@ class DocumentType(ObjectTypes):
     GOVERNMENT_TENDERS = "government_tenders"
     MANUALS = "manuals"
     PATENTS = "patents"
-    MARK = "mark"
 
 
 @object_types_registry.register("LayoutType")
@@ -132,6 +131,7 @@ class LayoutType(ObjectTypes):
     PAGE_NUMBER = "page_number"
     KEY_VALUE_AREA = "key_value_area"
     LIST_ITEM = "list_item"
+    MARK = "mark"
 
 
 @object_types_registry.register("TableType")