deepdoctection 0.39.1__tar.gz → 0.39.3__tar.gz

{deepdoctection-0.39.1 → deepdoctection-0.39.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: deepdoctection
-Version: 0.39.1
+Version: 0.39.3
 Summary: Repository for Document AI
 Home-page: https://github.com/deepdoctection/deepdoctection
 Author: Dr. Janis Meyer

{deepdoctection-0.39.1 → deepdoctection-0.39.3}/deepdoctection/__init__.py

@@ -25,7 +25,7 @@ from .utils.logger import LoggingRecord, logger
 
 # pylint: enable=wrong-import-position
 
-__version__ = "0.39.1"
+__version__ = "0.39.3"
 
 _IMPORT_STRUCTURE = {
     "analyzer": ["config_sanity_checks", "get_dd_analyzer", "ServiceFactory"],

{deepdoctection-0.39.1 → deepdoctection-0.39.3}/deepdoctection/analyzer/_config.py

@@ -40,6 +40,7 @@ cfg.TF.CELL.FILTER = None
 cfg.TF.ITEM.WEIGHTS = "item/model-1620000_inf_only.data-00000-of-00001"
 cfg.TF.ITEM.FILTER = None
 
+cfg.PT.ENFORCE_WEIGHTS = False
 cfg.PT.LAYOUT.WEIGHTS = "layout/d2_model_0829999_layout_inf_only.pt"
 cfg.PT.LAYOUT.WEIGHTS_TS = "layout/d2_model_0829999_layout_inf_only.ts"
 cfg.PT.LAYOUT.FILTER = None

{deepdoctection-0.39.1 → deepdoctection-0.39.3}/deepdoctection/analyzer/factory.py

@@ -98,7 +98,11 @@ class ServiceFactory:
         weights = (
             getattr(config.TF, mode).WEIGHTS
             if config.LIB == "TF"
-            else (getattr(config.PT, mode).WEIGHTS if detectron2_available() else getattr(config.PT, mode).WEIGHTS_TS)
+            else (
+                getattr(config.PT, mode).WEIGHTS
+                if detectron2_available() or config.PT.ENFORCE_WEIGHTS
+                else getattr(config.PT, mode).WEIGHTS_TS
+            )
         )
         filter_categories = (
             getattr(getattr(config.TF, mode), "FILTER")

{deepdoctection-0.39.1 → deepdoctection-0.39.3}/deepdoctection/datapoint/image.py

@@ -702,11 +702,11 @@ class Image:
         return get_uuid(self.image_id, *container_ids)
 
     def save(
-        self,
-        image_to_json: bool = True,
-        highest_hierarchy_only: bool = False,
-        path: Optional[PathLikeOrStr] = None,
-        dry: bool = False,
+            self,
+            image_to_json: bool = True,
+            highest_hierarchy_only: bool = False,
+            path: Optional[PathLikeOrStr] = None,
+            dry: bool = False,
     ) -> Optional[Union[ImageDict, str]]:
         """
         Export image as dictionary. As numpy array cannot be serialized `image` values will be converted into
@@ -719,6 +719,18 @@ class Image:
 
         :return: optional dict
         """
+
+        def set_image_keys_to_none(d):  # type: ignore
+            if isinstance(d, dict):
+                for key, value in d.items():
+                    if key == '_image':
+                        d[key] = None
+                    else:
+                        set_image_keys_to_none(value)
+            elif isinstance(d, list):
+                for item in d:
+                    set_image_keys_to_none(item)
+
         if path is None:
             path = Path(self.location)
         path = Path(path)
@@ -734,7 +746,7 @@ class Image:
         export_dict = self.as_dict()
         export_dict["location"] = fspath(export_dict["location"])
         if not image_to_json:
-            export_dict["_image"] = None
+            set_image_keys_to_none(export_dict)
         if dry:
             return export_dict
         with open(path_json, "w", encoding="UTF-8") as file:

{deepdoctection-0.39.1 → deepdoctection-0.39.3}/deepdoctection/datapoint/view.py

@@ -407,6 +407,35 @@ class Table(Layout):
         col_anns = self.base_page.get_annotation(annotation_ids=all_relation_ids, category_names=[LayoutType.COLUMN])
         return col_anns
 
+    def row(self, row_number: int) -> list[ImageAnnotationBaseView]:
+        """
+        Get a list of cells in a row.
+        """
+        all_relation_ids = self.get_relationship(Relationships.CHILD)
+        all_cells = self.base_page.get_annotation(
+            category_names=[LayoutType.CELL, CellType.SPANNING], annotation_ids=all_relation_ids
+        )
+        row_cells = list(
+            filter(lambda c: row_number in (c.row_number, c.row_number + c.row_span - 1), all_cells)  # type: ignore
+        )
+        row_cells.sort(key=lambda c: c.column_number)  # type: ignore
+        return row_cells  # type: ignore
+
+    def column(self, column_number: int) -> list[ImageAnnotationBaseView]:
+        """
+        Get a list of cells in a column.
+        """
+        all_relation_ids = self.get_relationship(Relationships.CHILD)
+        all_cells = self.base_page.get_annotation(
+            category_names=[LayoutType.CELL, CellType.SPANNING], annotation_ids=all_relation_ids
+        )
+        column_cells = list(
+            filter(lambda c: column_number in  # type: ignore
+                             (c.column_number, c.column_number + c.column_span - 1), all_cells)  # type: ignore
+        )
+        column_cells.sort(key=lambda c: c.row_number)  # type: ignore
+        return column_cells  # type: ignore
+
     @property
     def html(self) -> HTML:
         """

{deepdoctection-0.39.1 → deepdoctection-0.39.3}/deepdoctection/datasets/registry.py

@@ -18,6 +18,7 @@
 """
 Module for DatasetRegistry
 """
+import inspect
 
 import catalogue  # type: ignore
 from tabulate import tabulate
@@ -47,7 +48,10 @@ def get_dataset(name: str) -> DatasetBase:
     :param name: A dataset name
     :return: An instance of a dataset
     """
-    return dataset_registry.get(name)()
+    ds = dataset_registry.get(name)
+    if inspect.isclass(ds):
+        return ds()
+    return ds
 
 
 def print_dataset_infos(add_license: bool = True, add_info: bool = True) -> None:

{deepdoctection-0.39.1 → deepdoctection-0.39.3}/deepdoctection/mapper/cats.py

@@ -32,7 +32,7 @@ from .maputils import LabelSummarizer, curry
 @curry
 def cat_to_sub_cat(
     dp: Image,
-    categories_dict_names_as_key: dict[TypeOrStr, int],
+    categories_dict_names_as_key: Optional[dict[TypeOrStr, int]] = None,
     cat_to_sub_cat_dict: Optional[dict[TypeOrStr, TypeOrStr]] = None,
 ) -> Image:
     """
@@ -45,7 +45,8 @@ def cat_to_sub_cat(
     :param cat_to_sub_cat_dict: e.g. {'foo': 'sub_cat_1', 'bak': 'sub_cat_2'}
     :return: Image with updated Annotations
     """
-
+    if categories_dict_names_as_key is None:
+        categories_dict_names_as_key = {}
     if cat_to_sub_cat_dict is None:
         return dp
     cat_to_sub_cat_dict_obj_type = {get_type(key): get_type(value) for key, value in cat_to_sub_cat_dict.items()}
@@ -54,7 +55,7 @@ def cat_to_sub_cat(
         sub_cat = ann.get_sub_category(sub_cat_type)
         if sub_cat:
             ann.category_name = sub_cat.category_name
-            ann.category_id = categories_dict_names_as_key[ann.category_name]
+            ann.category_id = categories_dict_names_as_key.get(ann.category_name,DEFAULT_CATEGORY_ID)
 
     return dp
 

{deepdoctection-0.39.1 → deepdoctection-0.39.3}/deepdoctection/pipe/base.py

@@ -24,7 +24,7 @@ from __future__ import annotations
 from abc import ABC, abstractmethod
 from collections import defaultdict
 from dataclasses import dataclass, field
-from typing import Any, Mapping, Optional, Union, Callable
+from typing import Any, Callable, Mapping, Optional, Union
 from uuid import uuid1
 
 from ..dataflow import DataFlow, MapData
@@ -100,7 +100,7 @@ class PipelineComponent(ABC):
 
         :param filter_func: A function that takes an image datapoint and returns a boolean value
         """
-        self.filter_func = filter_func # type: ignore
+        self.filter_func = filter_func  # type: ignore
 
     @abstractmethod
     def serve(self, dp: Image) -> None:
@@ -122,7 +122,6 @@ class PipelineComponent(ABC):
         if not self.filter_func(dp):
             self.serve(dp)
 
-
     def pass_datapoint(self, dp: Image) -> Image:
         """
         Acceptance, handover to dp_manager, transformation and forwarding of dp. To measure the time, use

{deepdoctection-0.39.1 → deepdoctection-0.39.3}/deepdoctection/pipe/common.py

@@ -362,7 +362,7 @@ class AnnotationNmsService(PipelineComponent):
             self.threshold = [thresholds for _ in self.nms_pairs]
         else:
             assert len(self.nms_pairs) == len(thresholds), "Sequences of nms_pairs and thresholds must have same length"
-            self.threshold = thresholds # type: ignore
+            self.threshold = thresholds  # type: ignore
         if priority:
             assert len(self.nms_pairs) == len(priority), "Sequences of nms_pairs and priority must have same length"
 

{deepdoctection-0.39.1 → deepdoctection-0.39.3}/deepdoctection/pipe/lm.py

@@ -265,7 +265,7 @@ class LMSequenceClassifierService(PipelineComponent):
         padding: Literal["max_length", "do_not_pad", "longest"] = "max_length",
         truncation: bool = True,
         return_overflowing_tokens: bool = False,
-        use_other_as_default_category: bool = False
+        use_other_as_default_category: bool = False,
     ) -> None:
         """
         :param tokenizer: Tokenizer, typing allows currently anything. This will be changed in the future
@@ -309,11 +309,10 @@ class LMSequenceClassifierService(PipelineComponent):
         lm_output = None
         if lm_input is None:
             if self.use_other_as_default_category:
-                class_id = self.language_model.categories.get_categories(as_dict=True,
-                                                                         name_as_key=True).get(TokenClasses.OTHER, 1)
-                lm_output = SequenceClassResult(class_name=TokenClasses.OTHER,
-                                                class_id = class_id,
-                                                score=-1.)
+                class_id = self.language_model.categories.get_categories(as_dict=True, name_as_key=True).get(
+                    TokenClasses.OTHER, 1
+                )
+                lm_output = SequenceClassResult(class_name=TokenClasses.OTHER, class_id=class_id, score=-1.0)
         else:
             lm_output = self.language_model.predict(**lm_input)
         if lm_output:

{deepdoctection-0.39.1 → deepdoctection-0.39.3}/deepdoctection/pipe/segment.py

@@ -1190,14 +1190,16 @@ class PubtablesSegmentationService(PipelineComponent):
                             for key, value in cell_rn_cn_to_ann_id.items():
                                 if key[idx] == item_number:
                                     cell_ann = dp.get_annotation(annotation_ids=value)[0]
-                                    self.dp_manager.set_category_annotation(
-                                        item_header_cell_name, None, item_header_cell_name, cell_ann.annotation_id
-                                    )
+                                    if item_header_cell_name not in cell_ann.sub_categories:
+                                        self.dp_manager.set_category_annotation(
+                                            item_header_cell_name, None, item_header_cell_name, cell_ann.annotation_id
+                                        )
                                 else:
                                     cell_ann = dp.get_annotation(annotation_ids=value)[0]
-                                    self.dp_manager.set_category_annotation(
-                                        item_header_cell_name, None, CellType.BODY, cell_ann.annotation_id
-                                    )
+                                    if CellType.BODY not in cell_ann.sub_categories:
+                                        self.dp_manager.set_category_annotation(
+                                            item_header_cell_name, None, CellType.BODY, cell_ann.annotation_id
+                                        )
 
             # TODO: the summaries should be sub categories of the underlying ann
             self.dp_manager.set_summary_annotation(

{deepdoctection-0.39.1 → deepdoctection-0.39.3}/deepdoctection/train/hf_layoutlm_train.py

@@ -499,9 +499,7 @@ def train_hf_layoutlm(
         )
         pipeline_component_cls = pipeline_component_registry.get(pipeline_component_name)
         if dataset_type == DatasetType.SEQUENCE_CLASSIFICATION:
-            pipeline_component = pipeline_component_cls(tokenizer_fast,
-                                                        dd_model,
-                                                        use_other_as_default_category=True)
+            pipeline_component = pipeline_component_cls(tokenizer_fast, dd_model, use_other_as_default_category=True)
         else:
             pipeline_component = pipeline_component_cls(
                 tokenizer_fast,

{deepdoctection-0.39.1 → deepdoctection-0.39.3}/deepdoctection.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: deepdoctection
-Version: 0.39.1
+Version: 0.39.3
 Summary: Repository for Document AI
 Home-page: https://github.com/deepdoctection/deepdoctection
 Author: Dr. Janis Meyer

{deepdoctection-0.39.1 → deepdoctection-0.39.3}/setup.py

@@ -90,7 +90,6 @@ _DEPS = [
     "tensorflow-addons>=0.17.1",
     "tf2onnx>=1.9.2",
     "python-doctr==0.8.1",
-    "fasttext==0.9.2",
     "fasttext-wheel",
     # dev dependencies
     "python-dotenv==1.0.0",