deepdoctection 0.31__py3-none-any.whl → 0.33__py3-none-any.whl

deepdoctection/pipe/order.py

@@ -18,11 +18,14 @@
 """
 Module for ordering text and layout segments pipeline components
 """
+from __future__ import annotations
+
 import os
+from abc import ABC
 from copy import copy
 from itertools import chain
 from logging import DEBUG
-from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
+from typing import Any, Optional, Sequence, Union
 
 import numpy as np
 
@@ -32,9 +35,8 @@ from ..datapoint.image import Image
 from ..datapoint.view import IMAGE_DEFAULTS
 from ..extern.base import DetectionResult
 from ..extern.tp.tpfrcnn.utils.np_box_ops import ioa as np_ioa
-from ..pipe.base import PipelineComponent
+from ..pipe.base import MetaAnnotation, PipelineComponent
 from ..pipe.registry import pipeline_component_registry
-from ..utils.detection_types import JsonDict
 from ..utils.logger import LoggingRecord, logger
 from ..utils.settings import LayoutType, ObjectTypes, Relationships, TypeOrStr, get_type
 
@@ -67,7 +69,7 @@ class OrderGenerator:
     @staticmethod
     def group_words_into_lines(
         word_anns: Sequence[ImageAnnotation], image_id: Optional[str] = None
-    ) -> List[Tuple[int, int, str]]:
+    ) -> list[tuple[int, int, str]]:
         """Arranging words into horizontal text lines and sorting text lines vertically in order to give
         an enumeration of words that is used for establishing the reading order. Using this reading order arragement
         makes only sense for words within a rectangle and needs to be revised in more complex appearances.
@@ -75,7 +77,7 @@ class OrderGenerator:
         id)`.
         """
         reading_lines = []
-        rows: List[Dict[str, float]] = []
+        rows: list[dict[str, float]] = []
         for word in word_anns:
             bounding_box = word.get_bounding_box(image_id)
             row_found = False
@@ -114,13 +116,13 @@ class OrderGenerator:
     @staticmethod
     def group_lines_into_lines(
         line_anns: Sequence[ImageAnnotation], image_id: Optional[str] = None
-    ) -> List[Tuple[int, int, str]]:
+    ) -> list[tuple[int, int, str]]:
         """
         Sorting reading lines. Returns for a list of `ImageAnnotation` an list of tuples (each tuple containing the
         reading order and the `annotation_id` for each list element.
         :param line_anns: text line `ImageAnnotation`
         :param image_id: image_id of underyling image (to find get the bounding boxes)
-        :return: `List[(reading_order, reading_order,annotation_id)]`
+        :return: `list[(reading_order, reading_order,annotation_id)]`
         """
         reading_lines = []
         for ann in line_anns:
@@ -131,9 +133,9 @@ class OrderGenerator:
         return [(idx + 1, idx + 1, line[1]) for idx, line in enumerate(reading_lines)]
 
     @staticmethod
-    def _connected_components(columns: List[BoundingBox]) -> List[Dict[str, Any]]:
+    def _connected_components(columns: list[BoundingBox]) -> list[dict[str, Any]]:
         # building connected components of columns
-        connected_components: List[Dict[str, Any]] = []
+        connected_components: list[dict[str, Any]] = []
         for idx, col in enumerate(columns):
             col_dict = {"id": idx, "box": col}
             component_found = False
@@ -168,8 +170,8 @@ class OrderGenerator:
         return connected_components
 
     def order_blocks(
-        self, anns: List[ImageAnnotation], image_width: float, image_height: float, image_id: Optional[str] = None
-    ) -> Sequence[Tuple[int, str]]:
+        self, anns: list[ImageAnnotation], image_width: float, image_height: float, image_id: Optional[str] = None
+    ) -> Sequence[tuple[int, str]]:
         """
         Determining a text ordering of text blocks. These text blocks should be larger sections than barely words.
         It will first try to detect columns, then try to consolidate columns and finally try to detecting connected
@@ -181,12 +183,12 @@ class OrderGenerator:
         :param image_width: image width (to re-calculate bounding boxes into relative coords)
         :param image_height: image height (to re-calculate bounding boxes into relative coords)
         :param image_id: image id
-        :return: List of tuples with reading order position and `annotation_id`
+        :return: list of tuples with reading order position and `annotation_id`
         """
         if not anns:
             return []
         reading_blocks = []
-        columns: List[BoundingBox] = []
+        columns: list[BoundingBox] = []
         anns.sort(
             key=lambda x: (
                 x.bounding_box.transform(image_width, image_height).cy,  # type: ignore
@@ -267,7 +269,7 @@ class OrderGenerator:
         blocks.sort(key=lambda x: x[0])  # type: ignore
         sorted_blocks = []
         max_block_number = max(list(columns_dict.values()))
-        filtered_blocks: Sequence[Tuple[int, str]]
+        filtered_blocks: Sequence[tuple[int, str]]
         for idx in range(max_block_number + 1):
             filtered_blocks = list(filter(lambda x: x[0] == idx, blocks))  # type: ignore # pylint: disable=W0640
             sorted_blocks.extend(self._sort_anns_grouped_by_blocks(filtered_blocks, anns, image_width, image_height))
@@ -286,7 +288,7 @@ class OrderGenerator:
             )
         return reading_blocks
 
-    def _consolidate_columns(self, columns: List[BoundingBox]) -> Dict[int, int]:
+    def _consolidate_columns(self, columns: list[BoundingBox]) -> dict[int, int]:
         if not columns:
             return {}
         np_boxes = np.array([col.to_list(mode="xyxy") for col in columns])
@@ -307,8 +309,8 @@ class OrderGenerator:
 
     @staticmethod
     def _sort_anns_grouped_by_blocks(
-        block: Sequence[Tuple[int, str]], anns: Sequence[ImageAnnotation], image_width: float, image_height: float
-    ) -> List[Tuple[int, str]]:
+        block: Sequence[tuple[int, str]], anns: Sequence[ImageAnnotation], image_width: float, image_height: float
+    ) -> list[tuple[int, str]]:
         if not block:
             return []
         anns_and_blocks_numbers = list(zip(*block))
@@ -326,14 +328,14 @@ class OrderGenerator:
     @staticmethod
     def _make_column_detect_results(columns: Sequence[BoundingBox]) -> Sequence[DetectionResult]:
         column_detect_result_list = []
-        if os.environ.get("LOG_LEVEL") == "DEBUG":
+        if os.environ.get("LOG_LEVEL", "INFO") == "DEBUG":
             for box in columns:
                 column_detect_result_list.append(
                     DetectionResult(
                         box=box.to_list(mode="xyxy"),
                         absolute_coords=box.absolute_coords,
                         class_id=99,
-                        class_name=LayoutType.column,
+                        class_name=LayoutType.COLUMN,
                     )
                 )
         return column_detect_result_list
@@ -349,10 +351,11 @@ class TextLineGenerator:
         self, make_sub_lines: bool, line_category_id: Union[int, str], paragraph_break: Optional[float] = None
     ):
         """
-        :param make_sub_lines: Whether to build sub lines from lines
+        :param make_sub_lines: Whether to build sub lines from lines.
         :param line_category_id: category_id to give a text line
-        :param paragraph_break: threshold of two consecutive words. If distance is larger than threshold, two sublines
-                                will be built
+        :param paragraph_break: threshold of two consecutive words. If distance is larger than threshold, two sub-lines
+                                will be built. We use relative coordinates to calculate the distance between two
+                                consecutive words. A reasonable value is 0.035
         """
         if make_sub_lines and paragraph_break is None:
             raise ValueError("You must specify paragraph_break when setting make_sub_lines to True")
@@ -360,10 +363,10 @@ class TextLineGenerator:
         self.make_sub_lines = make_sub_lines
         self.paragraph_break = paragraph_break
 
-    def _make_detect_result(self, box: BoundingBox, relationships: Dict[str, List[str]]) -> DetectionResult:
+    def _make_detect_result(self, box: BoundingBox, relationships: dict[str, list[str]]) -> DetectionResult:
         return DetectionResult(
             box=box.to_list(mode="xyxy"),
-            class_name=LayoutType.line,
+            class_name=LayoutType.LINE,
             class_id=self.line_category_id,
             absolute_coords=box.absolute_coords,
             relationships=relationships,
@@ -375,6 +378,7 @@ class TextLineGenerator:
         image_width: float,
         image_height: float,
         image_id: Optional[str] = None,
+        highest_level: bool = True,
     ) -> Sequence[DetectionResult]:
         """
         Creating detecting result of lines (or sub lines) from given word type `ImageAnnotation`.
@@ -392,6 +396,8 @@ class TextLineGenerator:
         # list of  (word index, text line, word annotation_id)
         word_order_list = OrderGenerator.group_words_into_lines(word_anns, image_id)
         number_rows = max(word[1] for word in word_order_list)
+        if number_rows == 1 and not highest_level:
+            return []
         detection_result_list = []
         for number_row in range(1, number_rows + 1):
             # list of  (word index, text line, word annotation_id) for text line equal to number_row
@@ -423,29 +429,139 @@ class TextLineGenerator:
                     if current_box.absolute_coords:
                         current_box = current_box.transform(image_width, image_height)
 
-                    # If distance between boxes is lower than paragraph break, same sub line
+                    # If distance between boxes is lower than paragraph break, same sub-line
                     if current_box.ulx - prev_box.lrx < self.paragraph_break:  # type: ignore
                         sub_line.append(ann)
                         sub_line_ann_ids.append(ann.annotation_id)
                     else:
-                        boxes = [ann.get_bounding_box(image_id) for ann in sub_line]
-                        merge_box = merge_boxes(*boxes)
-                        detection_result = self._make_detect_result(merge_box, {"child": sub_line_ann_ids})
-                        detection_result_list.append(detection_result)
-                        sub_line = [ann]
-                        sub_line_ann_ids = [ann.annotation_id]
+                        # We need to iterate maybe more than one time, because sub-lines may have more than one line
+                        # if having been split. Take fore example a multi-column layout where a sub-line has
+                        # two lines because of a column break and fonts twice as large as the other column.
+                        detection_results = self.create_detection_result(
+                            sub_line, image_width, image_height, image_id, False
+                        )
+                        if detection_results:
+                            detection_result_list.extend(detection_results)
+                        else:
+                            boxes = [ann.get_bounding_box(image_id) for ann in sub_line]
+                            merge_box = merge_boxes(*boxes)
+                            detection_result = self._make_detect_result(merge_box, {"child": sub_line_ann_ids})
+                            detection_result_list.append(detection_result)
+                            sub_line = [ann]
+                            sub_line_ann_ids = [ann.annotation_id]
 
                     if idx == len(anns_per_row) - 1:
-                        boxes = [ann.get_bounding_box(image_id) for ann in sub_line]
-                        merge_box = merge_boxes(*boxes)
-                        detection_result = self._make_detect_result(merge_box, {"child": sub_line_ann_ids})
-                        detection_result_list.append(detection_result)
+                        detection_results = self.create_detection_result(
+                            sub_line, image_width, image_height, image_id, False
+                        )
+                        if detection_results:
+                            detection_result_list.extend(detection_results)
+                        else:
+                            boxes = [ann.get_bounding_box(image_id) for ann in sub_line]
+                            merge_box = merge_boxes(*boxes)
+                            detection_result = self._make_detect_result(merge_box, {"child": sub_line_ann_ids})
+                            detection_result_list.append(detection_result)
 
         return detection_result_list
 
 
+class TextLineServiceMixin(PipelineComponent, ABC):
+    """
+    This class is used to create text lines similar to TextOrderService.
+    It uses the logic of the TextOrderService but modifies it to suit its needs.
+    It specifically uses the _create_lines_for_words method and modifies the serve method.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        line_category_id: int = 1,
+        include_residual_text_container: bool = True,
+        paragraph_break: Optional[float] = None,
+    ):
+        """
+        Initialize the TextLineService with a line_category_id and a TextLineGenerator instance.
+        """
+        self.line_category_id = line_category_id
+        self.include_residual_text_container = include_residual_text_container
+        self.text_line_generator = TextLineGenerator(
+            self.include_residual_text_container, self.line_category_id, paragraph_break
+        )
+        super().__init__(name)
+
+    def _create_lines_for_words(self, word_anns: Sequence[ImageAnnotation]) -> Sequence[ImageAnnotation]:
+        """
+        This method creates lines for words using the TextLineGenerator instance.
+        """
+        detection_result_list = self.text_line_generator.create_detection_result(
+            word_anns,
+            self.dp_manager.datapoint.width,
+            self.dp_manager.datapoint.height,
+            self.dp_manager.datapoint.image_id,
+        )
+        line_anns = []
+        for detect_result in detection_result_list:
+            ann_id = self.dp_manager.set_image_annotation(detect_result)
+            if ann_id:
+                line_ann = self.dp_manager.get_annotation(ann_id)
+                child_ann_id_list = detect_result.relationships["child"]  # type: ignore
+                for child_ann_id in child_ann_id_list:
+                    line_ann.dump_relationship(Relationships.CHILD, child_ann_id)
+                line_anns.append(line_ann)
+        return line_anns
+
+
+class TextLineService(TextLineServiceMixin):
+    """
+    Some OCR systems do not identify lines of text but only provide text boxes for words. This is not sufficient
+    for certain applications. This service determines rule-based text lines based on word boxes. One difficulty is
+    that text lines are not continuous but are interrupted, for example in multi-column layouts.
+    These interruptions are taken into account insofar as the gap between two words on almost the same page height
+    must not be too large.
+
+    The service constructs new ImageAnnotation of the category `LayoutType.line` and forms relations between the
+    text lines and the words contained in the text lines. The reading order is not arranged.
+    """
+
+    def __init__(self, line_category_id: int = 1, paragraph_break: Optional[float] = None):
+        """
+        Initialize `TextLineService`
+
+        :param line_category_id: category_id to give a text line
+        :param paragraph_break: threshold of two consecutive words. If distance is larger than threshold, two sublines
+                                will be built
+        """
+        super().__init__(
+            name="text_line",
+            line_category_id=line_category_id,
+            include_residual_text_container=True,
+            paragraph_break=paragraph_break,
+        )
+
+    def clone(self) -> TextLineService:
+        """
+        This method returns a new instance of the class with the same configuration.
+        """
+        return self.__class__(self.line_category_id, self.text_line_generator.paragraph_break)
+
+    def serve(self, dp: Image) -> None:
+        text_container_anns = dp.get_annotation(category_names=LayoutType.WORD)
+        self._create_lines_for_words(text_container_anns)
+
+    def get_meta_annotation(self) -> MetaAnnotation:
+        """
+        This method returns metadata about the annotations created by this pipeline component.
+        """
+        return MetaAnnotation(
+            image_annotations=(LayoutType.LINE,),
+            sub_categories={LayoutType.LINE: {Relationships.CHILD}},
+            relationships={},
+            summaries=(),
+        )
+
+
 @pipeline_component_registry.register("TextOrderService")
-class TextOrderService(PipelineComponent):
+class TextOrderService(TextLineServiceMixin):
     """
     Reading order of words within floating text blocks as well as reading order of blocks within simple text blocks.
     To understand the difference between floating text blocks and simple text blocks consider a page containing an
@@ -470,7 +586,8 @@ class TextOrderService(PipelineComponent):
     A category annotation per word is generated, which fixes the order per word in the block, as well as a category
     annotation per block, which saves the reading order of the block per page.
 
-    The blocks are defined in `_floating_text_block_names` and text blocks in `_floating_text_block_names`.
+    The blocks are defined in `text_block_categories` and text blocks that should be considered when generating
+    narrative text must be added in `floating_text_block_categories`.
 
         order = TextOrderService(text_container="word",
                                  text_block_categories=["title", "text", "list", "cell",
@@ -517,23 +634,28 @@ class TextOrderService(PipelineComponent):
         """
         self.text_container = get_type(text_container)
         if isinstance(text_block_categories, (str, ObjectTypes)):
-            text_block_categories = [text_block_categories]
+            text_block_categories = (get_type(text_block_categories),)
         if text_block_categories is None:
             text_block_categories = IMAGE_DEFAULTS["text_block_categories"]
-        self.text_block_categories = [get_type(category) for category in text_block_categories]
+        self.text_block_categories = tuple((get_type(category) for category in text_block_categories))
         if isinstance(floating_text_block_categories, (str, ObjectTypes)):
-            floating_text_block_categories = [floating_text_block_categories]
+            floating_text_block_categories = (get_type(floating_text_block_categories),)
         if floating_text_block_categories is None:
             floating_text_block_categories = IMAGE_DEFAULTS["floating_text_block_categories"]
-        self.floating_text_block_categories = [get_type(category) for category in floating_text_block_categories]
+        self.floating_text_block_categories = tuple((get_type(category) for category in floating_text_block_categories))
         if include_residual_text_container:
-            self.floating_text_block_categories.append(LayoutType.line)
+            self.floating_text_block_categories = self.floating_text_block_categories + (LayoutType.LINE,)
         self.include_residual_text_container = include_residual_text_container
         self.order_generator = OrderGenerator(starting_point_tolerance, broken_line_tolerance, height_tolerance)
         self.text_line_generator = TextLineGenerator(
             self.include_residual_text_container, line_category_id, paragraph_break
         )
-        super().__init__("text_order")
+        super().__init__(
+            name="text_order",
+            line_category_id=line_category_id,
+            include_residual_text_container=include_residual_text_container,
+            paragraph_break=paragraph_break,
+        )
         self._init_sanity_checks()
 
     def serve(self, dp: Image) -> None:
@@ -541,12 +663,12 @@ class TextOrderService(PipelineComponent):
         text_block_anns = dp.get_annotation(category_names=self.text_block_categories)
         if self.include_residual_text_container:
             mapped_text_container_ids = list(
-                chain(*[text_block.get_relationship(Relationships.child) for text_block in text_block_anns])
+                chain(*[text_block.get_relationship(Relationships.CHILD) for text_block in text_block_anns])
             )
             residual_text_container_anns = [
                 ann for ann in text_container_anns if ann.annotation_id not in mapped_text_container_ids
             ]
-            if self.text_container == LayoutType.word:
+            if self.text_container == LayoutType.WORD:
                 text_block_anns.extend(self._create_lines_for_words(residual_text_container_anns))
             else:
                 text_block_anns.extend(residual_text_container_anns)
@@ -564,27 +686,9 @@ class TextOrderService(PipelineComponent):
                 annotation_id = self.dp_manager.set_image_annotation(detect_result)
                 if annotation_id:
                     self.dp_manager.set_category_annotation(
-                        Relationships.reading_order, idx, Relationships.reading_order, annotation_id
+                        Relationships.READING_ORDER, idx, Relationships.READING_ORDER, annotation_id
                     )
 
-    def _create_lines_for_words(self, word_anns: Sequence[ImageAnnotation]) -> Sequence[ImageAnnotation]:
-        detection_result_list = self.text_line_generator.create_detection_result(
-            word_anns,
-            self.dp_manager.datapoint.width,
-            self.dp_manager.datapoint.height,
-            self.dp_manager.datapoint.image_id,
-        )
-        line_anns = []
-        for detect_result in detection_result_list:
-            ann_id = self.dp_manager.set_image_annotation(detect_result)
-            if ann_id:
-                line_ann = self.dp_manager.get_annotation(ann_id)
-                child_ann_id_list = detect_result.relationships["child"]  # type: ignore
-                for child_ann_id in child_ann_id_list:
-                    line_ann.dump_relationship(Relationships.child, child_ann_id)
-                line_anns.append(line_ann)
-        return line_anns
-
     def order_text_in_text_block(self, text_block_ann: ImageAnnotation) -> None:
         """
         Order text within a text block. It will take all child-like text containers (determined by a
@@ -592,11 +696,11 @@ class TextOrderService(PipelineComponent):
 
         :param text_block_ann: text block annotation (category one of `text_block_categories`).
         """
-        text_container_ids = text_block_ann.get_relationship(Relationships.child)
+        text_container_ids = text_block_ann.get_relationship(Relationships.CHILD)
         text_container_ann = self.dp_manager.datapoint.get_annotation(
             annotation_ids=text_container_ids, category_names=self.text_container
         )
-        if self.text_container == LayoutType.word:
+        if self.text_container == LayoutType.WORD:
             word_order_list = self.order_generator.group_words_into_lines(
                 text_container_ann, self.dp_manager.datapoint.image_id
             )
@@ -606,10 +710,10 @@ class TextOrderService(PipelineComponent):
             )
         for word_order in word_order_list:
             self.dp_manager.set_category_annotation(
-                Relationships.reading_order, word_order[0], Relationships.reading_order, word_order[2]
+                Relationships.READING_ORDER, word_order[0], Relationships.READING_ORDER, word_order[2]
             )
 
-    def order_blocks(self, text_block_anns: List[ImageAnnotation]) -> None:
+    def order_blocks(self, text_block_anns: list[ImageAnnotation]) -> None:
         """
         Ordering of text blocks. Will use the internal order generator.
 
@@ -620,42 +724,40 @@ class TextOrderService(PipelineComponent):
         )
         for word_order in block_order_list:
             self.dp_manager.set_category_annotation(
-                Relationships.reading_order, word_order[0], Relationships.reading_order, word_order[1]
+                Relationships.READING_ORDER, word_order[0], Relationships.READING_ORDER, word_order[1]
             )
 
     def _init_sanity_checks(self) -> None:
-        assert self.text_container in (LayoutType.word, LayoutType.line), (
-            f"text_container must be either {LayoutType.word} or " f"{LayoutType.line}"
+        assert self.text_container in (LayoutType.WORD, LayoutType.LINE), (
+            f"text_container must be either {LayoutType.WORD} or " f"{LayoutType.LINE}"
         )
         add_category = []
         if self.include_residual_text_container:
-            add_category.append(LayoutType.line)
+            add_category.append(LayoutType.LINE)
 
         assert set(self.floating_text_block_categories) <= set(
-            self.text_block_categories + add_category  # type: ignore
+            self.text_block_categories + tuple(add_category)
         ), "floating_text_block_categories must be a subset of text_block_categories"
 
-    def get_meta_annotation(self) -> JsonDict:
+    def get_meta_annotation(self) -> MetaAnnotation:
         add_category = [self.text_container]
-        image_annotations = []
-        if self.include_residual_text_container and self.text_container == LayoutType.word:
-            add_category.append(LayoutType.line)
-            image_annotations.append(LayoutType.line)
+        image_annotations: list[ObjectTypes] = []
+        if self.include_residual_text_container and self.text_container == LayoutType.WORD:
+            add_category.append(LayoutType.LINE)
+            image_annotations.append(LayoutType.LINE)
         anns_with_reading_order = list(copy(self.floating_text_block_categories)) + add_category
-        return dict(
-            [
-                ("image_annotations", image_annotations),
-                ("sub_categories", {category: {Relationships.reading_order} for category in anns_with_reading_order}),
-                ("relationships", {}),
-                ("summaries", []),
-            ]
+        return MetaAnnotation(
+            image_annotations=tuple(image_annotations),
+            sub_categories={category: {Relationships.READING_ORDER} for category in anns_with_reading_order},
+            relationships={},
+            summaries=(),
         )
 
-    def clone(self) -> PipelineComponent:
+    def clone(self) -> TextOrderService:
         return self.__class__(
-            copy(self.text_container),
-            copy(self.text_block_categories),
-            copy(self.floating_text_block_categories),
+            self.text_container,
+            self.text_block_categories,
+            self.floating_text_block_categories,
             self.include_residual_text_container,
             self.order_generator.starting_point_tolerance,
             self.order_generator.broken_line_tolerance,
@@ -663,3 +765,6 @@ class TextOrderService(PipelineComponent):
             self.text_line_generator.paragraph_break,
             self.text_line_generator.line_category_id,
         )
+
+    def clear_predictor(self) -> None:
+        pass