groupmes 1.1.0__py3-none-any.whl

groupmes/__init__.py

@@ -0,0 +1,21 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================
+
+import importlib.metadata
+import os
+
+import groupmes.__patch__
+import groupmes.path
+
+PACKAGE = 'groupmes'
+PROCESS = 'groupme'
+
+__version__ = importlib.metadata.version(PACKAGE)
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))

groupmes/__main__.py

@@ -0,0 +1,13 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================
+
+from groupmes.cli import main
+
+if __name__ == "__main__":
+    main()

groupmes/__patch__.py

@@ -0,0 +1,8 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================

groupmes/border/__init__.py

@@ -0,0 +1,8 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================

groupmes/border/leftright.py

@@ -0,0 +1,315 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""left right border detector
+==========================
+
+The `left right border detector` (lrbd) separates borders of the left
+and right page especially which are used for books with different border
+width for left and right page.
+
+As a result of different left-right borders there are alternating border
+widths which we detect. On single pages there are no alternating border.
+
+Currently there are two strategies to detect different page border:
+
+* simple approach
+* raising edge
+
+Simple Approach
+~~~~~~~~~~~~~~~
+
+Some documents have exceptions on some pages. We handle this via allowed
+errors defined with the HolyValues `FIRSTSECOND_ERROR_COUNT_MAX` and
+`MIXED_ERROR_MIN`.
+
+Raising Edge
+~~~~~~~~~~~~
+
+The change of the text feed creates an raising edge between pages. This
+approach detects these edges to determine left and right page border
+width.
+"""
+
+import dataclasses
+import math
+import statistics
+import typing
+
+import configos
+import iamraw
+import utilo
+
+# max diff to match in common group.
+SIDE_DIFF_MAX = configos.HV_INT_PLUS(default=2)
+# exceptions which are allowed cause of user defined error.
+FIRSTSECOND_ERROR_COUNT_MAX = configos.HolyTable(
+    items=(
+        (2, 0.5),
+        (3, 0.35),
+        (5, 0.26),
+        (10, .21),
+        (15, 0.05),
+        (200, 0.01),
+    ),
+    right_outranges_none=False,
+)
+# errors which are a result of handle alternating border as single border.
+MIXED_ERROR_MIN = configos.HV_PERCENT_PLUS(default=15)
+# area where left border can be located.
+LEFT_PERCENT = configos.HV_PERCENT_PLUS(default=30)
+# area where right border can be located.
+RIGHT_PERCENT = configos.HV_PERCENT_PLUS(default=30)
+
+RAISING_EDGE_MIN = configos.HV_PERCENT_PLUS(default=75)
+
+# TODO: SHOULD WE DISABLE ALGO ON BIG FAIL COUNT?
+RAISING_FAILRATE = configos.HolyTable(
+    items=(
+        (5, 1 / 5),
+        (7, 2 / 7),
+        (10, 3 / 10),
+        (40, 10 / 40),
+        (200, 40 / 200),
+    ),
+    right_outranges_none=False,
+)
+
+LeftRight = tuple[float, float]
+
+DetectedBorder = typing.TypeVar('DetectedBorder', tuple[float], float)
+
+
+@dataclasses.dataclass
+class LeftRightDetected:
+    left: DetectedBorder = None
+    right: DetectedBorder = None
+    confidence: float = 0.0
+
+    @property
+    def valid(self):
+        return isinstance(self.left, tuple)
+
+
+def run(
+    textpositions: iamraw.PageContentTextPositions,
+    pagesizes: iamraw.PageSizeBorderList,
+) -> LeftRightDetected:
+    """Run LeftRight-Strategy to determine that document contains
+    special leftright-border or a normal equal border for every page."""
+    left, right = determine_pageborder(textpositions, pagesizes)
+    left, right = handle_emptypage(left, right)
+
+    result = simple(left, right)
+    if result:
+        return result
+
+    result = raising(left, right)
+    if result:
+        return result
+
+    leftborder = utilo.mode(left, minimize=True)
+    rightborder = utilo.mode(right, minimize=False)
+    return LeftRightDetected(
+        left=leftborder,
+        right=rightborder,
+        confidence=1.0,
+    )
+
+
+def simple(left: utilo.Numbers, right: utilo.Numbers) -> LeftRightDetected:
+    """Determine LeftRight border based on changing text feed. We use
+    the even numbers to determine the left page and the odd numbers to
+    determine the right text feed.
+
+    This appraoch is limited when one page was missing and therefore
+    these pages are mixed and therefore wrong classified.
+
+    Args:
+        left: ordered list of left text feed
+        right: ending of right text content border
+    Returns:
+        None if no valid LeftRightDetected was detected
+        LeftRightDetected if failrate is not to hight
+    """
+    if len(left) <= 1:
+        # TODO: INVESTIGATE HERE
+        utilo.error('could not run simple approach')
+        return None
+    mixed = utilo.diff_mode(left, max_diff=SIDE_DIFF_MAX)
+    # first side
+    first = left[::2]
+    first_matched = utilo.diff_mode(first, max_diff=SIDE_DIFF_MAX)
+    # second side
+    second = left[1::2]
+    second_matched = utilo.diff_mode(second, max_diff=SIDE_DIFF_MAX)
+
+    mixed_error = 1 - len(mixed) / len(left)
+    first_error = 1 - len(first_matched) / len(first)
+    second_error = 1 - len(second_matched) / len(second)
+
+    utilo.debug(f'mixed: {mixed_error}')
+    utilo.debug(f'first: {first_error}')
+    utilo.debug(f'second: {second_error}')
+
+    # left right
+    # TODO: DEFINE BETTER CONFIDENCE APPROACH
+    max_firstsecond_error = FIRSTSECOND_ERROR_COUNT_MAX(len(first))
+    if mixed_error > MIXED_ERROR_MIN.value and all([
+            first_error < max_firstsecond_error,
+            second_error < max_firstsecond_error,
+    ]):
+        leftborder = (
+            utilo.mode(first, minimize=True),
+            utilo.mode(second, minimize=True),
+        )
+        rightborder = (
+            utilo.mode(right[::2]),
+            utilo.mode(right[1::2]),
+        )
+        return LeftRightDetected(
+            left=leftborder,
+            right=rightborder,
+            confidence=1.0,
+        )
+    return None
+
+
+def raising(left: utilo.Numbers, right: utilo.Numbers) -> LeftRightDetected:
+    """Determine border depending on changing text feed on left page
+    border.
+
+    This approach has no problems when one or more leftright pages are
+    missing. The limit of problems is defined in lookup table
+    `RAISING_FAILRATE`.
+
+    Args:
+        left: ordered list of left text feed
+        right: ending of right text content border
+    Returns:
+        None if no valid LeftRightDetected was detected
+        LeftRightDetected if `failrate` is not too high
+    """
+    longest_left = longest_two(left)
+    longest_right = longest_two(right)
+
+    if longest_left is None or longest_right is None:
+        # single page document which does not contain left-right-pages
+        return None
+
+    first_left = statistics.mean(longest_left[0])
+    second_left = statistics.mean(longest_left[1])
+    edge = math.fabs(first_left - second_left)
+
+    first_right = statistics.mean(longest_right[0])
+    second_right = statistics.mean(longest_right[1])
+
+    edges = utilo.diffs(left)
+    failures = [
+        index for index, item in enumerate(edges)
+        if item < edge * RAISING_EDGE_MIN.value
+    ]
+    failrate = len(failures) / len(edges)
+    max_failrate = RAISING_FAILRATE(len(edges))
+
+    if failrate > max_failrate:
+        return None
+
+    first_left, second_left = utilo.roundme(first_left, second_left)
+    first_right, second_right = utilo.roundme(first_right, second_right)
+
+    leftborder = (
+        min([first_left, second_left]),
+        max([first_left, second_left]),
+    )
+    rightborder = (
+        min([first_right, second_right]),
+        max([first_right, second_right]),
+    )
+
+    return LeftRightDetected(
+        left=leftborder,
+        right=rightborder,
+        confidence=1.0,
+    )
+
+
+def handle_emptypage(left, right):
+    # TODO: THINK ABOUT IF THIS IS ENOUGH
+    # ignore empty pages
+    left_none = 0.0
+    left = [item if item is not None else left_none for item in left]
+    # TODO: Is default=0 a good one?
+    right_none = max((item for item in right if item is not None), default=0)
+    # NOTE: Determine more pages as large than it realy are - is this a
+    # problem?
+    right = [item if item is not None else right_none for item in right]
+    return left, right
+
+
+def determine_pageborder(textpositions, pagesizes):
+    left = []
+    right = []
+    before = -1
+    for current, (page, size) in utilo.sync_pages([textpositions, pagesizes]):
+        assert current > before, f'{before} < {current}'
+        before = current
+        if not page or not size:
+            left.append(None)
+            right.append(None)
+            continue
+        bounding = [item for item, _ in page.content.values()]
+        leftright = maximize_leftright(bounding, size)
+        left.append(leftright[0])
+        right.append(leftright[1])
+    return left, right
+
+
+def maximize_leftright(
+    boundings: utilo.Rectangles,
+    size: iamraw.PageSizeBorder,
+) -> LeftRight:
+    """Determine the left and right border of a page based on `mode`
+    selection in `size`.
+
+    Minimize the left and maximize the right position. The area where
+    mode is used to determine the most common border which is assumed as
+    correct border is limit by `size` configuration.
+
+    Args:
+        boundings: textpositions of defined page
+        size: width and height of current page
+    Returns:
+        tuple with left and right content bounding
+    """
+    left_max = size.size.width * LEFT_PERCENT
+    right_min = size.size.width * (1 - RIGHT_PERCENT)
+    left_max, right_min = utilo.roundme(left_max, right_min)
+    assert left_max <= right_min, 'left and right bounds are flipped'
+    left = [item[0] for item in boundings if item[0] <= left_max]
+    right = [item[2] for item in boundings if item[2] >= right_min]
+    # TODO: DO WE RELAY NEED THIS?
+    left = utilo.mode(left, minimize=True) if left else 0.0
+    right = utilo.mode(right, minimize=False) if right else size.size.width
+    return left, right
+
+
+CLUSTER_CANDIAT_DIFF_MAX = configos.HV_FLOAT_PLUS(default=2.0)
+
+
+def longest_two(items: utilo.Numbers) -> tuple[float, float]:
+
+    def close(candidat, clusteritem) -> bool:
+        diff = math.fabs(candidat - clusteritem)
+        return diff < CLUSTER_CANDIAT_DIFF_MAX
+
+    clustered = utilo.determine_cluster(items, close)
+    result = sorted(clustered, key=len, reverse=True)
+    if len(result) < 2:
+        return None
+    return result[0], result[1]

groupmes/border/most.py

@@ -0,0 +1,70 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import dataclasses
+import math
+
+import iamraw
+import utilo
+
+
+@dataclasses.dataclass
+class MostBoundingDetected:
+    left: float = None
+    right: float = None
+    top: float = None
+    bottom: float = None
+    confidence: float = 0.0
+
+
+def run(sizeandborder: iamraw.PageSizeBorderList) -> MostBoundingDetected:
+    borders = [item.border for item in sizeandborder]
+    most = most_boundingbox(borders)
+    # x0, y0, x1, y1
+    # left right
+    assert most[0] < most[1]
+    # top bottom
+    assert most[2] < most[3]
+    result = MostBoundingDetected(
+        left=most[0],
+        right=most[1],
+        top=most[2],
+        bottom=most[3],
+    )
+    return result
+
+
+def most_boundingbox(
+    boxes: utilo.Rectangles,
+    roundme: bool = False,
+) -> utilo.Rectangle:
+    """Extract bounding box of most common occurence for every side.
+
+    Round detected boundingbox to full number to make approach more
+    robust. Round numbers in direction to the border end to increase the
+    detected rectangle that fit more items in it.
+    """
+    # TODO: Think about right and left, maybe search for the top 2 borders?
+    # Filter None entries
+    # left, right, top, bottom
+    rounding = [math.floor, math.ceil, math.floor, math.ceil]
+    result = []
+    for index, method in enumerate(rounding):
+        # remove None items
+        filtered = [item[index] for item in boxes if item[index] is not None]
+        # round to have a more robust grouping
+        rounded = [method(item) for item in filtered] if roundme else filtered
+        # determine most occured border to determine them as required border
+        # support multiple border options.
+        minimize = method is math.floor
+        # TODO: REQUIRE A BETTER TY-BREAKER
+        mode = utilo.mode(rounded, minimize=minimize)
+        result.append(mode)
+    # (x0, y0, x1, y1)
+    return tuple(result)

groupmes/cli.py

@@ -0,0 +1,81 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================
+
+import utilo
+
+import groupmes
+
+DESCRIPTION = 'TODO'
+
+WORKPLAN = [
+    utilo.create_step(
+        'area',
+        inputs=[
+            utilo.ResultFile(producer='rawmaker', name='boxes_boxes'),
+            utilo.ResultFile(producer='tablero',
+                             name='result_result',
+                             optional=True),
+            utilo.ResultFile(producer='rawmaker', name='text_text'),
+            utilo.ResultFile(producer='rawmaker', name='text_positions'),
+        ],
+        output=('area',),
+    ),
+    utilo.create_step(
+        'border',
+        inputs=[
+            utilo.ResultFile(producer='rawmaker', name='border_pages'),
+            utilo.ResultFile(producer='rawmaker', name='text_positions'),
+        ],
+        output=('leftright',),
+    ),
+    utilo.create_step(
+        'distance',
+        inputs=[
+            utilo.ResultFile(producer=groupmes.PROCESS, name='area_area'),
+            utilo.ResultFile(producer='rawmaker', name='text_text'),
+            utilo.ResultFile(producer='rawmaker', name='text_positions'),
+        ],
+        output=('distance',),
+    ),
+    utilo.create_step(
+        'content',
+        inputs=[
+            utilo.ResultFile(producer='rawmaker', name='text_text'),
+            utilo.ResultFile(producer='rawmaker', name='text_positions'),
+            utilo.ResultFile(producer='rawmaker', name='border_pages'),
+            utilo.ResultFile(producer='footnote', name='result_result'),
+        ],
+        output=('content',),
+    ),
+    utilo.create_step(
+        'hefopa',
+        inputs=[
+            utilo.ResultFile(producer='headnote', name='result_result'),
+            utilo.ResultFile(producer='footnote', name='result_result'),
+            utilo.ResultFile(producer='pagenumber', name='result_result'),
+            utilo.ResultFile(producer='rawmaker', name='border_pages'),
+        ],
+        output=('result',),
+    )
+]
+
+
+def main():
+    utilo.featurepack(
+        workplan=WORKPLAN,
+        root=groupmes.ROOT,
+        featurepackage='groupmes.feature',
+        config=utilo.FeaturePackConfig(
+            description=DESCRIPTION,
+            multiprocessed=True,
+            name=groupmes.PROCESS,
+            pages=True,
+            version=groupmes.__version__,
+        ),
+    )

groupmes/feature/__init__.py

@@ -0,0 +1,8 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================

groupmes/feature/area.py

@@ -0,0 +1,221 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""Textual Area
+============
+
+TODO:
+ * table of content
+ * images
+ * reference table
+
+"""
+
+import collections
+import os
+
+import configos
+import serializeraw
+import utilo
+
+RECTANGLE_DIFF_MAX = configos.HV_FLOAT_PLUS(default=10.0)
+
+RequiredResources = collections.namedtuple(
+    'RequiredResources',
+    'textnavigator, tables, boxes',
+)
+
+PageContentTextualArea = collections.namedtuple(
+    'PageContentTextualArea',
+    'page, textual, outside, border',
+)
+PageContentTextualAreas = list[PageContentTextualArea]
+
+
+def work(
+    boxes: str,
+    tables: str,
+    text: str,
+    textpositions: str,
+    pages: tuple = None,
+) -> str:
+    """Extract different areas out of given data.
+
+    Args:
+        boxes(str): path to extract `rawmaker` content boxes
+        tables(str): path to extracted `tablero` tables
+        text(str): extracted `rawmaker` text
+        textpositions(str): positions of extracted text
+        pages(tuple): tuple of pages to process
+    Returns:
+        Dumped extracted areas.
+    """
+    loaded = load(
+        boxes=boxes,
+        pages=pages,
+        tables=tables,
+        text=text,
+        textpositions=textpositions,
+    )
+
+    grouped = group_areas(loaded=loaded)
+
+    dumped = dump_area(grouped)
+    return dumped
+
+
+def group_areas(loaded: RequiredResources) -> PageContentTextualAreas:
+    result = []
+    for navigator in loaded.textnavigator:
+        page = navigator.page
+
+        tables = utilo.select_page(loaded.tables, page)
+
+        boxes = utilo.select_page(loaded.boxes, page)
+        boxes = boxes.content if boxes else None
+
+        grouped = group_page(navigator, tables=tables, boxes=boxes)
+        result.append(grouped)
+    return result
+
+
+def group_page(navigator, tables, boxes) -> PageContentTextualArea:
+    if tables:
+        tables = table_checker(tables)
+
+    if boxes:
+        boxes = boxed_checker(boxes)
+
+    textual = []
+    inside_tables = []
+    inside_boxes = []
+    for text in navigator:
+        bounding = tuple(text.bounding)
+        if tables and tables.contains(*bounding):
+            inside_tables.append(bounding)
+        if boxes and boxes.contains(*bounding):
+            inside_boxes.append(bounding)
+        else:
+            textual.append(bounding)
+
+    # optimize rectangles
+    textual = utilo.rect_merge(textual)
+    inside_tables = utilo.rect_merge(inside_tables)
+    inside_boxes = utilo.rect_merge(inside_boxes)
+    outside = {
+        'boxes': inside_boxes,
+        'tables': inside_tables,
+    }
+    border = {
+        key: list(value) for key, value in (
+            ('boxes', boxes.content if boxes else []),
+            ('tables', tables.content if tables else []),
+        )
+    }
+    result = PageContentTextualArea(
+        page=navigator.page,
+        textual=textual,
+        outside=outside,
+        border=border,
+    )
+    return result
+
+
+def boxed_checker(items) -> utilo.RectangleCheck:
+    result = utilo.RectangleCheck(max_diff=RECTANGLE_DIFF_MAX)
+    for item in items:
+        result.extend(*item.box)
+    return result
+
+
+def table_checker(items) -> utilo.RectangleCheck:
+    result = utilo.RectangleCheck(max_diff=RECTANGLE_DIFF_MAX)
+    for item in items:
+        result.extend(*item.bounding)
+    return result
+
+
+def load(
+    boxes: str,
+    tables: str,
+    text: str,
+    textpositions: str,
+    pages: tuple = None,
+) -> RequiredResources:
+    # TODO: SHOULD WE REMOVE HIDDEN ITEMS?
+    textnavigator = serializeraw.ptn_fromfile(
+        text=text,
+        textpositions=textpositions,
+        pages=pages,
+        state=None,  # load hidden items
+    )
+    boxes = serializeraw.load_boxes(boxes, pages=pages)
+    if os.path.exists(tables):
+        tables = serializeraw.load_tables(tables, pages=pages)
+    else:
+        utilo.log(f'skip using tablero: {tables}, generation is required')
+        tables = []
+    result = RequiredResources(
+        boxes=boxes,
+        tables=tables,
+        textnavigator=textnavigator,
+    )
+    return result
+
+
+def dump_area(items) -> str:
+    raw = []
+    for page in items:
+        outside = {
+            key: [utilo.from_tuple(item) for item in value] if value else value
+            for key, value in page.outside.items()
+        }
+        border = {
+            key: [utilo.from_tuple(item) for item in border]
+                 if border else border for key, border in page.border.items()
+        }
+        textual = page.textual
+        if textual:
+            textual = [utilo.from_tuple(item) for item in textual]
+
+        content = {
+            'border': border,
+            'outside': outside,
+            'page': page.page,
+            'textual': textual,
+        }
+        raw.append(content)
+    dumped = utilo.yaml_dump(raw)
+    return dumped
+
+
+def load_area(content: str, pages: tuple = None) -> PageContentTextualAreas:
+    loaded = utilo.yaml_load(content)
+    result = []
+    for page in loaded:
+        pagenumber = int(page['page'])
+        if utilo.should_skip(pagenumber, pages):
+            continue
+        textual = [utilo.parse_tuple(item) for item in page['textual']
+                  ] if page['textual'] else page['textual']
+        outside = {
+            key: [utilo.parse_tuple(item) for item in values] if values else
+                 values for key, values in page['outside'].items()
+        }
+        border = {
+            key: [utilo.parse_tuple(item) for item in values] if values else
+                 values for key, values in page['border'].items()
+        }
+        result.append(
+            PageContentTextualArea(
+                border=border,
+                outside=outside,
+                page=pagenumber,
+                textual=textual,
+            ))
+    return result

groupmes/feature/border.py

@@ -0,0 +1,101 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import configos
+import iamraw
+import serializeraw
+import utilo
+
+import groupmes.border.leftright
+import groupmes.border.most
+
+
+def work(
+    sizeandborder: str,
+    textpositions: str,
+    pages: tuple = None,
+) -> tuple[str]:
+    sizeandborder = serializeraw.load_pageborders(sizeandborder, pages=pages)
+    textpositions = serializeraw.load_textpositions(textpositions, pages=pages)
+
+    result = determine_border(textpositions, sizeandborder)
+
+    dumped = serializeraw.dump_leftright_border(result)
+    return dumped
+
+
+def determine_border(
+    textpositions: iamraw.PageContentTextPositions,
+    pagesizes: iamraw.PageSizeBorderList,
+):
+    clustered = pagecluster(pagesizes)
+    result = []
+    for pages_incluster in clustered:
+        border = cluster_border(textpositions, pagesizes, pages_incluster)
+        result.append(border)
+    result = utilo.flat(result)
+    # sort by page number
+    result = sorted(result, key=lambda x: x[0])
+    return result
+
+
+def cluster_border(textpositions, pagesizes, pages_incluster):
+    textpositions = utilo.select_pages(textpositions, pages_incluster)
+    pagesizes = utilo.select_pages(pagesizes, pages_incluster)
+
+    textpositions = utilo.notnone(textpositions)
+    pagesizes = utilo.notnone(pagesizes)
+
+    most = groupmes.border.most.run(pagesizes)
+    leftright = groupmes.border.leftright.run(textpositions, pagesizes)
+
+    result = [(page, *expected_border(leftright, most, pagesizes, page))
+              for page in pages_incluster]
+    return result
+
+
+def expected_border(leftright, most, pagesizes, page: int):
+    # left, right, top, down
+    # TODO: CHECK THAT PAGE CALL IS CORRECT
+    left = leftright.left
+    if isinstance(left, tuple):
+        left = left[page % 2]  # pylint:disable=E1136
+
+    right = leftright.right
+    if isinstance(right, tuple):
+        right = right[page % 2]  # pylint:disable=E1136
+
+    pagesize = utilo.select_page(pagesizes, page).size
+    rightborder = pagesize.width - right
+    bottomborder = pagesize.height - most.bottom
+
+    result = (left, rightborder, most.top, bottomborder)
+    result = utilo.roundme(result)
+    return result
+
+
+PAGE_CLUSTER_SIZE_MIN = configos.HV_INT_PLUS(default=3)
+
+PAGE_CLUSTER_DIFF_MAX = configos.HV_FLOAT_PLUS(default=10.0)
+
+
+def pagecluster(pagesizes) -> list:
+
+    def equal_size(candidat, clusteritem) -> bool:
+        diff = utilo.norms(candidat[0], clusteritem[0])
+        return diff < PAGE_CLUSTER_DIFF_MAX
+
+    grouped = utilo.determine_cluster(
+        pagesizes,
+        classifier=equal_size,
+        min_elements=PAGE_CLUSTER_SIZE_MIN,
+    )
+
+    pages = [sorted(item.page for item in cluster) for cluster in grouped]
+    return pages

groupmes/feature/content.py

@@ -0,0 +1,52 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import iamraw
+import serializeraw
+import utilo
+
+
+def work(
+    text: str,
+    textpositions: str,
+    sizeandborder: str,
+    headerfooter: str,
+    pages: tuple | None = None,
+) -> str:
+    """Extract table of content out of `document`.
+
+    Args:
+        text(str): path to load document
+        textpositions(str): path to load document textpositions
+        sizeandborder(str): path with page sizes and content border
+        headerfooter(str): path with header and footer to determine
+                           content border.
+        pages(tuple): tuple of selected pages
+    Returns:
+        dump of extracted content bounding boxes
+    """
+    navigators = serializeraw.ptcn_fromfile(
+        text,
+        textpositions,
+        sizeandborder=sizeandborder,
+        headerfooter=headerfooter,
+        pages=pages,
+    )
+    result = []
+    for page in navigators:
+        top, bottom = page.content.top, page.content.bottom
+        top, bottom = utilo.roundme((top, bottom))
+        result.append(
+            iamraw.ContentBoundingBox(
+                page=page.page,
+                top=top,
+                bottom=bottom,
+            ))
+    dumped = serializeraw.dump_contentboundingbox(result)
+    return dumped

groupmes/feature/distance.py

@@ -0,0 +1,237 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""Distance detector
+=================
+
+Compute the distance between textual and non textual elements.
+
+There are two differences for every non textual elements. The distance
+before(negative) and the distance after(positive). If the page starts or
+ends with an non textual element, the distance is None.
+
+TODO: SUPPORT LEFT RIGHT DISTANCE
+
+"""
+
+import collections
+
+import serializeraw
+import texmex
+import utilo
+
+import groupmes.feature.area
+
+RequiredResources = collections.namedtuple(
+    'RequiredResources',
+    'area, textnavigator',
+)
+
+AreaDistance = collections.namedtuple(
+    'AreaDistance',
+    'index, before, after',
+)
+AreaDistances = list[AreaDistance]
+
+PageContentAreaDistance = collections.namedtuple(
+    'PageContentAreaDistance',
+    'page, content',
+)
+PageContentAreaDistances = list[PageContentAreaDistance]
+
+
+def work(
+    areas: str,
+    text: str,
+    textpositions: str,
+    pages: tuple = None,
+) -> str:
+    loaded = load(areas, text, textpositions, pages=pages)
+    distances = determine_distances(loaded)
+    dumped = dump_distance(distances)
+    return dumped
+
+
+def determine_distances(loaded: RequiredResources) -> PageContentAreaDistances:
+    result = []
+    for navigator in loaded.textnavigator:
+        page = navigator.page
+        areas = utilo.select_page(loaded.area, page)
+        grouped = group_page(navigator, areas)
+        if not grouped:
+            continue
+        result.append(PageContentAreaDistance(content=grouped, page=page))
+    return result
+
+
+def group_page(navigator, areas) -> AreaDistances:
+    if areas is None:
+        return []
+    distance = create_distance(areas.border)
+    distances = [distance.distance(line.bounding) for line in navigator]
+
+    collected = collections.defaultdict(list)
+    for item in distances:
+        if not item:
+            continue
+        values, index = item
+        if isinstance(values, float):
+            collected[index].append(values)
+        else:
+            collected[index].append(values[0])
+            collected[index + 1].append(values[1])
+
+    final = []
+    for key, value in collected.items():
+        negative = max((item for item in value if item < 0), default=None)
+        negative = utilo.roundme(negative) if negative is not None else None
+        positive = min((item for item in value if item >= 0), default=None)
+        positive = utilo.roundme(positive) if positive is not None else None
+        final.append(AreaDistance(index=key, before=negative, after=positive))
+    return final
+
+
+class Distance:
+
+    def __init__(self, diff: float = 10.0):
+        self.content = []
+        self.sorted = True
+        self.diff = diff
+
+    def distance(self, bounding):  # pylint:disable=R1260,R0911
+        if not self:
+            return None
+        if not self.sorted:
+            self.sort()
+        top, bottom = bounding[1], bounding[3]
+        if len(self) == 1:
+            if utilo.rect_inside(self[0], bounding, diff=self.diff):
+                return None
+            top_current = self[0][1]
+            bottom_current = self[0][3]
+            if bottom <= top_current:
+                # content is above
+                return (utilo.roundme(bottom - top_current), 0)
+            # content is below
+            return (utilo.roundme(top - bottom_current), 0)
+
+        if self[-1][3] <= bounding[1]:
+            # after
+            return (utilo.roundme(bounding[1] - self[-1][3]), len(self) - 1)
+
+        # in the middle
+        for index, (before, after) in enumerate(zip(self[0:-1], self[1:])):
+            bottom_before = before[3]
+            top_after = after[1]
+            if utilo.rect_inside(before, bounding):
+                return None
+            if utilo.rect_inside(after, bounding):
+                return None
+            if bottom_before <= top <= bottom <= top_after:
+                diff_top = top - bottom_before
+                diff_bottom = bottom - top_after
+                return (diff_top, diff_bottom), index
+        return None
+
+    def append(self, item):
+        self.content.append(item)
+        self.sorted = False
+
+    def sort(self):
+        self.content = sorted(self.content, key=lambda item: item[1])
+        self.sorted = True
+
+    def __getitem__(self, index):
+        return self.content[index]
+
+    def __len__(self):
+        return len(self.content)
+
+
+def create_distance(items) -> Distance:
+    result = Distance()
+    for values in items.values():
+        for item in values:
+            result.append(item)
+    result.sort()
+    return result
+
+
+def load(
+    area: str,
+    text: str,
+    textpositions: str,
+    pages: tuple = None,
+) -> RequiredResources:
+    try:
+        area = groupmes.feature.area.load_area(area, pages=pages)
+    except FileNotFoundError as err:
+        area = []
+        utilo.error(err)
+    text = serializeraw.load_document(text, pages=pages)
+    textpositions = serializeraw.load_textpositions(textpositions, pages=pages)
+    textnavigator = texmex.create_ptns(
+        text,
+        textpositions=textpositions,
+        fill_empty=False,
+    )
+    result = RequiredResources(
+        area=area,
+        textnavigator=textnavigator,
+    )
+    return result
+
+
+def dump_distance(items: PageContentAreaDistances) -> str:
+    raw = []
+    for page in items:
+        content = []
+        for item in page.content:
+            before = utilo.roundme(item.before) if item.before is not None else 'None' # yapf:disable
+            after = utilo.roundme(item.after) if item.after is not None else 'None' # yapf:disable
+            content.append(f'{item.index} {before} {after}')
+        raw.append({'page': page.page, 'content': content})
+    dumped = utilo.yaml_dump(raw)
+    return dumped
+
+
+def load_distance(
+    content: str,
+    pages: tuple = None,
+) -> PageContentAreaDistances:
+    # TODO: MOVE TO SERIALIZERAW
+    loaded = utilo.yaml_load(content)
+    result = []
+    for page in loaded:
+        pagenumber = int(page['page'])
+        if utilo.should_skip(pagenumber, pages):
+            continue
+        pagecontent = []
+        for line in page['content']:
+            index, before, after = line.split()
+            try:
+                before = float(before)
+            except ValueError:
+                before = None
+            try:
+                after = float(after)
+            except ValueError:
+                after = None
+            index = int(index)
+            pagecontent.append(
+                AreaDistance(
+                    index=index,
+                    before=before,
+                    after=after,
+                ))
+        result.append(
+            PageContentAreaDistance(
+                page=pagenumber,
+                content=pagecontent,
+            ))
+    return result

groupmes/feature/hefopa.py

@@ -0,0 +1,126 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2022-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import iamraw
+import serializeraw
+import texmex
+import utilo
+
+
+def work(
+    headnote: str,
+    footnote: str,
+    pagenumber: str,
+    borders: str,
+    pages: int | tuple = None,
+) -> str:
+    try:
+        headnote = serializeraw.load_headerfooter(headnote, pages=pages)
+    except FileNotFoundError as err:
+        utilo.error(err)
+        headnote: list = []
+    try:
+        footnote = serializeraw.load_headerfooter(footnote, pages=pages)
+    except FileNotFoundError as err:
+        utilo.error(err)
+        footnote: list = []
+
+    borders = serializeraw.load_pageborders(borders, pages=pages)
+    pagenumber = load_pagenumbers(pagenumber, borders, pages=pages)
+    merged = merge(headnote, footnote, pagenumber)
+    dumped = serializeraw.dump_headerfooter(merged)
+    return dumped
+
+
+def merge(headnotes, footnotes, pagenumbers) -> list:
+    result = iamraw.PageContentFooterHeaders(content=[])
+    result.__strategy__ = 'hefopa'
+    for page, (headnote, footnote, pagenumber) in utilo.sync_pages(
+        (headnotes, footnotes, pagenumbers)):
+        if not any((headnote, footnote, pagenumber)):
+            continue
+        item = iamraw.PageContentFooterHeader(page=page)
+        if footnote and footnote.footer:
+            item.footer = footnote.footer
+        if headnote:
+            if headnote.header:
+                item.header = headnote.header
+            if headnote.footer:
+                if not item.footer:
+                    item.footer = headnote.footer
+        if pagenumber and pagenumber.footer:
+            if not item.header and not item.footer:
+                item.header = pagenumber.header
+                item.footer = pagenumber.footer
+        result.content.append(item)
+    return result
+
+
+def load_pagenumbers(
+    pagenumber,
+    borders,
+    pages: tuple,
+) -> iamraw.PageContentFooterHeaders:
+    result = iamraw.PageContentFooterHeaders(content=[])
+    result.__stategy__ = 'pagenumber'
+    loaded = serializeraw.load_pagenumbers(pagenumber, pages=pages)
+    single = utilo.Single()
+    for item in loaded:
+        pdfpage = item.pdfpage  # pylint:disable=E1101
+        pageborder = utilo.select_page(borders, page=pdfpage)
+        # TODO: MAY REMOVE LATER
+        if single.contains(pdfpage):
+            utilo.error(f'duplicated pagenumber/pdfpage: {item}')
+            continue
+        page = create(
+            item,
+            pdfpage,
+            pageborder,
+        )
+        result.content.append(page)
+    return result
+
+
+def create(item, pdfpage, pageborder) -> iamraw.PageContentFooterHeader:
+    pageinfo = iamraw.PageInformation(value=item.detected)  # pylint:disable=E1101
+    header, footer = None, None
+    begin, end = head_foot_area(pageborder, item.bounding)  # pylint:disable=E1101
+    isheader = begin == texmex.START
+    if isheader:
+        header = iamraw.FixedHeaderInfo(page=pageinfo)
+        header.begin = begin
+        header.end = end
+    else:
+        footer = iamraw.FixedFooterInfo(page=pageinfo)
+        footer.begin = begin
+        footer.end = end
+    result = iamraw.PageContentFooterHeader(
+        page=pdfpage,
+        header=header,
+        footer=footer,
+    )
+    return result
+
+
+def head_foot_area(pageborder, pagenumber_bounding) -> float:
+    pageheight = pageborder.size.height
+    if not pageheight:
+        utilo.error(f'missing page height: {pageborder} {pagenumber_bounding}')
+        return texmex.END
+    pagenumber_y0 = pagenumber_bounding.y0
+    pagenumber_y1 = pagenumber_bounding.y1
+    header = pagenumber_y1 < 350
+    if header:
+        begin = texmex.START
+        end = utilo.roundme(pagenumber_y1 / pageheight + 0.00)  # TOL
+    else:
+        # footer
+        begin = utilo.roundme(pagenumber_y0 / pageheight - 0.01)  # TOL
+        end = texmex.END
+    return begin, end

groupmes/path.py

@@ -0,0 +1,34 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import utilo
+
+import groupmes
+
+
+def area(path: str, prefix: str = '') -> str:
+    return utilo.pathconnector(path, groupmes.PACKAGE, 'area_area', prefix)
+
+
+def border_leftright(path: str, prefix: str = '') -> str:
+    return utilo.pathconnector(
+        path,
+        groupmes.PACKAGE,
+        'border_leftright',
+        prefix,
+    )
+
+
+def distance(path: str, prefix: str = '') -> str:
+    return utilo.pathconnector(
+        path,
+        groupmes.PACKAGE,
+        'distance_distance',
+        prefix,
+    )

groupmes-1.1.0.dist-info/METADATA

@@ -0,0 +1,28 @@
+Metadata-Version: 2.4
+Name: groupmes
+Version: 1.1.0
+Author-email: Helmut Konrad Schewe <helmutus@outlook.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/anaticulae/groupmes
+Project-URL: Repository, https://github.com/anaticulae/groupmes
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: iamraw<5.0.0,>=4.91.5
+Requires-Dist: utilo<3.0.0,>=2.109.0
+Requires-Dist: configos<2.0.0,>=1.0.4
+Requires-Dist: konradus<2.0.0,>=1.0.1
+Provides-Extra: dev
+Requires-Dist: utilotest<2.0.0,>=1.0.4; extra == "dev"
+Requires-Dist: hoverpower==1.4.3; extra == "dev"
+Requires-Dist: gennex==1.0.3; extra == "dev"
+Requires-Dist: rawmaker==2.40.3; extra == "dev"
+Requires-Dist: pagenumber==1.0.0; extra == "dev"
+
+# groupmes
+
+* chapter: split document text by chapter
+* structure: divide document in head(toc) and tail(rest)
+* toc: create table of content out of raw text data

groupmes-1.1.0.dist-info/RECORD

@@ -0,0 +1,19 @@
+groupmes/__init__.py,sha256=h23h6l8V-tYm0m8M2D21L8ZbZGOOuKymxMJ2OHAtayo,801
+groupmes/__main__.py,sha256=XGxs_W_jWiuac2zJS9cmc_YQ4_06X2-6insjQwP5AwY,622
+groupmes/__patch__.py,sha256=TM9PAYPmQhbW1HY3Vx-bS5rXe132Z02TPBxFMad80Dk,552
+groupmes/cli.py,sha256=QuNRQDf4Kvs6qQhyCLBT2aGWtCH45BzQzYAXJ9vM9cc,2812
+groupmes/path.py,sha256=Nlxte_pPtVXEbFkCpSk8L50GJ_oKDJ7NRlTtKZ9YCms,1064
+groupmes/border/__init__.py,sha256=TM9PAYPmQhbW1HY3Vx-bS5rXe132Z02TPBxFMad80Dk,552
+groupmes/border/leftright.py,sha256=i_iSLvXoWe3HKctvOb_jarDsO202tWrwwCAURE9bM5s,10029
+groupmes/border/most.py,sha256=-82HuzurQC7xrwhOCgCquN-1XtKP1FePb-tweYcvm_o,2411
+groupmes/feature/__init__.py,sha256=kxNXE6nGBAEes4-tV7jYOvjrGAKCZ_bMBKXrB56Hgho,552
+groupmes/feature/area.py,sha256=AslnKHbh9mS_RhScrAKRShuMje3vXAwz0elag9DKqpU,6227
+groupmes/feature/border.py,sha256=qiQhnxid14Rff8osBNFhNDJZNBViqMy4WiVNufVntUY,3227
+groupmes/feature/content.py,sha256=os0Nwycs8EAY8U4UdapqdmIy3BVELNf63To8_NiGsZw,1779
+groupmes/feature/distance.py,sha256=a4gDdrwHRQkKwTzcEZP80UVI9LJuC2-UgqeO3HKJbbM,7267
+groupmes/feature/hefopa.py,sha256=-8QVippCqHvSZhZ7C7eRpD11pn1xlmLMa8lpn0pAjqY,4337
+groupmes-1.1.0.dist-info/METADATA,sha256=e22TPdVWcijKql3FYdiuFEdR8DNdXTBHxB9PfzUOYFk,1057
+groupmes-1.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+groupmes-1.1.0.dist-info/entry_points.txt,sha256=uweQthSsMHckhQ-yFmjsFhJK8sZslWMylWxAeCTOcTs,46
+groupmes-1.1.0.dist-info/top_level.txt,sha256=RqsOvZPqxlp_Pu6Ic-F6Z1H3MnsIIJpw3UEf7VZkJAU,9
+groupmes-1.1.0.dist-info/RECORD,,

groupmes-1.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

groupmes-1.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+groupme = groupmes.cli:main

groupmes-1.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+groupmes