groupmes 1.1.0__tar.gz

groupmes-1.1.0/PKG-INFO

@@ -0,0 +1,28 @@
+Metadata-Version: 2.4
+Name: groupmes
+Version: 1.1.0
+Author-email: Helmut Konrad Schewe <helmutus@outlook.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/anaticulae/groupmes
+Project-URL: Repository, https://github.com/anaticulae/groupmes
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: iamraw<5.0.0,>=4.91.5
+Requires-Dist: utilo<3.0.0,>=2.109.0
+Requires-Dist: configos<2.0.0,>=1.0.4
+Requires-Dist: konradus<2.0.0,>=1.0.1
+Provides-Extra: dev
+Requires-Dist: utilotest<2.0.0,>=1.0.4; extra == "dev"
+Requires-Dist: hoverpower==1.4.3; extra == "dev"
+Requires-Dist: gennex==1.0.3; extra == "dev"
+Requires-Dist: rawmaker==2.40.3; extra == "dev"
+Requires-Dist: pagenumber==1.0.0; extra == "dev"
+
+# groupmes
+
+* chapter: split document text by chapter
+* structure: divide document in head(toc) and tail(rest)
+* toc: create table of content out of raw text data

groupmes-1.1.0/README

@@ -0,0 +1,5 @@
+# groupmes
+
+* chapter: split document text by chapter
+* structure: divide document in head(toc) and tail(rest)
+* toc: create table of content out of raw text data

groupmes-1.1.0/groupmes/__init__.py

@@ -0,0 +1,21 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================
+
+import importlib.metadata
+import os
+
+import groupmes.__patch__
+import groupmes.path
+
+PACKAGE = 'groupmes'
+PROCESS = 'groupme'
+
+__version__ = importlib.metadata.version(PACKAGE)
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))

groupmes-1.1.0/groupmes/__main__.py

@@ -0,0 +1,13 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================
+
+from groupmes.cli import main
+
+if __name__ == "__main__":
+    main()

groupmes-1.1.0/groupmes/__patch__.py

@@ -0,0 +1,8 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================

groupmes-1.1.0/groupmes/border/__init__.py

@@ -0,0 +1,8 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================

groupmes-1.1.0/groupmes/border/leftright.py

@@ -0,0 +1,315 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""left right border detector
+==========================
+
+The `left right border detector` (lrbd) separates borders of the left
+and right page especially which are used for books with different border
+width for left and right page.
+
+As a result of different left-right borders there are alternating border
+widths which we detect. On single pages there are no alternating border.
+
+Currently there are two strategies to detect different page border:
+
+* simple approach
+* raising edge
+
+Simple Approach
+~~~~~~~~~~~~~~~
+
+Some documents have exceptions on some pages. We handle this via allowed
+errors defined with the HolyValues `FIRSTSECOND_ERROR_COUNT_MAX` and
+`MIXED_ERROR_MIN`.
+
+Raising Edge
+~~~~~~~~~~~~
+
+The change of the text feed creates an raising edge between pages. This
+approach detects these edges to determine left and right page border
+width.
+"""
+
+import dataclasses
+import math
+import statistics
+import typing
+
+import configos
+import iamraw
+import utilo
+
+# max diff to match in common group.
+SIDE_DIFF_MAX = configos.HV_INT_PLUS(default=2)
+# exceptions which are allowed cause of user defined error.
+FIRSTSECOND_ERROR_COUNT_MAX = configos.HolyTable(
+    items=(
+        (2, 0.5),
+        (3, 0.35),
+        (5, 0.26),
+        (10, .21),
+        (15, 0.05),
+        (200, 0.01),
+    ),
+    right_outranges_none=False,
+)
+# errors which are a result of handle alternating border as single border.
+MIXED_ERROR_MIN = configos.HV_PERCENT_PLUS(default=15)
+# area where left border can be located.
+LEFT_PERCENT = configos.HV_PERCENT_PLUS(default=30)
+# area where right border can be located.
+RIGHT_PERCENT = configos.HV_PERCENT_PLUS(default=30)
+
+RAISING_EDGE_MIN = configos.HV_PERCENT_PLUS(default=75)
+
+# TODO: SHOULD WE DISABLE ALGO ON BIG FAIL COUNT?
+RAISING_FAILRATE = configos.HolyTable(
+    items=(
+        (5, 1 / 5),
+        (7, 2 / 7),
+        (10, 3 / 10),
+        (40, 10 / 40),
+        (200, 40 / 200),
+    ),
+    right_outranges_none=False,
+)
+
+LeftRight = tuple[float, float]
+
+DetectedBorder = typing.TypeVar('DetectedBorder', tuple[float], float)
+
+
+@dataclasses.dataclass
+class LeftRightDetected:
+    left: DetectedBorder = None
+    right: DetectedBorder = None
+    confidence: float = 0.0
+
+    @property
+    def valid(self):
+        return isinstance(self.left, tuple)
+
+
+def run(
+    textpositions: iamraw.PageContentTextPositions,
+    pagesizes: iamraw.PageSizeBorderList,
+) -> LeftRightDetected:
+    """Run LeftRight-Strategy to determine that document contains
+    special leftright-border or a normal equal border for every page."""
+    left, right = determine_pageborder(textpositions, pagesizes)
+    left, right = handle_emptypage(left, right)
+
+    result = simple(left, right)
+    if result:
+        return result
+
+    result = raising(left, right)
+    if result:
+        return result
+
+    leftborder = utilo.mode(left, minimize=True)
+    rightborder = utilo.mode(right, minimize=False)
+    return LeftRightDetected(
+        left=leftborder,
+        right=rightborder,
+        confidence=1.0,
+    )
+
+
+def simple(left: utilo.Numbers, right: utilo.Numbers) -> LeftRightDetected:
+    """Determine LeftRight border based on changing text feed. We use
+    the even numbers to determine the left page and the odd numbers to
+    determine the right text feed.
+
+    This appraoch is limited when one page was missing and therefore
+    these pages are mixed and therefore wrong classified.
+
+    Args:
+        left: ordered list of left text feed
+        right: ending of right text content border
+    Returns:
+        None if no valid LeftRightDetected was detected
+        LeftRightDetected if failrate is not to hight
+    """
+    if len(left) <= 1:
+        # TODO: INVESTIGATE HERE
+        utilo.error('could not run simple approach')
+        return None
+    mixed = utilo.diff_mode(left, max_diff=SIDE_DIFF_MAX)
+    # first side
+    first = left[::2]
+    first_matched = utilo.diff_mode(first, max_diff=SIDE_DIFF_MAX)
+    # second side
+    second = left[1::2]
+    second_matched = utilo.diff_mode(second, max_diff=SIDE_DIFF_MAX)
+
+    mixed_error = 1 - len(mixed) / len(left)
+    first_error = 1 - len(first_matched) / len(first)
+    second_error = 1 - len(second_matched) / len(second)
+
+    utilo.debug(f'mixed: {mixed_error}')
+    utilo.debug(f'first: {first_error}')
+    utilo.debug(f'second: {second_error}')
+
+    # left right
+    # TODO: DEFINE BETTER CONFIDENCE APPROACH
+    max_firstsecond_error = FIRSTSECOND_ERROR_COUNT_MAX(len(first))
+    if mixed_error > MIXED_ERROR_MIN.value and all([
+            first_error < max_firstsecond_error,
+            second_error < max_firstsecond_error,
+    ]):
+        leftborder = (
+            utilo.mode(first, minimize=True),
+            utilo.mode(second, minimize=True),
+        )
+        rightborder = (
+            utilo.mode(right[::2]),
+            utilo.mode(right[1::2]),
+        )
+        return LeftRightDetected(
+            left=leftborder,
+            right=rightborder,
+            confidence=1.0,
+        )
+    return None
+
+
+def raising(left: utilo.Numbers, right: utilo.Numbers) -> LeftRightDetected:
+    """Determine border depending on changing text feed on left page
+    border.
+
+    This approach has no problems when one or more leftright pages are
+    missing. The limit of problems is defined in lookup table
+    `RAISING_FAILRATE`.
+
+    Args:
+        left: ordered list of left text feed
+        right: ending of right text content border
+    Returns:
+        None if no valid LeftRightDetected was detected
+        LeftRightDetected if `failrate` is not too high
+    """
+    longest_left = longest_two(left)
+    longest_right = longest_two(right)
+
+    if longest_left is None or longest_right is None:
+        # single page document which does not contain left-right-pages
+        return None
+
+    first_left = statistics.mean(longest_left[0])
+    second_left = statistics.mean(longest_left[1])
+    edge = math.fabs(first_left - second_left)
+
+    first_right = statistics.mean(longest_right[0])
+    second_right = statistics.mean(longest_right[1])
+
+    edges = utilo.diffs(left)
+    failures = [
+        index for index, item in enumerate(edges)
+        if item < edge * RAISING_EDGE_MIN.value
+    ]
+    failrate = len(failures) / len(edges)
+    max_failrate = RAISING_FAILRATE(len(edges))
+
+    if failrate > max_failrate:
+        return None
+
+    first_left, second_left = utilo.roundme(first_left, second_left)
+    first_right, second_right = utilo.roundme(first_right, second_right)
+
+    leftborder = (
+        min([first_left, second_left]),
+        max([first_left, second_left]),
+    )
+    rightborder = (
+        min([first_right, second_right]),
+        max([first_right, second_right]),
+    )
+
+    return LeftRightDetected(
+        left=leftborder,
+        right=rightborder,
+        confidence=1.0,
+    )
+
+
+def handle_emptypage(left, right):
+    # TODO: THINK ABOUT IF THIS IS ENOUGH
+    # ignore empty pages
+    left_none = 0.0
+    left = [item if item is not None else left_none for item in left]
+    # TODO: Is default=0 a good one?
+    right_none = max((item for item in right if item is not None), default=0)
+    # NOTE: Determine more pages as large than it realy are - is this a
+    # problem?
+    right = [item if item is not None else right_none for item in right]
+    return left, right
+
+
+def determine_pageborder(textpositions, pagesizes):
+    left = []
+    right = []
+    before = -1
+    for current, (page, size) in utilo.sync_pages([textpositions, pagesizes]):
+        assert current > before, f'{before} < {current}'
+        before = current
+        if not page or not size:
+            left.append(None)
+            right.append(None)
+            continue
+        bounding = [item for item, _ in page.content.values()]
+        leftright = maximize_leftright(bounding, size)
+        left.append(leftright[0])
+        right.append(leftright[1])
+    return left, right
+
+
+def maximize_leftright(
+    boundings: utilo.Rectangles,
+    size: iamraw.PageSizeBorder,
+) -> LeftRight:
+    """Determine the left and right border of a page based on `mode`
+    selection in `size`.
+
+    Minimize the left and maximize the right position. The area where
+    mode is used to determine the most common border which is assumed as
+    correct border is limit by `size` configuration.
+
+    Args:
+        boundings: textpositions of defined page
+        size: width and height of current page
+    Returns:
+        tuple with left and right content bounding
+    """
+    left_max = size.size.width * LEFT_PERCENT
+    right_min = size.size.width * (1 - RIGHT_PERCENT)
+    left_max, right_min = utilo.roundme(left_max, right_min)
+    assert left_max <= right_min, 'left and right bounds are flipped'
+    left = [item[0] for item in boundings if item[0] <= left_max]
+    right = [item[2] for item in boundings if item[2] >= right_min]
+    # TODO: DO WE RELAY NEED THIS?
+    left = utilo.mode(left, minimize=True) if left else 0.0
+    right = utilo.mode(right, minimize=False) if right else size.size.width
+    return left, right
+
+
+CLUSTER_CANDIAT_DIFF_MAX = configos.HV_FLOAT_PLUS(default=2.0)
+
+
+def longest_two(items: utilo.Numbers) -> tuple[float, float]:
+
+    def close(candidat, clusteritem) -> bool:
+        diff = math.fabs(candidat - clusteritem)
+        return diff < CLUSTER_CANDIAT_DIFF_MAX
+
+    clustered = utilo.determine_cluster(items, close)
+    result = sorted(clustered, key=len, reverse=True)
+    if len(result) < 2:
+        return None
+    return result[0], result[1]

groupmes-1.1.0/groupmes/border/most.py

@@ -0,0 +1,70 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import dataclasses
+import math
+
+import iamraw
+import utilo
+
+
+@dataclasses.dataclass
+class MostBoundingDetected:
+    left: float = None
+    right: float = None
+    top: float = None
+    bottom: float = None
+    confidence: float = 0.0
+
+
+def run(sizeandborder: iamraw.PageSizeBorderList) -> MostBoundingDetected:
+    borders = [item.border for item in sizeandborder]
+    most = most_boundingbox(borders)
+    # x0, y0, x1, y1
+    # left right
+    assert most[0] < most[1]
+    # top bottom
+    assert most[2] < most[3]
+    result = MostBoundingDetected(
+        left=most[0],
+        right=most[1],
+        top=most[2],
+        bottom=most[3],
+    )
+    return result
+
+
+def most_boundingbox(
+    boxes: utilo.Rectangles,
+    roundme: bool = False,
+) -> utilo.Rectangle:
+    """Extract bounding box of most common occurence for every side.
+
+    Round detected boundingbox to full number to make approach more
+    robust. Round numbers in direction to the border end to increase the
+    detected rectangle that fit more items in it.
+    """
+    # TODO: Think about right and left, maybe search for the top 2 borders?
+    # Filter None entries
+    # left, right, top, bottom
+    rounding = [math.floor, math.ceil, math.floor, math.ceil]
+    result = []
+    for index, method in enumerate(rounding):
+        # remove None items
+        filtered = [item[index] for item in boxes if item[index] is not None]
+        # round to have a more robust grouping
+        rounded = [method(item) for item in filtered] if roundme else filtered
+        # determine most occured border to determine them as required border
+        # support multiple border options.
+        minimize = method is math.floor
+        # TODO: REQUIRE A BETTER TY-BREAKER
+        mode = utilo.mode(rounded, minimize=minimize)
+        result.append(mode)
+    # (x0, y0, x1, y1)
+    return tuple(result)

groupmes-1.1.0/groupmes/cli.py

@@ -0,0 +1,81 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================
+
+import utilo
+
+import groupmes
+
+DESCRIPTION = 'TODO'
+
+WORKPLAN = [
+    utilo.create_step(
+        'area',
+        inputs=[
+            utilo.ResultFile(producer='rawmaker', name='boxes_boxes'),
+            utilo.ResultFile(producer='tablero',
+                             name='result_result',
+                             optional=True),
+            utilo.ResultFile(producer='rawmaker', name='text_text'),
+            utilo.ResultFile(producer='rawmaker', name='text_positions'),
+        ],
+        output=('area',),
+    ),
+    utilo.create_step(
+        'border',
+        inputs=[
+            utilo.ResultFile(producer='rawmaker', name='border_pages'),
+            utilo.ResultFile(producer='rawmaker', name='text_positions'),
+        ],
+        output=('leftright',),
+    ),
+    utilo.create_step(
+        'distance',
+        inputs=[
+            utilo.ResultFile(producer=groupmes.PROCESS, name='area_area'),
+            utilo.ResultFile(producer='rawmaker', name='text_text'),
+            utilo.ResultFile(producer='rawmaker', name='text_positions'),
+        ],
+        output=('distance',),
+    ),
+    utilo.create_step(
+        'content',
+        inputs=[
+            utilo.ResultFile(producer='rawmaker', name='text_text'),
+            utilo.ResultFile(producer='rawmaker', name='text_positions'),
+            utilo.ResultFile(producer='rawmaker', name='border_pages'),
+            utilo.ResultFile(producer='footnote', name='result_result'),
+        ],
+        output=('content',),
+    ),
+    utilo.create_step(
+        'hefopa',
+        inputs=[
+            utilo.ResultFile(producer='headnote', name='result_result'),
+            utilo.ResultFile(producer='footnote', name='result_result'),
+            utilo.ResultFile(producer='pagenumber', name='result_result'),
+            utilo.ResultFile(producer='rawmaker', name='border_pages'),
+        ],
+        output=('result',),
+    )
+]
+
+
+def main():
+    utilo.featurepack(
+        workplan=WORKPLAN,
+        root=groupmes.ROOT,
+        featurepackage='groupmes.feature',
+        config=utilo.FeaturePackConfig(
+            description=DESCRIPTION,
+            multiprocessed=True,
+            name=groupmes.PROCESS,
+            pages=True,
+            version=groupmes.__version__,
+        ),
+    )

groupmes-1.1.0/groupmes/feature/__init__.py

@@ -0,0 +1,8 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================