doctextstyle 5.20.5__py3-none-any.whl

doctextstyle/__init__.py

@@ -0,0 +1,23 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import os
+
+import doctextstyle.__patch__
+import hey
+from doctextstyle.vector.extract import run as extract_headlines  # pylint:disable=C0412
+from doctextstyle.vector.extract import run_fromdata as headlines_fromdata  # pylint:disable=C0412
+
+__version__ = hey.__version__
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+PROCESS = 'doctextstyle'
+
+JUSTIFIED = 1
+NOT_JUSTIFIED = 0

doctextstyle/__patch__.py

@@ -0,0 +1,8 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================

doctextstyle/cli.py

@@ -0,0 +1,72 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import os
+
+import serializeraw
+import utilo
+import utilo.cli
+
+import doctextstyle
+import doctextstyle.extractor
+
+DEFAULT_OUTPUT_FILE = 'doctextstyle__textstyle.yaml'
+
+DESCRIPTION = """\
+Extracts common text layout properties for text, headlines(h1, h2, h3),
+pagenumber, footnotes, paragraphs and lists.
+
+Requires: `rawmaker__oneline` data to create pagetextnavigators
+          `rawmaker__oneline` data to create pagecontenttextnavigators
+
+Writes: `doctextstyle__textstyle.yaml`
+"""
+
+
+@utilo.saveme
+def main() -> int:
+    commands = []
+    parser = utilo.cli.create_parser(
+        todo=commands,
+        description=DESCRIPTION,
+        config=utilo.ParserConfiguration(
+            inputparameter=True,
+            multiprocessed=True,
+            outputparameter=True,
+            pages=True,
+            prefix=False,
+        ),
+        version=doctextstyle.__version__,
+        prog=doctextstyle.PROCESS,
+    )
+    args = utilo.parse(parser)
+    utilo.log('run doctextstyle')
+    selected_pages = utilo.pages_fromargs(args)
+
+    inpath, outpath = utilo.cli.sources(args, singleinput=True)  # pylint:disable=W0632
+    inpath = inpath[0]
+
+    if os.path.isdir(outpath):
+        outpath = os.path.join(outpath, DEFAULT_OUTPUT_FILE)
+
+    # ensure that output directory exists
+    parent, _ = os.path.split(outpath)
+    os.makedirs(parent, exist_ok=True)
+
+    try:
+        extracted = doctextstyle.extractor.extract(inpath, pages=selected_pages)
+    except FileNotFoundError as error:
+        utilo.error('missing input location')
+        utilo.error(f'{error}')
+        return utilo.FAILURE
+
+    dumped = serializeraw.dump_doctextstyle(extracted)
+    utilo.file_replace(outpath, dumped)
+    utilo.log('completed')
+    return utilo.SUCCESS

doctextstyle/cluster.py

@@ -0,0 +1,166 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import collections
+import enum
+import functools
+
+import configos
+import iamraw
+import utilo
+
+
+class ClusterProperty(enum.Enum):
+    SIZE = enum.auto()
+    FONT = enum.auto()
+    BEFORE = enum.auto()
+    AFTER = enum.auto()
+    YPOS = enum.auto()
+    LEFT = enum.auto()
+
+
+ClusterPropertySelection = list[ClusterProperty]
+Tol = collections.namedtuple('Tol', 'abs, rel')
+NO_TOLERANCE = Tol(0.0, 0.0)
+
+CLUSTER_SIZE_MIN = configos.HV_INT_PLUS(default=5)
+
+CLUSTER_SIZE_DIFF_ABS = configos.HV_FLOAT_PLUS(default=0.5)
+
+CLUSTER_SIZE_DIFF_REL = configos.HV_FLOAT_PLUS(default=0.1)
+
+CLUSTER_AFTER_DIFF_ABS = configos.HV_FLOAT_PLUS(default=2.0)
+
+CLUSTER_AFTER_DIFF_REL = configos.HV_FLOAT_PLUS(default=0.1)
+
+CLUSTER_BEFORE_DIFF_ABS = configos.HV_FLOAT_PLUS(default=2.0)
+
+CLUSTER_BEFORE_DIFF_REL = configos.HV_FLOAT_PLUS(default=0.1)
+
+CLUSTER_LEFT_DIFF_ABS = configos.HV_FLOAT_PLUS(default=15.0)
+
+CLUSTER_LEFT_DIFF_REL = configos.HV_FLOAT_PLUS(default=0.15)
+
+
+def cluster(
+        items: iamraw.TextProperties,
+        selection: ClusterPropertySelection = None,
+        validator: callable = None,
+        *,
+        minsize: int = CLUSTER_SIZE_MIN,
+        unique_content: bool = False,
+        max_size_diff=Tol(CLUSTER_SIZE_DIFF_ABS, CLUSTER_SIZE_DIFF_REL),
+        max_after_diff=Tol(CLUSTER_AFTER_DIFF_ABS, CLUSTER_AFTER_DIFF_REL),
+        max_before_diff=Tol(CLUSTER_BEFORE_DIFF_ABS, CLUSTER_BEFORE_DIFF_REL),
+        max_left_diff=Tol(CLUSTER_LEFT_DIFF_ABS, CLUSTER_LEFT_DIFF_REL),
+):
+    if selection:
+        selection = set(selection)
+    if validator:
+        items = [item for item in items if validator(item)]
+
+    decider = functools.partial(
+        classifier,
+        selection=selection,
+        max_size_diff=max_size_diff,
+        max_after_diff=max_after_diff,
+        max_before_diff=max_before_diff,
+        max_left_diff=max_left_diff,
+    )
+    clustered = utilo.determine_cluster(
+        todo=items,
+        classifier=decider,
+        min_elements=minsize,
+    )
+    if unique_content:
+        clustered = [item for item in clustered if iscontent_unique(item)]
+    return clustered
+
+
+def classifier(  # pylint:disable=R1260
+    candidat,
+    clusteritem,
+    selection,
+    max_size_diff=None,
+    max_after_diff=None,
+    max_before_diff=None,
+    max_left_diff=None,
+) -> bool:
+    if selection is None or ClusterProperty.SIZE in selection:
+        if not utilo.pnear(
+                candidat.size,
+                clusteritem.size,
+                abs_tol=max_size_diff.abs,
+                rel_tol=max_size_diff.rel,
+        ):
+            return False
+    if selection is None or ClusterProperty.BEFORE in selection:
+        if not utilo.pnear(
+                candidat.before,
+                clusteritem.before,
+                abs_tol=max_before_diff.abs,
+                rel_tol=max_before_diff.rel,
+        ):
+            return False
+    if selection is None or ClusterProperty.AFTER in selection:
+        if not utilo.pnear(
+                candidat.after,
+                clusteritem.after,
+                abs_tol=max_after_diff.abs,
+                rel_tol=max_after_diff.rel,
+        ):
+            return False
+    if selection is None or ClusterProperty.LEFT in selection:
+        if not utilo.pnear(
+                candidat.left,
+                clusteritem.left,
+                abs_tol=max_left_diff.abs,
+                rel_tol=max_left_diff.rel,
+        ):
+            return False
+    if selection is None or ClusterProperty.FONT in selection:
+        if candidat.font != clusteritem.font:
+            return False
+    return True
+
+
+def iscontent_unique(current) -> bool:
+    expected = len(current.content)
+    current = {item.hashed for item in current.content}
+    return len(current) == expected
+
+
+def bestmatch(clustered, number: int = 0):
+    try:
+        largest_cluster_first_item = clustered[number].content[0]
+    except IndexError:
+        # no clusted content
+        return None
+    size = largest_cluster_first_item.size
+    font = largest_cluster_first_item.font
+    # TODO: PLUS INF OR MINUS INF
+    before = [item.before for item in clustered[number].content]
+    before = [utilo.INF if item is None else item for item in before]
+    before = utilo.mode(before)
+    after = [item.after for item in clustered[number].content]
+    after = [utilo.INF if item is None else item for item in after]
+    after = utilo.mode(after)
+    length = len(clustered[0])
+
+    if before is utilo.INF:
+        before = None
+    if after is utilo.INF:
+        after = None
+    return (size, font, length, (before, after))
+
+
+def remove(flats, toremove: list):
+    toremove = set(toremove)
+    flats = [item for item in flats if item not in toremove]
+    return flats

doctextstyle/extractor.py

@@ -0,0 +1,178 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import contextlib
+import os
+
+import iamraw
+import serializeraw
+import texmex
+import utilo
+
+import doctextstyle
+import doctextstyle.features
+import doctextstyle.features.blockquote
+import doctextstyle.features.content
+import doctextstyle.features.footnote
+import doctextstyle.features.headline
+import doctextstyle.features.pagesize
+import doctextstyle.features.textbounding as dtt
+import doctextstyle.parser
+import doctextstyle.utils
+
+
+def extract(path: str, pages: tuple = None) -> iamraw.DocTextStyle:  # pylint:disable=R0914,R0915
+    navigator, cnavigators, magic = load_data(path, pages=pages)
+
+    parsed = doctextstyle.parser.parses(navigator, magic)
+    flat = doctextstyle.utils.flatten(parsed)
+
+    text = doctextstyle.features.text(flat)
+    if not text or len(text) < 4:
+        utilo.error('not enough text data')
+        return iamraw.DocTextStyle()
+    text_after = text[3][1]
+
+    result = iamraw.DocTextStyle(
+        text_size=text[0],
+        text_distance=text_after,
+        text_family=text[1],
+    )
+
+    extract_headlines(result, flat)
+
+    pagenumber = doctextstyle.features.pagenumber(flat)
+    if pagenumber:
+        result.pagenumber_size = pagenumber[0]
+        result.pagenumber_family = pagenumber[1]
+
+    extract_footnotes(result, flat)
+
+    pagesizes = doctextstyle.features.pagesize.pagesizes(path, pages=pages)
+
+    if pagesizes:
+        result.page_width, result.page_height = pagesizes[0][0]
+        with contextlib.suppress(IndexError):
+            result.page_rotated_width, result.page_rotated_height = pagesizes[1][0] # yapf:disable
+    else:
+        utilo.error('no pagesize, too few pages to run feature')
+
+    extract_contentborder(result, path, pages)
+    extract_textdimension(result, navigator, cnavigators)
+    extract_blockquote(result, flat)
+    return result
+
+
+def load_data(path: str, pages: tuple = None):
+    navigator = serializeraw.ptn_frompath(
+        path,
+        prefix='oneline',
+        pages=pages,
+    )
+    try:
+        cnavigators = serializeraw.ptcn_frompath(
+            path,
+            prefix='oneline',
+            pages=pages,
+        )
+    except FileNotFoundError as error:
+        cnavigators = None
+        utilo.error(f'missing page text content navigator: {error}')
+
+    magic = iamraw.path.magic_content(path)
+    magic = serializeraw.load_types(magic) if os.path.exists(magic) else []
+    return navigator, cnavigators, magic
+
+
+def extract_footnotes(result, flat):
+    footnotes = doctextstyle.features.footnote.footnote(flat)
+    if footnotes:
+        footnote_after = footnotes[3][1]
+        result.footnote_size = footnotes[0]
+        result.footnote_family = footnotes[1]
+        result.footnote_distance = footnote_after
+
+
+def extract_contentborder(result, path, pages):
+    leftright = iamraw.path.groupme_border_leftright(path)
+    content = doctextstyle.features.content.content(leftright, pages=pages)
+    if content:
+        normal = content[0][0]
+        result.content_left = normal[0]
+        result.content_right = normal[1]
+        result.content_top = normal[2]
+        result.content_bottom = normal[3]
+    if len(content) > 1:
+        rotated = content[1][0]
+        result.content_rotated_left = rotated[0]
+        result.content_rotated_right = rotated[1]
+        result.content_rotated_top = rotated[2]
+        result.content_rotated_bottom = rotated[3]
+
+
+def extract_headlines(result, flat):
+    headlines = doctextstyle.features.headline.headlines(flat)
+    if not headlines:
+        return
+
+    result.h1_size = headlines[0][0]
+    result.h1_family = headlines[0][1]
+    result.h1_before = headlines[0][3][0]
+    result.h1_after = headlines[0][3][1]
+
+    if len(headlines) == 1:
+        return
+    result.h2_size = headlines[1][0]
+    result.h2_family = headlines[1][1]
+    result.h2_before = headlines[1][3][0]
+    result.h2_after = headlines[1][3][1]
+
+    if len(headlines) == 2:
+        return
+    result.h3_size = headlines[2][0]
+    result.h3_family = headlines[2][1]
+    result.h3_before = headlines[2][3][0]
+    result.h3_after = headlines[2][3][1]
+
+
+def extract_textdimension(result, navigators, cnavigators):
+    if not cnavigators:
+        return
+    twidth = dtt.text_width(cnavigators)
+    twidth_min = dtt.text_width_min(cnavigators)
+    twidth_max = dtt.text_width_max(cnavigators)
+
+    result.text_width = twidth
+    result.text_width_min = twidth_min
+    result.text_width_max = twidth_max
+
+    result.text_left = texmex.document_textfeed(cnavigators)
+    result.text_right = texmex.document_textfeed(navigators, left=False)
+
+    result.text_left = utilo.roundme(result.text_left)
+    result.text_right = utilo.roundme(result.text_right)
+
+    if result.content_right is None:
+        return
+    # content_right is None for very short documents, cause there are
+    # a minimum number of pages required to extract this properly.
+    right = result.page_width - result.content_right
+    result.text_alignment = dtt.justified(cnavigators, right)
+
+
+def extract_blockquote(result, flat):
+    style = doctextstyle.features.blockquote.blockquote_style(flat)
+    if not style:
+        return
+    font, size, left, right = style
+
+    result.block_size = size
+    result.block_family = font
+    result.block_left = left
+    result.block_right = right

doctextstyle/features/__init__.py

@@ -0,0 +1,51 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import doctextstyle.cluster
+
+
+def text(flats, returncluster: bool = False):
+    clustered = doctextstyle.cluster.cluster(
+        flats,
+        (
+            doctextstyle.cluster.ClusterProperty.SIZE,
+            doctextstyle.cluster.ClusterProperty.FONT,
+        ),
+    )
+    result = doctextstyle.cluster.bestmatch(clustered)
+    if not result:
+        # too few data to determine text style information
+        return None
+    if returncluster:
+        return result, clustered[0] if clustered else []
+    return result
+
+
+def pagenumber(flats, returncluster: bool = False):
+
+    def validator(item) -> bool:
+        if item.top >= 100 and item.bottom >= 100:
+            # page number is not in the middle of the page. The page
+            # number is located at the top or bottom of the page.
+            return False
+        return item.length <= 6
+
+    clustered = doctextstyle.cluster.cluster(
+        flats,
+        (
+            doctextstyle.cluster.ClusterProperty.SIZE,
+            doctextstyle.cluster.ClusterProperty.FONT,
+        ),
+        validator=validator,
+    )
+    # assert len(clustered) == 1, len(clustered)
+    result = doctextstyle.cluster.bestmatch(clustered)
+    if returncluster:
+        return result, clustered[0] if clustered else []
+    return result

doctextstyle/features/blockquote.py

@@ -0,0 +1,69 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import configos
+import utilo
+
+BLOCKQUOTE_CLUSTER_SIZE_MIN = configos.HV_INT_PLUS(default=30)
+
+BLOCKQUOTE_LEFT_DIFF_MAX = configos.HV_FLOAT_PLUS(default=15.0)
+
+BLOCKQUOTE_RIGHT_DIFF_MAX = configos.HV_FLOAT_PLUS(default=15.0)
+
+BLOCKQUOTE_TEXT_LENGTH_MIN = configos.HV_INT_PLUS(default=45)
+
+
+def blockquote_style(flats) -> tuple:
+    # TODO: REMOVE TEXT CLUSTER FIRST
+    flats = [item for item in flats if item.hashed.count('.') < 10]
+    # skip very short text
+    flats = [
+        item for item in flats if len(item.hashed) > BLOCKQUOTE_TEXT_LENGTH_MIN
+    ]
+    # skip item which start too right
+    flats = [item for item in flats if item.left <= 160]
+    # cluster text
+    clustered = utilo.determine_cluster(
+        flats,
+        classifier=classifier,
+        min_elements=BLOCKQUOTE_CLUSTER_SIZE_MIN,
+        strategy=utilo.MatchStrategy.MIN,
+    )
+    if len(clustered) < 2:
+        # too few cluster
+        return None
+    # largest cluster is text cluster
+    blockquote = clustered[1].center
+    result = (
+        blockquote.font,
+        blockquote.size,
+        blockquote.left,
+        blockquote.right,
+    )
+    return result
+
+
+def classifier(candidat, clusteritem):
+    # expected
+    x0 = clusteritem.left
+    x1 = clusteritem.right
+    # current
+    x00 = candidat.left
+    x11 = candidat.right
+    # compare font size
+    if candidat.size != clusteritem.size:
+        return False
+    leftdiff = utilo.near(x00, x0, diff=BLOCKQUOTE_LEFT_DIFF_MAX)
+    if not leftdiff:
+        return False
+    rightdiff = utilo.near(x11, x1, diff=BLOCKQUOTE_RIGHT_DIFF_MAX)
+    if not rightdiff:
+        return False
+    # merge candidat into cluster
+    return True

doctextstyle/features/content.py

@@ -0,0 +1,34 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import configos
+import serializeraw
+import utilo
+
+CONTENT_DIFF_MAX = configos.HV_FLOAT_PLUS(default=5.0)
+
+CONTENT_ELEMENTS_MIN = configos.HV_INT_PLUS(default=3)
+
+
+def content(path, pages: tuple = None):
+    leftright = serializeraw.load_leftright_border(path, pages)
+
+    def equals(candidat, clusteritem):
+        # left, right, top, down
+        distance = utilo.norms(candidat, clusteritem)
+        return distance < CONTENT_DIFF_MAX
+
+    clustered = utilo.determine_cluster(
+        leftright.values(),
+        classifier=equals,
+        min_elements=CONTENT_ELEMENTS_MIN,
+    )
+    # TODO: SUPPORT LEFT AND RIGHT DIFFERENT PAGE?
+    result = [(cluster[0], len(cluster)) for cluster in clustered]
+    return result

doctextstyle/features/footnote.py

@@ -0,0 +1,64 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import configos
+import iamraw
+import utilo
+
+import doctextstyle.cluster
+import doctextstyle.features
+import doctextstyle.features.headline
+
+FOOTNOTES_COUNT_MIN = configos.HV_INT_PLUS(default=10)
+
+VALIDATOR_COUNT_MIN = configos.HV_INT_PLUS(default=25)
+
+
+def footnote(flats: iamraw.TextProperties):
+    _text = doctextstyle.features.text(flats, returncluster=True)
+    _pagenumber = doctextstyle.features.pagenumber(
+        flats,
+        returncluster=True,
+    )
+    _headlines = doctextstyle.features.headline.headlines(
+        flats,
+        returncluster=True,
+    )
+    if _text:
+        flats = doctextstyle.cluster.remove(flats, _text[1])
+    else:
+        utilo.debug('footnote: no text style')
+
+    if _pagenumber:
+        flats = doctextstyle.cluster.remove(flats, _pagenumber[1])
+    else:
+        utilo.debug('footnote: no pagenumber style')
+
+    if _headlines:
+        for item in _headlines[1]:
+            flats = doctextstyle.cluster.remove(flats, item)
+    else:
+        utilo.debug('footnote: no headline style')
+
+    def validator(item) -> bool:
+        # Shrink footnotes to bottom area
+        return item.bottom < 150 and item.length >= VALIDATOR_COUNT_MIN
+
+    clustered = doctextstyle.cluster.cluster(
+        flats,
+        (doctextstyle.cluster.ClusterProperty.SIZE,),
+        validator=validator,
+        minsize=FOOTNOTES_COUNT_MIN,
+        unique_content=True,
+        max_size_diff=doctextstyle.cluster.NO_TOLERANCE,
+        max_before_diff=doctextstyle.cluster.NO_TOLERANCE,
+        max_after_diff=doctextstyle.cluster.NO_TOLERANCE,
+    )
+    result = doctextstyle.cluster.bestmatch(clustered)
+    return result