ibidem 1.0.0__py3-none-any.whl

ibidem/__init__.py

@@ -0,0 +1,16 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================
+
+import importlib.metadata
+import os
+
+__version__ = importlib.metadata.version('ibidem')
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+PROCESS = 'footnote'

ibidem/cli.py

@@ -0,0 +1,78 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2019-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================
+
+import utilo
+
+import ibidem
+
+DESCRIPTION = 'TODO'
+
+WORKPLAN = [
+    utilo.create_step(
+        'plain',
+        inputs=[
+            utilo.ResultFile(producer='rawmaker', name='text_text'),
+            utilo.ResultFile(producer='rawmaker', name='text_positions'),
+            utilo.ResultFile('rawmaker', name='horizontals_horizontals'),
+        ],
+        output=('plain',),
+    ),
+    utilo.create_step(
+        'highnote',
+        inputs=[
+            utilo.ResultFile(producer='rawmaker', name='text_text'),
+            utilo.ResultFile(producer='rawmaker', name='text_positions'),
+            utilo.ResultFile('rawmaker', name='horizontals_horizontals'),
+        ],
+        output=('highnote',),
+    ),
+    utilo.create_step(
+        'result',
+        inputs=[
+            utilo.ResultFile(producer='footnote', name='highnote_highnote'),
+            utilo.ResultFile(producer='footnote', name='plain_plain'),
+        ],
+        output=('result',),
+    ),
+    utilo.create_step(
+        'legacy',
+        inputs=[
+            utilo.ResultFile(producer='footnote', name='result_result'),
+        ],
+        output=('legacy',),
+    ),
+]
+
+
+def rename(path):
+    if not isinstance(path, str):
+        path = [rename(item) for item in path]
+        return path
+    path = utilo.rreplace(
+        path,
+        pattern='footnote__legacy_legacy',
+        replace='groupme__footer_footerheader',
+    )
+    return path
+
+
+def main():
+    utilo.featurepack(
+        workplan=WORKPLAN,
+        root=ibidem.ROOT,
+        featurepackage='ibidem.feature',
+        config=utilo.FeaturePackConfig(
+            description=DESCRIPTION,
+            multiprocessed=True,
+            name=ibidem.PROCESS,
+            pages=True,
+            rename=rename,
+            version=ibidem.__version__,
+        ),
+    )

ibidem/config.py

@@ -0,0 +1,15 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import configos
+import texmex
+
+FOOTER_SEPARATOR_WIDTH_MIN = configos.HV_INT_PLUS(default=70)
+
+VISIBLE = texmex.TextState.VISIBLE | texmex.TextState.FOOTNOTE

ibidem/feature/__init__.py

@@ -0,0 +1,8 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================

ibidem/feature/highnote.py

@@ -0,0 +1,42 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import serializeraw
+
+import ibidem.config
+import ibidem.strategy.highnote
+import ibidem.utils
+
+
+def work(
+    text: str,
+    textpositions: str,
+    horizontals: str,
+    pages=None,
+) -> str:
+    # load
+    horizontals = serializeraw.load_horizontals(
+        horizontals,
+        pages=pages,
+        width_min=ibidem.config.FOOTER_SEPARATOR_WIDTH_MIN,
+    )
+    ptns = serializeraw.ptn_fromfile(
+        text,
+        textpositions,
+        pages=pages,
+        state=ibidem.config.VISIBLE,
+    )
+    ptns = ibidem.utils.rotate_ifrequired(ptns)
+    strategy = ibidem.strategy.highnote.HighnoteStrategy(
+        horizontals=horizontals,
+        ptns=ptns,
+    )
+    result = strategy.result()
+    dumped = serializeraw.dump_headerfooter(result)
+    return dumped

ibidem/feature/legacy.py

@@ -0,0 +1,14 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import utilo
+
+
+def work(xresult) -> str:
+    return utilo.file_read(xresult)

ibidem/feature/plain.py

@@ -0,0 +1,41 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import serializeraw
+
+import ibidem.config
+import ibidem.strategy.plainmoving
+
+
+def work(
+    text: str,
+    textpositions: str,
+    horizontals: str,
+    pages=None,
+) -> str:
+    # load
+    horizontals = serializeraw.load_horizontals(
+        horizontals,
+        pages=pages,
+        width_min=ibidem.config.FOOTER_SEPARATOR_WIDTH_MIN,
+    )
+    ptns = serializeraw.ptn_fromfile(
+        text,
+        textpositions,
+        pages=pages,
+        state=ibidem.config.VISIBLE,
+    )
+    ptns = ibidem.utils.rotate_ifrequired(ptns)
+    strategy = ibidem.strategy.plainmoving.PlainMovingStrategy(
+        horizontals=horizontals,
+        ptns=ptns,
+    )
+    result = strategy.result()
+    dumped = serializeraw.dump_headerfooter(result)
+    return dumped

ibidem/feature/result.py

@@ -0,0 +1,142 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2019-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""Footer Extraction Step
+=============================
+
+TODO:
+    what should we do with empty header/footer
+"""
+
+import collections
+
+import iamraw
+import serializeraw
+import utilo
+
+
+def work(
+    xhighnote: str,
+    xplain: str,
+    pages=None,
+) -> str:
+    """Extract footer and header area out of horizontal lines.
+
+    Returns:
+        Dumped list with top and bottom border, which separates the
+        content from the footer and or header, for every page
+    """
+    highnote = serializeraw.load_headerfooter(
+        xhighnote,
+        pages=pages,
+    )
+    plain = serializeraw.load_headerfooter(
+        xplain,
+        pages=pages,
+    )
+    # select the best one
+    result = judge_strategy((
+        highnote,
+        plain,
+    ))
+    validate(result)
+    # dump
+    dumped = serializeraw.dump_headerfooter(result)
+    return dumped
+
+
+def judge_strategy(
+    results: list[iamraw.PageContentFooterHeaders],
+) -> iamraw.PageContentFooterHeaders:
+    """Decide which results fits best.
+
+    Zip result of different strategies. Sometimes there are multiple
+    options, therefore we have to use the priorities below.
+
+    Sources/Concept:
+
+        - MovingFooter:                footer (first prio)
+        - Pages:                       footer (second prio)
+        - FixedFooter:      header and footer (third prio)
+        - Common:           header            (last prio)
+        - PlainMoving:
+
+    Args:
+        results: lists of `ibidem.FootnoteDetectionStrategy`.result
+    Returns:
+        list of zipped result
+    """
+    assert results is not None, 'require list of strategy results'
+    result = []
+    for pdfpage, (
+            moving,
+            plainmoving,
+    ) in utilo.sync_pages(results):
+        footer = moving.footer if moving else None
+        footer_best = 'moving' if moving else None
+        # strategy: moving
+        if moving and moving.footer and moving.footer.notes:
+            footer = moving.footer
+            footer_best = 'moving'
+        # strategy: plain
+        if not (moving and moving.footer) and plainmoving and plainmoving.footer: # yapf:disable
+            # use plain moving only if no other strategy works
+            footer = plainmoving.footer
+            footer_best = 'plain'
+        # log footer best
+        if footer_best:
+            utilo.verbose(f'footer: {pdfpage} {footer_best}')
+        current = iamraw.PageContentFooterHeader(
+            footer=footer,
+            page=pdfpage,
+        )
+        result.append(current)
+
+    page_order = [item.page for item in result]
+    assert utilo.isascending(page_order), page_order
+    return result
+
+
+def quality(results: list) -> tuple:
+    """Determine quality[0.0, 1.0] of every extraction strategy."""
+    # count number of page
+    pages = set()
+    # count result for every strategy
+    counter = collections.defaultdict(int)
+    for pdfpage, data in utilo.sync_pages(results):
+        pages.add(pdfpage)
+        for index, item in enumerate(data):
+            if not item:
+                continue
+            counter[index] += 1
+    result = tuple(counter[index] / len(pages) if pages else 0
+                   for index, _ in enumerate(results))
+    return result
+
+
+def validate(items: list):
+    """Validate list of pageable items.
+
+    If some `page` attribute is duplicated, raise ValueError.
+
+    Args:
+        items(list): list of objects with <page,content>
+    Raises:
+        ValueError: if some page attribute is duplicated.
+    """
+    # TODO: REMOVE AFTER UPGRADING IAMRAW
+    counter = collections.Counter()
+    for item in items:
+        counter[item.page] += 1
+    msg = []
+    for page, value in counter.most_common():
+        if value <= 1:
+            continue
+        msg.append(f'duplicated page: {page} ({value})')
+    if msg:
+        raise ValueError(utilo.NEWLINE.join(msg))

ibidem/layout.py

@@ -0,0 +1,307 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2021-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import dataclasses
+import math
+
+import configos
+import iamraw
+import texmex
+import utilo
+
+import ibidem.utils
+
+HIGHNOTE_VERTICAL_LINE_DIFF_MAX = configos.HV_FLOAT_PLUS(default=15.0)
+
+FOOTNOTE_X0_MAX = configos.HolyTable(items=(
+    (440, 100),  # TODO: US Letter?
+    (550, 150),  # DINA4
+))
+FOOTNOTE_X1_MAX = configos.HolyTable(items=(
+    (440, 100 + 200),  # TODO: US Letter?
+    (550, 150 + 200),  # DINA4
+))
+
+FOOTNOTE_RATE_MIN = configos.HV_PERCENT_PLUS(default=50)
+
+
+def group_footnote_area(content) -> list:
+    connected_neighbors = connect_neighbors(content)
+    if not connected_neighbors:
+        return []
+    result = []
+    has_highnote = 0
+    for group in connected_neighbors:
+        splitted = split_textinfo(group)
+        if any(high for high, _ in splitted):
+            has_highnote += 1
+        merged = merge_online(splitted)
+        result.extend(merged)
+    result = merge_after(result)
+    rate = utilo.rate_rel(has_highnote, len(connected_neighbors))
+    if rate < FOOTNOTE_RATE_MIN:
+        if len(result) == 1:
+            return result
+        utilo.debug(f'no highnotes: {rate} detected, skip footnote result')
+        utilo.verbose(result)
+        return []
+    return result
+
+
+def merge_after(items: list) -> list:
+    """Make merger more robust against false formatted footnotes.
+
+    Merge footnote text which is under highnote on the left side.
+
+    See:
+        - bachelor041a p14
+        - master091b   p54
+    """
+    if not items:
+        return []
+    result = [items[0]]
+    for highnote, content in items[1:]:
+        if highnote is not None:
+            # normal footnote
+            result.append((highnote, content))
+            continue
+        highnote_before = result[-1][0] is not None
+        if highnote_before:
+            # merge them
+            for item in content.style.content:
+                item.start += len(result[-1][1].text)
+                item.end += len(result[-1][1].text)
+            result[-1][1].text += content.text
+            result[-1][1].style.content.extend(content.style.content)
+            continue
+        # footnote without highnote
+        result.append((None, content))
+    return result
+
+
+def connect_neighbors(items) -> list:
+    if not items:
+        return []
+    result = [[items[0]]]
+    for item in items[1:]:
+        before = result[-1][-1]
+        if connected(before.bounding, item.bounding):
+            result[-1].append(item)
+        else:
+            result.append([item])
+    return result
+
+
+def split_textinfo(content) -> list:
+    """Split text by `hightnote` and preserve TextInfo.
+
+    Go line by line from top to bottom. Collect lines till highnote
+    occurs. If highnote occurs merge content and add to result.
+
+    Returns:
+        list of a tuple of highnote and content
+    """
+    assert isinstance(content, list), type(content)
+    result = []
+    highnote = None
+    collected = []
+    for item in content:
+        for style in item.style.content:
+            if ishighnote(style, item.text):
+                if collected:
+                    result.append((highnote, union(collected)))
+                    collected = []
+                style = style.copy()
+                highnote = texmex.TextInfo(
+                    text=item.text[style.start:style.end],
+                    style=style,
+                    bounding=char_bounding(item.bounding, item.text, style),
+                )
+                style.start = 0
+                style.end = len(highnote.text)
+            else:
+                bounding = iamraw.split_x(
+                    bounding=item.bounding,
+                    part=style.start,
+                    parts=style.end,
+                    width=len(item.text),
+                )
+                collected.append(TextChunk(
+                    item.text,
+                    style,
+                    bounding=bounding,
+                ))
+    if highnote or collected:
+        # there is always content left at the end
+        result.append((highnote, union(collected)))
+    return result
+
+
+def merge_online(items) -> list:
+    """Ensure that high notes are located on a vertical line.
+
+    Therefore we have to ignore highnotes which are located inside the
+    text and not part of the text flow.
+
+    Steps:
+        1. Determine the most left highnotes
+        2. Adjust highnotes on most left one
+        3. Merge other highnotes into text
+    """
+    if not items:
+        return []
+    result = []
+    # skip None-Highnotes => if item
+    with_highnote = [high.bounding.x0 for high, _ in items if high]
+    mostleft = min(with_highnote) if with_highnote else None
+    high, collected = None, []
+    for highnote, content in items:
+        if not highnote:
+            result.append((None, content))
+            continue
+        diff = math.fabs(highnote.bounding.x0 - mostleft)
+        if diff > HIGHNOTE_VERTICAL_LINE_DIFF_MAX:
+            # highnote in content
+            collected.append(
+                shrink_tostyle(
+                    highnote.text,
+                    highnote.style,
+                    bounding=highnote.bounding,
+                ))
+            collected.extend([
+                shrink_tostyle(content.text, style, bounding=content.bounding)
+                for style in content.style
+            ])
+        else:
+            # new highnotes
+            if high:
+                result.append((high, union(collected)))
+            high = highnote
+            collected = [
+                shrink_tostyle(content.text, style, bounding=content.bounding)
+                for style in content.style
+            ]
+    notempty = collected or high
+    if notempty:
+        # do not add empty items
+        result.append((high, union(collected)))
+    return result
+
+
+# increase word diff at the start of the line to merge huge gap between
+# footnote number and footnote text.
+LEFTRIGHT_DIFF_MAX = configos.HolyTable(items=(
+    (0, 40),
+    (100, 40),
+    (120, 40),
+    (150, 30),
+    (200, 20),
+    (300, 20),
+    (400, 20),
+    (500, 20),
+))
+
+SAMEORIGIN_DIFF_MAX = configos.HV_FLOAT_PLUS(default=35.0)
+
+SAMELINE_DIFF_MAX = configos.HV_FLOAT_PLUS(default=5.0)
+
+UNDERFIRST_DIFF_MAX = configos.HV_FLOAT_PLUS(default=10.0)
+
+
+def connected(first: tuple, second: tuple) -> bool:
+    leftright_diff_max = LEFTRIGHT_DIFF_MAX(first.x0)
+    # words are neighbors
+    leftright = utilo.near(
+        first.x1,
+        second.x0,
+        diff=leftright_diff_max,
+    )
+    # plus indention
+    sameorigin = utilo.near(
+        first.x0,
+        second.x0,
+        diff=SAMEORIGIN_DIFF_MAX,
+    )
+    sameline = utilo.near(
+        first.y0,
+        second.y0,
+        diff=SAMELINE_DIFF_MAX,
+    )
+    underfirst = utilo.near(
+        first.y1,
+        second.y0,
+        diff=UNDERFIRST_DIFF_MAX,
+    )
+    result = (leftright or sameorigin) and (sameline or underfirst)
+    return result
+
+
+@dataclasses.dataclass
+class TextChunk:
+    text: str = None
+    style: texmex.TextStyle = None
+    bounding: iamraw.BoundingBox = None
+
+
+TextChunks = list[TextChunk]
+
+
+def shrink_tostyle(text: str, style, bounding=None) -> TextChunk:
+    text = text[style.start:style.end]
+    style = style.copy()
+    style.start, style.end = 0, len(text)
+    # TODO: ADD BOUNDING ADJUSTMENT, IN THE CURRENT CASE, WE DO NOT SHRINK
+    # THE BOUNDING
+    return TextChunk(text, style, bounding)
+
+
+def union(chunks: TextChunks) -> texmex.TextInfo:
+    raw = ''
+    content = []
+    for chunk in chunks:  # pylint:disable=W0612
+        start = len(raw)
+        raw += chunk.text[chunk.style.start:chunk.style.end]
+        end = len(raw)
+        section_style = chunk.style.copy()
+        section_style.start, section_style.end = start, end
+        content.append(section_style)
+    bounding = [item.bounding for item in chunks if item.bounding]
+    bounding = utilo.rect_max(bounding) if bounding else None
+    result = texmex.TextInfo(
+        text=raw,
+        style=texmex.TextStyle(content=content),
+        bounding=bounding,
+    )
+    return result
+
+
+def char_bounding(
+    bounding: iamraw.BoundingBox,
+    text: str,
+    style: texmex.TextStyle,
+) -> iamraw.BoundingBox:
+    width = bounding.x1 - bounding.x0
+    char_width = width / len(text)
+    x0 = bounding.x0 + char_width * style.start
+    x1 = bounding.x0 + char_width * style.end
+    result = iamraw.BoundingBox(x0, bounding.y0, x1, bounding.y1)
+    return result
+
+
+HIGHNOTE_RISE_MIN = configos.HV_FLOAT_PLUS(default=3.0)
+
+
+def ishighnote(style, text: str) -> bool:
+    highnote_occurs = style.rise >= HIGHNOTE_RISE_MIN
+    if not highnote_occurs:
+        return False
+    text = text[style.start:style.end].strip()
+    if ibidem.utils.NUMBER.match(text):
+        return True
+    return False

ibidem/parser/__init__.py

@@ -0,0 +1,8 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================