ibidem 1.0.0__tar.gz

ibidem-1.0.0/PKG-INFO

@@ -0,0 +1,29 @@
+Metadata-Version: 2.4
+Name: ibidem
+Version: 1.0.0
+Author-email: Helmut Konrad Schewe <helmutus@outlook.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/anaticulae/ibidem
+Project-URL: Repository, https://github.com/anaticulae/ibidem
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: iamraw<5.0.0,>=4.91.7
+Requires-Dist: utilo<3.0.0,>=2.109.0
+Requires-Dist: configos<2.0.0,>=1.0.4
+Requires-Dist: geostrat<2.0.0,>=1.5.1
+Requires-Dist: konradus<2.0.0,>=1.0.1
+Requires-Dist: elementae<2.0.0,>=1.0.1
+Requires-Dist: germania<2.0.0,>=1.32.0
+Provides-Extra: dev
+Requires-Dist: utilotest==1.0.4; extra == "dev"
+Requires-Dist: hoverpower==1.5.1; extra == "dev"
+Requires-Dist: gennex<2.0.0,>=1.0.3; extra == "dev"
+Requires-Dist: rawmaker==2.40.5; extra == "dev"
+Requires-Dist: pagenumber<2.0.0,>=1.0.0; extra == "dev"
+Requires-Dist: groupmes<2.0.0,>=1.1.0; extra == "dev"
+Requires-Dist: resinf<2.0.0,>=1.0.4; extra == "dev"
+
+# footnote

ibidem-1.0.0/README

@@ -0,0 +1 @@
+# footnote

ibidem-1.0.0/ibidem/__init__.py

@@ -0,0 +1,16 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================
+
+import importlib.metadata
+import os
+
+__version__ = importlib.metadata.version('ibidem')
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+PROCESS = 'footnote'

ibidem-1.0.0/ibidem/cli.py

@@ -0,0 +1,78 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2019-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================
+
+import utilo
+
+import ibidem
+
+DESCRIPTION = 'TODO'
+
+WORKPLAN = [
+    utilo.create_step(
+        'plain',
+        inputs=[
+            utilo.ResultFile(producer='rawmaker', name='text_text'),
+            utilo.ResultFile(producer='rawmaker', name='text_positions'),
+            utilo.ResultFile('rawmaker', name='horizontals_horizontals'),
+        ],
+        output=('plain',),
+    ),
+    utilo.create_step(
+        'highnote',
+        inputs=[
+            utilo.ResultFile(producer='rawmaker', name='text_text'),
+            utilo.ResultFile(producer='rawmaker', name='text_positions'),
+            utilo.ResultFile('rawmaker', name='horizontals_horizontals'),
+        ],
+        output=('highnote',),
+    ),
+    utilo.create_step(
+        'result',
+        inputs=[
+            utilo.ResultFile(producer='footnote', name='highnote_highnote'),
+            utilo.ResultFile(producer='footnote', name='plain_plain'),
+        ],
+        output=('result',),
+    ),
+    utilo.create_step(
+        'legacy',
+        inputs=[
+            utilo.ResultFile(producer='footnote', name='result_result'),
+        ],
+        output=('legacy',),
+    ),
+]
+
+
+def rename(path):
+    if not isinstance(path, str):
+        path = [rename(item) for item in path]
+        return path
+    path = utilo.rreplace(
+        path,
+        pattern='footnote__legacy_legacy',
+        replace='groupme__footer_footerheader',
+    )
+    return path
+
+
+def main():
+    utilo.featurepack(
+        workplan=WORKPLAN,
+        root=ibidem.ROOT,
+        featurepackage='ibidem.feature',
+        config=utilo.FeaturePackConfig(
+            description=DESCRIPTION,
+            multiprocessed=True,
+            name=ibidem.PROCESS,
+            pages=True,
+            rename=rename,
+            version=ibidem.__version__,
+        ),
+    )

ibidem-1.0.0/ibidem/config.py

@@ -0,0 +1,15 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import configos
+import texmex
+
+FOOTER_SEPARATOR_WIDTH_MIN = configos.HV_INT_PLUS(default=70)
+
+VISIBLE = texmex.TextState.VISIBLE | texmex.TextState.FOOTNOTE

ibidem-1.0.0/ibidem/feature/__init__.py

@@ -0,0 +1,8 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================

ibidem-1.0.0/ibidem/feature/highnote.py

@@ -0,0 +1,42 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import serializeraw
+
+import ibidem.config
+import ibidem.strategy.highnote
+import ibidem.utils
+
+
+def work(
+    text: str,
+    textpositions: str,
+    horizontals: str,
+    pages=None,
+) -> str:
+    # load
+    horizontals = serializeraw.load_horizontals(
+        horizontals,
+        pages=pages,
+        width_min=ibidem.config.FOOTER_SEPARATOR_WIDTH_MIN,
+    )
+    ptns = serializeraw.ptn_fromfile(
+        text,
+        textpositions,
+        pages=pages,
+        state=ibidem.config.VISIBLE,
+    )
+    ptns = ibidem.utils.rotate_ifrequired(ptns)
+    strategy = ibidem.strategy.highnote.HighnoteStrategy(
+        horizontals=horizontals,
+        ptns=ptns,
+    )
+    result = strategy.result()
+    dumped = serializeraw.dump_headerfooter(result)
+    return dumped

ibidem-1.0.0/ibidem/feature/legacy.py

@@ -0,0 +1,14 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import utilo
+
+
+def work(xresult) -> str:
+    return utilo.file_read(xresult)

ibidem-1.0.0/ibidem/feature/plain.py

@@ -0,0 +1,41 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import serializeraw
+
+import ibidem.config
+import ibidem.strategy.plainmoving
+
+
+def work(
+    text: str,
+    textpositions: str,
+    horizontals: str,
+    pages=None,
+) -> str:
+    # load
+    horizontals = serializeraw.load_horizontals(
+        horizontals,
+        pages=pages,
+        width_min=ibidem.config.FOOTER_SEPARATOR_WIDTH_MIN,
+    )
+    ptns = serializeraw.ptn_fromfile(
+        text,
+        textpositions,
+        pages=pages,
+        state=ibidem.config.VISIBLE,
+    )
+    ptns = ibidem.utils.rotate_ifrequired(ptns)
+    strategy = ibidem.strategy.plainmoving.PlainMovingStrategy(
+        horizontals=horizontals,
+        ptns=ptns,
+    )
+    result = strategy.result()
+    dumped = serializeraw.dump_headerfooter(result)
+    return dumped

ibidem-1.0.0/ibidem/feature/result.py

@@ -0,0 +1,142 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2019-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""Footer Extraction Step
+=============================
+
+TODO:
+    what should we do with empty header/footer
+"""
+
+import collections
+
+import iamraw
+import serializeraw
+import utilo
+
+
+def work(
+    xhighnote: str,
+    xplain: str,
+    pages=None,
+) -> str:
+    """Extract footer and header area out of horizontal lines.
+
+    Returns:
+        Dumped list with top and bottom border, which separates the
+        content from the footer and or header, for every page
+    """
+    highnote = serializeraw.load_headerfooter(
+        xhighnote,
+        pages=pages,
+    )
+    plain = serializeraw.load_headerfooter(
+        xplain,
+        pages=pages,
+    )
+    # select the best one
+    result = judge_strategy((
+        highnote,
+        plain,
+    ))
+    validate(result)
+    # dump
+    dumped = serializeraw.dump_headerfooter(result)
+    return dumped
+
+
+def judge_strategy(
+    results: list[iamraw.PageContentFooterHeaders],
+) -> iamraw.PageContentFooterHeaders:
+    """Decide which results fits best.
+
+    Zip result of different strategies. Sometimes there are multiple
+    options, therefore we have to use the priorities below.
+
+    Sources/Concept:
+
+        - MovingFooter:                footer (first prio)
+        - Pages:                       footer (second prio)
+        - FixedFooter:      header and footer (third prio)
+        - Common:           header            (last prio)
+        - PlainMoving:
+
+    Args:
+        results: lists of `ibidem.FootnoteDetectionStrategy`.result
+    Returns:
+        list of zipped result
+    """
+    assert results is not None, 'require list of strategy results'
+    result = []
+    for pdfpage, (
+            moving,
+            plainmoving,
+    ) in utilo.sync_pages(results):
+        footer = moving.footer if moving else None
+        footer_best = 'moving' if moving else None
+        # strategy: moving
+        if moving and moving.footer and moving.footer.notes:
+            footer = moving.footer
+            footer_best = 'moving'
+        # strategy: plain
+        if not (moving and moving.footer) and plainmoving and plainmoving.footer: # yapf:disable
+            # use plain moving only if no other strategy works
+            footer = plainmoving.footer
+            footer_best = 'plain'
+        # log footer best
+        if footer_best:
+            utilo.verbose(f'footer: {pdfpage} {footer_best}')
+        current = iamraw.PageContentFooterHeader(
+            footer=footer,
+            page=pdfpage,
+        )
+        result.append(current)
+
+    page_order = [item.page for item in result]
+    assert utilo.isascending(page_order), page_order
+    return result
+
+
+def quality(results: list) -> tuple:
+    """Determine quality[0.0, 1.0] of every extraction strategy."""
+    # count number of page
+    pages = set()
+    # count result for every strategy
+    counter = collections.defaultdict(int)
+    for pdfpage, data in utilo.sync_pages(results):
+        pages.add(pdfpage)
+        for index, item in enumerate(data):
+            if not item:
+                continue
+            counter[index] += 1
+    result = tuple(counter[index] / len(pages) if pages else 0
+                   for index, _ in enumerate(results))
+    return result
+
+
+def validate(items: list):
+    """Validate list of pageable items.
+
+    If some `page` attribute is duplicated, raise ValueError.
+
+    Args:
+        items(list): list of objects with <page,content>
+    Raises:
+        ValueError: if some page attribute is duplicated.
+    """
+    # TODO: REMOVE AFTER UPGRADING IAMRAW
+    counter = collections.Counter()
+    for item in items:
+        counter[item.page] += 1
+    msg = []
+    for page, value in counter.most_common():
+        if value <= 1:
+            continue
+        msg.append(f'duplicated page: {page} ({value})')
+    if msg:
+        raise ValueError(utilo.NEWLINE.join(msg))