capita 1.25.1__tar.gz

capita-1.25.1/PKG-INFO

@@ -0,0 +1,28 @@
+Metadata-Version: 2.4
+Name: capita
+Version: 1.25.1
+Author-email: Helmut Konrad Schewe <helmutus@outlook.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/anaticulae/capita
+Project-URL: Repository, https://github.com/anaticulae/capita
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: configos<2.0.0,>=1.0.4
+Requires-Dist: elementae<2.0.0,>=1.0.1
+Requires-Dist: geostrat<2.0.0,>=1.5.1
+Requires-Dist: germania<2.0.0,>=1.32.0
+Requires-Dist: iamraw<5.0.0,>=4.91.6
+Requires-Dist: layouta<2.0.0,>=1.0.0
+Requires-Dist: utilo<3.0.0,>=2.109.0
+Provides-Extra: dev
+Requires-Dist: gennex==1.0.3; extra == "dev"
+Requires-Dist: groupmes<2.0.0,>=1.1.0; extra == "dev"
+Requires-Dist: hoverpower==1.4.3; extra == "dev"
+Requires-Dist: pagenumber==1.0.0; extra == "dev"
+Requires-Dist: rawmaker==2.40.3; extra == "dev"
+Requires-Dist: utilotest<2.0.0,>=1.0.4; extra == "dev"
+
+# sections

capita-1.25.1/README

@@ -0,0 +1 @@
+# sections

capita-1.25.1/capita/__init__.py

@@ -0,0 +1,79 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================
+"""Sections
+========
+
+The `sections` tool is a very lightwight tool to determine the
+likelihood of a feature on a page very quickly. Sections is runned to
+determine which feature extraction should runned on which pages.
+
+Plan
+----
+
+The result of the section tool is a plan for the feature extractor.
+Further plans for judgements and informations for the user are not
+provided by sections tool. The section tool extracts only features but
+do not judges the result.
+
+Sources
+-------
+
+Data Provider:
+
+* Rawmaker
+* Linero
+* Imagero (planned)
+* Figero (planned)
+* Tabelero (planned)
+
+
+Feature Provider:
+
+* Groupme
+* Detector
+
+Planned Features
+----------------
+
+The following sections are planned to be supported:
+
+.. code-block:: none
+
+  * Introduction
+      * Titlepage
+      * Thank you
+      * Copyright etc.
+      * Erklaerung
+  * Table-Area
+      * Table of content
+      * Short cuts
+      * Figure table
+  * Content
+      * Chapter
+          * Figure
+          * Text
+          * Headlines
+  * Table-Area-B
+  * Appendix
+      * Resources
+      * Link
+      * Bibliography
+"""
+
+import importlib.metadata
+import os
+
+import capita.__patch__
+import capita.path
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+
+PACKAGE = 'capita'
+PROCESS = 'sections'
+__version__ = importlib.metadata.version(PACKAGE)

capita-1.25.1/capita/__main__.py

@@ -0,0 +1,13 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================
+
+from capita.cli import main
+
+if __name__ == "__main__":
+    main()

capita-1.25.1/capita/__patch__.py

@@ -0,0 +1,70 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import re
+
+import germania
+import utilo
+
+
+def authors(raw: str, verbose: bool = False) -> list:
+    """\
+    >>> authors('PEREIRA, M.G., VOLCHAN, E., SOUZA, G. G. DE, OLIVEIRA, L.,'
+    ... 'CAMPAGNOLI, R. R., PINHEIRO, W. M., & PESSOA, L. ')
+    ['PEREIRA, M.G.', 'VOLCHAN, E.', 'SOUZA, G. G. DE', 'OLIVEIRA, L.', 'CAMPAGNOLI, R. R.', 'PINHEIRO, W. M.', 'PESSOA, L.']
+    """
+    pattern = r"""
+        [a-zA-Z]{4,}\,
+        [ ]{0,2}
+        ([a-zA-Z]\.[ ]{0,2}){1,3}
+        [ ]{0,2}
+        (DE)?
+    """
+    result = []
+    for item in re.finditer(pattern, raw, re.VERBOSE):
+        item = utilo.extract_match(item).strip()
+        if verbose:
+            item = (item, item)
+        result.append(item)
+    return result
+
+
+germania.authors = authors
+
+
+def a_minus_b(a: set | list, b: set | list) -> set | list:
+    result = []
+    for item in a:
+        if item in b:
+            continue
+        result.append(item)
+    if isinstance(a, set):
+        result: set = set(result)
+    return result
+
+
+utilo.a_minus_b = a_minus_b
+
+
+def union(*items: set | list):
+    """\
+    >>> union([1, 2, 3])
+    [1, 2, 3]
+    >>> union([1, 2, 3], [2, 2, 2])
+    [1, 2, 3]
+    """
+    result = []
+    for item in items:
+        for it in item:
+            result.append(it)
+    result = utilo.make_unique(result)
+    return result
+
+
+utilo.union = union

capita-1.25.1/capita/biblio/__init__.py

@@ -0,0 +1,8 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================

capita-1.25.1/capita/biblio/utils.py

@@ -0,0 +1,76 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import configos
+import iamraw
+import utilo
+
+DIFF_MAX = configos.HolyTable(
+    items=(
+        (1, 0),
+        (5, 0),
+        (6, 1),
+        (10, 2),
+        (12, 2),
+    ),
+    right_outranges_none=False,
+)
+
+
+def cluster_bibpages(
+    items,
+    likelihood_name='bibliography_table',
+):
+    """Select hugest(max sum likelihood value) group."""
+    if not items:
+        return []
+    maxdiff = DIFF_MAX(len(items)) + 1
+    grouped = utilo.groupby_diff(
+        items,
+        maxdiff=maxdiff,
+        selector=lambda x: x.page,
+    )
+    if not grouped:
+        return []
+    hugest = sorted(
+        grouped,
+        key=lambda x: sum(item.content.value for item in x),
+    )
+    hugest = hugest[-1]
+    avg = sum((item.content.value for item in hugest)) / len(hugest)
+    avg = utilo.roundme(avg)
+    for item in hugest:
+        # every item of the group should have the same likelihood
+        item.content.value = avg
+    result = fill_empty(
+        hugest,
+        likelihood_name=likelihood_name,
+    )
+    return result
+
+
+def fill_empty(items: list, likelihood_name: str) -> list:
+    """Fill holes inside connected bib."""
+    result = list(items)
+    start, end = result[0].page, result[-1].page
+    done = {item.page for item in result}
+    avg = result[0].content.value
+    for page in utilo.rlist(start, end):
+        if page in done:
+            continue
+        result.append(
+            iamraw.PageContentLikelihood(
+                page=page,
+                content=iamraw.Likelihood(
+                    value=avg,
+                    name=likelihood_name,
+                ),
+            ))
+    result.sort(key=lambda x: x.page)
+    return result

capita-1.25.1/capita/chapter/__init__.py

@@ -0,0 +1,8 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2022-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================

capita-1.25.1/capita/chapter/outlines.py

@@ -0,0 +1,129 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2022-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""Chapter: Outlines
+=================
+
+Try to find text on the page which matches with lines from the outlines.
+
+In most cases, if outlines are given, there represent the headlines
+inside the document.
+"""
+
+import re
+
+import configos
+import elementae.headline.lookup
+import utilo
+
+import capita.chapter.utils
+
+TOCS_COUNT_MIN = configos.HV_INT_PLUS(default=3)
+
+
+def rate_from_outlines(tocs, pagestart: list) -> float:
+    """Try to find potential headline inside toc from outlines.
+
+    If no toc is given, use elementae.headlines-list as backup strategy.
+    """
+    chapter_rate = 0.0
+    if len(tocs) >= TOCS_COUNT_MIN:
+        chapter_rate = contains_outline(pagestart, tocs)
+        return chapter_rate
+    # disable feature if no toc is given
+    utilo.info('chapter: no toc provided')
+    # try backup
+    matched = any(
+        utilo.similar(
+            expected=HEADLINES_BACKUP,
+            current=item.text,
+            maxdiff=0.9,
+        ) for item in pagestart)
+    if matched:
+        return 1.0
+    return 0.0
+
+
+HEADLINES_BACKUP = elementae.headline.lookup.CHAPTER
+
+
+def contains_outline(content, toc) -> float:
+    """Check that content starts with a parsed headline entree.
+
+    Supported Pattern; line starts with:
+        * 3. Headline text
+        * Headline text
+    """
+    flat_toc = level_remove(toc)
+    flat_toc = toc_shrink(flat_toc)
+    if not flat_toc:
+        # no table of content was extracted
+        return 0.0
+    flat_toc = [item for item in flat_toc if item.lower() not in NOHEADLINES]
+    for line in content:
+        line = utilo.normalize_whitespaces(line.text.strip())
+        if utilo.issinglechar(line):
+            # I N T R O D U C T I O N 1
+            line = line.replace(' ', '')
+        # remove numbered headline pattern and potential white spaces
+        without_number = FIRSTLEVEL_DOT_PATTERN.sub('', line)
+        for headline in flat_toc:
+            if all((
+                    not capita.chapter.utils.startswith(line, headline),
+                    not capita.chapter.utils.startswith(without_number,
+                                                        headline),
+            )):
+                continue
+            rate = len(without_number) / len(headline)
+            if not 0.2 < rate < 2.0:
+                # headline is too long or too short, could not match with
+                # detected toc
+                continue
+            return 1.0
+    return -0.5
+
+
+NOHEADLINES = utilo.a_minus_b(  #pylint:disable=no-member
+    elementae.headline.lookup.HEADLINES,
+    elementae.headline.lookup.CHAPTER,
+)
+
+FIRSTLEVEL_DOT_PATTERN = re.compile(r'^\d{1,2}\.{0,1}\s+')
+
+
+def level_remove(toc):
+    flat = []
+    for item in toc:
+        if not item.title.strip():
+            # invalid outlines can break further processing
+            utilo.debug('empty outline element')
+            continue
+        flat.append(FIRSTLEVEL_DOT_PATTERN.sub('', item.title))
+    # remove roman level
+    result = []
+    for item in flat:
+        splitted = item.split(maxsplit=1)
+        if len(splitted) == 1:
+            result.append(item)
+            continue
+        if utilo.isroman(splitted[0]):
+            result.append(splitted[1])
+            continue
+        result.append(item)
+    return result
+
+
+def toc_shrink(items):
+    """Remove tocs after Anhang."""
+    result = []
+    for item in items:
+        if 'anhang' in item.lower():
+            # TODO: DANGEROUS! ANHANG CAN BE PART OF ANY OTHER HEADLINE
+            break
+        result.append(item)
+    return result

capita-1.25.1/capita/chapter/run.py

@@ -0,0 +1,123 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2022-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import statistics
+
+import configos
+import iamraw
+import texmex
+import utilo
+
+import capita.chapter.outlines
+import capita.chapter.starter
+
+FIRST_QUARTER = configos.HV_PERCENT_PLUS(default=45.0)
+
+
+def extract_chapter(
+    navigators: texmex.PTNs,
+    outlines,
+) -> iamraw.PageContentLikelihoods:
+    result = []
+    for page in navigators:
+        if nochapter(page):
+            continue
+        # prepare content
+        pagestart = page.before(FIRST_QUARTER)
+        # run strategies
+        rate = capita.chapter.starter.contain_chapter(pagestart)
+        rate += capita.chapter.outlines.rate_from_outlines(
+            outlines,
+            pagestart,
+        )
+        if rate <= 0.0:
+            # TODO: VERY VERBOSE, CHECK LATER
+            utilo.verbose(f'    no chapter {page.page}: chapterate {rate}')
+            continue
+        # convert result to percents
+        rate_in_percent = chaptervalue_to_percent(rate, outlines)
+        result.append(
+            iamraw.PageContentLikelihood(
+                page=page.page,
+                content=iamraw.Likelihood(rate_in_percent, 'chapter'),
+            ))
+        # TODO: There is the possiblity that header and start of chapter
+        # are together, support later
+        # result.append(0.0)
+    return result
+
+
+def nochapter(page) -> bool:
+    if not page:
+        # empty page
+        return True
+    if page.rotated:
+        utilo.verbose(f'no chapter {page.page}: rotated')
+        return True
+    pagestart = page.before(FIRST_QUARTER)
+    if no_textcontent(pagestart):
+        utilo.verbose(f'no chapter {page.page}: no textcontent')
+        return True
+    if contains_listof(pagestart):
+        if not capita.chapter.starter.startwith_hugenumber(pagestart):
+            utilo.verbose(f'no chapter {page.page}: list of dots')
+            # TODO: See todo below XXX???
+            # chapter_rate = 0
+            return True
+    return False
+
+
+def no_textcontent(content: list) -> bool:
+    """A chapter start requires some text line at the start.
+
+    This is required to skip false positive chapter starts which are
+    just a table or so.
+    """
+    if len(content) < 5:
+        return False
+    avg = statistics.mean((len(item.text) for item in content))
+    if avg < 35:
+        # TODO: HOLY VALUE
+        return True
+    return False
+
+
+def chaptervalue_to_percent(chaptervalue: float, hastoc: bool) -> float:
+    """Convert `chaptervalue` to percent.
+
+    Args:
+        chaptervalue(float): value of detected features
+        hastoc(bool): if no toc is provided, some features can not be
+                       processed.
+    Returns:
+        Likelihood of beeing a chapter start.?
+    """
+    # TODO: HOLY VALUES
+    # TODO: IMPROVE THIS CONCEPT
+    if not hastoc and chaptervalue >= 1.0:
+        return 1.0
+    if chaptervalue >= 2.5:
+        return 1.0
+    if chaptervalue >= 0.5:
+        return 0.5
+    return 0.0
+
+
+def contains_listof(content: list) -> bool:
+    raw = rawcontent(content)
+    dots_with_spaces = raw.count('. . . .')
+    connected_dots = raw.count('....')
+    result = dots_with_spaces > 4 or connected_dots >= 3
+    return result
+
+
+def rawcontent(content) -> str:
+    raw = utilo.NEWLINE.join([item.text for item in content])
+    raw = raw.lower()
+    return raw