reftable 1.0.0__py3-none-any.whl

reftable/__init__.py

@@ -0,0 +1,19 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2022-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================
+
+import importlib.metadata
+import os
+
+import reftable.path
+
+PROCESS = 'reftable'
+
+__version__ = importlib.metadata.version(PROCESS)
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))

reftable/__main__.py

@@ -0,0 +1,13 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2022-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+from reftable.cli import main
+
+if __name__ == "__main__":
+    main()

reftable/abbrev/__init__.py

@@ -0,0 +1,41 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""Abbreviation
+============
+
+Examples:
+
+* homework/page_50_math.pdf:page6
+* master/page_116_images_toc_formular.pdf:95
+
+TODO: Abbreviation creation tool
+TODO: Symbol collector
+"""
+
+import abc
+import dataclasses
+
+import iamraw
+import texmex
+
+
+@dataclasses.dataclass
+class AbbreviationData:
+    normal: texmex.PTNs = None
+    oneline: texmex.PTNs = None
+
+
+class AbbreviationExtractorStrategy(abc.ABC):
+
+    def __init__(self, loaded: AbbreviationData):
+        self.loaded = loaded
+
+    @abc.abstractmethod
+    def result(self) -> iamraw.AbbreviationResult:
+        pass

reftable/abbrev/geometry.py

@@ -0,0 +1,69 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""Abbreviation Parser: Geometry Strategy
+======================================
+
+This approach splits the page in two columns and detect a left shortcut
+and a right description column. It is required that the distance between
+left and right column is not ?too? tight.
+
+Working Examples
+----------------
+
+* bachelor37: page 2
+
+Not working
+-----------
+
+* homework50: columns are to tight together
+
+Nearly working
+--------------
+
+* master116: improve layout parser
+"""
+
+import geostrat
+import iamraw
+import texmex
+import utilo
+
+import reftable.abbrev
+
+
+class GeometryAbbreviationParser(reftable.abbrev.AbbreviationExtractorStrategy):
+
+    def result(self) -> iamraw.AbbreviationResult:
+        ready = iamraw.AbbreviationResult()
+        for page in self.loaded.normal:
+            parsed = parse_page(page)
+            if parsed is None:
+                utilo.info(f'could not parse page: {page.page}')
+                continue
+            for item in parsed:
+                ready.append(item)
+            ready.pdfpages.append(page.page)
+        return ready
+
+
+def parse_page(page: texmex.PTN) -> iamraw.Abbreviations:
+    parsed = geostrat.dc_parse_page(page)
+    if not parsed:
+        return None
+    result = []
+    for short, description in parsed:
+        short, description = short.strip(), description.strip()
+        description = description.lstrip('-')
+        result.append(
+            iamraw.Abbreviation(
+                short=short,
+                description=description,
+                position=iamraw.AbbreviationPosition(page=page.page),
+            ))
+    return result

reftable/abbrev/parser.py

@@ -0,0 +1,36 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import iamraw
+
+import reftable.abbrev
+import reftable.abbrev.geometry
+import reftable.abbrev.simple
+
+STRATEGIES = [
+    reftable.abbrev.simple.SimpleAbbreviationParser,
+    reftable.abbrev.geometry.GeometryAbbreviationParser,
+]
+
+
+def parse(data: reftable.abbrev.AbbreviationData) -> iamraw.AbbreviationResult:
+    assert isinstance(data.normal, list), type(data)
+    assert isinstance(data.oneline, list), type(data)
+    parsed = [strategy(data).result() for strategy in STRATEGIES]
+    judged = judge(parsed)
+    return judged
+
+
+def judge(results) -> iamraw.AbbreviationResult:
+    simple = results[0]
+    geometry = results[1]
+    more_than_double_parsed = (len(geometry) * 2) < len(simple)
+    if more_than_double_parsed:
+        return simple
+    return geometry

reftable/abbrev/simple.py

@@ -0,0 +1,66 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import elementae.headline.lookup
+import iamraw
+import utilo
+
+import reftable.abbrev
+
+
+class SimpleAbbreviationParser(reftable.abbrev.AbbreviationExtractorStrategy):
+
+    def result(self) -> iamraw.AbbreviationResult:
+        ready = iamraw.AbbreviationResult()
+        for page in self.loaded.oneline:
+            parsed = parse_page(page)
+            for item in parsed:
+                ready.append(item)
+            if parsed:
+                ready.pdfpages.append(page.page)
+        return ready
+
+
+def parse_page(content):
+    result = []
+    for line in content:
+        raw = line.text
+        utilo.debug(f'parse: {raw}')
+        try:
+            short, description = raw.split(maxsplit=1)
+        except ValueError:
+            if not isexcluded(raw):
+                utilo.debug(f'could not parse abbreviation: {raw}')
+            continue
+        if isexcluded(short) or isexcluded(description):
+            utilo.info(f'skip: {short}, {description}')
+            continue
+        short, description = short.strip(), description.strip()
+        parsed = iamraw.Abbreviation(
+            short=short,
+            description=description,
+        )
+        utilo.debug(f'parsed: {parsed}')
+        result.append(parsed)
+    return result
+
+
+def isexcluded(item: str) -> bool:
+    # TODO: ADD MORE SOPHISTICATED DOUBLE COLUMN CHECKER
+    item = item.strip().lower()
+    if item in elementae.headline.lookup.HEADLINES:
+        return True
+    if item in NO_ABBREVIATION:
+        return True
+    return False
+
+
+NO_ABBREVIATION = utilo.splitlines("""\
+BESCHREIBUNG
+""")

reftable/cli.py

@@ -0,0 +1,76 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================
+
+import utilo
+
+import reftable
+
+DESCRIPTION = ''
+
+WORKPLAN = [
+    utilo.create_step(
+        'abbrev',
+        inputs=[
+            utilo.ResultFile(producer='rawmaker', name='text_text'),
+            utilo.ResultFile(producer='rawmaker', name='text_positions'),
+            utilo.ResultFile(producer='rawmaker', name='oneline_text_text'),
+            utilo.ResultFile('rawmaker', name='oneline_text_positions'),
+            utilo.ResultFile(producer='groupme', name='footer_footerheader'),
+            utilo.ResultFile(producer='rawmaker', name='border_pages'),
+        ],
+        output=('abbrev',),
+    ),
+    utilo.create_step(
+        'toc',
+        inputs=[
+            utilo.ResultFile(producer='rawmaker', name='oneline_text_text'),
+            utilo.ResultFile('rawmaker', name='oneline_text_positions'),
+            utilo.ResultFile(producer='groupme', name='footer_footerheader'),
+            utilo.ResultFile(producer='rawmaker', name='border_pages'),
+        ],
+        output=('toc',),
+    ),
+    utilo.create_step(
+        'figure',
+        inputs=[
+            utilo.ResultFile(producer='rawmaker', name='text_text'),
+            utilo.ResultFile(producer='rawmaker', name='text_positions'),
+            utilo.ResultFile(producer='rawmaker', name='oneline_text_text'),
+            utilo.ResultFile('rawmaker', name='oneline_text_positions'),
+            utilo.ResultFile(producer='groupme', name='footer_footerheader'),
+            utilo.ResultFile(producer='rawmaker', name='border_pages'),
+        ],
+        output=('figure',),
+    ),
+    utilo.create_step(
+        'table',
+        inputs=[
+            utilo.ResultFile(producer='rawmaker', name='oneline_text_text'),
+            utilo.ResultFile('rawmaker', name='oneline_text_positions'),
+            utilo.ResultFile(producer='groupme', name='footer_footerheader'),
+            utilo.ResultFile(producer='rawmaker', name='border_pages'),
+        ],
+        output=('table',),
+    ),
+]
+
+
+def main():
+    utilo.featurepack(
+        workplan=WORKPLAN,
+        root=reftable.ROOT,
+        featurepackage='reftable.feature',
+        config=utilo.FeaturePackConfig(
+            description=DESCRIPTION,
+            multiprocessed=True,
+            name=reftable.PROCESS,
+            pages=True,
+            version=reftable.__version__,
+        ),
+    )

reftable/feature/__init__.py

@@ -0,0 +1,8 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2022-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================

reftable/feature/abbrev.py

@@ -0,0 +1,67 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import serializeraw
+
+import reftable.abbrev
+import reftable.abbrev.parser
+
+
+def work(
+    text: str,
+    textposition: str,
+    text_oneline: str,
+    textposition_oneline: str,
+    headerfooter: str,
+    sizeandborder: str,
+    pages: tuple = None,
+) -> str:
+    data = load_data(
+        text,
+        textposition,
+        text_oneline,
+        textposition_oneline,
+        headerfooter,
+        sizeandborder,
+        pages,
+    )
+    parsed = reftable.abbrev.parser.parse(data)
+    # dump result
+    dumped = serializeraw.dump_abbreviation_table(parsed)
+    return dumped
+
+
+def load_data(
+    text: str,
+    textposition: str,
+    text_oneline: str,
+    textposition_oneline: str,
+    headerfooter: str,
+    sizeandborder: str,
+    pages: tuple = None,
+) -> reftable.abbrev.AbbreviationData:
+    normal = serializeraw.ptcn_fromfile(
+        text=text,
+        textpositions=textposition,
+        sizeandborder=sizeandborder,
+        headerfooter=headerfooter,
+        pages=pages,
+    )
+    oneline = serializeraw.ptcn_fromfile(
+        text=text_oneline,
+        textpositions=textposition_oneline,
+        sizeandborder=sizeandborder,
+        headerfooter=headerfooter,
+        pages=pages,
+    )
+    data = reftable.abbrev.AbbreviationData(
+        normal=normal,
+        oneline=oneline,
+    )
+    return data

reftable/feature/figure.py

@@ -0,0 +1,51 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""Table of figures extractor
+==========================
+"""
+
+import serializeraw
+
+import reftable.figure.strategy
+
+
+def work(
+    text: str,
+    textpositions: str,
+    oneline_text: str,
+    oneline_textpositions: str,
+    headerfooter: str,
+    sizeandborder: str,
+    pages: tuple = None,
+) -> str:
+    """Extract table of figures out of `document`.
+
+    Args:
+        text(str): path to load document
+        textpositions(str): path to load document textpositions
+        oneline_text(str): oneline document
+        oneline_textpositions(str): oneline document positions
+        headerfooter(str): path with header and footer to determine
+                           content border.
+        sizeandborder(str): path with page sizes and content border
+        pages(tuple): tuple of selected pages
+    Returns:
+        dump of extracted table of content
+    """
+    result = reftable.figure.strategy.run(
+        text=text,
+        textpositions=textpositions,
+        oneline_text=oneline_text,
+        oneline_textpositions=oneline_textpositions,
+        headerfooter=headerfooter,
+        sizeandborder=sizeandborder,
+        pages=pages,
+    )
+    dumped = serializeraw.dump_toc(result)
+    return dumped

reftable/feature/table.py

@@ -0,0 +1,89 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""Table of figures extractor
+==========================
+"""
+
+import configos
+import elementae
+import iamraw
+import serializeraw
+import utilo
+
+import reftable.pageselector
+import reftable.toc.create
+import reftable.toc.run
+
+# minimal percentage of tabletable lines per page
+TOFS_PER_PAGE_MIN = configos.HV_PERCENT_PLUS(default=20, limit=100)
+
+
+def work(
+    text: str,
+    textpositions: str,
+    headerfooter: str,
+    sizeandborder: str,
+    pages: tuple = None,
+) -> str:
+    """Extract table of figures out of `document`.
+
+    Args:
+        text(str): path to load document
+        textpositions(str): path to load document textpositions
+        headerfooter(str): path with header and footer to determine
+                           content border.
+        sizeandborder(str): path with page sizes and content border
+        pages(tuple): tuple of selected pages
+    Returns:
+        dump of extracted table of content
+    """
+    navigators = serializeraw.ptcn_fromfile(
+        text,
+        textpositions,
+        sizeandborder=sizeandborder,
+        headerfooter=headerfooter,
+        pages=pages,
+    )
+    selected = reftable.pageselector.select_contentpages(
+        navigators,
+        wrong_table=NO_TABLES,
+        valid_lines_perpage_min=TOFS_PER_PAGE_MIN,
+    )
+    if not selected:
+        return EMPTY
+    # select toc pages only
+    navigators = utilo.select_pages(navigators, pages=selected)
+    if not headline_start(navigators[0]):
+        utilo.error(f'no valid table headline start: {selected}')
+        return EMPTY
+    # run
+    extracted = reftable.toc.run.extract(navigators)
+    # prepare
+    flat = utilo.flat(extracted.content)
+    leveled = reftable.toc.create.groupby_level(flat)
+    # dump
+    dumped = serializeraw.dump_toc(leveled)
+    return dumped
+
+
+EMPTY = serializeraw.dump_toc(iamraw.Toc())
+NO_TABLES = utilo.unique(elementae.ABBREVIATION + elementae.TOC +
+                         elementae.FIGURETABLE + elementae.SYMBOLTABLE)
+
+
+def headline_start(ptn) -> bool:
+    """Verify that the first ptn starts with a valid figure table headline."""
+    # TODO: INTEGRATE INTO SELECT_CONTENTPAGES
+    for line in ptn[0:8]:
+        parsed = elementae.headline.parser.parse_headline(line.text)
+        if not parsed:
+            continue
+        if utilo.verysimilar(parsed[0], expected=elementae.TABLETABLE):
+            return True
+    return False

reftable/feature/toc.py

@@ -0,0 +1,103 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""Table of content extractor
+==========================
+
+Outdated approaches
+-------------------
+
+- collect title and check if sequence exists again in the document
+
+"""
+
+import configos
+import elementae
+import elementae.headline.lookup
+import serializeraw
+import utilo
+
+import reftable.feature
+import reftable.pageselector
+import reftable.toc
+import reftable.toc.create
+import reftable.toc.run
+import reftable.toc.strategy
+
+# minimal percentage of toc lines per page
+TOCS_PER_PAGE_MIN = configos.HV_PERCENT_PLUS(default=30, limit=100)
+
+# limit possible toc to the first 15 pages
+POSSIBLE_PAGES = utilo.make_tuple(15)
+
+TOC_COUNT_MIN = configos.HV_INT_PLUS(default=4)
+
+
+def work(
+    text: str,
+    textpositions: str,
+    headerfooter: str,
+    sizeandborder: str,
+    pages: tuple = None,
+) -> str:
+    """Extract table of content out of `document`.
+
+    Args:
+        text(str): path to load document
+        textpositions(str): path to load document textpositions
+        headerfooter(str): path with header and footer to determine
+                           content border.
+        sizeandborder(str): path with page sizes and content border
+        pages(tuple): tuple of selected pages
+    Returns:
+        dump of extracted table of content
+    """
+    pages = POSSIBLE_PAGES if pages is None else pages
+    navigators = serializeraw.ptcn_fromfile(
+        text,
+        textpositions,
+        sizeandborder=sizeandborder,
+        headerfooter=headerfooter,
+        pages=pages,
+    )
+    extracted = run(navigators)
+    dumped = dump(extracted)
+    return dumped
+
+
+def run(navigators):
+    selected = reftable.pageselector.select_contentpages(
+        textnavigators=navigators,
+        wrong_table=NO_TOC,
+        skip_higherqual_level_three=False,
+        valid_lines_perpage_min=TOCS_PER_PAGE_MIN,
+    )
+    navigators = utilo.select_pages(
+        navigators,
+        pages=selected,
+    )
+    extracted = reftable.toc.run.extract(
+        navigators,
+        min_detection_count=TOC_COUNT_MIN,
+    )
+    return extracted
+
+
+def dump(extracted):
+    flat = utilo.flat(extracted.content)
+    leveled = reftable.toc.create.groupby_level(flat)
+    leveled.__strategy__ = extracted.strategy
+    dumped = serializeraw.dump_toc(leveled)
+    return dumped
+
+
+# NO_TOC = elementae.headline.lookup.HEADLINES - elementae.headline.lookup.TOC
+NO_TOC = [
+    item for item in elementae.headline.lookup.HEADLINES
+    if item not in elementae.headline.lookup.TOC
+]

reftable/figure/__init__.py

@@ -0,0 +1,19 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2022-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import configos
+import elementae
+import utilo
+
+# minimal percentage of figure lines per page
+TOFS_PER_PAGE_MIN = configos.HV_PERCENT_PLUS(default=20, limit=100.0)
+
+NO_FIGURES = utilo.unique(elementae.ABBREVIATION + elementae.BIBLIOGRAPHY +
+                          elementae.GLOSSARY + elementae.SYMBOLTABLE +
+                          elementae.TABLETABLE + elementae.TOC)