rawmaker 2.40.3__py3-none-any.whl

letty/__init__.py

@@ -0,0 +1,46 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""Layout Estimator Tool t.y.
+==========================
+
+Variation Strategy
+------------------
+
+Static
+~~~~~~
+
+Dynamic
+~~~~~~~
+
+Result: Points
+~~~~~~~~~~~~~~
+
+Judger
+-------
+
+Optimizer
+---------
+
+.. code-block:: none
+
+    Optimizer -> Strategy -> Points -> Tool(Points) -> Result -> Judger(Result)
+         !                     !                                     |
+         !                     !                                     |
+         < < < < < < < < < < <  < < < < < < < < < < < < < < < < < < </
+
+"""
+
+import os
+
+import rawmaker
+
+__version__ = rawmaker.__version__
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+
+PROCESS = 'letty'

letty/cli.py

@@ -0,0 +1,63 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import utilo
+import utilo.cli
+
+import letty
+import letty.quality.whitespace
+
+
+@utilo.saveme
+def main() -> int:
+    parser = create_parser()
+    args = utilo.parse(parser)  # pylint:disable=W0612
+    inpath, pages, whitespace = parse_args(args)
+    if whitespace:
+        white_spaces = letty.quality.whitespace.determine(inpath, pages=pages)
+        utilo.log(white_spaces)
+        return utilo.SUCCESS
+    parser.print_help()
+    return utilo.FAILURE
+
+
+def parse_args(args) -> tuple:
+    """\
+    >>> parse_args({})
+    ('...', None, False)
+    >>> parse_args(dict(pages=['3:10']))
+    ('...', (3, 4, 5, 6, 7, 8, 9), False)
+    """
+    inpath, _ = utilo.cli.sources(args, singleinput=True)  # pylint:disable=W0632
+    inpath = inpath[0]
+    pages = None
+    if args.get('pages', None) is not None:
+        pages = utilo.parse_pages(','.join(args['pages']))
+    whitespace = args.get('whitespace', False)
+    result = (inpath, pages, whitespace)
+    return result
+
+
+def create_parser():
+    commands = [
+        utilo.cli.Flag('--whitespace', message='evalute number of whitespaces'),
+    ]
+    parser = utilo.cli.create_parser(
+        todo=commands,
+        config=utilo.ParserConfiguration(
+            inputparameter=True,
+            outputparameter=False,
+            pages=True,
+            prefix=False,
+            verboseflag=True,
+        ),
+        version=letty.__version__,
+        prog=letty.PROCESS,
+    )
+    return parser

letty/optimizer.py

@@ -0,0 +1,138 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import collections
+import concurrent
+import itertools
+import math
+import os
+import sys
+
+import configos
+import utilo
+
+import letty.quality.whitespace
+
+OptimizerResult = collections.namedtuple('OptimizerResult', 'value, config')
+WORKER = 12
+
+
+def run(
+    path: str,
+    pages: tuple = None,
+    boxes: int = 1,
+    chars: int = 10,
+    lines: int = 1,
+    words: int = 1,
+    *,
+    multicore: bool = True,
+) -> OptimizerResult:
+    todo = strategy(chars=chars, words=words, lines=lines, boxes=boxes)
+    runner = threadpool if multicore else singlecore
+    result = runner(todo, path, pages)
+    judged = judge(result)
+    return judged
+
+
+def singlecore(todo: list, path: str, pages: tuple):
+    result = []
+    for config in todo:
+        quality = run_single(path, pages, config)
+        result.append(quality)
+    return result
+
+
+def threadpool(todo: list, path: str, pages: tuple):
+    result = []
+    with concurrent.futures.ThreadPoolExecutor(max_workers=WORKER) as executor:
+        futures = {
+            executor.submit(run_single, path, pages, config): config
+            for config in todo
+        }
+        for future in concurrent.futures.as_completed(futures):
+            try:
+                quality = future.result()
+                result.append(quality)
+            except Exception as error:  # pylint:disable=broad-except
+                utilo.error(f'{future} failed.')
+                utilo.error(error)
+    return result
+
+
+def run_single(path: str, pages: tuple, config: dict):
+    config = ' '.join([f'--{key}={value}' for key, value in config.items()])
+    pages_raw = ','.join([str(item) for item in pages])
+    pages_raw = f'--pages={pages_raw}' if pages is not None else ''
+    with utilo.make_tmpdir(root=configos.tmp()) as cwd:
+        cmd = f'rawmaker -i {path} -o {cwd} {pages_raw} --text {config}'
+        config_outpath = os.path.join(cwd, 'layout.ini')
+        utilo.file_create(config_outpath, config)
+        completed = utilo.run(cmd, cwd=cwd)
+        if completed.returncode:
+            utilo.error(f'could not run: {cmd}')
+            utilo.error(completed.stdout)
+            utilo.error(completed.stderr)
+            sys.exit(utilo.FAILURE)
+    quality = letty.quality.whitespace.determine(cwd, pages=pages)
+    return OptimizerResult(quality, config)
+
+
+def judge(result):
+    ratio, best = result[0]
+    utilo.log(result[0])
+    for item in result[1:]:
+        utilo.log(item)
+        if item[0] < ratio:
+            ratio, best = item
+    return ratio, best
+
+
+def strategy(
+    chars: int = 10,
+    words: int = 1,
+    lines: int = 1,
+    boxes: int = 1,
+):
+    boxes_flow = ranges(0.5, 1.0, boxes)
+    char_margin = ranges(0.5, 20.0, chars)
+    line_margin = ranges(0.01, 5.0, lines)
+    word_margin = ranges(1.5, 5.0, words)
+    result = []
+    for char, word, box, line, in itertools.product(
+            char_margin,
+            word_margin,
+            boxes_flow,
+            line_margin,
+    ):
+        result.append({
+            'boxes_flow': box,
+            'char_margin': char,
+            'word_margin': word,
+            'line_margin': line,
+        })
+    return result
+
+
+# TODO: REPLACE WITH utilo CODE
+def ranges(mini: float, maxi: float, steps: int = 15):
+    """Compute parameter.
+
+    >>> utilo.roundme(ranges(0.1, 100, steps=10))
+    [0.1, 0.12, 0.18, 0.34, 0.76, 1.92, 5.06, 13.61, 36.84, 99.99]
+    >>> utilo.roundme(ranges(0.1, 20, steps=5))
+    [0.1, 0.73, 2.43, 7.06, 19.64]
+    """
+    func = math.exp
+    maxed = func(steps - 1) / (maxi - mini)
+    result = []
+    for index in range(steps):
+        value = mini + (math.exp(index) - 1) / maxed
+        value = utilo.roundme(value, digits=5)  # pylint:disable=R0204
+        result.append(value)
+    return result

letty/quality/__init__.py

@@ -0,0 +1,8 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================

letty/quality/whitespace.py

@@ -0,0 +1,50 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import collections
+import re
+
+import serializeraw
+import utilo
+
+PageWhitespace = collections.namedtuple('PageWhitespaces', 'page, content')
+PageWhitespaces = list[PageWhitespace]
+
+
+def determine(path: str, pages: tuple = None) -> PageWhitespaces:
+    data = serializeraw.ptn_frompath(
+        path,
+        pages=pages,
+        logging=False,
+    )
+    analyzed = [analyse_page(item) for item in data]
+    result = quality(analyzed)
+    return result
+
+
+INNER_WHITESPACE = r'\b\s{2,}\b'
+CONTENT_ENDING = r'\b\n'
+
+COMMON = '(' + INNER_WHITESPACE + '|' + CONTENT_ENDING + ')'
+
+
+def analyse_page(page) -> PageWhitespace:
+    counter = collections.Counter()
+    for line in page:
+        text = line.text
+        for item in re.finditer(COMMON, text):
+            counter[len(utilo.extract_match(item))] += 1
+    counter[2] += len(page)
+    result = PageWhitespace(page=page.page, content=counter.most_common())
+    return result
+
+
+def quality(pages) -> int:
+    result = sum([page.content[0][1] for page in pages if page.content], 0)
+    return result

letty/strategy.py

@@ -0,0 +1,8 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================

rawmaker/__init__.py

@@ -0,0 +1,29 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================
+"""The rawmaker converts pdf to kiwi-internal project format
+
+Hint: Pay attention to the public API on this file!
+      Breaking changes are breaking!
+"""
+
+import os
+
+import configos
+
+# pylint:disable=W0613
+import rawmaker.__patch__
+from rawmaker.parameter import LAYOUT
+from rawmaker.parameter import ONELINE
+
+__version__ = '2.40.2'
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+PROCESS = 'rawmaker'
+
+configos.cloud_lookup(PROCESS)

rawmaker/__main__.py

@@ -0,0 +1,13 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================
+
+from rawmaker.cli import main
+
+if __name__ == "__main__":
+    main()

rawmaker/__patch__.py

@@ -0,0 +1,36 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import sys
+
+import pdfminer.glyphlist
+import pdfminer.pdfpage
+import utilo
+
+before = pdfminer.pdfpage.PDFPage.create_pages  # pylint:disable=C0103
+
+
+def create_pages(document):
+    try:
+        yield from before(document)
+    except IndexError:
+        utilo.error('pdfminer parsing error: IndexError')
+        sys.exit(1)
+    except RecursionError:
+        utilo.error('pdfminer parsing error: RecursionError')
+        sys.exit(1)
+
+
+pdfminer.pdfpage.PDFPage.create_pages = create_pages
+
+# TODO HACK HACK HACK
+# bachelor090  REGISTERED SIGN
+# circlecopyrt
+# pdfminer.glyphlist.glyphname2unicode['circlecopyrt'] = '\u25CF'
+pdfminer.glyphlist.glyphname2unicode['circlecopyrt'] = '\r'

rawmaker/cli.py

@@ -0,0 +1,206 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================
+"""The `rawmaker` takes pdf's from the input folder or direct file and parse
+the raw structure of the pdf and provide them as yaml file for further
+analyze-processes.
+
+- toc:    tableofcontent
+- text:   text content from pdf file
+- border: determine page size and bounding boxes from page content
+
+"""
+
+import contextlib
+import os
+import sys
+
+import protoerror
+import utilo
+
+import rawmaker
+import rawmaker.error
+import rawmaker.features
+
+PDF = utilo.Pattern('*', 'pdf')
+
+CHAR_MARGIN = utilo.Value('char_margin', float, defaultvar=2.0, minimum=0.1)
+LINE_OVERLAP = utilo.Value('line_overlap', float, defaultvar=0.5, minimum=0.1)
+LINE_MARGIN = utilo.Value('line_margin', float, defaultvar=0.5, minimum=0.1)
+WORD_MARGIN = utilo.Value('word_margin', float, defaultvar=0.1, minimum=0.1)
+BOXES_FLOW = utilo.Value('boxes_flow', float, defaultvar=0.5, minimum=0.1)
+NOSTRIP = utilo.Bool('nostrip')
+DETECT_VERTICAL = utilo.Bool('detect_vertical')
+
+PDF_INPUT = [PDF]
+
+CONFIG_INPUTS = [
+    BOXES_FLOW,
+    CHAR_MARGIN,
+    LINE_MARGIN,
+    LINE_OVERLAP,
+    WORD_MARGIN,
+    NOSTRIP,
+    DETECT_VERTICAL,
+]
+
+WORKPLAN = [
+    utilo.create_step(
+        'annotation',
+        inputs=PDF_INPUT,
+        output=('annotation',),
+    ),
+    utilo.create_step(
+        'border',
+        inputs=PDF_INPUT,
+        output=(
+            'pages',
+            'boundingboxes',
+        ),
+    ),
+    utilo.create_step(
+        'boxes',
+        inputs=[
+            utilo.ResultFile(producer='rawmaker', name='line_line'),
+        ],
+        output=('boxes',),
+    ),
+    utilo.create_step(
+        'figures',
+        inputs=PDF_INPUT + [
+            utilo.ResultFile(producer='rawmaker', name='boxes_boxes'),
+            utilo.Pattern('rawmaker__images_images/*', 'yaml'),
+        ],
+        output=[
+            ('figures/{FILEHASH_1}', 'yaml'),
+            ('figures/{FILEHASHS}', 'png'),
+        ],
+    ),
+    utilo.create_step(
+        'horizontals',
+        inputs=[
+            utilo.ResultFile(producer='rawmaker', name='line_line'),
+        ],
+        output=('horizontals',),
+    ),
+    utilo.create_step(
+        'fonts',
+        inputs=[PDF] + CONFIG_INPUTS,
+        output=(
+            'header',
+            'content',
+        ),
+    ),
+    utilo.create_step(
+        'formula',
+        inputs=PDF_INPUT,
+        output=('formula',),
+    ),
+    utilo.create_step(
+        'images',
+        inputs=PDF_INPUT,
+        output=[
+            ('images/{FILEHASH_1}', 'yaml'),
+            ('images/{FILEHASHS}', '???'),
+        ],
+    ),
+    utilo.create_step(
+        'line',
+        inputs=[
+            PDF,
+            utilo.ResultFile(producer='rawmaker', name='annotation_annotation'),
+        ],
+        output=('line',),
+    ),
+    utilo.create_step(
+        'text',
+        inputs=[PDF] + [
+            utilo.ResultFile(
+                producer='rawmaker',
+                name='horizontals_horizontals',
+            ),
+        ] + CONFIG_INPUTS,
+        output=(
+            'text',
+            'positions',
+        ),
+    ),
+    utilo.create_step(
+        'outlines',
+        inputs=PDF_INPUT,
+        output=('outlines',),
+    ),
+]
+
+RAWMAKER_DESCRIPTION = """
+Extract features from pdf document.
+"""
+
+LINTER_FLAG = 'linter'
+SUPERFAST_FLAG = 'sf'
+
+
+def main():
+    flags = [
+        (LINTER_FLAG, 'write linter result'),
+        (SUPERFAST_FLAG, 'use superfast to fork processes and merge results'),
+    ]
+    config = utilo.FeaturePackConfig(
+        configflag=True,
+        description=RAWMAKER_DESCRIPTION,
+        errorhook=errorhook,
+        flags=flags,
+        multiprocessed=True,
+        name=rawmaker.PROCESS,
+        pages=True,
+        profileflag=True,
+        singleinput=True,
+        verboseflag=True,
+        version=rawmaker.__version__,
+    )
+    with linter():
+        utilo.featurepack(
+            workplan=WORKPLAN,
+            config=config,
+            root=rawmaker.ROOT,
+            featurepackage='rawmaker.features',
+        )
+
+
+def errorhook(exception, source):  # pylint:disable=W0613
+    logger = errorhook.linter
+
+    if isinstance(exception, rawmaker.error.InvalidPDF):
+        logger.add_finding(msgid='F0000', confidence=1.0)
+
+
+def superfast() -> bool:
+    return '--sf' in sys.argv
+
+
+@contextlib.contextmanager
+def linter():
+    """Write result of linting when using `--linter` parameter."""
+    # path to write error report
+    root = str(os.getcwd())
+    # setup linter
+    solver = protoerror.Solver()
+    for msg, solution in protoerror.solution.SOLUTION.items():
+        solver.add_solution(msg, solution)
+
+    active = [protoerror.MessageStatus(msgid='F0000', active=True)]
+
+    # init linter
+    errorhook.linter = protoerror.Linter(solver=solver, active=active)
+
+    try:
+        yield
+    except SystemExit as exc:
+        if f'--{LINTER_FLAG}' in sys.argv:
+            errorhook.linter.write(root)
+        raise exc

rawmaker/cli_automate.py

@@ -0,0 +1,69 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2021-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import argparse
+import os
+import sys
+
+import utilo
+import utilotest
+
+DESCRIPTION = """\
+Collect pdf files of defined folders and use them to run rawmaker.
+"""
+
+
+@utilo.saveme
+def main():
+    parameter = user_input()
+    run(*parameter)
+    sys.exit(utilo.SUCCESS)
+
+
+def user_input() -> tuple:
+    parser = argparse.ArgumentParser(description=DESCRIPTION)
+    parser.add_argument(
+        '-i',
+        dest='inpath',
+        default=os.getcwd(),
+    )
+    parser.add_argument(
+        '-o',
+        dest='outpath',
+        default=os.path.join(os.getcwd(), 'outpath'),
+    )
+    parser.add_argument(
+        '-n',
+        default=1,
+        type=int,
+        dest='cores',
+        help='count of used cores',
+    )
+    args = parser.parse_args()
+    inpath, outpath, cores = args.inpath, args.outpath, args.cores
+    return inpath, outpath, cores
+
+
+def run(inpath: str, outpath: str, cores: int = 1):
+    os.makedirs(outpath, exist_ok=True)
+    files = utilo.file_list(inpath, include='pdf', absolute=True)
+    for item in files:
+        utilo.log(item)
+    cmds = []
+    for item in files:
+        _, name = os.path.split(item)
+        # use quotation marks to encapsulate file path white spaces
+        item = f'"{item}"' if ' ' in str(item) else item
+        name = utilotest.simple(name)  # TODO: REPLACE WITH utilo CODE
+        out = os.path.join(outpath, name)
+        cmd = f'rawmaker -i {item} -o {out} -j4'
+        cmds.append(cmd)
+    for cmd in cmds:
+        utilo.log(cmd, preserve_newlines=False)
+    utilo.run_parallel(cmds, worker=cores, verbose=True)

rawmaker/converter/__init__.py

@@ -0,0 +1,8 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================