rawmaker 2.40.3__py3-none-any.whl

rawmaker/miner/underline.py

@@ -0,0 +1,66 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2022-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import iamraw
+import serializeraw
+import utilo
+
+
+def underline_chars(
+    document: iamraw.Document,
+    underlinex: str = None,
+    pages: tuple = None,
+):
+    # TODO: MARK HORIZONTAL AS TEXT UNDERLINE HORIZONTAL!
+    # TODO: SUPPORT PARTIAL UNDERLINES
+    # TODO: UPDATE STYLE RANGE AFTER SETTING ONLY SOME CHARS AS UNDERLINED
+    # TODO: REPLACE UNDERLINE WITH STYLE(NONE, UNDERLINE, CROSSED, OVERLINED)
+    if not utilo.exists(underlinex):
+        utilo.log(f'missing underlines: {underlinex}, skipping char underline')
+        return document
+    underlinex = serializeraw.load_horizontals(
+        underlinex,
+        pages=pages,
+    )
+    for pdfpage in underlinex:
+        underlines, pagenumber = pdfpage.content, pdfpage.page
+        current_page = utilo.select_page(document.pages, page=pagenumber)
+        if not current_page:
+            continue
+        for underline in underlines:
+            for textcontainer in current_page:
+                if not underlined(textcontainer.box, underline.box):
+                    continue
+                # TODO: REMOVE APPEND AFTER SHRINKING TEXTCONTAINER TO
+                # SINGLE LINE
+                # update chars
+                for char in utilo.flat(textcontainer, append=True):
+                    char.underline = True
+                break
+    return document
+
+
+def underlined(text: utilo.Rectangle, horizontal: utilo.Rectangle) -> bool:
+    # TODO: SUPPORT CROSSED ETC.
+    hline_inside = text[1] < horizontal[1] < text[3]
+    if not hline_inside:
+        return False
+    near_bottom = utilo.near(
+        expected=text[3],
+        current=horizontal[1],
+        diff=3.0,
+    )
+    if not near_bottom:
+        return False
+    # start and end of horizontal and text matches
+    leftright = utilo.near(text[0], horizontal[0], diff=5.0)
+    leftright &= utilo.near(text[2], horizontal[2], diff=5.0)
+    if not leftright:
+        return False
+    return True

rawmaker/parameter.py

@@ -0,0 +1,130 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import dataclasses
+
+import configos
+import pdfminer.layout
+import utilo
+
+STRIP = True
+
+ONELINE = pdfminer.layout.LAParams(
+    boxes_flow=1.0,
+    char_margin=100.0,
+    line_margin=0.001,
+)
+
+
+@dataclasses.dataclass
+class ParsingConfiguration:
+    boxes_flow: float = 0.5
+    char_margin: float = 2.0
+    line_margin: float = 0.5
+    line_overlap: float = 0.5
+    word_margin: float = 0.1
+    detect_vertical: bool = True
+    strip: bool = STRIP
+
+    def cmdline(self) -> str:
+        """Convert configuration to `linix` command line parameter syntax."""
+        parameter = []
+        for item, value in vars(self).items():
+            if item == 'strip':
+                if not value:
+                    parameter.append('--nostrip')
+                continue
+            if isinstance(value, bool):
+                if value:
+                    parameter.append(f'--{item}')
+                continue
+            parameter.append(f'--{item}={value}')
+        return utilo.from_tuple(parameter)
+
+    def laparams(self) -> pdfminer.layout.LAParams:
+        result = pdfminer.layout.LAParams(
+            boxes_flow=self.boxes_flow,
+            char_margin=self.char_margin,
+            detect_vertical=self.detect_vertical,
+            line_margin=self.line_margin,
+            line_overlap=self.line_overlap,
+            word_margin=self.word_margin,
+        )
+        return result
+
+    @classmethod
+    def from_dict(cls, **kwargs):
+        """\
+        >>> ParsingConfiguration.from_dict()
+        --boxes_flow=...--char_margin=...--line_margin=...--line_overlap=...
+        """
+        instance = cls()
+        for key, value in kwargs.items():
+            if key == 'nostrip':
+                instance.strip = not value
+                continue
+            if hasattr(instance, key):
+                setattr(instance, key, value)
+        return instance
+
+    def __str__(self):
+        return self.cmdline()
+
+    def __repr__(self):
+        return str(self)
+
+
+LAYOUT_CHAR_MARGIN = configos.HV_FLOAT_PLUS(default=1.1)
+
+LAYOUT = ParsingConfiguration(char_margin=LAYOUT_CHAR_MARGIN)
+
+
+def from_config(config: ParsingConfiguration) -> pdfminer.layout.LAParams:
+    boxes_flow: float = 0.5
+    char_margin: float = 2.0
+    line_margin: float = 0.5
+    line_overlap: float = 0.5
+    word_margin: float = 0.1
+    detect_vertical = True
+    if config:
+        boxes_flow = config.boxes_flow
+        char_margin = config.char_margin
+        detect_vertical = config.detect_vertical
+        line_margin = config.line_margin
+        line_overlap = config.line_overlap
+        word_margin = config.word_margin
+    result = pdfminer.layout.LAParams(
+        boxes_flow=boxes_flow,
+        char_margin=char_margin,
+        detect_vertical=detect_vertical,
+        line_margin=line_margin,
+        line_overlap=line_overlap,
+        word_margin=word_margin,
+    )
+    return result
+
+
+def print_layout(layout: ParsingConfiguration = None):
+    assert layout, 'missing layout'
+    layout = from_config(layout)
+    utilo.log('   layout:', end=' ', level=utilo.Level.INFORMATION)
+    information = [
+        ('boxes_flow', layout.boxes_flow),
+        ('char_margin', layout.char_margin),
+        ('line_margin', layout.line_margin),
+        ('line_overlap', layout.line_overlap),
+        ('word_margin', layout.word_margin),
+    ]
+    for (text, value) in information:
+        utilo.log(
+            '%s %.2f' % (text, value),  # pylint:disable=C0209
+            end=' ',
+            level=utilo.Level.INFORMATION,
+        )
+    utilo.log(level=utilo.Level.INFORMATION)  # newline

rawmaker/patch/__init__.py

@@ -0,0 +1,8 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================

rawmaker/patch/ltchar.py

@@ -0,0 +1,79 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import contextlib
+
+import pdfminer.layout
+import pdfminer.pdffont
+
+
+class PatchedLTChar(pdfminer.layout.LTChar):
+
+    # pylint:disable=R0913
+    def __init__(self, matrix, font, fontsize, scaling, rise, text, textwidth,
+                 textdisp, ncs, graphicstate):
+        super().__init__(matrix, font, fontsize, scaling, rise, text, textwidth,
+                         textdisp, ncs, graphicstate)
+        if fontsize == 1:
+            # HACK: CLARIFY WHAT IS CORRECT, we need more generated pdf
+            # examples
+            # HQEOGA+Arial in bachelor37
+            # self.fontsize = self.size
+            # matrix[0] normal char
+            # matrix[1] rotated char
+            self.fontsize = matrix[0] if matrix[0] else matrix[1]
+            assert self.fontsize, matrix
+        else:
+            self.fontsize = fontsize
+        self.rise = rise
+        self.flags = font.flags
+
+    def __getitem__(self, index):
+        """Access bounding component."""
+        return self.bbox[index]
+
+    @property
+    def text(self):
+        return self.get_text()
+
+
+def vertical(item: pdfminer.layout.LTChar) -> bool:
+    """Check LTChar.upright flag."""
+    with contextlib.suppress(AttributeError):
+        if not item.upright:
+            return True
+        return False
+    return None
+
+
+def render_char(  # pylint:disable=W9015,W9016
+        self, matrix, font, fontsize, scaling, rise, cid, ncs,
+        graphicstate) -> PatchedLTChar:
+    """Patch LTChar to introduce font size hack.
+
+    Args:
+        cid(int): character number id - ASCII
+        ncs(PDFColorSpace): color space of document
+    Returns:
+        Patched char object.
+    """
+    try:
+        text = font.to_unichr(cid)
+        assert isinstance(text, str), type(text)
+    except pdfminer.pdffont.PDFUnicodeNotDefined:
+        text = self.handle_undefined_char(font, cid)
+
+    textwidth = font.char_width(cid)
+    textdisp = font.char_disp(cid)
+
+    # patch to document font size and rise
+    item = PatchedLTChar(matrix, font, fontsize, scaling, rise, text, textwidth,
+                         textdisp, ncs, graphicstate)
+    self.cur_item.add(item)
+    return item.adv

rawmaker/reader.py

@@ -0,0 +1,97 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================
+
+import sys
+from contextlib import contextmanager
+from os.path import exists
+from os.path import isfile
+
+import utilo
+from pdfminer.pdfdocument import PDFDocument
+from pdfminer.pdfdocument import PDFEncryptionError
+from pdfminer.pdfdocument import PDFSyntaxError
+from pdfminer.pdfparser import PDFParser
+
+# from rawmaker.error import InvalidPDF
+# from rawmaker.error import PDFParserImplementationError
+
+
+@contextmanager
+def read(path: str, password: str = None, verify: bool = True) -> PDFDocument:
+    """Open pdf from `path`.
+
+    Args:
+        path(str): path to pdf-file
+        password(str): optional password to extract encrypted data
+        verify(bool): ensure that file starts with `%PDF-`
+    Raises:
+        TextExtractNotAllowed: if no extraction is allowed - currently disabled
+        FileNotFoundError: `path` does not exists
+        ValueError: `path` is not a file
+    Yields:
+        PDFDocument: open pdf file
+    """
+    if not exists(path):
+        raise FileNotFoundError(f'Path does not exists: {path}')
+    if not isfile(path):
+        raise ValueError(f'Read requires an pdf document, not {path}')
+    if verify:
+        header = open(path, 'rb').read(5)
+        if header != b'%PDF-':
+            # TODO: MOVE TO def before() method after upgrading utilo
+            utilo.error('invalid pdf header')
+            sys.exit(1)
+    with open(path, 'rb') as fp:
+        # Create a PDF parser object associated with the file object.
+        parser = PDFParser(fp)
+        # Create a PDF document object that stores the document structure.
+        # Supply the password for initialization.
+        document = open_document(parser, password)
+        yield document
+
+
+def open_document(parser: PDFParser, password: str) -> PDFDocument:
+    """Open pdf document base on selected `parser`.
+
+    Hint:
+        Using fallback as default mode is very slow. Therefore we try
+        without fallback and if this does not work, we try it with
+        fallback again.
+        Try first without using fallback because this is much faster on
+        valid documents. If the run without fallback fails, start it
+        with fallback again.
+    """
+    password = password if password is not None else ''
+    try:
+        document = PDFDocument(parser, password, fallback=False)
+    except PDFSyntaxError:
+        pass  # try with fallback again
+    except PDFEncryptionError as encryption:
+        utilo.error('encryption not supported')
+        utilo.debug(encryption)
+        sys.exit(1)
+    except Exception:  # pylint:disable=broad-except
+        utilo.print_stacktrace()
+        sys.exit(2)
+        # raise PDFParserImplementationError(path) from exc
+    else:
+        return document
+
+    try:
+        utilo.info('try to use `fallback` pdf loader')
+        document = PDFDocument(parser, password, fallback=True)
+    except PDFSyntaxError:
+        utilo.print_stacktrace()
+        sys.exit(3)
+        # raise InvalidPDF(path) from exc
+    except Exception:  # pylint:disable=broad-except
+        # raise PDFParserImplementationError(path) from exc
+        utilo.print_stacktrace()
+        sys.exit(2)
+    return document

rawmaker/text/__init__.py

@@ -0,0 +1,8 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================

rawmaker/text/chars.py

@@ -0,0 +1,24 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import iamraw
+
+import rawmaker.features.text
+import rawmaker.miner.char
+import rawmaker.reader
+
+
+def extract_chars(document: str, pages: tuple = None) -> iamraw.Document:
+    assert isinstance(document, str), str(document)
+    document = rawmaker.features.text.extract_document(
+        document,
+        pages=pages,
+        converter=rawmaker.miner.char.CharPDFConvert,
+    )
+    return document

rawmaker/text/data.py

@@ -0,0 +1,47 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import dataclasses
+
+import iamraw
+import utilo
+
+
+@dataclasses.dataclass
+class WordBoxPage:
+
+    content: iamraw.BoundingBoxes = dataclasses.field(default_factory=list)
+    page: int = None
+
+    def __getitem__(self, index):
+        return self.content[index]  # pylint:disable=E1136
+
+    def __len__(self):
+        return len(self.content)  # pylint:disable=E1136
+
+
+WordBoxPages = list[WordBoxPage]
+
+
+@dataclasses.dataclass
+class PageLines:
+
+    lines: list = dataclasses.field(default_factory=list)
+
+    def __getitem__(self, index):
+        return self.lines[index]  # pylint:disable=E1136
+
+    def __len__(self):
+        return len(self.lines)  # pylint:disable=E1136
+
+    def __str__(self):
+        word = lambda x: ''.join([char.text for char in x])  # pylint:disable=C3001
+        line = lambda x: ' '.join([word(item) for item in x])  # pylint:disable=C3001
+        lines = utilo.NEWLINE.join(line(item) for item in self.lines)  # pylint:disable=E1133
+        return f'PageLines: {len(self.lines)}\n{lines}'

rawmaker/text/superfast.py

@@ -0,0 +1,91 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2021-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import os
+
+import iamraw
+import pdflog
+import serializeraw
+import utilo
+
+import rawmaker.parameter
+
+
+def superfast(
+    document: str,
+    config: rawmaker.parameter.ParsingConfiguration,
+    workdir: str,
+    pages: list = None,
+) -> iamraw.Document:
+    if pages is None:
+        pagecount = pdflog.pagecount(document)
+        pages = utilo.make_tuple(pagecount)
+    chunks = utilo.chunks(pages, size=10)
+    parameter = config.cmdline()
+    todo = []
+    for index, chunk in enumerate(chunks):
+        joined_pages = utilo.from_tuple(chunk, separator=',')
+        cmd = (f'rawmaker -i {document} -o {workdir} --prefix {index}'
+               f' --text --pages {joined_pages} {parameter}')
+        utilo.log(cmd)
+        todo.append(cmd)
+    # run in parallel
+    completed = utilo.run_parallel(todo, workdir, worker=12)
+    assert completed == utilo.SUCCESS, completed
+    # merge document
+    document = merge_document(workdir, len(chunks))
+    return document
+
+
+def merge_document(path: str, size: int) -> iamraw.Document:
+    """Merge chunks of extract document.
+
+    A little bit diry, but ok for now. XXX
+    """
+    text_files = [
+        os.path.join(path, f'rawmaker__{item}_text_text.yaml')
+        for item in range(size)
+    ]
+    posi_files = [
+        os.path.join(path, f'rawmaker__{item}_text_positions.yaml')
+        for item in range(size)
+    ]
+
+    text = [serializeraw.load_document(item) for item in text_files]
+    positions = [serializeraw.load_textpositions(item) for item in posi_files]
+
+    for item in text_files + posi_files:
+        utilo.info(f'remove {item}')
+        utilo.file_remove(item)
+
+    for docs, pos in zip(text, positions):
+        for page in docs:
+            index = 0
+            for item in page:
+                if not isinstance(item, iamraw.TextContainer):
+                    continue
+                # bounding, mean
+                bounding, mean = utilo.select_page(pos, page.page).content[index] # yapf:disable
+                fake_text_mean_height(item, bounding, mean)
+                item.box = bounding
+                index += 1
+
+    document = iamraw.Document(dimension=text[0].dimension)
+    for chunk in text:
+        for page in chunk:
+            document.append(page)
+    return document
+
+
+def fake_text_mean_height(item, bounding, mean):
+    # TODO: REMOVE THIS HACK LATER
+    for line in item.lines:
+        for char in line:
+            # Fake mean char height
+            char.box = iamraw.BoundingBox(0, bounding.y1 - mean, 0, bounding.y1)

rawmaker/text/wordbox.py

@@ -0,0 +1,95 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import configos
+import utilo
+
+import rawmaker.text.chars
+import rawmaker.text.data
+
+DIFF_MAX = configos.HolyTable(items=[
+    (6, 1.0),
+    (12, 1.0),
+    (16, 1.0),
+    (22, 10.0),
+    (24, 10.0),
+    (48, 15.0),
+])
+
+
+def parses(
+    source: str,
+    pages: tuple,
+    word_length_min: int = 1,
+    difftable: configos.HolyTable = DIFF_MAX,
+) -> rawmaker.text.data.WordBoxPages:
+    extracted = rawmaker.text.chars.extract_chars(source, pages)
+    pages = [
+        extract_page(
+            page,
+            word_length_min=word_length_min,
+            difftable=difftable,
+        ) for page in extracted
+    ]
+    boundings = [wordbox_boundings(page) for page in pages]
+
+    # adjust page numbers
+    for page, bounding in zip(extracted, boundings):
+        bounding.page = page.page
+    return boundings
+
+
+def extract_page(
+    page,
+    word_length_min: int = 1,
+    difftable: configos.HolyTable = DIFF_MAX,
+) -> rawmaker.text.data.PageLines:
+    # remove white space
+    page = [item for item in page if item.get_text().strip()]
+
+    lines = utilo.same_line_cluster(
+        page,
+        min_elements=word_length_min,  # support single chars
+        matcher=lambda x: x.bbox[3],
+    )
+    # ensure top bottom
+    lines = sorted(lines, key=lambda bounding: bounding.center[3])  # y1
+
+    result = []
+    for line in lines:
+        # ensure left right
+        line = sorted(line, key=lambda x: x[0])  # x0
+        merged = merge_line(line, difftable=difftable)
+        result.append(merged)
+    return rawmaker.text.data.PageLines(lines=result)
+
+
+def merge_line(line, difftable: configos.HolyTable):
+    if not line:
+        return []
+    diffs = [
+        after[0] - current[2] for current, after in zip(line[0:-1], line[1:])
+    ]
+    result = [[line[0]]]
+    for char, diff in zip(line[1:], diffs):
+        if diff > difftable(char.fontsize):
+            result.append([char])
+        else:
+            result[-1].append(char)
+    return result
+
+
+def wordbox_boundings(page) -> rawmaker.text.data.WordBoxPage:
+    result = []
+    for line in page:
+        for word in line:
+            bounding = utilo.rect_max([char.bbox for char in word])
+            result.append(bounding)
+    wordbox = rawmaker.text.data.WordBoxPage(content=result)
+    return wordbox

rawmaker/utils.py

@@ -0,0 +1,44 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2021-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import contextlib
+
+
+def resolve(reference):
+    with contextlib.suppress(AttributeError):
+        return reference.resolve()
+    return reference
+
+
+ENCODINGS = 'ascii cp1252 utf8 '.split()
+
+
+def guess_decoding(text: bytes) -> str:
+    r"""\
+    >>> guess_decoding(b'http://road.cc/measure-\x96-smart-street')
+    'http://road.cc/measure-–-smart-street'
+    """
+    text = resolve(text)
+    for encoding in ENCODINGS:
+        try:
+            text = text.decode(encoding)
+        except UnicodeDecodeError:
+            continue
+        return text
+    return None
+
+
+def guess_encoding(text: bytes) -> str:
+    for encoding in ENCODINGS:
+        try:
+            text = text.encode(encoding)
+        except UnicodeEncodeError:
+            continue
+        return text
+    return None