rawmaker 2.40.3__py3-none-any.whl

rawmaker/features/boxes.py

@@ -0,0 +1,153 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""Boxes
+=====
+"""
+
+import functools
+import operator
+
+import configos
+import iamraw
+import pdfminer.layout
+import serializeraw
+import utilo
+
+# width of box
+RECTANGLE_WIDTH_MIN = configos.HV_FLOAT_PLUS(default=50.0)
+# height of box
+RECTANGLE_HEIGHT_MIN = configos.HV_FLOAT_PLUS(default=50.0)
+# distance of two merging boxes/rectangles
+ENDING_DISTANCE_MAX = configos.HV_FLOAT_PLUS(default=3)
+
+
+def work(lines: str, pages: tuple) -> str:
+    """Extract content boxes from given `document`.
+
+    Args:
+        lines(str): path to lines
+        pages(tuple): pages to analyze
+    Returns:
+        dumped parsed boxes, dumped parsed horizontals
+    """
+    assert isinstance(lines, str), type(lines)
+    lines = serializeraw.load_lines(lines, pages=pages)
+    boxes = determine_clusteritem(lines, determine_pageboxes)
+    dumped_boxes = serializeraw.dump_boxes(boxes)
+    return dumped_boxes
+
+
+def determine_boxes(
+    lines,
+    rectangle_width_min=RECTANGLE_WIDTH_MIN,
+    rectangle_height_min=RECTANGLE_HEIGHT_MIN,
+):
+    collect = functools.partial(
+        determine_pageboxes,
+        rectangle_width_min=rectangle_width_min,
+        rectangle_height_min=rectangle_height_min,
+    )
+    boxes = determine_clusteritem(lines, collect)
+    return boxes
+
+
+def determine_clusteritem(
+    lines: iamraw.PageContentLines,
+    collector: callable,
+):
+    result = []
+    for paged in lines:
+        lines_in_page, page = paged.content, paged.page
+        # remove lines which are to short and represent a dot
+        lines_in_page = [
+            item for item in lines_in_page if not utilo.isdot(item)
+        ]
+        # remove duplicated lines
+        lines_in_page = utilo.unique_lines(lines_in_page)
+        grouped = determine_cluster(lines_in_page)
+        collected = collector(
+            grouped,
+            page,
+            rotated=paged.rotated,
+        )
+        result.append(collected)
+    return result
+
+
+def determine_pageboxes(
+    clusters: list[pdfminer.layout.LTLine],
+    page: int,
+    rotated: bool = False,  # pylint:disable=W0613
+    rectangle_width_min=RECTANGLE_WIDTH_MIN,
+    rectangle_height_min=RECTANGLE_HEIGHT_MIN,
+) -> iamraw.PageContentBoxes:
+    result = []
+    for cluster in clusters:
+        count = len(cluster)
+        if count != 4:
+            continue
+        x0 = min([line[0] for line in cluster] + [line[2] for line in cluster])
+        x1 = max([line[0] for line in cluster] + [line[2] for line in cluster])
+        y0 = min([line[1] for line in cluster] + [line[3] for line in cluster])
+        y1 = max([line[1] for line in cluster] + [line[3] for line in cluster])
+        width, height = x1 - x0, y1 - y0
+        if width < rectangle_width_min:
+            # small boxes are mostly a result of bad parsed figures or
+            # tables, we do not want them.
+            continue
+        if height < rectangle_height_min:
+            continue
+        box = iamraw.Box(box=iamraw.BoundingBox(x0, y0, x1, y1))
+        result.append(box)
+        # ensure to sort items top to bottom and left to right
+        result = sorted(result, key=operator.attrgetter('box.y0', 'box.x0'))
+    return iamraw.PageContentBoxes(content=result, page=page)
+
+
+def determine_cluster(items: iamraw.BoundingBoxes) -> iamraw.BoundingBoxes:  # pylint:disable=R1260
+    # TODO: REPLACE THIS CODE
+    if not items:
+        return []
+    # a single element is a cluster
+    result = [[item] for item in items]
+
+    def match(result, current):
+        for clusterindex, cluster in enumerate(result):
+            for clusteritem in cluster:
+                for test in current:
+                    if utilo.intersecting_ending(
+                            clusteritem,
+                            test,
+                            tol=ENDING_DISTANCE_MAX,
+                    ):
+                        return clusterindex
+        return None
+
+    def cluster(result):
+        result, todo = result[0], result[1:]
+        if not isinstance(result[0], list):
+            result = [result]
+        while todo:  # pylint:disable=W0149
+            current = todo.pop()
+            index = match(result, current)
+            if index is None:
+                # No match, create new cluster
+                result.insert(0, current)
+            else:
+                result[index].extend(current)
+        return result
+
+    single = utilo.Single()
+    while True:  # pylint:disable=W0149
+        # Break when cluster does not change result Cluster till cluster
+        # move does not change the result.
+        result = cluster(result)
+        if single.contains(result):
+            break
+    return result

rawmaker/features/figures.py

@@ -0,0 +1,24 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""Figure Extractor
+
+Extract figures and convert to images
+"""
+
+DumpedFigureInformation = list[tuple[str, bytes]]
+
+
+def work(  # pylint:disable=keyword-arg-before-vararg,W0613
+    path: str,
+    boxes: str = None,  # pylint:disable=W0613
+    *images: list,
+    pages: tuple = None,
+) -> DumpedFigureInformation:
+    dumped = []
+    return dumped

rawmaker/features/fonts.py

@@ -0,0 +1,229 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================
+"""Extract fonts out of pdf document to gather information
+
+Stored format:
+
+    (
+        container,
+        line,
+        char,
+        fontkey
+    )
+
+Stored item is the first different item.
+
+The font container indexing indexes only on text-container, other pages
+objects are ignored.
+"""
+
+import functools
+import math
+
+import iamraw
+import serializeraw
+import utilo
+
+import rawmaker.features
+import rawmaker.features.text
+import rawmaker.fonts.parser
+import rawmaker.miner.rawchar
+import rawmaker.parameter
+import rawmaker.reader
+
+
+def work(  # pylint:disable=W9015,W0613
+    document: str,
+    boxes_flow: float = 0.5,
+    char_margin: float = 2.0,
+    line_margin: float = 0.5,
+    line_overlap: float = 0.5,
+    word_margin: float = 0.1,
+    nostrip: bool = not rawmaker.parameter.STRIP,
+    detect_vertical: bool = False,
+    pages: list = None,
+) -> tuple[str, str]:
+    """Extract structured text out of document
+
+    Args:
+        document: pdf-document to run parsing
+        char_margin(float): XXX:5.0 why?
+        pages: limit analyzed area, if None every page is analyzed
+    Returns:
+        parsed document as yaml output
+    """
+    # TODO: CHANGE BEHAVIOR OF --detect_vertical. Convert to PARAMETER
+    # with True as default.
+    detect_vertical = True
+    assert isinstance(document, str), str(document)
+    config = rawmaker.parameter.ParsingConfiguration.from_dict(**locals())
+    document = rawmaker.features.text.extract_document(
+        document,
+        config=config,
+        pages=pages,
+    )
+    header, content = parse_fonts(document)
+    header, content = (
+        serializeraw.dump_font_header(header),
+        serializeraw.dump_font_content(content),
+    )
+    return header, content
+
+
+class FontStore:
+
+    def __init__(self, parser=None):
+        self.parser = parser if parser else rawmaker.fonts.parser.font_fromraw
+        self.data = {}
+
+    @functools.lru_cache(maxsize=128)
+    def font_key(self, raw_font: str, scale: float, flags: int) -> int:
+        parsed = self.parser(raw_font, scale, flags)
+        hashed = hash(parsed)
+        try:
+            self.data[hashed]
+        except KeyError:
+            self.data[hashed] = parsed
+        return hashed
+
+    def font(self, hashed: int):
+        return self.data[hashed]
+
+    def fonts(self):
+        return list(self.data.values())
+
+
+def process_page(  # pylint:disable=R0914
+    page: iamraw.Page,
+    fontstore: FontStore,
+) -> iamraw.PageFontContent:
+    """Iterate throw text container and extract the different fonts and
+    positions.
+
+    There are three indexs describing the position where the font-size
+    or font-rises changes. The text container, the line in the
+    container, and the char in line. The position of change is oriented
+    on python range/indexing. We note the change one char after the
+    change. Container and line are equal. Therefore on line endings, the
+    change is noted on a char position which does not exists.
+
+    Args:
+        page(Page): current pdf page
+        fontstore(FontStore): fontstore to store full information of used font
+    Returns:
+        Page with font information of the page text content.
+    """
+    assert isinstance(page, iamraw.Page), type(page)
+    position = (0, 0, 0)  # container, line, char
+    current_font, current_scale = None, None
+    current_flags = None
+    textcontainer = utilo.select_type(page.children, iamraw.TextContainer)
+    result = []
+    for container_index, container in enumerate(textcontainer):
+        rotated = isinstance(container, iamraw.VerticalTextContainer)
+        for line_index, line in enumerate(container.lines):
+            for char_index, char in enumerate(line):
+                try:
+                    font = char.font
+                except AttributeError:
+                    # Virtual chars have no fonts, but newlines are part
+                    # of font definition.
+                    position = (container_index, line_index, char_index)
+                    continue
+                scale = scale_fromchar(char, vertical=rotated)
+                flags = flags_fromchar(char)
+                # No font type or size is selected
+                if current_font is None:
+                    current_font, current_scale = (font, scale)
+                    current_flags = flags
+                    continue
+                # Font type, size or flags changed
+                if any((
+                        current_font != font,
+                        current_scale != scale,
+                        current_flags != flags,
+                )):
+                    fontid = add_font(
+                        current_font,
+                        current_scale,
+                        flags=current_flags,
+                        position=position,
+                        fontstore=fontstore,
+                    )
+                    result.append(fontid)
+                    # Reset current front
+                    current_font, current_scale = font, scale
+                    current_flags = flags
+                # update last index of current font
+                position = (container_index, line_index, char_index)
+    # add last text line of a page, because there is nothing changing
+    if current_font:
+        fontid = add_font(
+            current_font,
+            current_scale,
+            flags=current_flags,
+            position=position,
+            fontstore=fontstore,
+        )
+        result.append(fontid)
+    return iamraw.PageFontContent(content=result, page=page.page)
+
+
+def parse_fonts(document: iamraw.Document):
+    fontstore = FontStore(rawmaker.fonts.parser.font_fromraw)
+    content = [process_page(page, fontstore) for page in document.pages]
+    # Run header after content is important. DO NOT CHANGE ORDER. If
+    # running .fonts() first, content will be empty cause no fonts where
+    # processed.
+    header = fontstore.fonts()
+    return header, content
+
+
+def add_font(font, scale, flags, *, fontstore, position):
+    # position = (container, line, chars + 1)
+    container, line, char = position
+    # store position after the change happend
+    char = char + 1
+    fontkey = fontstore.font_key(font, scale, flags)
+    return (container, line, char, fontkey)
+
+
+def flags_fromchar(char) -> tuple:
+    try:
+        # LTChar
+        flags = char.ltchar.flags
+    except AttributeError:
+        # Char
+        flags = char.flags
+    return flags
+
+
+def upright_fromchar(char) -> bool:
+    try:
+        upright = char.ltchar.upright
+    except AttributeError:
+        upright = True
+    return upright
+
+
+def scale_fromchar(char, vertical: bool = False) -> float:
+    # TODO: INVESTIGATE 1.34??
+    # NOTE: This works for POSTSCRIPT_14_DEFAULT's but not for
+    # Calibri.
+    scale = utilo.roundme(char.size / 1.34005)
+    # TODO: THINK ABOUT VERTICAL HACK
+    if scale < 0:
+        rotated = not upright_fromchar(char)
+        rotated |= vertical
+        absolute = math.fabs(scale)
+        if rotated and absolute > 4.0:  # TODO: HOLY VALUE
+            # rotated char which is printed top down
+            return absolute
+        utilo.debug(f'negative font size: {scale} {char}')
+    return scale

rawmaker/features/formula.py

@@ -0,0 +1,16 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import serializeraw
+
+
+def work(path: str, pages: tuple = None) -> str:  # pylint:disable=W0613
+    formulas = []
+    dumped = serializeraw.dump_rawformulas(formulas)
+    return dumped

rawmaker/features/horizontals.py

@@ -0,0 +1,132 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""Horizontals
+===========
+
+Whats the difference between `boxes_horizontals` and `lines`?
+`boxes_horizontals` contain only vertical lines. `lines` can contain
+every lines in every direction.
+
+Why do we cluster for horizontal lines?
+To ignore lines which are part of a box and can not be a horizontal line.
+"""
+
+import functools
+import operator
+
+import configos
+import iamraw
+import pdfminer.layout
+import serializeraw
+import utilo
+
+import rawmaker.features
+import rawmaker.features.border
+import rawmaker.features.boxes
+import rawmaker.reader
+
+# TODO: LTLine - replace with own data structure to reduce dependencies to
+# rawmaker
+LineClusters = list[list[pdfminer.layout.LTLine]]
+
+# minimal length of a horizontal line
+HORIZONTAL_WIDTH_MIN = configos.HV_FLOAT(default=0.2)
+# maximal difference in x-component
+HORIZONTAL_VERTICAL_DIFF_MAX = configos.HV_FLOAT_PLUS(default=5.0)
+
+
+def work(lines: str, pages: tuple) -> str:
+    """Extract content horizontal lines from given `document`
+
+    Args:
+        lines(str): path to document
+        pages(tuple): pages to analyze
+    Returns:
+        dumped parsed boxes, dumped parsed horizontals
+    """
+    assert isinstance(lines, str), type(lines)
+    lines = serializeraw.load_lines(lines, pages=pages)
+    horizontal = determine_horizontal(lines)
+    dumped = serializeraw.dump_horizontals(horizontal)
+    return dumped
+
+
+def determine_horizontal(lines, pagewidth=500):
+    worker = functools.partial(determine_pagehorizontals, page_width=pagewidth)
+    # run worker
+    result = rawmaker.features.boxes.determine_clusteritem(
+        lines,
+        worker,
+    )
+    return result
+
+
+def determine_pagehorizontals(  # pylint:disable=R0914
+    cluster: LineClusters,
+    page: int,
+    *,
+    page_width: float,
+    rotated: bool = False,
+    vertical_maxerror: float = HORIZONTAL_VERTICAL_DIFF_MAX,
+    horizontal_minwidth: float = HORIZONTAL_WIDTH_MIN,
+) -> iamraw.PageContentHorizontals:
+    """Collect single line which are expanded horizontal
+
+    Args:
+        cluster: list of line cluster
+        page(int): current analyzed page
+
+        page_width(float): width of page page
+        rotated(bool): if True pdfpage is rotated
+        vertical_maxerror(float): maximal vertical difference of the left and
+                                  right y-component [0.0,1.0].
+        horizontal_minwidth(float): minimum distance between left and right
+                                    x-component [0.0,1.0].
+    Returns:
+        list with horizontal line
+    """
+    horizontal_minwidth = horizontal_minwidth * page_width
+    collected = []
+    for merged in cluster:
+        if len(merged) != 1:
+            # ignore boxed lines
+            continue
+        # convert from BoundingBox
+        x0, y0, x1, y1 = utilo.roundme(tuple(merged[0]))
+        height = abs(y1 - y0)
+        width = abs(x1 - x0)
+        # check roated
+        if rotated:
+            width, height = height, width  # flip
+        if height > vertical_maxerror:
+            utilo.debug(f'no horizontal line {x0} {y0} {x1} {y1}; page: {page}'
+                        f' vertical error: {height} > {vertical_maxerror}')
+            continue
+        if width < horizontal_minwidth:
+            utilo.debug(f'no horizontal line {x0} {y0} {x1} {y1}; page: {page}'
+                        f' too short: {width} < {horizontal_minwidth}')
+            continue
+        if not rotated:
+            y0 = y1 = utilo.roundme((y0 + y1) / 2)
+        else:
+            x0 = x1 = utilo.roundme((x0 + x1) / 2)
+        box = iamraw.BoundingBox(x0, y0, x1, y1)
+        horizontal = iamraw.HorizontalLine(box=box)
+        collected.append(horizontal)
+    # ensure to sort items top to bottom and left to right
+    collected = sorted(
+        collected,
+        key=operator.attrgetter('box.y0', 'box.x0'),
+    )
+    result = iamraw.PageContentHorizontals(
+        content=collected,
+        page=page,
+        rotated=rotated,
+    )
+    return result