rawmaker 2.40.3__py3-none-any.whl

rawmaker/features/images.py

@@ -0,0 +1,155 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""ImageExtractor
+
+The ImageExtractor provides the possibility to extract all images out of
+a pdf file.
+
+Support formats:
+    - png?
+    - jpg?
+"""
+
+import collections
+import io
+import os
+
+import PIL.Image
+import serializeraw
+import ughost
+import utilo
+
+import rawmaker
+import rawmaker.images.info
+import rawmaker.miner.images
+import rawmaker.reader
+
+PageContentImages = collections.namedtuple('PageContentImages', 'content, page')
+PageContentImagesList = list[PageContentImages]
+
+DumpedImageInformations = list[tuple[str, bytes]]
+
+
+def work(document: str, pages: tuple = None) -> DumpedImageInformations:
+    extracted = extract_images(document, pages=pages)
+    extracted = beautify_images(extracted, document)
+    result = []
+    for page in extracted:
+        for info, (rawimage, ext) in page.content:
+            info = serializeraw.dump_image_info(info)
+            result.append((info, (rawimage, ext)))
+    return result
+
+
+def extract_images(
+    document: str,
+    outputfolder: str = None,
+    pages=None,
+) -> PageContentImagesList:
+    # TODO: REPLACE AFTER UPGRADING utilo
+    if outputfolder is None:
+        outputfolder = utilo.tmpfile(rawmaker.ROOT)
+    with rawmaker.reader.read(document) as loaded:
+        extracted = rawmaker.miner.images.extract_images(
+            loaded,
+            outputfolder=outputfolder,
+            pages=pages,
+        )
+    result = convert_images(
+        extracted,
+        outputfolder,
+        pages=pages,
+    )
+    return result
+
+
+def convert_images(
+    extracted: dict,
+    outputfolder: str,
+    pages: tuple = None,
+) -> list:
+    result = []
+    for page, images in extracted.items():
+        # convert selected pages to global pages
+        page = convert_pages(page, pages)
+        pagecontent = []
+        for parsed in images:
+            bounding = parsed.bounding
+            path = os.path.join(outputfolder, parsed.filename)
+            if not os.path.exists(path):
+                # TODO: FIX IMAGE EXTRACTION
+                utilo.error(f'missing image: {path}')
+                continue
+            loaded = utilo.file_read_binary(path)
+            info = rawmaker.images.info.imageinfo(path, page, bounding)
+            if info is None:
+                utilo.error(f'could not extract {path}, {page}, {bounding}')
+                continue
+            ext = utilo.file_ext(path)
+            pagecontent.append((info, (loaded, ext)))
+        if not pagecontent:
+            continue
+        result.append(PageContentImages(page=page, content=pagecontent))
+    return result
+
+
+def beautify_images(images, path: str):
+    """Use ghost to render pdf and crop image area."""
+    result = []
+    for page in images:
+        boundings = [item[0] for item in page.content]
+        extracted = run_ghost(path, boundings)
+        content = []
+        for raw, bounding in zip(extracted, boundings):
+            content.append((bounding, (raw, 'png')))
+        result.append(PageContentImages(content=content, page=page.page))
+    return result
+
+
+def run_ghost(path: str, boundings: list) -> list:
+    """Extract images out of pdf.
+
+    If ghost is not installed, we return a empty white image.
+    """
+    if ughost.HAS_GHOST:
+        extracted = ughost.images(path, boundings)
+        return extracted
+    utilo.error('could not beautify images: install ghost')
+    result = []
+    for bounding in boundings:
+        size = (int(bounding.width), int(bounding.height))
+        # white image backup bock box
+        raw = PIL.Image.new('RGB', size, color=1)
+        png = convert_topng(raw)
+        result.append(png)
+    return result
+
+
+def convert_topng(image) -> bytes:
+    raw = io.BytesIO()
+    image.save(raw, format='png')
+    # rewind the buffer
+    raw.seek(0)
+    # convert to bytes
+    result = raw.getvalue()
+    return result
+
+
+def convert_pages(page: int, pages: tuple) -> int:
+    """Pdfminer produces directly ascending pages.
+
+    If we select pages=('0:5,28') pdfminer produces 0, 1, 2, 3, 4, 5.
+    This method convert this to 0, 1, 2, 3, 4, 28.
+    """
+    # TODO: INVESTIGATE HERE
+    if pages is None:
+        return page
+    # starting with starting offset
+    offset = min(pages, default=0)
+    return pages[page - offset]

rawmaker/features/line.py

@@ -0,0 +1,337 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""Line Extractor
+==============
+
+This module aims to extract lines out of pdf document.
+
+Furthermore the lines are:
+    * fixed in x0/x1 and y0/y1
+    * sorted from top to bottom and left to right
+    * if required merged together.
+"""
+
+import operator
+
+import configos
+import iamraw
+import pdfminer.layout
+import pdfminer.pdfdocument
+import serializeraw
+import utilo
+
+import rawmaker.reader
+
+# maximal difference in y-component
+HORIZONTAL_DIFF_MAX = configos.HV_FLOAT_PLUS(default=2.0)
+# maximal difference in x-component
+VERTICAL_DIFF_MAX = configos.HV_FLOAT_PLUS(default=2.0)
+# minimal number of minus signs which build a horizontal line
+REQUIRED_MINUS_SIGNS = configos.HV_INT_PLUS(default=40)
+
+
+def work(document: str, annotations: str, pages: tuple = None) -> str:
+    if utilo.exists(annotations):
+        annotations = serializeraw.load_annotations(annotations, pages=pages)
+    else:
+        utilo.debug(f'missing {annotations} could not skip underlines')
+        annotations = []
+    with rawmaker.reader.read(document) as pdf:
+        extracted = determine_lines(pdf, pages=pages)
+    extracted = skip_lines(extracted, annotations)
+    dumped = serializeraw.dump_lines(extracted)
+    return dumped
+
+
+def determine_lines(
+    document: pdfminer.pdfdocument.PDFDocument,
+    pages: tuple = None,
+) -> iamraw.PageContentLines:
+    lines_ = lines(document, pages=pages)
+    rotates = rotated(document, pages=pages)
+    result = []
+    for content, number in lines_:
+        # left point is left above from right down point
+        content = [utilo.rect_ensure_bounding(item) for item in content]
+        # top down, left right
+        content.sort(key=operator.itemgetter(1, 0))
+        # merge lines which are divided by pdf printer
+        merged = utilo.merge_lines(content)
+        contentline = iamraw.PageContentLine(
+            content=merged,
+            page=number,
+            rotated=number in rotates,
+        )
+        result.append(contentline)
+    return result
+
+
+def rotated(
+    document: pdfminer.pdfdocument.PDFDocument,
+    pages: tuple = None,
+) -> set:
+    """Determine rotated pages."""
+    result = set()
+    for page in rawmaker.features.process_pagecontent(
+            document,
+            pages=pages,
+    ):
+        width, height = page.content.width, page.content.height
+        if width < height:
+            continue
+        result.add(page.page)
+    return result
+
+
+def skip_lines(linex, annotation) -> list:
+    result = []
+    for page in linex:
+        anno = utilo.select_page(annotation, page.page)
+        if not anno:
+            result.append(page)
+            continue
+        invalid_area = [item.bounds for item in anno.hyperlinks]
+        # remove annotated lines. This lines are the underlines of
+        # hyperlinks which are produced by cray pdf printer.
+        linex = [
+            item for item in page.content
+            if not utilo.rectangles_intersecting(invalid_area, item)
+        ]
+        result.append(iamraw.PageContentLine(content=linex, page=page.page))
+    return result
+
+
+# do not merge near horizontal: '_______________' to text container below.
+LAYOUT_LINES = pdfminer.layout.LAParams(line_margin=0.0000001)
+
+
+def lines(
+    pdf: pdfminer.pdfdocument.PDFDocument,
+    pages: tuple = None,
+) -> list:
+    """Extract all `LTLine` out of `PDFDocument` page wise
+
+    Support 3 different types of pdf layout elements:
+        LTLine:
+        LTRect: small difference between oposite lines
+        LTTextBoxHorizontal:
+
+    Args:
+        pdf: pdf document to collect lines
+        pages: select pages to run anlaysis on
+    Returns:
+        list of line objects[LTLine, LTRect, LTTextBoxHorizontal]
+    """
+    utilo.asserts(pdf, pdfminer.pdfdocument.PDFDocument)
+    possible_lines = type_in_document(
+        pdf,
+        datatype=(
+            pdfminer.layout.LTTextBoxHorizontal,
+            pdfminer.layout.LTLine,
+            pdfminer.layout.LTRect,
+            pdfminer.layout.LTFigure,
+            pdfminer.layout.LTCurve,
+        ),
+        layout=LAYOUT_LINES,
+        pages=pages,
+    )
+    strategy = {
+        pdfminer.layout.LTLine: accept_ltline,
+        pdfminer.layout.LTRect: accept_ltrect,
+        pdfminer.layout.LTTextBoxHorizontal: accept_text_as_line,
+        pdfminer.layout.LTCurve: accept_curve_as_line,
+        pdfminer.layout.LTFigure: accept_figure_as_line,
+    }
+    result = []
+    for content, pagenumber in possible_lines:
+        page = []
+        for item in content:
+            # check item against strategy. If no stategy is supported, the
+            # element is skipped.
+            try:
+                if not strategy[type(item)](item):
+                    continue
+                page.append(item)
+            except KeyError:
+                utilo.error(f'unsupported strategy {item}')
+        # convert bounding
+        page = [(item.bbox[0], item.bbox[1], item.bbox[2], item.bbox[3])
+                for item in page]
+        # round bounding
+        page = [utilo.roundme(item) for item in page]
+        # remove very short lines/dots
+        page = [item for item in page if not utilo.isdot(item, max_length=5.0)]
+        # ensure left, top, right, down bounding
+        page = [utilo.rect_ensure_bounding(item) for item in page]
+        # sort item top down; left right
+        page.sort(key=operator.itemgetter(1, 0))
+        # merges divided lines
+        page = utilo.merge_lines(page)
+        # remove duplicated lines which mainly produces out of bad figure
+        # extraction
+        # TODO: ADD LINE DENSITY CHECK?
+        page = utilo.unique_lines(page, max_diff=3.0)
+        result.append((page, pagenumber))
+    return result
+
+
+def accept_text_as_line(item: pdfminer.layout.LTTextBoxHorizontal):
+    symbols = '_-='
+    text = item.get_text()
+    if len(text) < REQUIRED_MINUS_SIGNS:
+        return False
+    for symbol in symbols:
+        if text.count(symbol) >= REQUIRED_MINUS_SIGNS:
+            # update bounding to pass vertical error test.
+            # use vertical centric position
+            # TODO: CHECK THIS: Make it symbol dependend?
+            if symbol in '_':
+                ypos = utilo.roundme(max((item.bbox[1], item.bbox[3])))
+            else:
+                ypos = utilo.roundme((item.bbox[1] + item.bbox[3]) / 2)
+            # update bounding box
+            item.bbox = (item.bbox[0], ypos, item.bbox[2], ypos)
+            return True
+    return False
+
+
+def accept_ltrect(item: pdfminer.layout.LTRect):
+    return accept_ltline(item)
+
+
+def accept_ltline(
+    item: pdfminer.layout.LTLine,
+    vertical_max_diff=VERTICAL_DIFF_MAX,
+    horizontal_max_diff=HORIZONTAL_DIFF_MAX,
+) -> bool:
+    """Accept horizontal or vertical lines
+
+    The lines must vary only little. A crossing line has vertical
+    and horizontal error. We want | or - not / or \\.
+    """
+    assert item.bbox[3] >= item.bbox[1], str(item.bbox)
+    assert item.bbox[0] <= item.bbox[2], str(item.bbox)
+
+    horizontal_error = item.bbox[3] - item.bbox[1] >= horizontal_max_diff
+    vertical_error = item.bbox[2] - item.bbox[0] >= vertical_max_diff
+
+    if horizontal_error and vertical_error:
+        return False
+
+    if vertical_error:
+        # HACK: WORKAROUND TODO:
+        # horizontal lines: There are lines in bachelor028 which are
+        try:
+            blueline = BLUE in (item.stroking_color, item.non_stroking_color)
+        except AttributeError:
+            blueline = False
+        if blueline:
+            utilo.debug('skip horizontal blue line which is may part of a '
+                        'hyperlink and destroys footnote detection')
+            utilo.debug(item)
+            return False
+    return True
+
+
+BLUE = [0, 0, 1]
+
+
+def accept_figure_as_line(figure: pdfminer.layout.LTFigure) -> bool:
+    """Some pdf renderer converts lines into images."""
+    content = figure._objs  # pylint:disable=W0212
+    if len(content) != 1:
+        return False
+    # Do we need a min width? I don't think so because thats the job of
+    # later running methods.
+    if accept_ltline(content[0]):
+        return True
+    if figure_special_line(figure):
+        return True
+    return False
+
+
+def accept_curve_as_line(curve: pdfminer.layout.LTCurve) -> bool:
+    pts = curve.pts
+    if not curve.linewidth and not curve.fill:
+        # invisible line
+        return False
+    if curve.stroke:
+        if curve.stroking_color is None and curve.non_stroking_color is None:
+            # TODO: DONT KNOW WHY
+            return False
+    if curve.fill:
+        # polygon?
+        if curve.height < 5.0 or curve.width < 5.0:
+            return True
+    if len(pts) == 2:
+        # start and end point
+        return True
+    # more than two points in a row, check if point are on a line
+    # [(437.04645, 259.38056), (437.04645, 293.26655), (437.04645, 269.60483999999997)]
+    items = [(*first, *second) for first, second in zip(pts[:-1], pts[1:])]
+    merged = utilo.merge_lines(items, diff=1.5)
+    if len(merged) == 1:
+        # all lines in a row
+        return True
+    return False
+
+
+# horizontal line which is rendered as a figure
+HORIZONTAL_FIGURE_LINE_WIDTH_MIN = configos.HV_FLOAT_PLUS(default=350.0)
+# the object have to be more width than height with this ratio
+HORIZONTAL_FIGURE_LINE_RATIO_MIN = configos.HV_FLOAT_PLUS(default=45.0)
+
+
+def figure_special_line(figure: pdfminer.layout.LTFigure) -> bool:
+    """Detect special line and update figure box if figure is special line."""
+    # TODO: THIS IS ONLY A HORIZONTAL?
+    # EXAMPLE: MASTER155
+    #  'width': 413.96, 'height': 8.54
+    # TODO: ANALYZE IMAGE
+    image = figure._objs[0]  # pylint:disable=W0212
+    height = utilo.rect_height(image.bbox)
+    width = utilo.rect_width(image.bbox)
+    ratio = width / height
+    if width <= HORIZONTAL_FIGURE_LINE_WIDTH_MIN:
+        return False
+    if ratio <= HORIZONTAL_FIGURE_LINE_RATIO_MIN:
+        return False
+    # adjust bounding of figure to middle line
+    # TODO: USE IMAGE INFORMATION
+    middle = utilo.roundme((figure.bbox[1] + figure.bbox[3]) / 2)
+    figure.bbox = (figure.bbox[0], middle, figure.bbox[2], middle)
+    return True
+
+
+def type_in_document(
+    document: pdfminer.pdfdocument.PDFDocument,
+    datatype: object,
+    layout=None,
+    pages: tuple = None,
+) -> list[tuple[pdfminer.layout.LTPage, int]]:
+    """Extract defined `datatype` out of `PDFDocument`
+
+    Args:
+        document(PDFDocument): pdf document to extract all types
+        datatype: selected item type
+        layout(Param): process with different layout
+        pages(tuple): select pages
+    Returns:
+        List with selected `datatype`.
+    """
+    utilo.asserts(document, pdfminer.pdfdocument.PDFDocument)
+    result = []
+    for page in rawmaker.features.process_pagecontent(
+            document,
+            layout=layout,
+            pages=pages,
+    ):
+        data = [item for item in page.content if isinstance(item, datatype)]
+        result.append((data, page.page))
+    return result

rawmaker/features/outlines.py

@@ -0,0 +1,123 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================
+"""Document Outlines
+=================
+
+See PDF2008: 12.3.3 Document Outline
+
+Basic structure of get_outlines: (level, title, args, children)
+
+Entries of outlines dict
+------------------------
+
+Dest(str, list): destination if item was clicked/activated, not present
+                 if A is present.
+A(dict): Action(launch application, play sound, chaning state) Shall
+         not be present if an DEST item is present
+SE(dict): Reference to structure element(see Structural Hierarchy)
+"""
+
+import iamraw
+import pdfminer.pdfdocument
+import pdfminer.pdfpage
+import serializeraw
+import utilo
+
+import rawmaker.destination
+import rawmaker.reader
+import rawmaker.utils
+
+
+def work(document: str) -> str:
+    """Extract outlines of a pdf document.
+
+    If there are no outlines provided dump empty list.
+    """
+    assert isinstance(document, str), str(document)
+    parsed = parse_outlines(document)
+    toc = iamraw.create_toc(parsed)
+    try:
+        # toc to yaml
+        dumped = serializeraw.dump_toc(toc)
+    except TypeError:
+        utilo.error('could not convert toc to YAML.')
+        utilo.error('The toc may contain indirect references, buffer, etc.')
+        utilo.error('Outline implementation seem not complete, yet.')
+        dumped = None
+    return dumped
+
+
+def parse_outlines(document: str) -> list:
+    result = []
+    with rawmaker.reader.read(document) as pdf:
+        try:
+            # extract all outlines from pdf
+            outlines = list(pdf.get_outlines())
+            pagelookup = rawmaker.destination.pageids(document)
+        except pdfminer.pdfdocument.PDFNoOutlines:
+            outlines = []
+            utilo.error('could not locatate any outlines')
+        for (level, title, dest, action, _) in outlines:
+            try:
+                page = pagenumber(action, dest, pdf)
+            except (AttributeError, ValueError) as error:
+                utilo.error('PDF NOT FULLY SUPPORTED')
+                utilo.print_stacktrace()
+                utilo.error(error)
+                continue
+            if not isinstance(page, int):
+                try:
+                    page = pagelookup[page.objid]
+                except KeyError:
+                    utilo.error(f'invalid page lookup: {page.objid} pdf is '
+                                'maybe an invalid extraction out of an other '
+                                f'file: {pagelookup}')
+                    continue
+            assert isinstance(page, int), f'require convertion: {type(page)}'
+            raw_section = iamraw.SectionRaw(
+                level,
+                title,
+                page=page,
+                raw='toc outline page',
+                raw_location=-1,
+            )
+            result.append(raw_section)
+    return result
+
+
+def pagenumber(action, dest, pdf) -> rawmaker.destination.ExplicitDestination:
+    parsed = None
+    if action:
+        parsed = rawmaker.destination.parse(action)
+        if isinstance(parsed, rawmaker.destination.NamedDestination):
+            try:
+                resolved = pdf.get_dest(parsed.pdf_reference)
+            except pdfminer.pdfdocument.PDFDestinationNotFound:
+                utilo.error(f'invald pdf reference: {parsed.pdf_reference}')
+                return -1
+            resolved = rawmaker.utils.resolve(resolved)
+            parsed = rawmaker.destination.parse(resolved)
+    if dest:
+        dest = rawmaker.utils.resolve(dest)
+        if isinstance(dest, list):
+            # pdf 1.5: [<PDFObjRef:13>, /'XYZ', 72.0, 769.89, None]
+            resolved = dest
+        else:
+            destname = dest if isinstance(dest, bytes) else dest.name
+            resolved = pdf.get_dest(destname)
+            if isinstance(resolved, list):
+                # pdf 1.4: [<PDFObjRef:4>, /'XYZ', 134.031754, 373.949829, None]
+                pass
+            else:
+                resolved = rawmaker.utils.resolve(resolved)
+        parsed = rawmaker.destination.parse(resolved)
+    assert parsed
+    if isinstance(parsed, rawmaker.destination.ExternalLinkDestination):
+        return -1
+    return parsed.page

rawmaker/features/text.py

@@ -0,0 +1,91 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================
+"""Extract text out of pdf document to gather information."""
+
+import os
+
+import iamraw
+import serializeraw
+
+import rawmaker.cli
+import rawmaker.features
+import rawmaker.miner.position
+import rawmaker.miner.text
+import rawmaker.miner.underline
+import rawmaker.parameter
+import rawmaker.reader
+import rawmaker.text.superfast
+
+
+def work(  # pylint:disable=W9015,W0613
+    document: str,
+    xhorizontals: str = None,
+    boxes_flow: float = 0.5,
+    char_margin: float = 2.0,
+    line_margin: float = 0.5,
+    line_overlap: float = 0.5,
+    word_margin: float = 0.1,
+    nostrip: bool = not rawmaker.parameter.STRIP,
+    detect_vertical: bool = False,
+    pages: tuple = None,
+) -> tuple[str, str]:
+    """Extract structured text out of document
+
+    Args:
+        document: pdf-document to run parsing
+        char_margin(float): XXX Why 5.0?
+        pages(list): List of processed pages.
+    Returns:
+        parsed document as yaml output
+        parsed positions of text container
+    """
+    # TODO: CHANGE BEHAVIOR OF --detect_vertical. Convert to PARAMETER
+    # with True as default.
+    detect_vertical = True  # TODO: REMOVE?
+    config = rawmaker.parameter.ParsingConfiguration.from_dict(**locals())
+    if rawmaker.cli.superfast():  # pylint:disable=W0160
+        document = rawmaker.text.superfast.superfast(
+            document,
+            config,
+            workdir=os.getcwd(),
+            pages=pages,
+        )
+    else:
+        document = extract_document(source=document, config=config, pages=pages)
+    document = rawmaker.miner.underline.underline_chars(
+        document,
+        xhorizontals,
+        pages=pages,
+    )
+    positions = rawmaker.miner.position.hash_positions(document, pages=pages)
+    # dump result
+    dumped_text = serializeraw.dump_document(document)
+    dumped_positions = serializeraw.dump_textpositions(positions)
+    return dumped_text, dumped_positions
+
+
+def extract_document(
+    source: str,
+    config: rawmaker.parameter.ParsingConfiguration = None,
+    converter=None,
+    pages: tuple = None,
+) -> iamraw.Document:
+    if config:
+        rawmaker.parameter.print_layout(config)
+    if converter is None:
+        converter = rawmaker.miner.text.PrecisePDFConverter
+    assert isinstance(source, str), str(source)
+    with rawmaker.reader.read(source) as pdf:
+        document = rawmaker.features.extract_content(
+            pdf,
+            config=config,
+            converter=converter,
+            pages=pages,
+        )
+    return document