rawmaker 2.40.3__py3-none-any.whl

rawmaker/features/__init__.py

@@ -0,0 +1,138 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================
+
+import collections
+
+import iamraw
+import pdfminer.converter
+import pdfminer.layout
+import pdfminer.pdfdocument
+import pdfminer.pdfinterp
+import pdfminer.pdfpage
+import utilo
+
+import rawmaker.converter.basic
+import rawmaker.miner.text
+import rawmaker.parameter
+
+PageContent = collections.namedtuple('PageContent', 'content, page')
+
+
+def create_interpreter(layout=None) -> pdfminer.pdfinterp.PDFPageInterpreter:
+    if not layout:
+        layout = rawmaker.parameter.ParsingConfiguration().laparams()
+    device = rawmaker.converter.basic.PageAggregator(laparams=layout)
+    interpreter = pdfminer.pdfinterp.PDFPageInterpreter(
+        device.resources,
+        device,
+    )
+    return interpreter, device
+
+
+def process_pdfpages(
+    document: pdfminer.pdfdocument.PDFDocument,
+    pages: tuple = None,
+) -> pdfminer.pdfpage.PDFPage:
+    """Contextmanager to yield `PDFPage` of every selected page of
+    `PDFDocument`.
+
+    Args:
+        document: open pdf file
+        pages: number of pages to procress, if None every page is processed
+    Yields:
+        PDFPage: tuple of page content and pdf page number
+    """
+    utilo.call('process_pdfpages')
+    utilo.asserts(document, pdfminer.pdfdocument.PDFDocument)
+    create_pages = pdfminer.pdfpage.PDFPage.create_pages
+    with utilo.SkipCollector(pages) as collector:
+        for number, page in enumerate(create_pages(document), start=0):
+            if collector.skip(number):
+                continue
+            page.pageid = number
+            yield (page, number)
+
+
+def process_document(
+    document: pdfminer.pdfdocument.PDFDocument,
+    layout=None,
+    pages=None,
+) -> tuple[int, pdfminer.layout.LTPage]:
+    """Yield (pagenumber, LTPage) for every selected page of `PDFDocument`"""
+    assert isinstance(
+        document,
+        pdfminer.pdfdocument.PDFDocument,
+    ), type(document)
+    interpreter, device = create_interpreter(layout=layout)
+    for content, number in process_pdfpages(document, pages=pages):
+        interpreter.process_page(content)
+        pagecontent = PageContent(content=device.get_result(), page=number)
+        yield (content, pagecontent)
+
+
+def process_pagecontent(
+    document: pdfminer.pdfdocument.PDFDocument,
+    layout=None,
+    pages=None,
+) -> pdfminer.layout.LTPage:
+    utilo.asserts(document, pdfminer.pdfdocument.PDFDocument)
+    for _, content in process_document(document, layout=layout, pages=pages):
+        yield content
+
+
+def page_selection(document: iamraw.Document, pages: tuple):
+    assert isinstance(document, iamraw.Document), type(document)
+    if pages:
+        assert isinstance(pages, (list, tuple)), '%s %s' % (pages, type(pages))  # pylint:disable=C0209
+        return pages
+    # if pages is None, every page must processed
+    return list(range(len(document.pages)))
+
+
+def extract_content(
+    document: pdfminer.pdfdocument.PDFDocument,
+    config: rawmaker.parameter.ParsingConfiguration = None,
+    converter=rawmaker.miner.text.PrecisePDFConverter,
+    pages: tuple = None,
+) -> iamraw.Document:
+    """Extract content from PDF file
+
+    Args:
+        document(PDFDocument): PDF file to process
+        config(ParsingConfiguration): parametrization for layout analysis.
+                                      This parameter defines how chars are
+                                      matched together in words and sentences.
+                                      See pdf reference documentation.
+        converter(pdfminer.converter.PDFLayoutAnalyzer): how to handle
+                                                         the layout extraction
+        pages: tuple of selected pages
+    Returns:
+        Document: parsed and layouted document
+    """
+    if config is None:
+        config = rawmaker.parameter.ParsingConfiguration()
+    utilo.asserts(config, rawmaker.parameter.ParsingConfiguration)
+    # prepare parser
+    device = converter(config=config)
+    device.new_document()
+    interpreter = pdfminer.pdfinterp.PDFPageInterpreter(device.rsrcmgr, device)
+    # Processing layout
+    create_pages = pdfminer.pdfpage.PDFPage.create_pages
+    with utilo.SkipCollector(pages) as collector:
+        for index, page in enumerate(create_pages(document)):
+            if collector.skip(index):
+                continue
+            interpreter.process_page(page)
+    result = device.finish_document()
+    # upgrade page number
+    pages = page_selection(result, pages)
+    # TODO: REPLACE PAGE WITH ENDLESS ITER AND CHANGE ZIP TO ZIP_LONGEST
+    for (page, pagenumber) in zip(result.pages, pages):
+        page.page = pagenumber
+    return result

rawmaker/features/annotation.py

@@ -0,0 +1,254 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""Add parser to parse non annotated links to inform user about
+broken/malformated links."""
+
+import iamraw
+import pdfminer.pdfdocument
+import serializeraw
+import utilo
+
+import rawmaker.features
+import rawmaker.reader
+import rawmaker.utils
+
+
+def work(document: str, pages=None) -> str:
+    assert isinstance(document, str), str(document)
+    with rawmaker.reader.read(document) as pdf:
+        annotations = extract_annotations(pdf, pages=pages)
+    dumped = serializeraw.dump_annotations(annotations)
+    return dumped
+
+
+def extract_annotations(
+    document: pdfminer.pdfdocument.PDFDocument,
+    pages=None,
+) -> iamraw.PageAnnotations:
+    result = []
+    for page, number in rawmaker.features.process_pdfpages(
+            document,
+            pages=pages,
+    ):
+        parsed = parse_page(page, pagenumber=number)
+        if not (parsed.hyperlinks or parsed.pagelinks):
+            # skip empty page
+            continue
+        result.append(parsed)
+    return result
+
+
+ANNOTATION_LABEL = 'Annot'
+
+
+def parse_page(
+    page: pdfminer.pdfpage.PDFPage,
+    pagenumber: int,
+) -> iamraw.PageAnnotation:
+    """Parse annotation from `PDFPage`.
+
+    Args:
+        page(PDFPage): pdf page to parse annotation
+        pagenumber(int): number of selected page
+    Returns:
+        parsed Annotations.
+
+    There are 2 different types of annotation, the internal and external
+    links:
+    *   The internal links, better called page links refer to a chapter or a
+        location in the document.
+    *   The external links, so called hyperlink refer to clickable weblinks.
+
+    # Internal reference
+    # {'A': {'S': /'GoTo', 'D': b'subsection.1.30.7'}}
+    # {'S': /'GoTo', 'D': b'chapter*.1'}
+    """
+    pageannotation = page.annots
+    if not pageannotation:
+        return iamraw.PageAnnotation(None, None, pagenumber)
+    getobj = page.doc.getobj
+    if not isinstance(pageannotation, list):
+        # WORKAROUND: THIS IS A FIX WHEN PAGE ANNOTATIONS ARE NESTED IN A
+        # SINGLE REFERENCE, DON'T KNOW WHY THIS CAN HAPPEN. TODO:
+        # INVESTIGATE LATER
+        pageannotation = list(getobj(page.annots.objid))
+    pagelinks, hyperlinks = [], []
+    for reference in pageannotation:
+        if isinstance(reference, dict):  # pylint:disable=W0160
+            # reference is already resolved
+            pageobject = reference
+        else:
+            pageobject = getobj(reference.objid)
+        reference = parse_reference(pageobject, getobj)
+        if reference:
+            pagelinks.append(reference)
+            continue
+        if annotation := parse_appereance_strean(pageobject):
+            pagelinks.append(annotation)
+            continue
+        external = parse_external(pageobject, getobj)
+        if external:
+            hyperlinks.append(external)
+            continue
+        utilo.error(f'Unhandeld annotation {pageobject}')
+    # flip boundings
+    pageheight = float(page.mediabox[3])
+    for item in pagelinks:
+        item.bounds = flip_bounding(item.bounds, pageheight)
+    for item in hyperlinks:
+        item.bounds = flip_bounding(item.bounds, pageheight)
+    return iamraw.PageAnnotation(pagelinks, hyperlinks, page=pagenumber)
+
+
+def flip_bounding(box, pageheight):
+    result = iamraw.BoundingBox(
+        box[0],
+        pageheight - box[3],
+        box[2],
+        pageheight - box[1],
+    )
+    return result
+
+
+def parse_reference(pageobject, getobj=None) -> iamraw.PageLink:
+    if link := parse_link(pageobject):
+        return link
+    if label := parse_label(pageobject, getobj):
+        return label
+    return None
+
+
+def parse_label(pageobject, getobj=None) -> iamraw.PageLink:
+    try:
+        typ = pageobject['Type'].name
+        if typ != ANNOTATION_LABEL:
+            return None
+    except KeyError:
+        return None
+    try:
+        annotated = pageobject['A']
+    except KeyError:
+        return None
+    if isinstance(annotated, pdfminer.pdftypes.PDFObjRef):
+        # TODO: add layer to automatically convert reference to object.
+        annotated = getobj(annotated.objid)
+    try:
+        pagelink = annotated['D']
+    except KeyError:
+        return None
+    bounds = determine_bounding(pageobject['Rect'])
+    pagelink = parse_pagelink(pagelink)
+    return iamraw.PageLink(bounds=bounds, goal=pagelink)
+
+
+def parse_link(pageobject) -> iamraw.PageLink:
+    # TODO: don't know what this element means
+    #{'Type': /'Annot', 'Border': [0, 0, 0], 'H': /'I', 'C': [0,
+    #0.5, 0.5], 'Rect': [348.517, 428.927, 431.794, 439.831],
+    #'Subtype': /'Link', 'A': {'F': b'distributions.pdf', 'S':
+    #/'GoToR', 'D': [0, /'Fit']}} [0, /'Fit']
+    try:
+        typ = pageobject['Subtype'].name
+        if typ != 'Link':
+            return None
+        assert typ == 'Link'
+    except KeyError:
+        return None
+    try:
+        pagelink = pageobject['Dest']
+    except KeyError:
+        return None
+    bounds = determine_bounding(pageobject['Rect'])
+    pagelink = parse_pagelink(pagelink)
+    return iamraw.PageLink(bounds=bounds, goal=pagelink)
+
+
+def parse_appereance_strean(pageobject) -> iamraw.PageLink:
+    # 12.5.5 Appereance Stream
+    # TODO: IMPROVE
+    try:
+        typ = pageobject['Type'].name
+    except KeyError:
+        return None
+    if typ != ANNOTATION_LABEL:
+        return None
+    try:
+        stream = rawmaker.utils.resolve(pageobject['AP'])  # pylint:disable=W0612
+        stream = rawmaker.utils.resolve(stream['N'])
+    except KeyError:
+        return None
+    bounds = determine_bounding(pageobject['Rect'])
+    result = iamraw.PageLink(
+        bounds=bounds,
+        goal='STREAM',
+    )
+    return result
+
+
+def parse_external(pageobject, getobj=None) -> iamraw.HyperLink:
+    # {'S': /'URI', 'URI': b'http://www.europarl.europa.eu/factsheets/de/sheet/92/allgemeine-steuerpolitik'}
+    # {'F': b'/C/Users/user/Downloads/MEMO-16-2265_DE.pdf', 'S': /'Launch'}
+    try:
+        annotated = pageobject['A']
+    except KeyError:
+        return None
+    if isinstance(annotated, pdfminer.pdftypes.PDFObjRef):
+        # TODO: add layer to automatically convert reference to object.
+        annotated = getobj(annotated.objid)
+    bounds = determine_bounding(pageobject['Rect'])
+    if 'URI' in annotated:
+        hyperlink = hyperlink_decode(annotated['URI'])
+        return iamraw.HyperLink(bounds=bounds, goal=hyperlink)
+    if 'F' in annotated:
+        hyperlink = hyperlink_decode(annotated['F'])
+        return iamraw.HyperLink(bounds=bounds, goal=hyperlink)
+    return None
+
+
+def hyperlink_decode(text: bytes) -> str:
+    r""""
+    text(bytes): 7-bit ASCII, see 12.6.4.7
+
+    Hint: ASCII must not always be correct cause of bad programmed printer.?
+    TODO: MAY A PDFMINER CONVERTION ERROR?
+    TODO: VERIFY LINUX/WIN DUE CP1252
+
+    >>> hyperlink_decode(b'http://road.cc/measure-\x96-smart-street')
+    'http://road.cc/measure-–-smart-street'
+    """
+    result = rawmaker.utils.guess_decoding(text)
+    if result is None:
+        utilo.error(f'annotation: could not decode: {text}')
+    return result
+
+
+def parse_pagelink(pagelink):
+    r"""\
+    >>> parse_pagelink(b'glo:glos:Glas\xfcbergangstemperatur')
+    'glo:glos:Glasübergangstemperatur'
+    """
+    if isinstance(pagelink, bytes):
+        decoded = rawmaker.utils.guess_decoding(pagelink)
+        if decoded:
+            return decoded
+    if isinstance(pagelink, list):
+        if isinstance(pagelink[0], pdfminer.pdftypes.PDFObjRef):
+            # internal link to pdf page
+            # resolve objid
+            pagelink[0] = f'objid: {pagelink[0].objid}'
+        pagelink = [str(item) for item in pagelink]
+    else:
+        pagelink = str(pagelink)
+    return pagelink
+
+
+def determine_bounding(bounding):
+    bounding = utilo.rect_ensure_bounding(bounding)
+    result = iamraw.BoundingBox.from_list(bounding)
+    return result

rawmaker/features/border.py

@@ -0,0 +1,172 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""The `border`-feature enables to detect the pdf page size in ?pixel?
+and locate the cropped box around the content.
+
+Features:
+ * page size
+ * content size
+
+"""
+
+import collections
+import contextlib
+
+import iamraw
+import pdfminer.pdfdocument
+import serializeraw
+import utilo
+
+import rawmaker.features
+import rawmaker.reader
+
+PagePageSize = collections.namedtuple('PagePageSize', 'size page')
+
+
+def work(document: str, pages: tuple = None) -> tuple[str, str]:
+    """Extract page size of `document` bounding boxes of page content.
+
+    Args:
+        document: path to document to run parsing
+        pages: tuple of processed pages
+    Returns:
+        tuple(pages, boxes): page size and list of bounding boxes for page
+                             content.
+    """
+    assert isinstance(document, str), str(document)
+    with rawmaker.reader.read(document) as pdf:
+        sizeandborders, boxes = determine_boundingboxes(pdf, pages=pages)
+
+    pages = serializeraw.dump_pageborders(sizeandborders)
+    boundingboxes = serializeraw.dump_boundingboxes(boxes)
+
+    return pages, boundingboxes
+
+
+def determine_boundingboxes(
+    document: pdfminer.pdfdocument.PDFDocument,
+    pages: tuple = None,
+) -> iamraw.PageBoundingsList:
+    """Extract page size, border and boundingboxes from `PDFDocument`.
+
+    Args:
+        document(PDFDocument): loaded document
+        pages: tuple of processed pages
+    Returns:
+        sizeandborder(List[PageSizeBorder]) a list for every page with page
+                border and a list of the BoundingBoxes of the objects on the
+                current page.
+        boxes(PageBoundings)
+    """
+    sizeborders, boxes = [], []
+    contentid = 0
+    for page, content in rawmaker.features.process_document(document, pages=pages): # yapf:disable
+        content, pagenumber = content.content, content.page
+        size = pagesize_from_page(page)
+
+        pagebounding = iamraw.PageBoundings(
+            boundings=boundingboxes_from_page(content, contentid),
+            page=pagenumber,
+        )
+        boxes.append(pagebounding)
+
+        contentid += len(content)
+        border = cropborder_from_page(content)
+        sizeborders.append(
+            iamraw.PageSizeBorder(
+                size=size,
+                border=border,
+                page=pagenumber,
+            ))
+    return sizeborders, boxes
+
+
+def pagesizes(
+    pdf: pdfminer.pdfdocument.PDFDocument,
+    pages: tuple = None,
+) -> list[iamraw.PageSize]:
+    """Extract page sizes of `PDFDocument`.
+
+    Args:
+        pdf(PDFDocument): load pdf document
+        pages: tuple of processed pages
+    Returns:
+        List of page sizes.
+    """
+    result = []
+    for page, content in rawmaker.features.process_document(pdf, pages=pages):
+        content, pagenumber = content.content, content.page
+        size = pagesize_from_page(page)
+        result.append(PagePageSize(size=size, page=pagenumber))
+    return result
+
+
+def boundingboxes_from_page(content: list, contentid: int) -> tuple:
+    """Extract bounding boxes from page `content`.
+
+    Args:
+        content: content of a single page
+        contentid: last id of the previous page
+    Returns:
+        Cropbox which contains all items of this page
+    """
+    result = []
+    for index, item in enumerate(content, start=contentid):
+        result.append((index, item.bbox))
+    return result
+
+
+def pagesize_from_page(page: pdfminer.pdfdocument.PDFDocument) -> iamraw.PageSize:  # yapf:disable
+    # x, y, width, height
+    pagewidth = utilo.roundme(page.mediabox[2])
+    pageheight = utilo.roundme(page.mediabox[3])
+
+    rotate = page.rotate
+    if rotate in {90, 270}:
+        # rotated page, flip page size
+        pagewidth, pageheight = pageheight, pagewidth
+    return iamraw.PageSize(width=pagewidth, height=pageheight)
+
+
+def cropborder_from_page(content) -> iamraw.Border:
+    """Determine bounding box which includes all page items except of
+    white space only text.
+
+
+    >>> cropborder_from_page([
+    ...     pdfminer.layout.LTLine(linewidth=1.0, p0=(50.520,78.540) , p1=(106.200,78.540)),
+    ...     pdfminer.layout.LTLine(linewidth=1.0, p0=(107.160,78.540), p1=(122.220,78.540)),
+    ... ])
+    Border(left=50.52, right=122.22, top=78.54, bottom=78.54)
+    """
+
+    def no_whitespace(items):
+        result = []
+        for item in items:
+            with contextlib.suppress(AttributeError):
+                if not item.get_text().strip():
+                    # skip white spaces
+                    continue
+            result.append(item)
+        return result
+
+    content = no_whitespace(content)
+    if not content:
+        return iamraw.Border(None, None, None, None)
+
+    # left, top, right, bottom
+    x0 = min((item.bbox[0] for item in content))
+    y0 = min((item.bbox[1] for item in content))
+    x1 = max((item.bbox[2] for item in content))
+    y1 = max((item.bbox[3] for item in content))
+    # left, right, top, bottom
+    x0, y0, x1, y1 = utilo.roundme((x0, y0, x1, y1))
+    assert x0 <= x1, f'{x0} <= {x1}'
+    assert y0 <= y1, f'{y0} <= {y1}'
+    return iamraw.Border(left=x0, right=x1, top=y0, bottom=y1)