rawmaker 2.40.3__py3-none-any.whl

rawmaker/converter/basic.py

@@ -0,0 +1,174 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import contextlib
+
+import iamraw
+import pdfminer.converter
+import pdfminer.layout
+import pdfminer.pdfinterp
+import pdfminer.pdfpage
+import utilo
+
+
+class FlippedLayoutAnalyzer(pdfminer.converter.PDFLayoutAnalyzer):
+
+    def __init__(self, laparams=None, pageno=0):
+        super().__init__(
+            rsrcmgr=pdfminer.pdfinterp.PDFResourceManager(),
+            pageno=pageno,
+            laparams=laparams,
+        )
+
+    def receive_layout(self, ltpage):
+        if content_inside_single_figure(ltpage):
+            # extract content out of a single figure container
+            ltpage._objs = ltpage._objs[0]._objs  # pylint:disable=W0212
+            params = self.laparams
+            if not params:
+                # use default layout for image extractor
+                params = pdfminer.layout.LAParams()
+            ltpage.analyze(params)
+        for item in ltpage:
+            flip_object(item, ltpage)
+        for item in ltpage:
+            item.bbox = figure_bounding(item)
+        # remove invisible objects
+        ltpage._objs = [item for item in ltpage if item.bbox is not None]  # pylint:disable=W0212
+
+    def handle_undefined_char(self, font, cid) -> str:
+        # TODO: CHECK AFTER UPGRADING PDFMINER
+        # TODO: FIX PAGE NUMBER
+        try:
+            char = MAPPING[cid]
+            utilo.debug(f'could not convert: {font!r}, {cid!r} use backup: '
+                        f'{char} on page: {self.pageno}')
+        except KeyError:
+            utilo.error(f'could not convert: {font!r}, {cid!r} '
+                        f'on page: {self.pageno} no backup char defined')
+            # use warning to log only once
+            utilo.warning(str(vars(font)))
+            char = chr(cid)
+        return char
+
+    @property
+    def resources(self):
+        return self.rsrcmgr
+
+
+def content_inside_single_figure(page) -> bool:
+    """Some pdf printer write all page content to a single figure.
+
+    If all content is in a single figure, no text extraction is possible.
+    """
+    objs = page._objs  # pylint:disable=W0212
+    if len(objs) != 1:
+        return False
+    figure = objs[0]
+    if not isinstance(figure, pdfminer.layout.LTFigure):
+        return False
+    if len(figure._objs) == 1:  # pylint:disable=W0212
+        # image container, see master116 page,2,3. This works fine.
+        return False
+    return True
+
+
+# REMOVE HACK LATER
+# UAZWCW+CMR10
+MAPPING = {
+    0: '−',
+    1: '·',
+    12: 'fi',
+    13: 'fl',
+    14: 'ffi',
+    # -Fern´andez,  ´Ecole  may support later, for now removing is a good match
+    # for more infos see master110p106
+    # 19: '´',
+    19: '',
+    20: '≤',
+    25: 'ß',
+    127: '¨',  # Umlaute, oe, ae, ue, use already implemented replace to
+    # support umlaute
+}
+
+
+def flip_object(item, page):
+    try:
+        box = list(item.bbox)
+    except AttributeError:
+        # VirtualChar for example
+        return
+    pageheight = page.height
+    box[1], box[3] = pageheight - box[3], pageheight - box[1]
+    box = utilo.roundme(box)  # pylint:disable=R0204
+    try:
+        item.bbox = iamraw.BoundingBox(*box)
+    except AssertionError:
+        utilo.debug(f'invalid bounding on page {page}: {box}')
+        utilo.debug(item)
+    item.x0, item.y0, item.x1, item.y1 = box
+    with contextlib.suppress(AttributeError):
+        for obj in item._objs:  # pylint:disable=W0212
+            flip_object(obj, page)
+
+
+class PageAggregator(FlippedLayoutAnalyzer):
+
+    def __init__(self, laparams=None):
+        super().__init__(laparams=laparams)
+        self.result = None
+
+    def receive_layout(self, ltpage):
+        super().receive_layout(ltpage)
+        self.result = ltpage
+
+    def get_result(self):
+        return self.result
+
+
+def figure_bounding(figure) -> tuple:
+    """Bounding of some bad printed figures where too large, we strip
+    this bounding to real content.
+
+    Empty figures must return None
+    >>> assert figure_bounding(pdfminer.layout.LTFigure('empty', (10, 10, 50, 50),
+    ... (1, 1, 1, 1, 1, 1))) is None
+    """
+    if not isinstance(figure, pdfminer.layout.LTFigure):
+        return figure.bbox
+    figure = [item for item in figure if visible(item)]
+    boundings = []
+    for item in figure:
+        if isinstance(item, pdfminer.layout.LTFigure):  # pylint:disable=W0160
+            # figure inside a figure
+            bounding = figure_bounding(item)
+        else:
+            bounding = item.bbox
+        if bounding is None:
+            # hidden item
+            continue
+        boundings.append(bounding)
+    if not boundings:
+        return None
+    result = utilo.rect_max(boundings)
+    return result
+
+
+def visible(item) -> bool:
+    with contextlib.suppress(AttributeError):
+        # TODO: INVESTIGATE THIS
+        if item.linewidth:
+            return True
+        if item.fill:
+            if not item.evenodd:
+                return True
+            return False
+        if not item.stroking_color and not item.non_stroking_color:
+            return False
+    return True

rawmaker/converter/images.py

@@ -0,0 +1,168 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import configos
+import pdfminer.converter
+import pdfminer.image
+import pdfminer.layout
+import pdfminer.pdfdocument
+import pdfminer.pdfinterp
+import pdfminer.pdftypes
+import pdfminer.psparser
+import utilo
+
+import rawmaker.converter.basic
+
+
+class ImageConverter(rawmaker.converter.basic.FlippedLayoutAnalyzer):
+
+    def __init__(self, imagewriter, firstpage: int):
+        super().__init__(pageno=firstpage)
+        assert callable(imagewriter), imagewriter
+        self.imagewriter = imagewriter
+        # TODO avoid duplicated parsed, check if we require this?
+        self.parsed = utilo.Single()
+
+    def receive_layout(self, ltpage):
+        super().receive_layout(ltpage)
+        for item in ltpage:
+            self.render_pagecontent(ltpage.pageid, item)
+
+    def render_pagecontent(self, pageid, item):
+        """Collect all imageable items"""
+        if isinstance(item, pdfminer.layout.LTImage):
+            self.render_result_image(item, pageid=pageid)
+        elif isinstance(item, pdfminer.layout.LTFigure):
+            self.render_figure(item, pageid=pageid)
+
+    def render_result_image(
+        self,
+        image: pdfminer.layout.LTImage,
+        pageid: int,
+    ):
+        # add pageid to ensure that equal image names from different pages
+        # are not handled as same same.
+        imagename = f'{pageid}_{image.name}'
+        if self.parsed.contains(imagename):
+            return
+        self.imagewriter(pageid, image)
+
+    def render_figure(
+        self,
+        item: pdfminer.layout.LTFigure,
+        pageid: int,
+    ):
+        # TODO: RENDER CURVES ETC.
+        images = item._objs  # pylint:disable=W0212
+        if len(images) == 1:
+            if isinstance(images[0], pdfminer.layout.LTFigure):
+                # image inside figure
+                images = images[0]._objs  # pylint:disable=W0212
+        images = [
+            item for item in images if isinstance(item, pdfminer.layout.LTImage)
+        ]
+        if not images:
+            return
+        assert len(images) == 1, str(images)
+        # TODO: Investigate with list
+        image = images[0]  # pylint:disable=W0212
+        if skipme(image):
+            return
+        self.render_result_image(image, pageid)
+
+
+SKIPME_RATE_MIN = configos.HV_PERCENT_PLUS(default=50.0)
+
+
+def skipme(image) -> bool:
+    """\
+    Master31Page10 Black/White image is printed under figure caption.
+    """
+    # TODO: INVESTIGATE THIS HACK
+    stream_raw = image.stream.rawdata
+    counted = stream_raw.count(b'\x00')
+    rate = counted / len(stream_raw)
+    if rate >= SKIPME_RATE_MIN:
+        return True
+    return False
+
+
+class FastImageInterpreter(pdfminer.pdfinterp.PDFPageInterpreter):
+    """Experimental, think about the sence of this ?optimization?."""
+
+    # TODO: SEE DOCSTRING
+
+    def __init__(self, rsrcmgr, device):
+        super().__init__(rsrcmgr, device)
+        self.fast = {
+            'CS': self.do_CS,
+            'Do': self.do_Do,
+            'EI': self.do_EI,
+            'MP': self.do_MP,
+            'Q': self.do_Q,
+            'SC': self.do_SC,
+            'SCN': self.do_SCN,
+            'cm': self.do_cm,
+            'cs': self.do_cs,
+            'sc': self.do_sc,
+            'scn': self.do_scn,
+        }
+
+    # pylint:disable=W0613,R0201
+    def render_char(self, matrix, font, fontsize, scaling, rise, cid, ncs,
+                    graphicstate):
+        # assert 0
+        return
+
+    def render_string(self, textstate, seq, ncs, graphicstate):
+        return
+
+    def do_TJ(self, seq):
+        return
+
+    def execute(self, streams):  # pylint:disable=R1260
+        try:
+            parser = pdfminer.pdfinterp.PDFContentParser(streams)
+        except pdfminer.psparser.PSEOF:
+            # empty page
+            return
+        while 1:  # pylint:disable=W0149
+            try:
+                (_, obj) = parser.nextobject()
+            except pdfminer.psparser.PSEOF:
+                break
+            if isinstance(obj, pdfminer.psparser.PSKeyword):
+                name = pdfminer.psparser.keyword_name(obj)
+                try:
+                    func = self.fast[name]
+                except KeyError:
+                    continue
+                nargs = func.__code__.co_argcount - 1
+                # nargs = six.get_function_code(func).co_argcount - 1
+                if nargs:
+                    args = self.pop(nargs)
+                    if len(args) == nargs:
+                        func(*args)
+                else:
+                    func()
+
+            else:
+                self.push(obj)
+
+
+def create_fastimageextractor(imagelistener, firstpage: int):
+    device = ImageConverter(
+        imagewriter=imagelistener,
+        firstpage=firstpage,
+    )
+    interpreter = pdfminer.pdfinterp.PDFPageInterpreter(
+        device.resources,
+        device,
+    )
+    return interpreter

rawmaker/date.py

@@ -0,0 +1,83 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""Date
+====
+
+See Adobe PDF 2008 REF 7.9.4.
+
+Format: (D:YYYYMMDDHHmmSSOHH'mm)
+YYYY: Year
+MM: Month(01-12)
+DD: Day(0-31)
+HH: Hour(0-23)
+mm: minute(00-59)
+SS: second(00-59)
+O: + or -
+HH: offset in hours
+'
+mm: offset in minutes
+
+Parse and generate raw data again.
+
+>>> time = "D:20160419072554+02'00"
+>>> parsed = parse(time)
+
+Convert parsed back to raw string
+
+>>> raw(parsed)
+"D:20160419072554+02'00"
+"""
+
+import dataclasses
+import re
+
+
+@dataclasses.dataclass
+class PDFDate:
+    year: int = None
+    month: int = None
+    day: int = None
+    hour: int = None
+    minute: int = None
+    second: int = None
+    utc_hour: int = None
+    utc_minute: int = None
+
+
+PATTERN = (r'D:(?P<year>\d{4})(?P<month>\d{2})(?P<day>\d{2})'
+           r'(?P<hour>\d{2})(?P<minute>\d{2})(?P<second>\d{2})(?P<sign>[+-])'
+           r'(?P<utc_hour>\d{2})\'(?P<utc_minute>\d{2})')
+
+
+def raw(date: PDFDate) -> str:
+    sign = '+' if date.utc_hour >= 0 else '-'
+    result = (f'D:{date.year:04d}{date.month:02d}{date.day:02d}'
+              f'{date.hour:02d}{date.minute:02d}{date.second:02d}'
+              f'{sign}{date.utc_hour:02d}\'{date.utc_minute:02d}')
+    return result
+
+
+def parse(item: str) -> PDFDate:
+    """Parse ASN.1 date pattern.
+
+    >>> parse("D:20160419072554+02'00")
+    PDFDate(year=2016, month=4, day=19, hour=7, minute=25, second=54, utc_hour=2, utc_minute=0)
+    """
+    matched = re.match(PATTERN, item)
+    if not matched:
+        return None
+    values = [
+        'day', 'hour', 'minute', 'month', 'second', 'year', 'utc_hour',
+        'utc_minute'
+    ]
+    data = {key: int(matched[key]) for key in values}
+    result = PDFDate(**data)
+    if matched['sign'] == '-':
+        result.hour = result * -1
+    return result

rawmaker/destination.py

@@ -0,0 +1,202 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""Destination
+===========
+
+Hint: There is no direct link between `Annotation` a the real pdf page.
+Therefore we have to extract the reference and link them with the
+page-header to determine the real pdf page. See: `pageids` and
+`solve_pageid.`
+
+Types:
+
+- Simple: Page Number is directly encoded
+- Named: Solve `Destination` reference to determine page number
+- Explicit: PDFPageReference is given
+
+"""
+
+import contextlib
+import dataclasses
+
+import pdfminer.pdfpage
+import pdfminer.pdftypes
+
+import rawmaker.utils
+
+
+class DestinationMixin:
+    pass
+
+
+@dataclasses.dataclass
+class ExplicitDestination(DestinationMixin):
+    page: int = None
+    left: float = None
+    top: float = None
+    zoom: float = None
+
+
+@dataclasses.dataclass
+class ExternalLinkDestination(DestinationMixin):
+    """Hyperlink to external web resource.
+
+    See: 12.6.4.7 URI Actions; PDF 2008
+    """
+    hyperlink: str = None
+
+
+@dataclasses.dataclass
+class NamedDestination(DestinationMixin):
+    reference: str = None
+
+    @property
+    def pdf_reference(self) -> bytes:
+        """Convert human readable reference to pdf reference.
+
+        >>> NamedDestination('Kapitel 1').pdf_reference
+        b'Kapitel 1'
+        """
+        encoded = rawmaker.utils.guess_encoding(self.reference)
+        return encoded
+
+
+def parse(item) -> DestinationMixin:  # pylint:disable=R1260
+    """\
+    A `null` value means that parameter shall be unchanged.
+
+    # TODO: Change null later
+    >>> parse([b'/null', 0.0, 0.0, 1.0]).page
+    0
+    """
+    item = rawmaker.utils.resolve(item)
+    hyperlink = parse_hyperlink(item)
+    if hyperlink:
+        return hyperlink
+    fitr = parse_fitr(item)
+    if fitr:
+        return fitr
+    simple = parse_simple(item)
+    if simple:
+        return simple
+    for method in (parse_explict, parse_named):
+        explicit = method(item)
+        if explicit:
+            return explicit
+    return None
+
+
+def parse_hyperlink(item) -> ExternalLinkDestination:
+    """\
+    >>> parse_hyperlink("{'S': /'URI', 'URI': b'http://www.helm.org/jst.pdf'}")
+    """
+    if not isinstance(item, dict):
+        return None
+    try:
+        hyperlink = item['URI']
+    except KeyError:
+        return None
+    return ExternalLinkDestination(hyperlink=hyperlink)
+
+
+def parse_simple(item) -> NamedDestination:
+    """Page number is directly encoded, therefore we can convert and
+    return.
+
+    >>> from pdfminer.psparser import PSLiteral as PS
+    >>> parse_simple({'S': PS('GoTo'), 'D': b'FF'}).reference
+    'FF'
+    """
+    if not isinstance(item, dict):
+        return None
+    if not item['S'].name == 'GoTo':
+        # 12.6.4.2 Go-To Actions
+        return None
+    reference = item['D']
+    reference = rawmaker.utils.guess_decoding(reference)
+    return NamedDestination(reference=reference)
+
+
+def parse_fitr(item) -> ExplicitDestination:
+    """\
+    >>> from pdfminer.psparser import PSLiteral as PS
+    >>> parse_fitr({'S': 'GoTo', 'D': [5, PS('FitR'), 0, 625, 440, 309]}).page
+    5
+    >>> parse_fitr({'D': [None, 'FitH', 3512], 'S': 'GoTo'})
+    ExplicitDestination(page=0, left=None, top=3512, zoom=None)
+    """
+    item = rawmaker.utils.resolve(item)
+    # {'S': /'GoTo', 'D': [0, /'FitR', 0, 625, 440, 309]}
+    with contextlib.suppress(TypeError):
+        item = item['D']
+    # [0, /'FitR', 0, 625, 440, 309]
+    if not isinstance(item, list):
+        return None
+    if isinstance(item[1], float):
+        # [/b'null', 0.0, 0.0, 1.0]
+        return ExplicitDestination(page=int(item[1]))  # TODO: HACK?
+    fit = ('Fit', 'FitH', 'FitR', 'XYZ')
+    if not item[1] in fit and not item[1].name in fit:
+        return None
+    pagenumber = item[0]
+    top = item[2] if len(item) >= 3 else None
+    if pagenumber is None:
+        # TODO: CHANGE TO UNCHANGED/NONE
+        pagenumber = 0
+    return ExplicitDestination(page=pagenumber, top=top)
+
+
+def parse_explict(item) -> ExplicitDestination:
+    item = rawmaker.utils.resolve(item)
+    with contextlib.suppress(KeyError, TypeError):
+        # KeyError: ? add docs here ?
+        # TypeError: item is already the requested list:
+        # [34, /'XYZ', 72.4799999, 532.319999, 0]
+        item = item['D']
+    if isinstance(item, bytes):
+        # {'S': /'GoTo', 'D': b'subsection.A.5.4'}
+        return None
+    try:
+        page, _, left, top, zoom = item  # TODO: FLIP Y-Coordinate
+    except ValueError:
+        return None
+    with contextlib.suppress(AttributeError):
+        # skip when zoom is already a float
+        # null means: do not change current zoom
+        if zoom.name == b'null':
+            zoom = 0.0
+    result = ExplicitDestination(
+        page=page,
+        left=left,
+        top=top,
+        zoom=zoom,
+    )
+    return result
+
+
+def parse_named(item) -> ExplicitDestination:
+    doc = item.doc
+    item = rawmaker.utils.resolve(item)
+    with contextlib.suppress(KeyError):
+        item = item['D']
+    resolved = doc.lookup_name('Dests', item)
+    resolved = rawmaker.utils.resolve(resolved)
+    return parse_explict(resolved)
+
+
+def pageids(path: str) -> dict:
+    result = {}
+    with open(path, mode='rb') as pdf:
+        pages = pdfminer.pdfpage.PDFPage.get_pages(
+            pdf,
+            check_extractable=False,
+        )
+        for index, page in enumerate(pages):
+            result[page.pageid] = index
+    return result

rawmaker/error.py

@@ -0,0 +1,34 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2019-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================
+
+from pdfminer.pdfdocument import PDFNoOutlines
+
+
+class RawMakerError(Exception):
+    """Parent class for user errors or input errors.
+
+    Exceptions of this type are handled by the command line tool
+    and result in clear error messages, as opposed to backtraces.
+    """
+
+
+class TextExtractionNotAllowed(RawMakerError):
+    pass
+
+
+class InvalidPDF(RawMakerError):
+    pass
+
+
+class PDFParserImplementationError(RawMakerError):
+    pass
+
+
+class MissingOutlines(RawMakerError, PDFNoOutlines):
+    pass