ocrmypdf 16.0.0__py310-none-any.whl

ocrmypdf/RELEASE.md

@@ -0,0 +1,38 @@
+<!-- SPDX-FileCopyrightText: 2022 James R. Barlow -->
+<!-- SPDX-License-Identifier: CC-BY-SA-4.0 -->
+
+# Release checklist
+
+## Patch release
+
+- Check `pytest`
+
+- Update release notes
+
+## Minor release
+
+## Major release
+
+- Run `pre-commit autoupdate`
+
+- Check README.md
+
+- Check pyproject.toml
+
+    - Are classifiers up to date?
+    - Is `python_requires` correct?
+    - Is it to drop support for older Pythons?
+    - Can we tighten any `install_requires` dependencies?
+
+- Search for old version shims we can remove
+
+    - "shim"
+    - ` pikepdf.__version__`
+
+- Search for deprecation: search all files for deprec*, etc.
+
+- Check requirements in setup.cfg
+
+- Delete `tests/cache`, do `pytest --runslow`, and update cache.
+
+- Do `pytest --cov-report html`

ocrmypdf/__init__.py

@@ -0,0 +1,70 @@
+# SPDX-FileCopyrightText: 2022 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+"""Adds OCR layer to PDFs."""
+
+from __future__ import annotations
+
+from pluggy import HookimplMarker as _HookimplMarker
+
+from ocrmypdf import helpers, hocrtransform, pdfa, pdfinfo
+from ocrmypdf._concurrent import Executor
+from ocrmypdf._jobcontext import PageContext, PdfContext
+from ocrmypdf._pipelines._common import (
+    configure_debug_logging,
+)
+from ocrmypdf._version import PROGRAM_NAME, __version__
+from ocrmypdf.api import (
+    Verbosity,
+    configure_logging,
+    ocr,
+)
+from ocrmypdf.exceptions import (
+    BadArgsError,
+    DpiError,
+    EncryptedPdfError,
+    ExitCode,
+    ExitCodeException,
+    InputFileError,
+    MissingDependencyError,
+    OutputFileAccessError,
+    PriorOcrFoundError,
+    SubprocessOutputError,
+    TesseractConfigError,
+    UnsupportedImageFormatError,
+)
+from ocrmypdf.pluginspec import OcrEngine, OrientationConfidence
+
+hookimpl = _HookimplMarker('ocrmypdf')
+
+
+__all__ = [
+    '__version__',
+    'BadArgsError',
+    'configure_debug_logging',
+    'configure_logging',
+    'DpiError',
+    'EncryptedPdfError',
+    'Executor',
+    'ExitCode',
+    'ExitCodeException',
+    'helpers',
+    'hocrtransform',
+    'hookimpl',
+    'InputFileError',
+    'MissingDependencyError',
+    'ocr',
+    'OcrEngine',
+    'OrientationConfidence',
+    'OutputFileAccessError',
+    'PageContext',
+    'pdfa',
+    'PdfContext',
+    'pdfinfo',
+    'PriorOcrFoundError',
+    'PROGRAM_NAME',
+    'SubprocessOutputError',
+    'TesseractConfigError',
+    'UnsupportedImageFormatError',
+    'Verbosity',
+]

ocrmypdf/__main__.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: 2022 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+"""ocrmypdf command line entrypoint."""
+
+from __future__ import annotations
+
+import logging
+import multiprocessing
+import os
+import signal
+import sys
+from contextlib import suppress
+
+from ocrmypdf import __version__
+from ocrmypdf._pipelines.ocr import run_pipeline_cli
+from ocrmypdf._plugin_manager import get_parser_options_plugins
+from ocrmypdf._validation import check_options
+from ocrmypdf.api import Verbosity, configure_logging
+from ocrmypdf.exceptions import (
+    BadArgsError,
+    ExitCode,
+    InputFileError,
+    MissingDependencyError,
+)
+
+log = logging.getLogger('ocrmypdf')
+
+
+def sigbus(*args):
+    """Handle SIGBUS signals.
+
+    pikepdf, depending on configuration, may use mmap so SIGBUS is a
+    possibility.
+    """
+    raise InputFileError("Lost access to the input file")
+
+
+def run(args=None):
+    """Run the ocrmypdf command line interface."""
+    _parser, options, plugin_manager = get_parser_options_plugins(args=args)
+
+    with suppress(AttributeError, PermissionError):
+        os.nice(5)
+
+    verbosity = options.verbose
+    if not os.isatty(sys.stderr.fileno()):
+        options.progress_bar = False
+    if options.quiet:
+        verbosity = Verbosity.quiet
+        options.progress_bar = False
+    configure_logging(
+        verbosity,
+        progress_bar_friendly=options.progress_bar,
+        manage_root_logger=True,
+        plugin_manager=plugin_manager,
+    )
+    log.debug('ocrmypdf %s', __version__)
+    try:
+        check_options(options, plugin_manager)
+    except ValueError as e:
+        log.error(e)
+        return ExitCode.bad_args
+    except BadArgsError as e:
+        log.error(e)
+        return e.exit_code
+    except MissingDependencyError as e:
+        log.error(e)
+        return ExitCode.missing_dependency
+
+    with suppress(AttributeError, OSError):
+        signal.signal(signal.SIGBUS, sigbus)
+
+    result = run_pipeline_cli(options=options, plugin_manager=plugin_manager)
+    return result
+
+
+if __name__ == '__main__':
+    multiprocessing.freeze_support()
+    if os.name == 'posix':
+        multiprocessing.set_start_method('forkserver')
+    sys.exit(run())

ocrmypdf/_concurrent.py

@@ -0,0 +1,129 @@
+# SPDX-FileCopyrightText: 2022 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+"""OCRmyPDF concurrency abstractions."""
+
+from __future__ import annotations
+
+import threading
+from abc import ABC, abstractmethod
+from collections.abc import Iterable
+from typing import Any, Callable, TypeVar
+
+from ocrmypdf._progressbar import NullProgressBar, ProgressBar
+
+T = TypeVar('T')
+
+
+def _task_noop(*_args, **_kwargs):
+    return
+
+
+def _task_finished_noop(_result: Any, pbar: ProgressBar):
+    pbar.update()
+
+
+class Executor(ABC):
+    """Abstract concurrent executor."""
+
+    pool_lock = threading.Lock()
+    pbar_class = NullProgressBar
+
+    def __init__(self, *, pbar_class=None):
+        if pbar_class:
+            self.pbar_class = pbar_class
+
+    def __call__(
+        self,
+        *,
+        use_threads: bool,
+        max_workers: int,
+        progress_kwargs: dict,
+        worker_initializer: Callable | None = None,
+        task: Callable[..., T] | None = None,
+        task_arguments: Iterable | None = None,
+        task_finished: Callable[[T, ProgressBar], None] | None = None,
+    ) -> None:
+        """Set up parallel execution and progress reporting.
+
+        Args:
+            use_threads: If ``False``, the workload is the sort that will benefit from
+                running in a multiprocessing context (for example, it uses Python
+                heavily, and parallelizing it with threads is not expected to be
+                performant).
+            max_workers: The maximum number of workers that should be run.
+            progress_kwargs: Arguments to set up the progress bar.
+            worker_initializer: Called when a worker is initialized, in the worker's
+                execution context. If the child workers are processes, it must be
+                possible to marshall/pickle the worker initializer.
+                ``functools.partial`` can be used to bind parameters.
+            task: Called when the worker starts a new task, in the worker's execution
+                context. Must be possible to marshall to the worker.
+            task_finished: Called when a worker finishes a task, in the parent's
+                context.
+            task_arguments: An iterable that generates a group of parameters for each
+                task. This runs in the parent's context, but the parameters must be
+                marshallable to the worker.
+        """
+        if not task_arguments:
+            return  # Nothing to do!
+        if not worker_initializer:
+            worker_initializer = _task_noop
+        if not task_finished:
+            task_finished = _task_finished_noop
+        if not task:
+            task = _task_noop
+
+        with self.pool_lock:
+            self._execute(
+                use_threads=use_threads,
+                max_workers=max_workers,
+                progress_kwargs=progress_kwargs,
+                worker_initializer=worker_initializer,
+                task=task,
+                task_arguments=task_arguments,
+                task_finished=task_finished,
+            )
+
+    @abstractmethod
+    def _execute(
+        self,
+        *,
+        use_threads: bool,
+        max_workers: int,
+        progress_kwargs: dict,
+        worker_initializer: Callable,
+        task: Callable,
+        task_arguments: Iterable,
+        task_finished: Callable,
+    ):
+        """Custom executors should override this method."""
+
+
+def setup_executor(plugin_manager) -> Executor:
+    pbar_class = plugin_manager.hook.get_progressbar_class()
+    return plugin_manager.hook.get_executor(progressbar_class=pbar_class)
+
+
+class SerialExecutor(Executor):
+    """Implements a purely sequential executor using the parallel protocol.
+
+    The current process/thread will be the worker that executes all tasks
+    in order. As such, ``worker_initializer`` will never be called.
+    """
+
+    def _execute(
+        self,
+        *,
+        use_threads: bool,
+        max_workers: int,
+        progress_kwargs: dict,
+        worker_initializer: Callable,
+        task: Callable,
+        task_arguments: Iterable,
+        task_finished: Callable,
+    ):  # pylint: disable=unused-argument
+        with self.pbar_class(**progress_kwargs) as pbar:
+            for args in task_arguments:
+                result = task(*args)
+                task_finished(result, pbar)

ocrmypdf/_exec/__init__.py

@@ -0,0 +1,6 @@
+# SPDX-FileCopyrightText: 2022 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+"""Manage third party executables."""
+
+from __future__ import annotations

ocrmypdf/_exec/ghostscript.py

@@ -0,0 +1,296 @@
+# SPDX-FileCopyrightText: 2022 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+"""Interface to Ghostscript executable."""
+
+from __future__ import annotations
+
+import logging
+import os
+import re
+from collections import deque
+from io import BytesIO
+from os import fspath
+from pathlib import Path
+from subprocess import PIPE, CalledProcessError
+
+from packaging.version import Version
+from PIL import Image, UnidentifiedImageError
+
+from ocrmypdf.exceptions import ColorConversionNeededError, SubprocessOutputError
+from ocrmypdf.helpers import Resolution
+from ocrmypdf.subprocess import get_version, run, run_polling_stderr
+
+COLOR_CONVERSION_STRATEGIES = frozenset(
+    [
+        'CMYK',
+        'Gray',
+        'LeaveColorUnchanged',
+        'RGB',
+        'UseDeviceIndependentColor',
+    ]
+)
+# Ghostscript executable - gswin32c is not supported
+GS = 'gswin64c' if os.name == 'nt' else 'gs'
+
+
+log = logging.getLogger(__name__)
+
+
+class DuplicateFilter(logging.Filter):
+    """Filter out duplicate log messages.
+
+    A context window of default 5 messages is used to determine if a message is a
+    duplicate. This is because some Ghostscript messages are word wrapped.
+    """
+
+    def __init__(self, logger: logging.Logger, context_window=5):
+        self.window: deque[str] = deque([], maxlen=context_window)
+        self.logger = logger
+        self.levelno = logging.DEBUG
+        self.count = 0
+
+    def filter(self, record):
+        if record.msg in self.window:
+            self.count += 1
+            self.levelno = record.levelno
+            return False
+        else:
+            if self.count >= 1:
+                rep_msg = f"(suppressed {self.count} repeated lines)"
+                self.count = 0  # Avoid infinite recursion
+                self.logger.log(self.levelno, rep_msg)
+                self.window.clear()
+            self.window.append(record.msg)
+            return True
+
+
+log.addFilter(DuplicateFilter(log))
+
+
+def version() -> Version:
+    return Version(get_version(GS))
+
+
+def _gs_error_reported(stream) -> bool:
+    match = re.search(r'error', stream, flags=re.IGNORECASE)
+    return bool(match)
+
+
+def _gs_devicen_reported(stream) -> bool:
+    """Did Ghostscript warn about a DeviceN with inappropriate alternate?
+
+    If so, we need the user to select a color conversion, or the resulting PDF will
+    not present correctly in some PDF viewers.
+    """
+    match = re.search(
+        r'DeviceN.*inappropriate alternate',
+        stream,
+        flags=re.IGNORECASE | re.MULTILINE,
+    )
+    return bool(match)
+
+
+def rasterize_pdf(
+    input_file: os.PathLike,
+    output_file: os.PathLike,
+    *,
+    raster_device: str,
+    raster_dpi: Resolution,
+    pageno: int = 1,
+    page_dpi: Resolution | None = None,
+    rotation: int | None = None,
+    filter_vector: bool = False,
+    stop_on_error: bool = False,
+):
+    """Rasterize one page of a PDF at resolution raster_dpi in canvas units."""
+    raster_dpi = raster_dpi.round(6)
+    if not page_dpi:
+        page_dpi = raster_dpi
+
+    args_gs = (
+        [
+            GS,
+            '-dQUIET',
+            '-dSAFER',
+            '-dBATCH',
+            '-dNOPAUSE',
+            '-dInterpolateControl=-1',
+            f'-sDEVICE={raster_device}',
+            f'-dFirstPage={pageno}',
+            f'-dLastPage={pageno}',
+            f'-r{raster_dpi.x:f}x{raster_dpi.y:f}',
+        ]
+        + (['-dFILTERVECTOR'] if filter_vector else [])
+        + (['-dPDFSTOPONERROR'] if stop_on_error else [])
+        + [
+            '-o',
+            '-',
+            '-sstdout=%stderr',  # Literal %s, not string interpolation
+            '-dAutoRotatePages=/None',  # Probably has no effect on raster
+            '-f',
+            fspath(input_file),
+        ]
+    )
+
+    try:
+        p = run(args_gs, stdout=PIPE, stderr=PIPE, check=True)
+    except CalledProcessError as e:
+        log.error(e.stderr.decode(errors='replace'))
+        raise SubprocessOutputError('Ghostscript rasterizing failed') from e
+    else:
+        stderr = p.stderr.decode(errors='replace')
+        if _gs_error_reported(stderr):
+            log.error(stderr)
+
+    try:
+        with Image.open(BytesIO(p.stdout)) as im:
+            if rotation is not None:
+                log.debug("Rotating output by %i", rotation)
+                # rotation is a clockwise angle and Image.ROTATE_* is
+                # counterclockwise so this cancels out the rotation
+                if rotation == 90:
+                    im = im.transpose(Image.Transpose.ROTATE_90)
+                elif rotation == 180:
+                    im = im.transpose(Image.Transpose.ROTATE_180)
+                elif rotation == 270:
+                    im = im.transpose(Image.Transpose.ROTATE_270)
+                if rotation % 180 == 90:
+                    page_dpi = page_dpi.flip_axis()
+            im.save(fspath(output_file), dpi=page_dpi)
+    except UnidentifiedImageError:
+        log.error(
+            f"Ghostscript (using {raster_device} at {raster_dpi} dpi) produced "
+            "an invalid page image file."
+        )
+        raise
+
+
+class GhostscriptFollower:
+    """Parses the output of Ghostscript and uses it to update the progress bar."""
+
+    re_process = re.compile(r"Processing pages \d+ through (\d+).")
+    re_page = re.compile(r"Page (\d+)")
+
+    def __init__(self, progressbar_class):
+        self.count = 0
+        self.progressbar_class = progressbar_class
+        self.progressbar = None
+
+    def __call__(self, line):
+        if not self.progressbar_class:
+            return
+        if not self.progressbar:
+            m = self.re_process.match(line.strip())
+            if m:
+                self.count = int(m.group(1))
+                self.progressbar = self.progressbar_class(
+                    total=self.count, desc="PDF/A conversion", unit='page'
+                )
+                return
+        else:
+            if self.re_page.match(line.strip()):
+                self.progressbar.update()
+
+
+def generate_pdfa(
+    pdf_pages,
+    output_file: os.PathLike,
+    *,
+    compression: str,
+    color_conversion_strategy: str,
+    pdf_version: str = '1.5',
+    pdfa_part: str = '2',
+    progressbar_class=None,
+    stop_on_error: bool = False,
+):
+    # Ghostscript's compression is all or nothing. We can either force all images
+    # to JPEG, force all to Flate/PNG, or let it decide how to encode the images.
+    # In most case it's best to let it decide.
+    compression_args = []
+    if compression == 'jpeg':
+        compression_args = [
+            "-dAutoFilterColorImages=false",
+            "-dColorImageFilter=/DCTEncode",
+            "-dAutoFilterGrayImages=false",
+            "-dGrayImageFilter=/DCTEncode",
+        ]
+    elif compression == 'lossless':
+        compression_args = [
+            "-dAutoFilterColorImages=false",
+            "-dColorImageFilter=/FlateEncode",
+            "-dAutoFilterGrayImages=false",
+            "-dGrayImageFilter=/FlateEncode",
+        ]
+    else:
+        compression_args = [
+            "-dAutoFilterColorImages=true",
+            "-dAutoFilterGrayImages=true",
+        ]
+
+    gs_version = version()
+    if gs_version == Version('9.56.0'):
+        # 9.56.0 breaks our OCR, should be fixed in 9.56.1
+        # https://bugs.ghostscript.com/show_bug.cgi?id=705187
+        compression_args.append('-dNEWPDF=false')
+
+    if os.name == 'nt':
+        # Windows has lots of fatal "permission denied" errors
+        stop_on_error = False
+
+    # nb no need to specify ProcessColorModel when ColorConversionStrategy
+    # is set; see:
+    # https://bugs.ghostscript.com/show_bug.cgi?id=699392
+    args_gs = (
+        [
+            GS,
+            "-dBATCH",
+            "-dNOPAUSE",
+            "-dSAFER",
+            f"-dCompatibilityLevel={str(pdf_version)}",
+            "-sDEVICE=pdfwrite",
+            "-dAutoRotatePages=/None",
+            f"-sColorConversionStrategy={color_conversion_strategy}",
+        ]
+        + (['-dPDFSTOPONERROR'] if stop_on_error else [])
+        + compression_args
+        + [
+            "-dJPEGQ=95",
+            f"-dPDFA={pdfa_part}",
+            "-dPDFACompatibilityPolicy=1",
+            "-o",
+            "-",
+            "-sstdout=%stderr",  # Literal %s, not string interpolation
+        ]
+    )
+    args_gs.extend(fspath(s) for s in pdf_pages)  # Stringify Path objs
+    try:
+        with Path(output_file).open('wb') as output:
+            p = run_polling_stderr(
+                args_gs,
+                stdout=output,
+                stderr=PIPE,
+                check=True,
+                text=True,
+                encoding='utf-8',
+                errors='replace',
+                callback=GhostscriptFollower(progressbar_class),
+            )
+    except CalledProcessError as e:
+        # Ghostscript does not change return code when it fails to create
+        # PDF/A - check PDF/A status elsewhere
+        log.error(e.stderr)
+        raise SubprocessOutputError('Ghostscript PDF/A rendering failed') from e
+    else:
+        stderr = p.stderr
+        # If there is an error we log the whole stderr, except for filtering
+        # duplicates.
+        if _gs_error_reported(stderr):
+            # Ghostscript outputs the pattern **** Error: ....  frequently.
+            # Occasionally the error message is spammed many times. We filter
+            # out duplicates of this message using the filter above. We use
+            # the **** pattern to split the stderr into parts.
+            for part in stderr.split('****'):
+                log.error(part)
+        if _gs_devicen_reported(stderr):
+            raise ColorConversionNeededError()

ocrmypdf/_exec/jbig2enc.py

@@ -0,0 +1,51 @@
+# SPDX-FileCopyrightText: 2022 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+"""Interface to jbig2 executable."""
+
+from __future__ import annotations
+
+from subprocess import PIPE
+
+from packaging.version import Version
+
+from ocrmypdf.exceptions import MissingDependencyError
+from ocrmypdf.subprocess import get_version, run
+
+
+def version() -> Version:
+    return Version(get_version('jbig2', regex=r'jbig2enc (\d+(\.\d+)*).*'))
+
+
+def available():
+    try:
+        version()
+    except MissingDependencyError:
+        return False
+    return True
+
+
+def convert_group(cwd, infiles, out_prefix, threshold):
+    args = [
+        'jbig2',
+        '-b',
+        out_prefix,
+        '--symbol-mode',  # symbol mode (lossy)
+        '-t',
+        str(threshold),  # threshold
+        # '-r', # refinement mode (lossless symbol mode, currently disabled in
+        # jbig2)
+        '--pdf',
+    ]
+    args.extend(infiles)
+    proc = run(args, cwd=cwd, stdout=PIPE, stderr=PIPE)
+    proc.check_returncode()
+    return proc
+
+
+def convert_single(cwd, infile, outfile, threshold):
+    args = ['jbig2', '--pdf', '-t', str(threshold), infile]
+    with open(outfile, 'wb') as fstdout:
+        proc = run(args, cwd=cwd, stdout=fstdout, stderr=PIPE)
+    proc.check_returncode()
+    return proc