pdfalyzer 1.16.13__py3-none-any.whl → 1.17.0__py3-none-any.whl

CHANGELOG.md

@@ -1,5 +1,13 @@
 # NEXT RELEASE
 
+# 1.17.0
+* Add `extract_pdf_pages` command line tool (imported from `clown_sort`)
+* Add `extract_text_from_pdfs` command line tool (imported from `clown_sort`)
+
+### 1.16.14
+* Bump `yaralyzer` to v1.0.9, handle `FileNotFoundError` which is now raised instead of `TypeError`
+* Drop support for python 3.9
+
 ### 1.16.13
 * Bump `yaralyzer` to v1.0.7 and fix reference to yaralyzer's renamed `prefix_with_style()` method
 

pdfalyzer/__init__.py

@@ -1,5 +1,6 @@
 import code
 import sys
+from argparse import Namespace
 from os import environ, getcwd, path
 
 from dotenv import load_dotenv
@@ -24,13 +25,14 @@ from yaralyzer.output.file_export import invoke_rich_export
 from yaralyzer.output.rich_console import console
 from yaralyzer.util.logging import log_and_print
 
+from pdfalyzer.decorators.pdf_file import PdfFile
 from pdfalyzer.helpers.filesystem_helper import file_size_in_mb, set_max_open_files
 from pdfalyzer.helpers.rich_text_helper import print_highlighted
 from pdfalyzer.output.pdfalyzer_presenter import PdfalyzerPresenter
 from pdfalyzer.output.styles.rich_theme import PDFALYZER_THEME_DICT
 from pdfalyzer.pdfalyzer import Pdfalyzer
 from pdfalyzer.util.argument_parser import (MAX_QUALITY, ask_to_proceed, output_sections, parse_arguments,
-     parse_combine_pdfs_args)
+     parse_combine_pdfs_args, parse_pdf_page_extraction_args, parse_text_extraction_args)
 from pdfalyzer.util.pdf_parser_manager import PdfParserManager
 
 # For the table shown by running pdfalyzer_show_color_theme
@@ -132,7 +134,19 @@ def combine_pdfs():
     print_highlighted(txt)
 
 
-# TODO: migrate this functionality from clown_sort
-# def extract_pages_from_pdf() -> None:
-#     args = parse_pdf_page_extraction_args()
-#     PdfFile(args.pdf_file).extract_page_range(args.page_range, destination_dir=args.destination_dir)
+def extract_pdf_pages() -> None:
+    args = parse_pdf_page_extraction_args()
+    PdfFile(args.pdf_file).extract_page_range(args.page_range, destination_dir=args.destination_dir)
+
+
+def extract_text_from_pdfs() -> None:
+    """
+    Extract text from a single file or from all files in a given directory. Can accept
+    multiple paths as arguments on the command line.
+    """
+    args: Namespace = parse_text_extraction_args()
+    console.line()
+
+    for file_path in args.files_to_process:
+        PdfFile(file_path).print_extracted_text(args.page_range, args.print_as_parsed)
+        console.line(2)

pdfalyzer/binary/binary_scanner.py

@@ -1,6 +1,5 @@
 """
-Class for handling binary data - scanning through it for various suspicious patterns as well as forcing
-various character encodings upon it to see what comes out.
+`BinaryScanner` class.
 """
 from collections import defaultdict
 from typing import Iterator, Optional, Tuple
@@ -28,8 +27,18 @@ from pdfalyzer.util.adobe_strings import CONTENTS, CURRENTFILE_EEXEC, FONT_FILE_
 
 
 class BinaryScanner:
+    """
+    Class for handling binary data - scanning through it for various suspicious patterns as well as forcing
+    various character encodings upon it to see what comes out.
+    """
+
     def __init__(self, _bytes: bytes, owner: PdfTreeNode, label: Optional[Text] = None):
-        """'owner' arg is an optional link back to the object containing this binary."""
+        """
+        Args:
+            _bytes (bytes): The binary data to be scanned.
+            owner (PdfTreeNode): The `PdfTreeNode` that contains this binary data.
+            label (Optional[Text]): A rich `Text` label for the binary data (e.g. the PDF object's address).
+        """
         self.bytes = _bytes
         self.label = label
         self.owner = owner
@@ -42,7 +51,7 @@ class BinaryScanner:
         self.regex_extraction_stats = defaultdict(lambda: RegexMatchMetrics())
 
     def check_for_dangerous_instructions(self) -> None:
-        """Scan for all the strings in DANGEROUS_INSTRUCTIONS list and decode bytes around them."""
+        """Scan for all the strings in `DANGEROUS_INSTRUCTIONS` list and decode bytes around them."""
         subheader = "Scanning Binary For Anything That Could Be Described As 'sus'..."
         print_section_sub_subheader(subheader, style=f"bright_red")
 
@@ -71,8 +80,8 @@ class BinaryScanner:
 
     def force_decode_quoted_bytes(self) -> None:
         """
-        Find all strings matching QUOTE_PATTERNS (AKA between quote chars) and decode them with various encodings.
-        The --quote-type arg will limit this decode to just one kind of quote.
+        Find all strings matching `QUOTE_PATTERNS` (AKA between quote chars) and decode them with various
+        encodings. The `--quote-type` arg will limit this decode to just one kind of quote.
         """
         quote_selections = PdfalyzerConfig._args.extract_quoteds
 
@@ -100,11 +109,11 @@ class BinaryScanner:
     # YARA rules are written on the fly and then YARA does the matching.
     # -------------------------------------------------------------------------------
     def extract_guillemet_quoted_bytes(self) -> Iterator[Tuple[BytesMatch, BytesDecoder]]:
-        """Iterate on all strings surrounded by Guillemet quotes, e.g. «string»"""
+        """Iterate on all strings surrounded by Guillemet quotes, e.g. «string»."""
         return self._quote_yaralyzer(QUOTE_PATTERNS[GUILLEMET], GUILLEMET).match_iterator()
 
     def extract_backtick_quoted_bytes(self) -> Iterator[Tuple[BytesMatch, BytesDecoder]]:
-        """Returns an interator over all strings surrounded by backticks"""
+        """Returns an interator over all strings surrounded by backticks."""
         return self._quote_yaralyzer(QUOTE_PATTERNS[BACKTICK], BACKTICK).match_iterator()
 
     def extract_front_slash_quoted_bytes(self) -> Iterator[Tuple[BytesMatch, BytesDecoder]]:
@@ -137,7 +146,14 @@ class BinaryScanner:
         console.line()
 
     def process_yara_matches(self, yaralyzer: Yaralyzer, pattern: str, force: bool = False) -> None:
-        """Decide whether to attempt to decode the matched bytes, track stats. force param ignores min/max length."""
+        """
+        Decide whether to attempt to decode the matched bytes and track stats.
+
+        Args:
+            yaralyzer (Yaralyzer): The `Yaralyzer` instance to use for finding matches.
+            pattern (str): The pattern being searched for (used for stats tracking).
+            force (bool): If `True`, decode all matches even if they are very short or very long.
+        """
         for bytes_match, decoder in yaralyzer.match_iterator():
             log.debug(f"Trackings match stats for {pattern}, bytes_match: {bytes_match}, is_decodable: {bytes_match.is_decodable()}")  # noqa: E501
 
@@ -162,7 +178,7 @@ class BinaryScanner:
         return self.bytes.split(CURRENTFILE_EEXEC)[1] if CURRENTFILE_EEXEC in self.bytes else self.bytes
 
     def _quote_yaralyzer(self, quote_pattern: str, quote_type: str):
-        """Helper method to build a Yaralyzer for a quote_pattern"""
+        """Helper method to build a Yaralyzer for a `quote_pattern`."""
         label = f"{quote_type}_Quoted"
 
         if quote_type == GUILLEMET:
@@ -177,7 +193,7 @@ class BinaryScanner:
         rules_label: Optional[str] = None,
         pattern_label: Optional[str] = None
     ) -> Yaralyzer:
-        """Build a yaralyzer to scan self.bytes"""
+        """Build a `yaralyzer` to scan `self.bytes`."""
         return Yaralyzer.for_patterns(
             patterns=[escape_yara_pattern(pattern)],
             patterns_type=pattern_type,
@@ -198,5 +214,5 @@ class BinaryScanner:
         self.suppression_notice_queue = []
 
     def _eexec_idx(self) -> int:
-        """Returns the location of CURRENTFILES_EEXEC within the binary stream data (or 0 if it's not there)."""
+        """Returns the location of `CURRENTFILES_EEXEC` within the binary stream data (or 0 if it's not there)."""
         return self.bytes.find(CURRENTFILE_EEXEC) if CURRENTFILE_EEXEC in self.bytes else 0

pdfalyzer/config.py

@@ -9,9 +9,10 @@ from os import environ, pardir, path
 from yaralyzer.config import YaralyzerConfig, is_env_var_set_and_not_false, is_invoked_by_pytest
 
 PDFALYZE = 'pdfalyze'
+PDFALYZER = f"{PDFALYZE}r"
 ALL_STREAMS = -1
 PYTEST_FLAG = 'INVOKED_BY_PYTEST'
-PROJECT_ROOT = path.join(str(importlib.resources.files('pdfalyzer')), pardir)
+PROJECT_ROOT = path.join(str(importlib.resources.files(PDFALYZER)), pardir)
 
 # 3rd part pdf-parser.py
 PDF_PARSER_EXECUTABLE_ENV_VAR = 'PDFALYZER_PDF_PARSER_PY_PATH'

pdfalyzer/decorators/indeterminate_node.py

@@ -1,11 +1,3 @@
-"""
-Some nodes cannot be placed until we have walked the rest of the tree. For instance
-if we encounter a /Page that relationships /Resources we need to know if there's a
-/Pages parent of the /Page before committing to a tree structure.
-
-This class handles choosing among the candidates for a given PDF object's parent node
-(AKA "figuring out where to place the node in the PDF object tree").
-"""
 from typing import Callable, List, Optional
 
 from rich.markup import escape
@@ -18,6 +10,14 @@ from pdfalyzer.util.adobe_strings import *
 
 
 class IndeterminateNode:
+    """
+    Class to handle choosing among the candidates for a given PDF object's parent node.
+
+    Some nodes cannot be placed until we have walked the rest of the tree. For instance
+    if we encounter a /Page that relationships /Resources we need to know if there's a
+    /Pages parent of the /Page before committing to a tree structure.
+    """
+
     def __init__(self, node: PdfTreeNode) -> None:
         self.node = node
 
@@ -56,7 +56,7 @@ class IndeterminateNode:
 
         self.node.set_parent(parent)
 
-    def find_node_with_most_descendants(self, list_of_nodes: List[PdfTreeNode] = None) -> PdfTreeNode:
+    def find_node_with_most_descendants(self, list_of_nodes: Optional[List[PdfTreeNode]] = None) -> PdfTreeNode:
         """Find node with a reference to this one that has the most descendants"""
         list_of_nodes = list_of_nodes or [r.from_node for r in self.node.non_tree_relationships]
         max_descendants = max([node.descendants_count() for node in list_of_nodes])
@@ -64,7 +64,7 @@ class IndeterminateNode:
 
     def _has_only_similar_relationships(self) -> bool:
         """
-        Returns True if all the nodes w/references to this one have the same type or if all the
+        Returns `True` if all the nodes w/references to this one have the same type or if all the
         reference_keys that point to this node are the same.
         """
         unique_refferer_labels = self.node.unique_labels_of_referring_nodes()
@@ -125,6 +125,6 @@ class IndeterminateNode:
 
 
 def find_node_with_lowest_id(list_of_nodes: List[PdfTreeNode]) -> PdfTreeNode:
-    """Find node in list_of_nodes_with_lowest ID."""
+    """Return node in `list_of_nodes` with lowest ID."""
     lowest_idnum = min([n.idnum for n in list_of_nodes])
     return next(n for n in list_of_nodes if n.idnum == lowest_idnum)

pdfalyzer/decorators/pdf_file.py

@@ -0,0 +1,212 @@
+import io
+from logging import Logger
+from os import path
+from pathlib import Path
+from typing import List, Optional, Union
+
+from pypdf import PdfReader, PdfWriter
+from pypdf.errors import DependencyError, EmptyFileError, PdfStreamError
+from rich.console import Console
+from rich.markup import escape
+from rich.panel import Panel
+from rich.text import Text
+from yaralyzer.output.rich_console import console
+from yaralyzer.util.logging import log as yaralyzer_log
+
+from pdfalyzer.helpers.filesystem_helper import create_dir_if_it_does_not_exist, insert_suffix_before_extension
+from pdfalyzer.helpers.image_helper import ocr_text
+from pdfalyzer.helpers.rich_text_helper import (attention_getting_panel, error_text, mild_warning,
+     print_error, stderr_console)
+from pdfalyzer.helpers.string_helper import exception_str
+from pdfalyzer.util.page_range import PageRange
+
+DEFAULT_PDF_ERRORS_DIR = Path.cwd().joinpath('pdf_errors')
+MIN_PDF_SIZE_TO_LOG_PROGRESS_TO_STDERR = 1024 * 1024 * 20
+
+
+class PdfFile:
+    """
+    Wrapper for a PDF file path that provides useful methods and properties.
+
+    Attributes:
+        file_path (Path): The path to the PDF file.
+        dirname (Path): The directory containing the PDF file.
+        basename (str): The base name of the PDF file (with extension).
+        basename_without_ext (str): The base name of the PDF file (without extension).
+        extname (str): The file extension of the PDF file.
+    """
+
+    def __init__(self, file_path: Union[str, Path]) -> None:
+        """
+        Args:
+            file_path (Union[str, Path]): Path to the PDF file.
+        """
+        self.file_path: Path = Path(file_path)
+
+        if not self.file_path.exists():
+            raise FileNotFoundError(f"File '{file_path}' does not exist.")
+
+        self.dirname = self.file_path.parent
+        self.basename: str = path.basename(file_path)
+        self.basename_without_ext: str = str(Path(self.basename).with_suffix(''))
+        self.extname: str = self.file_path.suffix
+        self.file_size = self.file_path.stat().st_size
+
+    def extract_page_range(
+            self,
+            page_range: PageRange,
+            destination_dir: Optional[Path] = None,
+            extra_file_suffix: Optional[str] = None
+        ) -> Path:
+        """
+        Extract a range of pages to a new PDF file.
+
+        Args:
+            page_range (PageRange): Range of pages to extract.
+            destination_dir (Optional[Path]): Directory to save the new PDF file. Defaults to the same
+                directory as the source PDF.
+            extra_file_suffix (Optional[str]): An optional suffix to append to the new PDF's filename.
+                Defaults to the page range suffix.
+
+        Returns:
+            Path: The path to the newly created PDF file containing the extracted pages.
+        """
+        destination_dir = destination_dir or self.dirname
+        create_dir_if_it_does_not_exist(destination_dir)
+
+        if extra_file_suffix is None:
+            file_suffix = page_range.file_suffix()
+        else:
+            file_suffix = f"{page_range.file_suffix()}__{extra_file_suffix}"
+
+        extracted_pages_pdf_basename = insert_suffix_before_extension(self.file_path, file_suffix).name
+        extracted_pages_pdf_path = destination_dir.joinpath(extracted_pages_pdf_basename)
+        console.print(f"Extracting {page_range.file_suffix()} from '{self.file_path}' to '{extracted_pages_pdf_path}'...")
+        pdf_writer = PdfWriter()
+
+        with open(self.file_path, 'rb') as source_pdf:
+            pdf_writer.append(fileobj=source_pdf, pages=page_range.to_tuple())
+
+            with open(extracted_pages_pdf_path, 'wb') as extracted_pages_pdf:
+                pdf_writer.write(extracted_pages_pdf)
+
+        console.print(f"Extracted pages to new PDF: '{extracted_pages_pdf_path}'.")
+        return extracted_pages_pdf_path
+
+    def extract_text(
+            self,
+            page_range: Optional[PageRange] = None,
+            logger: Optional[Logger] = None,
+            print_as_parsed: bool = False
+        ) -> Optional[str]:
+        """
+        Use PyPDF to extract text page by page and use Tesseract to OCR any embedded images.
+
+        Args:
+            page_range (Optional[PageRange]): If provided, only extract text from pages in this range.
+                Page numbers are 1-indexed. If not provided, extract text from all pages.
+            log (Optional[Logger]): If provided, log progress to this logger. Otherwise use default logger.
+            print_as_parsed (bool): If True, print each page's text to STDOUT as it is parsed.
+
+        Returns:
+            Optional[str]: The extracted text, or None if extraction failed.
+        """
+        from PIL import Image  # Imported here to avoid hard dependency if not using this method
+        log = logger or yaralyzer_log
+        log.debug(f"Extracting text from '{self.file_path}'...")
+        self._page_numbers_of_errors: List[int] = []
+        extracted_pages = []
+
+        try:
+            pdf_reader = PdfReader(self.file_path)
+            page_count = len(pdf_reader.pages)
+            log.debug(f"PDF Page count: {page_count}")
+
+            for page_number, page in enumerate(pdf_reader.pages, start=1):
+                if page_range and not page_range.in_range(page_number):
+                    self._log_to_stderr(f"Skipping page {page_number}...")
+                    continue
+
+                self._log_to_stderr(f"Parsing page {page_number}...")
+                page_buffer = Console(file=io.StringIO())
+                page_buffer.print(Panel(f"PAGE {page_number}", padding=(0, 15), expand=False))
+                page_buffer.print(escape(page.extract_text().strip()))
+                image_number = 1
+
+                # Extracting images is a bit fraught (lots of PIL and pypdf exceptions have come from here)
+                try:
+                    for image_number, image in enumerate(page.images, start=1):
+                        image_name = f"Page {page_number}, Image {image_number}"
+                        self._log_to_stderr(f"   Processing {image_name}...", "dim")
+                        page_buffer.print(Panel(image_name, expand=False))
+                        image_obj = Image.open(io.BytesIO(image.data))
+                        image_text = ocr_text(image_obj, f"{self.file_path} ({image_name})")
+                        page_buffer.print((image_text or '').strip())
+                except (OSError, NotImplementedError, TypeError, ValueError) as e:
+                    error_str = exception_str(e)
+                    msg = f"{error_str} while parsing embedded image {image_number} on page {page_number}..."
+                    mild_warning(msg)
+
+                    # Dump an error PDF and encourage user to report to pypdf team.
+                    if 'JBIG2Decode' not in str(e):
+                        stderr_console.print_exception()
+
+                        if page_number not in self._page_numbers_of_errors:
+                            self._handle_extraction_error(page_number, error_str)
+                            self._page_numbers_of_errors.append(page_number)
+
+                page_text = page_buffer.file.getvalue()
+                extracted_pages.append(page_text)
+                log.debug(page_text)
+
+                if print_as_parsed:
+                    print(f"{page_text}")
+        except DependencyError:
+            log.error("Pdfalyzer is missing an optional dependency required to extract text. Try 'pip install pdfalyzer[extract]'")
+        except EmptyFileError:
+            log.warning("Skipping empty file!")
+        except PdfStreamError as e:
+            print_error(f"Error parsing PDF file '{self.file_path}': {e}")
+            stderr_console.print_exception()
+
+        return "\n\n".join(extracted_pages).strip()
+
+    def print_extracted_text(self, page_range: Optional[PageRange] = None, print_as_parsed: bool = False) -> None:
+        """Fancy wrapper for printing the extracted text to the screen."""
+        console.print(Panel(str(self.file_path), expand=False, style='bright_white reverse'))
+        txt = self.extract_text(page_range=page_range, print_as_parsed=print_as_parsed)
+
+        if not print_as_parsed:
+            console.print(txt)
+
+    def _handle_extraction_error(self, page_number: int, error_msg: str) -> None:
+        """Rip the offending page to a new file and suggest that user report bug to PyPDF."""
+        destination_dir = DEFAULT_PDF_ERRORS_DIR
+
+        try:
+            extracted_file = self.extract_page_range(PageRange(str(page_number)), destination_dir, error_msg)
+        except Exception as e:
+            stderr_console.print(error_text(f"Failed to extract a page for submission to PyPDF team."))
+            extracted_file = None
+
+        blink_txt = Text('', style='bright_white')
+        blink_txt.append("An error (", style='blink color(154)').append(error_msg, style='color(11) blink')
+        blink_txt.append(') ', style='blink color(154)')
+        blink_txt.append("was encountered while processing a PDF file.\n\n", style='blink color(154)')
+
+        txt = Text(f"The error was of a type such that it probably came from a bug in ", style='bright_white')
+        txt.append('PyPDF', style='underline bright_green').append('. It was encountered processing the file ')
+        txt.append(str(self.file_path), style='file').append('. You should see a stack trace above this box.\n\n')
+
+        txt.append('The offending page will be extracted to ', style='bright_white')
+        txt.append(str(extracted_file), style='file').append('.\n\n')
+        txt.append(f"Please visit 'https://github.com/py-pdf/pypdf/issues' to report a bug. ", style='bold')
+        txt.append(f"Providing the devs with the extracted page and the stack trace help improve pypdf.")
+        stderr_console.print(attention_getting_panel(blink_txt + txt, title='PyPDF Error'))
+
+    def _log_to_stderr(self, msg: str, style: Optional[str] = None) -> None:
+        """When parsing very large PDFs it can be useful to log progress and other messages to STDERR."""
+        if self.file_size < MIN_PDF_SIZE_TO_LOG_PROGRESS_TO_STDERR:
+            return
+
+        stderr_console.print(msg, style=style or "")

pdfalyzer/decorators/pdf_object_properties.py

@@ -15,7 +15,7 @@ from pdfalyzer.util.adobe_strings import *
 
 
 class PdfObjectProperties:
-    """Simple class to extract critical features of a PdfObject."""
+    """Simple class to extract critical features of a `PdfObject`."""
 
     def __init__(
         self,
@@ -86,7 +86,7 @@ class PdfObjectProperties:
         obj: PdfObject,
         is_single_row_table: bool = False
     ) -> List[Union[Text, str]]:
-        """PDF object property at reference_key becomes a formatted 3-tuple for use in Rich tables."""
+        """PDF object property at `reference_key` becomes a formatted 3-tuple for use in Rich tables."""
         with_resolved_refs = cls.resolve_references(reference_key, obj)
 
         return [
@@ -101,7 +101,7 @@ class PdfObjectProperties:
     # TODO: this doesn't recurse...
     @classmethod
     def _obj_to_rich_text(cls, obj: Any) -> Text:
-        """Recurse through obj and build a Text object."""
+        """Recurse through `obj` and build a `Text` object."""
         if isinstance(obj, dict):
             key_value_pairs = [Text(f"{k}: ").append_text(cls._obj_to_rich_text(v)) for k, v in obj.items()]
             return Text('{').append_text(comma_join_txt(key_value_pairs)).append('}')

pdfalyzer/decorators/pdf_tree_node.py

@@ -1,10 +1,5 @@
 """
-PDF node decorator - wraps actual PDF objects to make them anytree nodes.
-Also adds decorators/generators for Rich text representation.
-
-Child/parent relationships should be set using the add_child() and set_parent()
-methods and not set directly. (TODO: this could be done better with anytree
-hooks)
+`PdfTreeNode` decorates a `PdfObject` with tree structure information.
 """
 from typing import Callable, List, Optional
 
@@ -27,11 +22,22 @@ DECODE_FAILURE_LEN = -1
 
 
 class PdfTreeNode(NodeMixin, PdfObjectProperties):
+    """
+    PDF node decorator - wraps actual PDF objects to make them `anytree` nodes.
+    Also adds decorators/generators for Rich text representation.
+
+    Child/parent relationships should be set using the `add_child()` and `set_parent()`
+    methods and not set directly.
+
+    TODO: this could be done better with anytree hooks.
+    """
+
     def __init__(self, obj: PdfObject, address: str, idnum: int):
         """
-        obj:     The underlying PDF object
-        address: The first address that points from some node to this one
-        idnum:   ID used in the reference
+        Args:
+            obj (PdfObject): The underlying PDF object
+            address (str): The first address that points from some node to this one
+            idnum (int): ID used in the reference
         """
         PdfObjectProperties.__init__(self, obj, address, idnum)
         self.non_tree_relationships: List[PdfObjectRelationship] = []
@@ -54,7 +60,7 @@ class PdfTreeNode(NodeMixin, PdfObjectProperties):
 
     @classmethod
     def from_reference(cls, ref: IndirectObject, address: str) -> 'PdfTreeNode':
-        """Builds a PdfTreeDecorator from an IndirectObject."""
+        """Alternate constructor to Build a `PdfTreeNode` from an `IndirectObject`."""
         try:
             return cls(ref.get_object(), address, ref.idnum)
         except PdfReadError as e:
@@ -82,7 +88,7 @@ class PdfTreeNode(NodeMixin, PdfObjectProperties):
             child.set_parent(self)
 
     def add_non_tree_relationship(self, relationship: PdfObjectRelationship) -> None:
-        """Add a relationship that points at this node's PDF object. TODO: doesn't include parent/child"""
+        """Add a relationship that points at this node's PDF object. TODO: doesn't include parent/child."""
         if relationship in self.non_tree_relationships:
             return
 

pdfalyzer/decorators/pdf_tree_verifier.py

@@ -11,6 +11,8 @@ from pdfalyzer.util.adobe_strings import *
 
 
 class PdfTreeVerifier:
+    """Class to verify that the PDF tree is complete/contains all the nodes in the PDF file."""
+
     def __init__(self, pdfalyzer: 'Pdfalyzer') -> None:
         self.pdfalyzer = pdfalyzer
 

pdfalyzer/detection/yaralyzer_helper.py

@@ -1,14 +1,17 @@
 """
-Class to help with the pre-configured YARA rules in the /yara directory.
+Functions to help with the pre-configured YARA rules in the /yara directory.
 """
 from importlib.resources import as_file, files
 from sys import exit
 from typing import Optional, Union
 
 from yaralyzer.config import YaralyzerConfig
+from yaralyzer.output.rich_console import print_fatal_error_and_exit
 from yaralyzer.yaralyzer import Yaralyzer
 
-YARA_RULES_DIR = files('pdfalyzer').joinpath('yara_rules')
+from pdfalyzer.config import PDFALYZER
+
+YARA_RULES_DIR = files(PDFALYZER).joinpath('yara_rules')
 
 YARA_RULES_FILES = [
     'didier_stevens.yara',
@@ -20,11 +23,12 @@ YARA_RULES_FILES = [
 
 
 def get_file_yaralyzer(file_path_to_scan: str) -> Yaralyzer:
-    """Get a yaralyzer for a file path"""
+    """Get a yaralyzer for a file path."""
     return _build_yaralyzer(file_path_to_scan)
 
 
 def get_bytes_yaralyzer(scannable: bytes, label: str) -> Yaralyzer:
+    """Get a yaralyzer for a `scannable` bytes."""
     return _build_yaralyzer(scannable, label)
 
 
@@ -44,10 +48,5 @@ def _build_yaralyzer(scannable: Union[bytes, str], label: Optional[str] = None)
 
                         try:
                             return Yaralyzer.for_rules_files(rules_paths, scannable, label)
-                        except ValueError as e:
-                            # TODO: use YARA_FILE_DOES_NOT_EXIST_ERROR_MSG variable
-                            if "it doesn't exist" in str(e):
-                                print(str(e))
-                                exit(1)
-                            else:
-                                raise e
+                        except FileNotFoundError as e:
+                            print_fatal_error_and_exit(str(e))

pdfalyzer/helpers/filesystem_helper.py

@@ -6,6 +6,7 @@ from pathlib import Path
 from typing import Optional, Union
 
 from yaralyzer.output.rich_console import console
+from yaralyzer.util.logging import log
 
 from pdfalyzer.helpers.rich_text_helper import print_highlighted
 
@@ -18,9 +19,20 @@ PDF_EXT = '.pdf'
 # type StrOrPath = Union[str, Path]
 
 
-def with_pdf_extension(file_path: Union[str, Path]) -> str:
-    """Append '.pdf' to 'file_path' if it doesn't already end with '.pdf'."""
-    return str(file_path) + ('' if is_pdf(file_path) else PDF_EXT)
+def create_dir_if_it_does_not_exist(dir: Path) -> None:
+    """Like it says on the tin."""
+    if dir.exists():
+        return
+
+    console.warning(f"Need to create '{dir}'")
+    dir.mkdir(parents=True, exist_ok=True)
+
+
+def insert_suffix_before_extension(file_path: Path, suffix: str, separator: str = '__') -> Path:
+    """Inserting 'page 1' suffix in 'path/to/file.jpg' -> '/path/to/file__page_1.jpg'."""
+    suffix = strip_bad_chars(suffix).replace(' ', '_')
+    file_path_without_extension = file_path.with_suffix('')
+    return Path(f"{file_path_without_extension}{separator}{suffix}{file_path.suffix}")
 
 
 def is_pdf(file_path: Union[str, Path]) -> bool:
@@ -100,3 +112,15 @@ def set_max_open_files(num_filehandles: int = DEFAULT_MAX_OPEN_FILES) -> tuple[O
                 soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
 
     return (soft, hard)
+
+
+def strip_bad_chars(text: str) -> str:
+    """Remove chars that don't work well in filenames."""
+    text = ' '.join(text.splitlines()).replace('\\s+', ' ')
+    text = re.sub('’', "'", text).replace('|', 'I').replace(',', ',')
+    return re.sub('[^-0-9a-zA-Z@.,?_:=#\'\\$" ()]+', '_', text).replace('  ', ' ')
+
+
+def with_pdf_extension(file_path: Union[str, Path]) -> str:
+    """Append `".pdf"` to `file_path` if it doesn't already end with `".pdf"`."""
+    return str(file_path) + ('' if is_pdf(file_path) else PDF_EXT)

pdfalyzer/helpers/image_helper.py

@@ -0,0 +1,31 @@
+from typing import Optional
+
+from PIL import Image
+from yaralyzer.output.rich_console import console
+
+from pdfalyzer.helpers.rich_text_helper import warning_text
+
+
+def ocr_text(image: Image.Image, image_name: str) -> Optional[str]:
+    """Use pytesseract to OCR the text in the image and return it as a string."""
+    import pytesseract
+    text = None
+
+    try:
+        text = pytesseract.image_to_string(image)
+    except pytesseract.pytesseract.TesseractError as e:
+        console.print_exception()
+        console.print(warning_text(f"Tesseract OCR failure '{image_name}'! No OCR text extracted..."))
+    except OSError as e:
+        if 'truncated' in str(e):
+            console.print(warning_text(f"Truncated image file '{image_name}'!"))
+        else:
+            console.print_exception()
+            console.print(f"Error while extracting '{image_name}'!", style='bright_red')
+            raise e
+    except Exception as e:
+        console.print_exception()
+        console.print(f"Error while extracting '{image_name}'!", style='bright_red')
+        raise e
+
+    return None if text is None else text.strip()