docp 0.0.0.dev1__py3-none-any.whl → 0.2.0__py3-none-any.whl

docp/parsers/_pdfbaseparser.py

@@ -0,0 +1,236 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+:Purpose:   This module provides generalised base functionality for
+            parsing PDF documents.
+
+:Platform:  Linux/Windows | Python 3.10+
+:Developer: J Berendt
+:Email:     development@s3dev.uk
+
+:Comments:  n/a
+
+.. attention::
+
+            This module is *not* designed to be interacted with
+            directly, only via the appropriate interface class(es).
+
+            Rather, please create an instance of a PDF document parsing
+            object using the following class:
+
+                - :class:`~docp.parsers.pdfparser.PDFParser`
+
+"""
+# pylint: disable=import-error
+# pylint: disable=protected-access
+# pylint: disable=wrong-import-order
+
+import os
+import pdfplumber
+from collections import Counter
+from unidecode import unidecode
+# locals
+try:
+    from .libs.utilities import utilities
+    from .objects.pdfobject import DocPDF
+except ImportError:
+    from libs.utilities import utilities
+    from objects.pdfobject import DocPDF
+
+
+class _PDFBaseParser:
+    """Base class containing generalised PDF parsing functionality."""
+
+    def __init__(self, path: str):
+        """Private base parser class initialiser.
+
+        Args:
+            path (str): Full path to the document to be parsed.
+
+        """
+        self._path = path
+        self._doc = DocPDF()
+        self._tbl_opath = None
+        self._set_paths()
+        self._open()
+
+    def __del__(self):
+        """Class deconstructor.
+
+        :Tasks:
+            - Ensure the PDF document is closed.
+
+        """
+        if hasattr(self._doc, '_parser'):
+            self._doc._parser.close()
+
+    @property
+    def doc(self) -> DocPDF:
+        """Accessor to the document object."""
+        return self._doc
+
+    def _get_crop_coordinates(self,
+                              skip_header: bool=False,
+                              skip_footer: bool=False) -> tuple[float]:
+        """Determine the bounding box coordinates.
+
+        These coordinates are used for removing the header and/or footer.
+
+        Args:
+            skip_header (bool, optional): If True, set the coordinates
+                such that the header is skipped. Defaults to False.
+            skip_footer (bool, optional): If True, set the coordinates
+                such that the footer is skipped. Defaults to False.
+
+        :Logic:
+            When excluding a header and/or footer, the following page
+            numbers are used for header/footer *position* detection,
+            given the length of the document:
+
+                - Number of pages [1]: 1
+                - Number of pages [2,10]: 2
+                - Number of pages [11,]: 5
+
+        Returns:
+            tuple: A bounding box tuple of the following form, to be
+            passed directly into the :func:`Page.crop` method::
+
+                (x0, top, x1, bottom)
+
+        """
+        npages = self._doc.npages
+        match npages:
+            case 1: num = 1
+            case _ if npages in range(2, 11): num = 2
+            case _: num = 5
+        pg = self._doc.parser.pages[num - 1]  # The parser does not have a page offset at [0].
+        # Default coordinates to the whole page.
+        coords = {'x0': 0, 'top': 0, 'x1': pg.width, 'bottom': pg.height}
+        # If the header and/or footer is to be skipped, find and iterate
+        # through the common lines and overwrite the coordinates as
+        # appropriate, given the key and the line's location on the page.
+        if skip_header or skip_footer:
+            lines = self._scan_common()
+            for line in lines:
+                s = pg.search(line)
+                if s:
+                    for key in coords:
+                        v = s[0][key]
+                        match key:
+                            case 'top' if v < pg.height/2 and skip_header:
+                                coords[key] = max(coords[key], v+2)
+                            case 'bottom' if v > pg.height/2 and skip_footer:
+                                coords[key] = min(coords[key], v-2)
+        return tuple(coords.values())
+
+    def _open(self) -> None:
+        """Open the PDF document for reading.
+
+        Before opening the file, a test is performed to ensure the PDF
+        is valid. The file must:
+
+            - exist
+            - be a valid PDF file, per the file signature
+            - have a .pdf file extension
+
+        :Other Operations:
+
+            - Store the ``pdfplumber`` parser object returned from the
+              :func:`pdfplumber.open` function into the
+              :attr:`self._doc._parser` attribute.
+            - Store the number of pages into the
+              :attr:`self._doc._npages` attribute.
+            - Store the document's meta data into the
+              :attr:`self._doc._meta` attribute.
+
+        Raises:
+            TypeError: Raised if the file type criteria above are not
+            met.
+
+        """
+        if all((os.path.exists(self._doc._fpath),
+                utilities.ispdf(self._doc._fpath),
+                os.path.splitext(self._doc._fpath)[1].lower() == '.pdf')):
+            self._doc._parser = pdfplumber.open(self._doc._fpath)
+            self._doc._npages = len(self._doc._parser.pages)
+            self._doc._meta = self._doc._parser.metadata
+        else:
+            msg = f'{self._doc._fname} is not a valid PDF file.'
+            raise TypeError(msg)
+
+    @staticmethod
+    def _prepare_row(row: list) -> str:
+        """Prepare the table row for writing a table to to CSV.
+
+        Args:
+            row (list): A list of strings, constituting a table row.
+
+        :Processing Tasks:
+
+            For each element in the row:
+
+                - Remove any double quote characters (ASCII and Unicode).
+                - Replace any empty values with ``'None'``.
+                - If the element contains a comma, wrap the element in
+                  double quotes.
+                - Attempt to convert any non-ASCII characters to an
+                  associated ASCII character. If the replacement cannot
+                  be made, the character is replaced with a ``'?'``.
+
+        Returns:
+            str: A processed comma-separated string, ready to be written
+            to a CSV file.
+
+        """
+        trans = {34: '', 8220: '', 8221: ''}  # Remove double quotes in Unicode.
+        row = [e.translate(trans) if e else 'None' for e in row]  # Cannot be a generator.
+        for idx, e in enumerate(row):
+            if ',' in e:
+                row[idx] = f'"{e}"'  # Escape comma-separation by quoting.
+        line = unidecode(','.join(row).replace('\n', ' '), errors='replace', replace_str='?')
+        return line
+
+    def _scan_common(self) -> list[str]:
+        """Scan the PDF document to find the most common lines.
+
+        :Rationale:
+            Generally, the most common lines in a document will be the
+            header and footer, as these are expected to be repeated on
+            each page of the document.
+
+            'Most common' is defined as line occurring on 90% of the
+            pages throughout the document. Therefore, only documents with
+            more than three pages are scanned. Otherwise, the 90% may
+            exclude relevant pieces of the document (as was discovered in
+            testing).
+
+        :Logic:
+            For documents with more than three pages, the entire PDF is
+            read through and each line extracted. The occurrence of each
+            line is counted, with the most common occurrences returned
+            to the caller.
+
+            The returned lines are to be passed into a page search to
+            determine the x/y coordinates of the header and footer.
+
+        Returns:
+            list: For documents with more than three pages, a list
+            containing the most common lines in the document. Otherwise,
+            an empty list if returned.
+
+        """
+        # Only scan if document has more than three pages.
+        if self._doc.npages < 4:
+            return []
+        if self._doc._common is None:
+            # Create a line generator for all pages.
+            lines = (l for p in self._doc.parser.pages for l in p.extract_text().split('\n'))
+            # Return the lines whose occurrence rate is 90% of document pages.
+            self._doc._common = [i[0] for i in Counter(lines).most_common()
+                                 if i[1] > self._doc.npages * 0.9]
+        return self._doc._common
+
+    def _set_paths(self) -> None:
+        """Set the document's file path attributes."""
+        self._doc._fpath = os.path.realpath(self._path)
+        self._doc._fname = os.path.basename(self._path)

docp/parsers/_pdftableparser.py

@@ -0,0 +1,272 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+:Purpose:   This module provides the logic for parsing tables from a PDF
+            document.
+
+:Platform:  Linux
+:Developer: J Berendt
+:Email:     jeremy.berendt@rolls-royce.com
+
+.. attention::
+
+            This module is *not* designed to be interacted with
+            directly, only via the appropriate interface class(es).
+
+            Rather, please create an instance of a PDF document parsing
+            object using the following:
+
+                - :class:`~docp.parsers.pdfparser.PDFParser`
+
+"""
+# pylint: disable=import-error
+# pylint: disable=protected-access
+# pylint: disable=wrong-import-order
+
+import io
+import os
+import pandas as pd
+import shutil
+# locals
+from parsers._pdfbaseparser import _PDFBaseParser
+
+# TODO: Move to a config file/class.  (TOML?)
+_SETTINGS = {'vertical_strategy': 'lines',
+             'horizontal_strategy':'lines',
+             'snap_x_tolerance': 12}
+
+
+class _PDFTableParser(_PDFBaseParser):
+    """Private PDF document table parser intermediate class.
+
+    Args:
+        path (str): Full path to the PDF document.
+
+    :Example:
+
+        Extract tables from a PDF file::
+
+            >>> from docp import PDFParser
+
+            >>> pdf = PDFParser(path='/path/to/myfile.pdf')
+            >>> pdf.extract_tables()
+
+            >>> tables = pdf.doc.tables
+
+    """
+
+    def extract_tables(self,
+                       table_settings: dict=None,
+                       as_dataframe: bool=False,
+                       to_csv: bool=True,
+                       verbose: bool=False):
+        """Extract tables from the document.
+
+        Before a table is extracted, a number of validation tests are
+        performed to verify what has been identified as a 'table' is
+        actually a table which might be useful to the user.
+
+        Each 'valid' table is written as a CSV file on the user's
+        desktop.
+
+        Additionally, the extracted table data is stored to the class'
+        :attr:`self.tables` attribute.
+
+        Args:
+            table_settings (dict, optional): Table settings to be used
+                for the table extraction. Defaults to None, which is
+                replaced by the value in the config.
+            as_dataframe (bool, optional): By default, the extracted
+                tables are returned as a list of (lists of lists), for
+                example: all_tables[table[rows[data]]]. However, if this
+                argument is ``True``, the table data is returned as a
+                list of ``pandas.DataFrame`` objects. In this case, the
+                first row of the table is used as the header, and all
+                remaining rows are treated as data. **Note:** This will
+                *not* work properly for all tables. Defaults to False.
+            to_csv (bool, optional): Dump extracted table data to a CSV
+                file, one per table. Defaults to True.
+            verbose (bool, optional): Display how many tables were
+                extracted, and the path to their location.
+
+        """
+        # pylint: disable=invalid-name
+        # pylint: disable=too-many-nested-blocks
+        # pylint: disable=unnecessary-dunder-call
+        if self._doc.tables:
+            # Reinitialise the doc object and reopen the document.
+            self.__init__(path=self._path)
+        c = 0
+        if to_csv:
+            self._create_table_directory_path()
+        if table_settings is None:
+            table_settings = _SETTINGS
+        for p in self._doc._pdf.pages:
+            tblno = 1
+            tables = self._filter_tables(tables=p.find_tables(), threshold=5000)
+            for table in tables:
+                pc = p.crop(table.bbox)
+                data = pc.extract_table(table_settings=table_settings)
+                if all(len(row) > 1 for row in data) and len(data) > 1:
+                    # Verify no table rows are found in the most common rows (header/footer).
+                    if not self._table_header_footer(table=data):
+                        if not as_dataframe:
+                            self._doc._tables.append(data)
+                        if to_csv or as_dataframe:
+                            buffer = self._to_buffer(data=data)
+                            if to_csv:
+                                c += self._to_csv(buffer=buffer,
+                                                  pageno=p.page_number,
+                                                  tableno=tblno)
+                            if as_dataframe:
+                                self._to_df(buffer=buffer)
+                            buffer.close()
+                        tblno += 1
+        if verbose and to_csv:
+            print('',
+                  'Complete.',
+                  f'{c} tables were extracted and stored at the path below.',
+                  f'Path: {self._tbl_opath}',
+                  sep='\n')
+
+    def _create_table_directory_path(self):
+        """Create the output directory for table data.
+
+        If the directory does not exist, it is created.
+
+        """
+        # Defined in parent class.
+        # pylint: disable=attribute-defined-outside-init
+        trans = {32: '_', 45: '_'}
+        path = (os.path.join(os.path.join(os.environ['HOME'], 'Desktop'),
+                             'docutils',
+                             'pdf_tables',
+                             (os.path.splitext(os.path.basename(self._path))[0]
+                              .lower()
+                              .translate(trans))))
+        self._tbl_opath = path
+        if not os.path.exists(path):
+            os.makedirs(path)
+
+    def _create_table_file_path(self, pageno: int, tblno: int) -> str:
+        """Create the filename for the table.
+
+        Args:
+            pageno (int): Page from which the table was extracted.
+            tblno (int): Number of the table on the page, starting at 1.
+
+        Returns:
+            str: Explicit path to the file to be written.
+
+        """
+        path = os.path.join(self._tbl_opath,
+                            f'pg{str(pageno).zfill(3)}_tb{str(tblno).zfill(3)}.csv')
+        return path
+
+    @staticmethod
+    def _filter_tables(tables: list, threshold: int=5000) -> list:
+        """Remove tables from the passed list which are deemed invalid.
+
+        Args:
+            tables (list): A list of tables as detected by the
+                :meth:`Page.find_table()` method.
+            threshold (int, optional): Minimum pixel area for a detected
+                table to be returned. Defaults to 5000.
+
+        :Rationale:
+            An 'invalid' table is determined by the number of pixels
+            which the table covered. Any table which is less than (N)
+            pixels is likely a block of text which has been categorised
+            as a 'table', but is not.
+
+        Returns:
+            list: A list of tables whose pixel area is greater than
+            ``threshold``.
+
+        """
+        # pylint: disable=invalid-name
+        t = []
+        for table in tables:
+            x0, y0, x1, y1 = table.bbox
+            if (x1-x0) * (y1-y0) > threshold:
+                t.append(table)
+        return t
+
+    def _table_header_footer(self, table: list[list]) -> bool:
+        """Verify a table is not a header or footer.
+
+        Args:
+            table (list[list]): Table (a list of lists) be a analysed.
+
+        :Rationale:
+            A table is determined to be a header or footer if any of the
+            line contained in the 'common lines list' are found in the
+            table.
+
+            If any of these lines are found, the table is determined to
+            be a header/footer, True is returned.
+
+        Returns:
+            bool: False if the table is *not* a header/footer, otherwise
+            True.
+
+        """
+        lines = self._scan_common()  # Only re-runs if not already run.
+        # r: row; c: cell; l: line
+        return any(l in c for l in lines for r in table for c in r if c)
+
+    def _to_buffer(self, data: list[list]) -> io.StringIO:
+        """Write the table data into a string buffer.
+
+        Args:
+            data (list[list]): The table data as a list of lists to be
+                written to a buffer.
+
+        Returns:
+            io.StringIO: A string buffer as an ``io.StringIO`` object.
+
+        """
+        b = io.StringIO()
+        for row in data:
+            line = self._prepare_row(row=row)
+            b.write(line)
+            b.write('\n')
+        b.seek(0)
+        return b
+
+    def _to_csv(self, buffer: io.StringIO, pageno: int, tableno: int) -> int:
+        """Write a table (from the buffer) to CSV.
+
+        Args:
+            buffer (io.StringIO): A pre-processed ``StringIO`` object
+                containing table data to be written.
+            pageno (int): Page number from the ``Page`` object.
+            tableno (int): Number of the table on the page, based at 1.
+
+        Returns:
+            int: 1 if the file was written, otherwise 0. This is used by
+            the caller to track the number of CSV files written.
+
+        """
+        if buffer.seek(0, os.SEEK_END):  # Test buffer is populated.
+            path = self._create_table_file_path(pageno=pageno, tblno=tableno)
+            with open(path, 'w', encoding='utf-8') as f:
+                buffer.seek(0)
+                shutil.copyfileobj(buffer, f)
+                return 1
+        return 0
+
+    def _to_df(self, buffer: io.StringIO):
+        """Write a table (from the buffer) to a DataFrame.
+
+        Once written, the DataFrame is appended to
+        :attr:`self._doc._tables` list of tables.
+
+        Args:
+            buffer (io.StringIO): A pre-processed ``StringIO`` object
+                containing table data to be written.
+
+        """
+        if buffer.seek(0, os.SEEK_END):
+            buffer.seek(0)
+            self._doc._tables.append(pd.read_csv(buffer))