sparrow-parse 0.2.9__tar.gz → 0.3.0__tar.gz

{sparrow_parse-0.2.9 → sparrow_parse-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sparrow-parse
-Version: 0.2.9
+Version: 0.3.0
 Summary: Sparrow Parse is a Python package for parsing and extracting information from documents.
 Home-page: https://github.com/katanaml/sparrow/tree/main/sparrow-data/parse
 License: GPL-3.0
@@ -15,8 +15,9 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Topic :: Software Development
+Requires-Dist: PyPDF2 (==3.0.1)
 Requires-Dist: numpy (==1.26.4)
-Requires-Dist: pymupdf4llm (==0.0.5)
+Requires-Dist: pymupdf4llm (==0.0.6)
 Requires-Dist: rich (>=13.7.1,<14.0.0)
 Requires-Dist: sentence-transformers (==3.0.1)
 Requires-Dist: torch (==2.2.2)
@@ -128,6 +129,27 @@ Example:
 
 *debug* - `True`
 
+## PDF optimization
+
+```
+from sparrow_parse.extractor.pdf_optimizer import PDFOptimizer
+
+pdf_optimizer = PDFOptimizer()
+
+num_pages, output_files, temp_dir = pdf_optimizer.split_pdf_to_pages(file_path,
+                                                                     output_directory,
+                                                                     convert_to_images)
+
+```
+
+Example:
+
+*file_path* - `/Users/andrejb/infra/shared/katana-git/sparrow/sparrow-ml/llm/data/invoice_1.pdf`
+
+*output_directory* - set to not `None`, for debug purposes only
+
+*convert_to_images* - default `False`, to split into PDF files
+
 ## Library build
 
 ```

{sparrow_parse-0.2.9 → sparrow_parse-0.3.0}/README.md

@@ -100,6 +100,27 @@ Example:
 
 *debug* - `True`
 
+## PDF optimization
+
+```
+from sparrow_parse.extractor.pdf_optimizer import PDFOptimizer
+
+pdf_optimizer = PDFOptimizer()
+
+num_pages, output_files, temp_dir = pdf_optimizer.split_pdf_to_pages(file_path,
+                                                                     output_directory,
+                                                                     convert_to_images)
+
+```
+
+Example:
+
+*file_path* - `/Users/andrejb/infra/shared/katana-git/sparrow/sparrow-ml/llm/data/invoice_1.pdf`
+
+*output_directory* - set to not `None`, for debug purposes only
+
+*convert_to_images* - default `False`, to split into PDF files
+
 ## Library build
 
 ```

{sparrow_parse-0.2.9 → sparrow_parse-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sparrow-parse"
-version = "0.2.9"
+version = "0.3.0"
 description = "Sparrow Parse is a Python package for parsing and extracting information from documents."
 authors = ["Andrej Baranovskij <andrejus.baranovskis@gmail.com>"]
 license = "GPL-3.0"
@@ -25,10 +25,11 @@ torch = {version = "2.2.2", source = "pypi"}
 unstructured = {version = "0.14.5", extras = ["all-docs"]}
 unstructured-inference = "0.7.33"
 rich = "^13.7.1"
-pymupdf4llm = "0.0.5"
+pymupdf4llm = "0.0.6"
 transformers = "4.41.2"
 sentence-transformers = "3.0.1"
 numpy = "1.26.4"
+PyPDF2 = "3.0.1"
 
 
 [tool.poetry.scripts]

sparrow_parse-0.3.0/sparrow_parse/__init__.py

@@ -0,0 +1 @@
+__version__ = '0.3.0'

{sparrow_parse-0.2.9 → sparrow_parse-0.3.0}/sparrow_parse/extractor/html_extractor.py

@@ -244,7 +244,7 @@ if __name__ == "__main__":
     #     ['deposits', 'account_number', 'od_limit', 'currency_balance', 'sgd_balance', 'transaction_date',
     #      'value_date', 'description', 'cheque', 'withdrawal', 'deposit', 'balance', 'maturity_date'],
     #     True,
-    #     True,
+    #     False,
     #     True,
     #     True)
     #

sparrow_parse-0.3.0/sparrow_parse/extractor/pdf_optimizer.py

@@ -0,0 +1,72 @@
+import PyPDF2
+from pdf2image import convert_from_path
+import os
+import tempfile
+import shutil
+
+
+class PDFOptimizer(object):
+    def __init__(self):
+        pass
+
+    def split_pdf_to_pages(self, file_path, output_dir=None, convert_to_images=False):
+        # Create a temporary directory
+        temp_dir = tempfile.mkdtemp()
+        output_files = []
+
+        if not convert_to_images:
+            # Open the PDF file
+            with open(file_path, 'rb') as pdf_file:
+                reader = PyPDF2.PdfReader(pdf_file)
+                number_of_pages = len(reader.pages)
+
+                # Split the PDF into separate files per page
+                for page_num in range(number_of_pages):
+                    writer = PyPDF2.PdfWriter()
+                    writer.add_page(reader.pages[page_num])
+
+                    output_filename = os.path.join(temp_dir, f'page_{page_num + 1}.pdf')
+                    with open(output_filename, 'wb') as output_file:
+                        writer.write(output_file)
+                        output_files.append(output_filename)
+
+                    if output_dir:
+                        # Save each page to the debug folder
+                        debug_output_filename = os.path.join(output_dir, f'page_{page_num + 1}.pdf')
+                        with open(debug_output_filename, 'wb') as output_file:
+                            writer.write(output_file)
+
+            # Return the number of pages, the list of file paths, and the temporary directory
+            return number_of_pages, output_files, temp_dir
+        else:
+            # Convert the PDF to images
+            images = convert_from_path(file_path, dpi=400)
+
+            # Save the images to the temporary directory
+            for i, image in enumerate(images):
+                output_filename = os.path.join(temp_dir, f'page_{i + 1}.jpg')
+                image.save(output_filename, 'JPEG')
+                output_files.append(output_filename)
+
+                if output_dir:
+                    # Save each image to the debug folder
+                    debug_output_filename = os.path.join(output_dir, f'page_{i + 1}.jpg')
+                    image.save(debug_output_filename, 'JPEG')
+
+            # Return the number of pages, the list of file paths, and the temporary directory
+            return len(images), output_files, temp_dir
+
+
+if __name__ == "__main__":
+    pdf_optimizer = PDFOptimizer()
+
+    # output_directory = "/Users/andrejb/Documents/work/bankstatement/output_pages"
+    # # Ensure the output directory exists
+    # os.makedirs(output_directory, exist_ok=True)
+    #
+    # # Split the optimized PDF into separate pages
+    # num_pages, output_files, temp_dir = pdf_optimizer.split_pdf_to_pages("/Users/andrejb/Documents/work/bankstatement/statement.pdf",
+    #                                                                      output_directory,
+    #                                                                      False)
+    #
+    # shutil.rmtree(temp_dir, ignore_errors=True)

{sparrow_parse-0.2.9 → sparrow_parse-0.3.0}/sparrow_parse/extractor/unstructured_processor.py

@@ -176,4 +176,3 @@ if __name__ == "__main__":
     #     ['tables', 'unstructured'],
     #     True,
     #     True)
-

sparrow_parse-0.3.0/sparrow_parse/temp.py

@@ -0,0 +1,27 @@
+# content, table_content = processor.extract_data(
+    #     '/Users/andrejb/Documents/work/schreiber/invoice_data/test/2618407.pdf',
+    #     'hi_res',
+    #     'yolox',
+    #     # 'detectron2_onnx',
+    #     ['tables', 'unstructured'],
+    #     True,
+    #     True)
+
+    # content, table_content = processor.extract_data(
+    #     '/Users/andrejb/Documents/work/epik/bankstatement/OCBC_1_1.pdf',
+    #     'hi_res',
+    #     'yolox',
+    #     ['tables', 'unstructured'],
+    #     True,
+    #     True)
+
+    # output_directory = "/Users/andrejb/Documents/work/epik/bankstatement/output_pages"
+    # # Ensure the output directory exists
+    # os.makedirs(output_directory, exist_ok=True)
+    #
+    # # Split the optimized PDF into separate pages
+    # num_pages, output_files, temp_dir = pdf_optimizer.split_pdf_to_pages("/Users/andrejb/Documents/work/epik/bankstatement/OCBC_1_statement.pdf",
+    #                                                                      output_directory,
+    #                                                                      False)
+    #
+    # shutil.rmtree(temp_dir, ignore_errors=True)

sparrow_parse-0.2.9/sparrow_parse/__init__.py

@@ -1 +0,0 @@
-__version__ = '0.2.9'

sparrow_parse-0.2.9/sparrow_parse/temp.py

@@ -1,16 +0,0 @@
-# content, table_content = processor.extract_data(
-    #     '/Users/andrejb/Documents/work/schreiber/invoice_data/test/2618407.pdf',
-    #     'hi_res',
-    #     'yolox',
-    #     # 'detectron2_onnx',
-    #     ['tables', 'unstructured'],
-    #     True,
-    #     True)
-
-    # content, table_content = processor.extract_data(
-    #     '/Users/andrejb/Documents/work/epik/bankstatement/OCBC_1_1.pdf',
-    #     'hi_res',
-    #     'yolox',
-    #     ['tables', 'unstructured'],
-    #     True,
-    #     True)