sparrow-parse 0.2.9__py3-none-any.whl → 0.3.0__py3-none-any.whl

sparrow_parse/__init__.py

@@ -1 +1 @@
-__version__ = '0.2.9'
+__version__ = '0.3.0'

sparrow_parse/extractor/html_extractor.py

@@ -244,7 +244,7 @@ if __name__ == "__main__":
     #     ['deposits', 'account_number', 'od_limit', 'currency_balance', 'sgd_balance', 'transaction_date',
     #      'value_date', 'description', 'cheque', 'withdrawal', 'deposit', 'balance', 'maturity_date'],
     #     True,
-    #     True,
+    #     False,
     #     True,
     #     True)
     #

sparrow_parse/extractor/pdf_optimizer.py

@@ -0,0 +1,72 @@
+import PyPDF2
+from pdf2image import convert_from_path
+import os
+import tempfile
+import shutil
+
+
+class PDFOptimizer(object):
+    def __init__(self):
+        pass
+
+    def split_pdf_to_pages(self, file_path, output_dir=None, convert_to_images=False):
+        # Create a temporary directory
+        temp_dir = tempfile.mkdtemp()
+        output_files = []
+
+        if not convert_to_images:
+            # Open the PDF file
+            with open(file_path, 'rb') as pdf_file:
+                reader = PyPDF2.PdfReader(pdf_file)
+                number_of_pages = len(reader.pages)
+
+                # Split the PDF into separate files per page
+                for page_num in range(number_of_pages):
+                    writer = PyPDF2.PdfWriter()
+                    writer.add_page(reader.pages[page_num])
+
+                    output_filename = os.path.join(temp_dir, f'page_{page_num + 1}.pdf')
+                    with open(output_filename, 'wb') as output_file:
+                        writer.write(output_file)
+                        output_files.append(output_filename)
+
+                    if output_dir:
+                        # Save each page to the debug folder
+                        debug_output_filename = os.path.join(output_dir, f'page_{page_num + 1}.pdf')
+                        with open(debug_output_filename, 'wb') as output_file:
+                            writer.write(output_file)
+
+            # Return the number of pages, the list of file paths, and the temporary directory
+            return number_of_pages, output_files, temp_dir
+        else:
+            # Convert the PDF to images
+            images = convert_from_path(file_path, dpi=400)
+
+            # Save the images to the temporary directory
+            for i, image in enumerate(images):
+                output_filename = os.path.join(temp_dir, f'page_{i + 1}.jpg')
+                image.save(output_filename, 'JPEG')
+                output_files.append(output_filename)
+
+                if output_dir:
+                    # Save each image to the debug folder
+                    debug_output_filename = os.path.join(output_dir, f'page_{i + 1}.jpg')
+                    image.save(debug_output_filename, 'JPEG')
+
+            # Return the number of pages, the list of file paths, and the temporary directory
+            return len(images), output_files, temp_dir
+
+
+if __name__ == "__main__":
+    pdf_optimizer = PDFOptimizer()
+
+    # output_directory = "/Users/andrejb/Documents/work/bankstatement/output_pages"
+    # # Ensure the output directory exists
+    # os.makedirs(output_directory, exist_ok=True)
+    #
+    # # Split the optimized PDF into separate pages
+    # num_pages, output_files, temp_dir = pdf_optimizer.split_pdf_to_pages("/Users/andrejb/Documents/work/bankstatement/statement.pdf",
+    #                                                                      output_directory,
+    #                                                                      False)
+    #
+    # shutil.rmtree(temp_dir, ignore_errors=True)

sparrow_parse/extractor/unstructured_processor.py

@@ -176,4 +176,3 @@ if __name__ == "__main__":
     #     ['tables', 'unstructured'],
     #     True,
     #     True)
-

sparrow_parse/temp.py

@@ -13,4 +13,15 @@
     #     'yolox',
     #     ['tables', 'unstructured'],
     #     True,
-    #     True)
+    #     True)
+
+    # output_directory = "/Users/andrejb/Documents/work/epik/bankstatement/output_pages"
+    # # Ensure the output directory exists
+    # os.makedirs(output_directory, exist_ok=True)
+    #
+    # # Split the optimized PDF into separate pages
+    # num_pages, output_files, temp_dir = pdf_optimizer.split_pdf_to_pages("/Users/andrejb/Documents/work/epik/bankstatement/OCBC_1_statement.pdf",
+    #                                                                      output_directory,
+    #                                                                      False)
+    #
+    # shutil.rmtree(temp_dir, ignore_errors=True)

{sparrow_parse-0.2.9.dist-info → sparrow_parse-0.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sparrow-parse
-Version: 0.2.9
+Version: 0.3.0
 Summary: Sparrow Parse is a Python package for parsing and extracting information from documents.
 Home-page: https://github.com/katanaml/sparrow/tree/main/sparrow-data/parse
 License: GPL-3.0
@@ -15,8 +15,9 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Topic :: Software Development
+Requires-Dist: PyPDF2 (==3.0.1)
 Requires-Dist: numpy (==1.26.4)
-Requires-Dist: pymupdf4llm (==0.0.5)
+Requires-Dist: pymupdf4llm (==0.0.6)
 Requires-Dist: rich (>=13.7.1,<14.0.0)
 Requires-Dist: sentence-transformers (==3.0.1)
 Requires-Dist: torch (==2.2.2)
@@ -128,6 +129,27 @@ Example:
 
 *debug* - `True`
 
+## PDF optimization
+
+```
+from sparrow_parse.extractor.pdf_optimizer import PDFOptimizer
+
+pdf_optimizer = PDFOptimizer()
+
+num_pages, output_files, temp_dir = pdf_optimizer.split_pdf_to_pages(file_path,
+                                                                     output_directory,
+                                                                     convert_to_images)
+
+```
+
+Example:
+
+*file_path* - `/Users/andrejb/infra/shared/katana-git/sparrow/sparrow-ml/llm/data/invoice_1.pdf`
+
+*output_directory* - set to not `None`, for debug purposes only
+
+*convert_to_images* - default `False`, to split into PDF files
+
 ## Library build
 
 ```

sparrow_parse-0.3.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+sparrow_parse/__init__.py,sha256=gTggO06fb2c9XKEwlQYUSPlUfy82yVlM9pzLMOUqVcY,21
+sparrow_parse/__main__.py,sha256=Xs1bpJV0n08KWOoQE34FBYn6EBXZA9HIYJKrE4ZdG78,153
+sparrow_parse/data/invoice_1_table.txt,sha256=dsWEASxlVNidpTCQDowCM7SjaUzSqwx7DuydTfaQ7xI,1115
+sparrow_parse/extractor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sparrow_parse/extractor/extractor_helper.py,sha256=n9M9NyZfesiCCj3ET9WoyqRcWIFJ4k-jyQlUAarKIhE,13658
+sparrow_parse/extractor/html_extractor.py,sha256=Y9c17epY6esn1lNGhOVpzgRuolFJUUZAfZ3G9fKcArU,9916
+sparrow_parse/extractor/markdown_processor.py,sha256=dC2WUdA-v2psh7oytruftxYkXdQi72FoEYxF30ROuO0,4506
+sparrow_parse/extractor/pdf_optimizer.py,sha256=cgXLY7JBtxZVU4KenNIhfN2ogcZXyvD0f7SitvkbJ4o,3016
+sparrow_parse/extractor/unstructured_processor.py,sha256=oonkB5ALaV1pVs0a-xr8yAf-kirIabmtugHMnnEILqo,6770
+sparrow_parse/temp.py,sha256=gy4_mtNW_KfXn9br_suu6jHx7JKYLKs9pIOBynh_JWY,1134
+sparrow_parse-0.3.0.dist-info/METADATA,sha256=rmt5FkqS9RLhEf0M3oYYu8A-r9VE7Yt_ZYlSnn7AcPw,6273
+sparrow_parse-0.3.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+sparrow_parse-0.3.0.dist-info/entry_points.txt,sha256=H507qotwq3VX4lv5pY9MZYtupKNE1RRb8gEQucPiGi0,52
+sparrow_parse-0.3.0.dist-info/RECORD,,

sparrow_parse-0.2.9.dist-info/RECORD

@@ -1,13 +0,0 @@
-sparrow_parse/__init__.py,sha256=e5Klz6yLU-4Ub9zxGUBo9wADAYsST38ZylTv9ze4i60,21
-sparrow_parse/__main__.py,sha256=Xs1bpJV0n08KWOoQE34FBYn6EBXZA9HIYJKrE4ZdG78,153
-sparrow_parse/data/invoice_1_table.txt,sha256=dsWEASxlVNidpTCQDowCM7SjaUzSqwx7DuydTfaQ7xI,1115
-sparrow_parse/extractor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sparrow_parse/extractor/extractor_helper.py,sha256=n9M9NyZfesiCCj3ET9WoyqRcWIFJ4k-jyQlUAarKIhE,13658
-sparrow_parse/extractor/html_extractor.py,sha256=rwtumbPrJoJ8UryjlrASAjAGSsu6bbu-TQm_6ouJods,9915
-sparrow_parse/extractor/markdown_processor.py,sha256=dC2WUdA-v2psh7oytruftxYkXdQi72FoEYxF30ROuO0,4506
-sparrow_parse/extractor/unstructured_processor.py,sha256=z46aXacMvfW_wmsACs0LtamoMc19eogGd5fVVAj4vIo,6771
-sparrow_parse/temp.py,sha256=Hl1wPOEytXnfbUobU8BJgEswPsfncibbQdwrpSHtlOo,513
-sparrow_parse-0.2.9.dist-info/METADATA,sha256=qgLxJOpKE9X_uutQhasniDYf6kX93p5exRUTZCAYts4,5622
-sparrow_parse-0.2.9.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-sparrow_parse-0.2.9.dist-info/entry_points.txt,sha256=H507qotwq3VX4lv5pY9MZYtupKNE1RRb8gEQucPiGi0,52
-sparrow_parse-0.2.9.dist-info/RECORD,,