sparrow-parse 0.2.8__tar.gz → 0.3.0__tar.gz

{sparrow_parse-0.2.8 → sparrow_parse-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sparrow-parse
-Version: 0.2.8
+Version: 0.3.0
 Summary: Sparrow Parse is a Python package for parsing and extracting information from documents.
 Home-page: https://github.com/katanaml/sparrow/tree/main/sparrow-data/parse
 License: GPL-3.0
@@ -15,8 +15,9 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Topic :: Software Development
+Requires-Dist: PyPDF2 (==3.0.1)
 Requires-Dist: numpy (==1.26.4)
-Requires-Dist: pymupdf4llm (==0.0.5)
+Requires-Dist: pymupdf4llm (==0.0.6)
 Requires-Dist: rich (>=13.7.1,<14.0.0)
 Requires-Dist: sentence-transformers (==3.0.1)
 Requires-Dist: torch (==2.2.2)
@@ -128,6 +129,27 @@ Example:
 
 *debug* - `True`
 
+## PDF optimization
+
+```
+from sparrow_parse.extractor.pdf_optimizer import PDFOptimizer
+
+pdf_optimizer = PDFOptimizer()
+
+num_pages, output_files, temp_dir = pdf_optimizer.split_pdf_to_pages(file_path,
+                                                                     output_directory,
+                                                                     convert_to_images)
+
+```
+
+Example:
+
+*file_path* - `/Users/andrejb/infra/shared/katana-git/sparrow/sparrow-ml/llm/data/invoice_1.pdf`
+
+*output_directory* - set to not `None`, for debug purposes only
+
+*convert_to_images* - default `False`, to split into PDF files
+
 ## Library build
 
 ```

{sparrow_parse-0.2.8 → sparrow_parse-0.3.0}/README.md

@@ -100,6 +100,27 @@ Example:
 
 *debug* - `True`
 
+## PDF optimization
+
+```
+from sparrow_parse.extractor.pdf_optimizer import PDFOptimizer
+
+pdf_optimizer = PDFOptimizer()
+
+num_pages, output_files, temp_dir = pdf_optimizer.split_pdf_to_pages(file_path,
+                                                                     output_directory,
+                                                                     convert_to_images)
+
+```
+
+Example:
+
+*file_path* - `/Users/andrejb/infra/shared/katana-git/sparrow/sparrow-ml/llm/data/invoice_1.pdf`
+
+*output_directory* - set to not `None`, for debug purposes only
+
+*convert_to_images* - default `False`, to split into PDF files
+
 ## Library build
 
 ```

{sparrow_parse-0.2.8 → sparrow_parse-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sparrow-parse"
-version = "0.2.8"
+version = "0.3.0"
 description = "Sparrow Parse is a Python package for parsing and extracting information from documents."
 authors = ["Andrej Baranovskij <andrejus.baranovskis@gmail.com>"]
 license = "GPL-3.0"
@@ -25,10 +25,11 @@ torch = {version = "2.2.2", source = "pypi"}
 unstructured = {version = "0.14.5", extras = ["all-docs"]}
 unstructured-inference = "0.7.33"
 rich = "^13.7.1"
-pymupdf4llm = "0.0.5"
+pymupdf4llm = "0.0.6"
 transformers = "4.41.2"
 sentence-transformers = "3.0.1"
 numpy = "1.26.4"
+PyPDF2 = "3.0.1"
 
 
 [tool.poetry.scripts]

sparrow_parse-0.3.0/sparrow_parse/__init__.py

@@ -0,0 +1 @@
+__version__ = '0.3.0'

{sparrow_parse-0.2.8 → sparrow_parse-0.3.0}/sparrow_parse/extractor/html_extractor.py

@@ -224,29 +224,29 @@ if __name__ == "__main__":
     # to run for debugging, navigate to sparrow_parse and run the following command:
     # python -m extractor.html_extractor
 
-    with open('data/invoice_1_table.txt', 'r') as file:
-        file_content = file.read()
-
-    file_content = file_content.strip()[1:-1].strip()
-    data_list = re.split(r"',\s*'", file_content)
-    data_list = [item.strip(" '") for item in data_list]
+    # with open('data/invoice_1_table.txt', 'r') as file:
+    #     file_content = file.read()
+    #
+    # file_content = file_content.strip()[1:-1].strip()
+    # data_list = re.split(r"',\s*'", file_content)
+    # data_list = [item.strip(" '") for item in data_list]
 
     extractor = HTMLExtractor()
 
-    answer, targets_unprocessed = extractor.read_data(
-        # ['description', 'qty', 'net_price', 'net_worth', 'vat', 'gross_worth'],
-        ['transaction_date', 'value_date', 'description', 'cheque', 'withdrawal', 'deposit', 'balance',
-         'deposits', 'account_number', 'od_limit', 'currency_balance', 'sgd_balance', 'maturity_date'],
-        data_list,
-        0.5,
-        0.3,
-        # None,
-        ['deposits', 'account_number', 'od_limit', 'currency_balance', 'sgd_balance', 'transaction_date',
-         'value_date', 'description', 'cheque', 'withdrawal', 'deposit', 'balance', 'maturity_date'],
-        True,
-        True,
-        True,
-        True)
-
-    print(answer)
-    print(targets_unprocessed)
+    # answer, targets_unprocessed = extractor.read_data(
+    #     # ['description', 'qty', 'net_price', 'net_worth', 'vat', 'gross_worth'],
+    #     ['transaction_date', 'value_date', 'description', 'cheque', 'withdrawal', 'deposit', 'balance',
+    #      'deposits', 'account_number', 'od_limit', 'currency_balance', 'sgd_balance', 'maturity_date'],
+    #     data_list,
+    #     0.5,
+    #     0.3,
+    #     # None,
+    #     ['deposits', 'account_number', 'od_limit', 'currency_balance', 'sgd_balance', 'transaction_date',
+    #      'value_date', 'description', 'cheque', 'withdrawal', 'deposit', 'balance', 'maturity_date'],
+    #     True,
+    #     False,
+    #     True,
+    #     True)
+    #
+    # print(answer)
+    # print(targets_unprocessed)

sparrow_parse-0.3.0/sparrow_parse/extractor/pdf_optimizer.py

@@ -0,0 +1,72 @@
+import PyPDF2
+from pdf2image import convert_from_path
+import os
+import tempfile
+import shutil
+
+
+class PDFOptimizer(object):
+    def __init__(self):
+        pass
+
+    def split_pdf_to_pages(self, file_path, output_dir=None, convert_to_images=False):
+        # Create a temporary directory
+        temp_dir = tempfile.mkdtemp()
+        output_files = []
+
+        if not convert_to_images:
+            # Open the PDF file
+            with open(file_path, 'rb') as pdf_file:
+                reader = PyPDF2.PdfReader(pdf_file)
+                number_of_pages = len(reader.pages)
+
+                # Split the PDF into separate files per page
+                for page_num in range(number_of_pages):
+                    writer = PyPDF2.PdfWriter()
+                    writer.add_page(reader.pages[page_num])
+
+                    output_filename = os.path.join(temp_dir, f'page_{page_num + 1}.pdf')
+                    with open(output_filename, 'wb') as output_file:
+                        writer.write(output_file)
+                        output_files.append(output_filename)
+
+                    if output_dir:
+                        # Save each page to the debug folder
+                        debug_output_filename = os.path.join(output_dir, f'page_{page_num + 1}.pdf')
+                        with open(debug_output_filename, 'wb') as output_file:
+                            writer.write(output_file)
+
+            # Return the number of pages, the list of file paths, and the temporary directory
+            return number_of_pages, output_files, temp_dir
+        else:
+            # Convert the PDF to images
+            images = convert_from_path(file_path, dpi=400)
+
+            # Save the images to the temporary directory
+            for i, image in enumerate(images):
+                output_filename = os.path.join(temp_dir, f'page_{i + 1}.jpg')
+                image.save(output_filename, 'JPEG')
+                output_files.append(output_filename)
+
+                if output_dir:
+                    # Save each image to the debug folder
+                    debug_output_filename = os.path.join(output_dir, f'page_{i + 1}.jpg')
+                    image.save(debug_output_filename, 'JPEG')
+
+            # Return the number of pages, the list of file paths, and the temporary directory
+            return len(images), output_files, temp_dir
+
+
+if __name__ == "__main__":
+    pdf_optimizer = PDFOptimizer()
+
+    # output_directory = "/Users/andrejb/Documents/work/bankstatement/output_pages"
+    # # Ensure the output directory exists
+    # os.makedirs(output_directory, exist_ok=True)
+    #
+    # # Split the optimized PDF into separate pages
+    # num_pages, output_files, temp_dir = pdf_optimizer.split_pdf_to_pages("/Users/andrejb/Documents/work/bankstatement/statement.pdf",
+    #                                                                      output_directory,
+    #                                                                      False)
+    #
+    # shutil.rmtree(temp_dir, ignore_errors=True)

{sparrow_parse-0.2.8 → sparrow_parse-0.3.0}/sparrow_parse/extractor/unstructured_processor.py

@@ -176,12 +176,3 @@ if __name__ == "__main__":
     #     ['tables', 'unstructured'],
     #     True,
     #     True)
-
-    content, table_content = processor.extract_data(
-        '/Users/andrejb/Documents/work/epik/bankstatement/OCBC_1_1.pdf',
-        'hi_res',
-        'yolox',
-        ['tables', 'unstructured'],
-        True,
-        True)
-

sparrow_parse-0.3.0/sparrow_parse/temp.py

@@ -0,0 +1,27 @@
+# content, table_content = processor.extract_data(
+    #     '/Users/andrejb/Documents/work/schreiber/invoice_data/test/2618407.pdf',
+    #     'hi_res',
+    #     'yolox',
+    #     # 'detectron2_onnx',
+    #     ['tables', 'unstructured'],
+    #     True,
+    #     True)
+
+    # content, table_content = processor.extract_data(
+    #     '/Users/andrejb/Documents/work/epik/bankstatement/OCBC_1_1.pdf',
+    #     'hi_res',
+    #     'yolox',
+    #     ['tables', 'unstructured'],
+    #     True,
+    #     True)
+
+    # output_directory = "/Users/andrejb/Documents/work/epik/bankstatement/output_pages"
+    # # Ensure the output directory exists
+    # os.makedirs(output_directory, exist_ok=True)
+    #
+    # # Split the optimized PDF into separate pages
+    # num_pages, output_files, temp_dir = pdf_optimizer.split_pdf_to_pages("/Users/andrejb/Documents/work/epik/bankstatement/OCBC_1_statement.pdf",
+    #                                                                      output_directory,
+    #                                                                      False)
+    #
+    # shutil.rmtree(temp_dir, ignore_errors=True)

sparrow_parse-0.2.8/sparrow_parse/__init__.py

@@ -1 +0,0 @@
-__version__ = '0.2.8'

sparrow_parse-0.2.8/sparrow_parse/temp.py

@@ -1,16 +0,0 @@
-# content, table_content = processor.extract_data(
-    #     '/Users/andrejb/Documents/work/schreiber/invoice_data/test/2618407.pdf',
-    #     'hi_res',
-    #     'yolox',
-    #     # 'detectron2_onnx',
-    #     ['tables', 'unstructured'],
-    #     True,
-    #     True)
-
-    # content, table_content = processor.extract_data(
-    #     '/Users/andrejb/Documents/work/epik/bankstatement/OCBC_1_1.pdf',
-    #     'hi_res',
-    #     'yolox',
-    #     ['tables', 'unstructured'],
-    #     True,
-    #     True)