sparrow-parse 0.3.4__py3-none-any.whl → 0.3.6__py3-none-any.whl

sparrow_parse/processors/markdown_processor.py

@@ -1,137 +0,0 @@
-import pymupdf4llm
-import pandas as pd
-import re
-from rich.progress import Progress, SpinnerColumn, TextColumn
-from rich import print
-from bs4 import BeautifulSoup
-
-
-class MarkdownProcessor(object):
-    def __init__(self):
-        pass
-
-    def extract_data(self, file_path, options, local=True, debug=False):
-        markdown_text = self.invoke_pipeline_step(
-            lambda: pymupdf4llm.to_markdown(file_path),
-            "Extracting markdown text from the document...",
-            local
-        )
-
-        content, table_content = self.invoke_pipeline_step(
-            lambda: self.load_text_data(markdown_text, options),
-            "Loading text data...",
-            local
-        )
-
-        if debug:
-            print("Data extracted from the document:")
-            print(content)
-            print("\n")
-            print("Table content extracted from the document:")
-            if table_content:
-                print(len(table_content))
-            print(table_content)
-
-        return content, table_content
-
-    def load_text_data(self, markdown_text, options):
-        content, table_content = None, None
-
-        if options is None:
-            content = markdown_text
-
-        if options and "tables" in options and "markdown" in options:
-            content = self.extract_form_data(markdown_text)
-            table_content = self.extract_tables(markdown_text)
-
-        return content, table_content
-
-    def extract_form_data(self, markdown_text):
-        return markdown_text
-
-    def extract_tables(self, markdown_text):
-        # Regular expression to match markdown tables
-        table_pattern = re.compile(r'(\|.+\|\n\|[-| ]+\|\n(?:\|.*\|\n)*?)(?=\|.*TOTAL)', re.MULTILINE)
-
-        # Find all tables in the markdown text
-        tables = table_pattern.findall(markdown_text)
-
-        html_tables = []
-        for table_text in tables:
-            # Split the table into lines
-            lines = table_text.strip().split('\n')
-
-            # Extract headers and rows
-            headers = [self.clean_column_name(header.strip()) for header in lines[0].split('|') if header]
-            rows = []
-            for line in lines[2:]:  # Skip header and separator lines
-                row = [cell.strip() for cell in line.split('|') if cell]
-                rows.append(row)
-
-            # Convert to Pandas DataFrame
-            df = pd.DataFrame(rows, columns=headers)
-
-            # Convert DataFrame to HTML and append to the list
-            html_table = df.to_html(index=False)
-            if self.table_has_header(html_table):
-                html_tables.append(html_table)
-
-            return html_tables
-
-    def clean_column_name(self, name):
-        """
-        Cleans the column name by removing spaces if the name is a single word with spaces between letters.
-
-        Args:
-        name (str): The column name to clean.
-
-        Returns:
-        str: The cleaned column name.
-        """
-        # Check if the name contains only letters and spaces
-        if all(char.isalpha() or char.isspace() for char in name):
-            # Check if it is a single word with spaces between letters
-            parts = name.split()
-            if len(parts) > 1 and all(len(part) == 1 for part in parts):
-                return ''.join(parts)
-        return name
-
-    def invoke_pipeline_step(self, task_call, task_description, local):
-        if local:
-            with Progress(
-                    SpinnerColumn(),
-                    TextColumn("[progress.description]{task.description}"),
-                    transient=False,
-            ) as progress:
-                progress.add_task(description=task_description, total=None)
-                ret = task_call()
-        else:
-            print(task_description)
-            ret = task_call()
-
-        return ret
-
-    def table_has_header(self, table_html):
-        soup = BeautifulSoup(table_html, 'html.parser')
-        table = soup.find('table')
-
-        # Check if the table contains a <thead> tag
-        if table.find('thead'):
-            return True
-
-        # Check if the table contains any <th> tags inside the table (in case there's no <thead>)
-        if table.find_all('th'):
-            return True
-
-        return False
-
-
-if __name__ == "__main__":
-    processor = MarkdownProcessor()
-
-    # content, table_content = processor.extract_data(
-    #     '/Users/andrejb/infra/shared/katana-git/sparrow/sparrow-ml/llm/data/invoice_1.pdf',
-    #     ['tables', 'markdown'],
-    #     True,
-    #     True)
-

sparrow_parse/processors/unstructured_processor.py

@@ -1,178 +0,0 @@
-import tempfile
-import os
-from unstructured.partition.pdf import partition_pdf
-from unstructured.partition.image import partition_image
-import json
-from unstructured.staging.base import elements_to_json
-from rich.progress import Progress, SpinnerColumn, TextColumn
-from rich import print
-from bs4 import BeautifulSoup
-
-
-class UnstructuredProcessor(object):
-    def __init__(self):
-        pass
-
-    def extract_data(self, file_path, strategy, model_name, options, local=True, debug=False):
-        # Extracts the elements from the PDF
-        elements = self.invoke_pipeline_step(
-            lambda: self.process_file(file_path, strategy, model_name),
-            "Extracting elements from the document...",
-            local
-        )
-
-        if debug:
-            new_extension = 'json'  # You can change this to any extension you want
-            new_file_path = self.change_file_extension(file_path, new_extension)
-
-            content, table_content = self.invoke_pipeline_step(
-                lambda: self.load_text_data(elements, new_file_path, options),
-                "Loading text data...",
-                local
-            )
-        else:
-            with tempfile.TemporaryDirectory() as temp_dir:
-                temp_file_path = os.path.join(temp_dir, "file_data.json")
-
-                content, table_content = self.invoke_pipeline_step(
-                    lambda: self.load_text_data(elements, temp_file_path, options),
-                    "Loading text data...",
-                    local
-                )
-
-        if debug:
-            print("Data extracted from the document:")
-            print(content)
-            print("\n")
-            print("Table content extracted from the document:")
-            if table_content:
-                print(len(table_content))
-            print(table_content)
-
-        return content, table_content
-
-    def process_file(self, file_path, strategy, model_name):
-        elements = None
-
-        if file_path.lower().endswith('.pdf'):
-            elements = partition_pdf(
-                filename=file_path,
-                strategy=strategy,
-                infer_table_structure=True,
-                hi_res_model_name=model_name,
-                languages=['en']
-            )
-        elif file_path.lower().endswith(('.jpg', '.jpeg', '.png')):
-            elements = partition_image(
-                filename=file_path,
-                strategy=strategy,
-                infer_table_structure=True,
-                hi_res_model_name=model_name,
-                languages=['en']
-            )
-
-        return elements
-
-    def change_file_extension(self, file_path, new_extension, suffix=None):
-        # Check if the new extension starts with a dot and add one if not
-        if not new_extension.startswith('.'):
-            new_extension = '.' + new_extension
-
-        # Split the file path into two parts: the base (everything before the last dot) and the extension
-        # If there's no dot in the filename, it'll just return the original filename without an extension
-        base = file_path.rsplit('.', 1)[0]
-
-        # Concatenate the base with the new extension
-        if suffix is None:
-            new_file_path = base + new_extension
-        else:
-            new_file_path = base + "_" + suffix + new_extension
-
-        return new_file_path
-
-    def load_text_data(self, elements, file_path, options):
-        elements_to_json(elements, filename=file_path)
-
-        content, table_content = None, None
-
-        if options is None:
-            content = self.process_json_file(file_path)
-
-        if options and "tables" in options and "unstructured" in options:
-            content = self.process_json_file(file_path, "form")
-
-            table_content = self.process_json_file(file_path, "table")
-
-        return content, table_content
-
-    def process_json_file(self, file_path, option=None):
-        # Read the JSON file
-        with open(file_path, 'r') as file:
-            data = json.load(file)
-
-        # Iterate over the JSON data and extract required elements
-        extracted_elements = []
-        for entry in data:
-            if entry["type"] == "Table" and (option is None or option == "table" or option == "form"):
-                table_data = entry["metadata"]["text_as_html"]
-                if option == "table" and self.table_has_header(table_data):
-                    extracted_elements.append(table_data)
-                if option is None or option == "form":
-                    extracted_elements.append(table_data)
-            elif entry["type"] == "Title" and (option is None or option == "form"):
-                extracted_elements.append(entry["text"])
-            elif entry["type"] == "NarrativeText" and (option is None or option == "form"):
-                extracted_elements.append(entry["text"])
-            elif entry["type"] == "UncategorizedText" and (option is None or option == "form"):
-                extracted_elements.append(entry["text"])
-            elif entry["type"] == "ListItem" and (option is None or option == "form"):
-                extracted_elements.append(entry["text"])
-            elif entry["type"] == "Image" and (option is None or option == "form"):
-                extracted_elements.append(entry["text"])
-
-        if option is None or option == "form":
-            # Convert list to single string with two new lines between each element
-            extracted_data = "\n\n".join(extracted_elements)
-            return extracted_data
-
-        return extracted_elements
-
-    def invoke_pipeline_step(self, task_call, task_description, local):
-        if local:
-            with Progress(
-                    SpinnerColumn(),
-                    TextColumn("[progress.description]{task.description}"),
-                    transient=False,
-            ) as progress:
-                progress.add_task(description=task_description, total=None)
-                ret = task_call()
-        else:
-            print(task_description)
-            ret = task_call()
-
-        return ret
-
-    def table_has_header(self, table_html):
-        soup = BeautifulSoup(table_html, 'html.parser')
-        table = soup.find('table')
-
-        # Check if the table contains a <thead> tag
-        if table.find('thead'):
-            return True
-
-        # Check if the table contains any <th> tags inside the table (in case there's no <thead>)
-        if table.find_all('th'):
-            return True
-
-        return False
-
-
-if __name__ == "__main__":
-    processor = UnstructuredProcessor()
-    # content, table_content = processor.extract_data(
-    #     '/Users/andrejb/infra/shared/katana-git/sparrow/sparrow-ml/llm/data/invoice_1.pdf',
-    #     'hi_res',
-    #     'yolox',
-    #     ['tables', 'unstructured'],
-    #     True,
-    #     True)

sparrow_parse/temp.py

@@ -1,27 +0,0 @@
-# content, table_content = processor.extract_data(
-    #     '/Users/andrejb/Documents/work/schreiber/invoice_data/test/2618407.pdf',
-    #     'hi_res',
-    #     'yolox',
-    #     # 'detectron2_onnx',
-    #     ['tables', 'unstructured'],
-    #     True,
-    #     True)
-
-    # content, table_content = processor.extract_data(
-    #     '/Users/andrejb/Documents/work/epik/bankstatement/OCBC_1_1.pdf',
-    #     'hi_res',
-    #     'yolox',
-    #     ['tables', 'unstructured'],
-    #     True,
-    #     True)
-
-    # output_directory = "/Users/andrejb/Documents/work/epik/bankstatement/output_pages"
-    # # Ensure the output directory exists
-    # os.makedirs(output_directory, exist_ok=True)
-    #
-    # # Split the optimized PDF into separate pages
-    # num_pages, output_files, temp_dir = pdf_optimizer.split_pdf_to_pages("/Users/andrejb/Documents/work/epik/bankstatement/OCBC_1_statement.pdf",
-    #                                                                      output_directory,
-    #                                                                      False)
-    #
-    # shutil.rmtree(temp_dir, ignore_errors=True)

sparrow_parse-0.3.4.dist-info/RECORD

@@ -1,23 +0,0 @@
-sparrow_parse/__init__.py,sha256=SH0xuWVUkyLHZJwWBZ8GJoeliTeYFcqA6TWJgrkLv-U,21
-sparrow_parse/__main__.py,sha256=Xs1bpJV0n08KWOoQE34FBYn6EBXZA9HIYJKrE4ZdG78,153
-sparrow_parse/temp.py,sha256=gy4_mtNW_KfXn9br_suu6jHx7JKYLKs9pIOBynh_JWY,1134
-sparrow_parse/extractors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sparrow_parse/extractors/html_extractor.py,sha256=qe9Oz7J-GiIE8G1kIDMOeh96xe6P59Gyh5SjgV3v2c8,9977
-sparrow_parse/extractors/vllm_extractor.py,sha256=Qwmf-SW4z_UstiiynX5TkyovlkokVhLuzcbUVZ16TXM,1540
-sparrow_parse/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sparrow_parse/helpers/html_extractor_helper.py,sha256=n9M9NyZfesiCCj3ET9WoyqRcWIFJ4k-jyQlUAarKIhE,13658
-sparrow_parse/helpers/pdf_optimizer.py,sha256=KI_EweGt9Y_rDH1uCpYD5wKCW3rdjSFFhoVtiPBxX8k,3013
-sparrow_parse/processors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sparrow_parse/processors/markdown_processor.py,sha256=dC2WUdA-v2psh7oytruftxYkXdQi72FoEYxF30ROuO0,4506
-sparrow_parse/processors/table_structure_processor.py,sha256=bG_6jx66n_KNdY_O6hrZD1D4DHX5Qy__RYcKHmrSGnc,23894
-sparrow_parse/processors/unstructured_processor.py,sha256=oonkB5ALaV1pVs0a-xr8yAf-kirIabmtugHMnnEILqo,6770
-sparrow_parse/vllm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sparrow_parse/vllm/huggingface_inference.py,sha256=Q2Ju65LDzbO-8RWW7cXzrR-pbZ1zKuPVODlKOTWKg_E,1114
-sparrow_parse/vllm/inference_base.py,sha256=W0N2khehGdF1XHzZACG3I1UZaydHMk6BZgWNvaJD4Ck,197
-sparrow_parse/vllm/inference_factory.py,sha256=r04e95uPWG5l8Q23yeDqKmvFxLyF991aA2m0hfBTNn8,993
-sparrow_parse/vllm/local_gpu_inference.py,sha256=I_uWYiFAQhRrykOKbVz69NzftDxuemDKtAye4kWhtnU,617
-sparrow_parse-0.3.4.dist-info/METADATA,sha256=L7qXKxktk42gUQlBlZAdzHQqfORoC6vBwRCd-VSwv3Y,7444
-sparrow_parse-0.3.4.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-sparrow_parse-0.3.4.dist-info/entry_points.txt,sha256=8CrvTVTTcz1YuZ8aRCYNOH15ZOAaYLlcbYX3t28HwJY,54
-sparrow_parse-0.3.4.dist-info/top_level.txt,sha256=n6b-WtT91zKLyCPZTP7wvne8v_yvIahcsz-4sX8I0rY,14
-sparrow_parse-0.3.4.dist-info/RECORD,,