sparrow-parse 0.1.10__tar.gz → 0.2.1__tar.gz

{sparrow_parse-0.1.10 → sparrow_parse-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sparrow-parse
-Version: 0.1.10
+Version: 0.2.1
 Summary: Sparrow Parse is a Python package for parsing and extracting information from documents.
 Home-page: https://github.com/katanaml/sparrow/tree/main/sparrow-data/parse
 License: GPL-3.0
@@ -29,7 +29,7 @@ Description-Content-Type: text/markdown
 
 ## Description
 
-This module implements Sparrow Parse [library](https://pypi.org/project/sparrow-parse/) with helpful methods for data pre-processing.
+This module implements Sparrow Parse [library](https://pypi.org/project/sparrow-parse/) with helpful methods for data pre-processing, parsing and extracting information.
 
 ## Install
 
@@ -37,21 +37,96 @@ This module implements Sparrow Parse [library](https://pypi.org/project/sparrow-
 pip install sparrow-parse
 ```
 
-## Use
+## Pre-processing
 
-Import
+### Unstructured
 
 ```
-from sparrow_parse.extractor.file_processor import FileProcessor
+from sparrow_parse.extractor.unstructured_processor import UnstructuredProcessor
+
+processor = UnstructuredProcessor()
+
+content, table_content = processor.extract_data(
+        file_path,  # file to process
+        strategy,  # data processing strategy supported by unstructured
+        model_name,  # model supported by unstructured
+        options,  # table extraction into HTML format
+        local,  # True if running from CLI, or False if running from FastAPI
+        debug)  # Debug
 ```
 
-Usage
+Example:
+
+*file_path* - `/Users/andrejb/infra/shared/katana-git/sparrow/sparrow-ml/llm/data/invoice_1.pdf`
+
+*strategy* - `hi_res`
+
+*model_name* - `yolox`
+
+*options* - `['tables', 'html']`
+
+*local* - `True`
+
+*debug* - `True`
+
+### Markdown
 
 ```
-processor = FileProcessor()
-content = processor.extract_data(file_path, strategy, model_name, options, local, debug)
+from sparrow_parse.extractor.markdown_processor import MarkdownProcessor
+
+processor = MarkdownProcessor()
+
+content, table_content = processor.extract_data(
+        file_path,  # file to process
+        options,  # table extraction into HTML format
+        local,  # True if running from CLI, or False if running from FastAPI
+        debug)  # Debug
 ```
 
+Example:
+
+*file_path* - `/Users/andrejb/infra/shared/katana-git/sparrow/sparrow-ml/llm/data/invoice_1.pdf`
+
+*options* - `['tables', 'markdown']`
+
+*local* - `True`
+
+*debug* - `True`
+
+## Parsing and extraction
+
+```
+from sparrow_parse.extractor.html_extractor import HTMLExtractor
+
+extractor = HTMLExtractor()
+
+answer, targets_unprocessed = extractor.read_data(
+        target_columns,  # list of table columns data to fetch
+        data, # list of HTML tables
+        column_keywords,  # list of valid column names, can be empty. Useful to filter junk content
+        group_by_rows,  # JSON result grouping
+        update_targets,  # Set to true, if page contains multiple tables with the same columns
+        local,  # True if running from CLI, or False if running from FastAPI
+        debug)  # Debug
+
+```
+
+Example:
+
+*target_columns* - `['description', 'qty', 'net_price', 'net_worth', 'vat', 'gross_worth']`
+
+*data* - `list of HTML tables`
+
+*column_keywords* - `None`
+
+*group_by_rows* - `True`
+
+*update_targets* - `True`
+
+*local* - `True`
+
+*debug* - `True`
+
 ## Library build
 
 ```

{sparrow_parse-0.1.10 → sparrow_parse-0.2.1}/README.md

@@ -2,7 +2,7 @@
 
 ## Description
 
-This module implements Sparrow Parse [library](https://pypi.org/project/sparrow-parse/) with helpful methods for data pre-processing.
+This module implements Sparrow Parse [library](https://pypi.org/project/sparrow-parse/) with helpful methods for data pre-processing, parsing and extracting information.
 
 ## Install
 
@@ -10,21 +10,96 @@ This module implements Sparrow Parse [library](https://pypi.org/project/sparrow-
 pip install sparrow-parse
 ```
 
-## Use
+## Pre-processing
 
-Import
+### Unstructured
 
 ```
-from sparrow_parse.extractor.file_processor import FileProcessor
+from sparrow_parse.extractor.unstructured_processor import UnstructuredProcessor
+
+processor = UnstructuredProcessor()
+
+content, table_content = processor.extract_data(
+        file_path,  # file to process
+        strategy,  # data processing strategy supported by unstructured
+        model_name,  # model supported by unstructured
+        options,  # table extraction into HTML format
+        local,  # True if running from CLI, or False if running from FastAPI
+        debug)  # Debug
 ```
 
-Usage
+Example:
+
+*file_path* - `/Users/andrejb/infra/shared/katana-git/sparrow/sparrow-ml/llm/data/invoice_1.pdf`
+
+*strategy* - `hi_res`
+
+*model_name* - `yolox`
+
+*options* - `['tables', 'html']`
+
+*local* - `True`
+
+*debug* - `True`
+
+### Markdown
 
 ```
-processor = FileProcessor()
-content = processor.extract_data(file_path, strategy, model_name, options, local, debug)
+from sparrow_parse.extractor.markdown_processor import MarkdownProcessor
+
+processor = MarkdownProcessor()
+
+content, table_content = processor.extract_data(
+        file_path,  # file to process
+        options,  # table extraction into HTML format
+        local,  # True if running from CLI, or False if running from FastAPI
+        debug)  # Debug
 ```
 
+Example:
+
+*file_path* - `/Users/andrejb/infra/shared/katana-git/sparrow/sparrow-ml/llm/data/invoice_1.pdf`
+
+*options* - `['tables', 'markdown']`
+
+*local* - `True`
+
+*debug* - `True`
+
+## Parsing and extraction
+
+```
+from sparrow_parse.extractor.html_extractor import HTMLExtractor
+
+extractor = HTMLExtractor()
+
+answer, targets_unprocessed = extractor.read_data(
+        target_columns,  # list of table columns data to fetch
+        data, # list of HTML tables
+        column_keywords,  # list of valid column names, can be empty. Useful to filter junk content
+        group_by_rows,  # JSON result grouping
+        update_targets,  # Set to true, if page contains multiple tables with the same columns
+        local,  # True if running from CLI, or False if running from FastAPI
+        debug)  # Debug
+
+```
+
+Example:
+
+*target_columns* - `['description', 'qty', 'net_price', 'net_worth', 'vat', 'gross_worth']`
+
+*data* - `list of HTML tables`
+
+*column_keywords* - `None`
+
+*group_by_rows* - `True`
+
+*update_targets* - `True`
+
+*local* - `True`
+
+*debug* - `True`
+
 ## Library build
 
 ```

{sparrow_parse-0.1.10 → sparrow_parse-0.2.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sparrow-parse"
-version = "0.1.10"
+version = "0.2.1"
 description = "Sparrow Parse is a Python package for parsing and extracting information from documents."
 authors = ["Andrej Baranovskij <andrejus.baranovskis@gmail.com>"]
 license = "GPL-3.0"

sparrow_parse-0.2.1/sparrow_parse/__init__.py

@@ -0,0 +1 @@
+__version__ = '0.2.1'

{sparrow_parse-0.1.10 → sparrow_parse-0.2.1}/sparrow_parse/extractor/html_extractor.py

@@ -3,8 +3,8 @@ from sentence_transformers import SentenceTransformer, util
 from bs4 import BeautifulSoup
 import json
 from rich.progress import Progress, SpinnerColumn, TextColumn
-from extractor_helper import merge_html_table_headers
-from extractor_helper import clean_html_table_header_names
+from .extractor_helper import merge_html_table_headers
+from .extractor_helper import clean_html_table_header_names
 import re
 
 
@@ -221,7 +221,10 @@ class HTMLExtractor(object):
 
 
 if __name__ == "__main__":
-    # with open('../data/invoice_1_table.txt', 'r') as file:
+    # to run for debugging, navigate to sparrow_parse and run the following command:
+    # python -m extractor.html_extractor
+
+    # with open('data/invoice_1_table.txt', 'r') as file:
     #     file_content = file.read()
     #
     # file_content = file_content.strip()[1:-1].strip()

{sparrow_parse-0.1.10 → sparrow_parse-0.2.1}/sparrow_parse/extractor/markdown_processor.py

@@ -128,18 +128,10 @@ class MarkdownProcessor(object):
 
 if __name__ == "__main__":
     processor = MarkdownProcessor()
+
     # content, table_content = processor.extract_data(
     #     '/Users/andrejb/infra/shared/katana-git/sparrow/sparrow-ml/llm/data/invoice_1.pdf',
     #     ['tables', 'markdown'],
     #     True,
     #     True)
-    content, table_content = processor.extract_data(
-        '/Users/andrejb/Documents/work/schreiber/invoice_data/test/2618407.pdf',
-        ['tables', 'markdown'],
-        True,
-        True)
-    # content, table_content = processor.extract_data(
-    #     '/Users/andrejb/Documents/work/epik/bankstatement/POSB_2_1.pdf',
-    #     ['tables', 'markdown'],
-    #     True,
-    #     True)
+

{sparrow_parse-0.1.10 → sparrow_parse-0.2.1}/sparrow_parse/temp.py

@@ -13,4 +13,16 @@
     #     'yolox',
     #     ['tables', 'html'],
     #     True,
-    #     True)
+    #     True)
+
+
+# content, table_content = processor.extract_data(
+#     '/Users/andrejb/Documents/work/schreiber/invoice_data/test/2618407.pdf',
+#     ['tables', 'markdown'],
+#     True,
+#     True)
+# content, table_content = processor.extract_data(
+#     '/Users/andrejb/Documents/work/epik/bankstatement/POSB_2_1.pdf',
+#     ['tables', 'markdown'],
+#     True,
+#     True)

sparrow_parse-0.1.10/sparrow_parse/__init__.py

@@ -1 +0,0 @@
-__version__ = '0.1.10'