sparrow-parse 0.3.0__tar.gz → 0.3.2__tar.gz

{sparrow_parse-0.3.0 → sparrow-parse-0.3.2}/PKG-INFO

@@ -1,30 +1,20 @@
 Metadata-Version: 2.1
 Name: sparrow-parse
-Version: 0.3.0
+Version: 0.3.2
 Summary: Sparrow Parse is a Python package for parsing and extracting information from documents.
 Home-page: https://github.com/katanaml/sparrow/tree/main/sparrow-data/parse
-License: GPL-3.0
-Keywords: llm,rag,vision
 Author: Andrej Baranovskij
 Author-email: andrejus.baranovskis@gmail.com
-Requires-Python: >=3.9,<3.12
-Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
+License: UNKNOWN
+Project-URL: Homepage, https://github.com/katanaml/sparrow/tree/main/sparrow-data/parse
+Project-URL: Repository, https://github.com/katanaml/sparrow
+Keywords: llm,rag,vision
+Platform: UNKNOWN
 Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
+Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: Topic :: Software Development
-Requires-Dist: PyPDF2 (==3.0.1)
-Requires-Dist: numpy (==1.26.4)
-Requires-Dist: pymupdf4llm (==0.0.6)
-Requires-Dist: rich (>=13.7.1,<14.0.0)
-Requires-Dist: sentence-transformers (==3.0.1)
-Requires-Dist: torch (==2.2.2)
-Requires-Dist: transformers (==4.41.2)
-Requires-Dist: unstructured-inference (==0.7.33)
-Requires-Dist: unstructured[all-docs] (==0.14.5)
-Project-URL: Repository, https://github.com/katanaml/sparrow
+Classifier: Programming Language :: Python :: 3.10
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 
 # Sparrow Parse
@@ -152,14 +142,30 @@ Example:
 
 ## Library build
 
+Create Python virtual environment
+
 ```
-poetry build
+python -m venv .env_sparrow_parse
 ```
 
-Publish to PyPi
+Install Python libraries
 
 ```
-poetry publish
+pip install -r requirements.txt
+```
+
+Build package
+
+```
+pip install setuptools wheel
+python setup.py sdist bdist_wheel
+```
+
+Upload to PyPI
+
+```
+pip install twine
+twine upload dist/*
 ```
 
 ## Commercial usage
@@ -180,3 +186,4 @@ If your organization is seeking to utilize Sparrow under a proprietary license,
 
 Licensed under the GPL 3.0. Copyright 2020-2024 Katana ML, Andrej Baranovskij. [Copy of the license](https://github.com/katanaml/sparrow/blob/main/LICENSE).
 
+

{sparrow_parse-0.3.0 → sparrow-parse-0.3.2}/README.md

@@ -123,14 +123,30 @@ Example:
 
 ## Library build
 
+Create Python virtual environment
+
+```
+python -m venv .env_sparrow_parse
+```
+
+Install Python libraries
+
+```
+pip install -r requirements.txt
+```
+
+Build package
+
 ```
-poetry build
+pip install setuptools wheel
+python setup.py sdist bdist_wheel
 ```
 
-Publish to PyPi
+Upload to PyPI
 
 ```
-poetry publish
+pip install twine
+twine upload dist/*
 ```
 
 ## Commercial usage

sparrow-parse-0.3.2/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

sparrow-parse-0.3.2/setup.py

@@ -0,0 +1,37 @@
+from setuptools import setup, find_packages
+
+with open("README.md", "r", encoding="utf-8") as fh:
+    long_description = fh.read()
+
+with open("requirements.txt", "r", encoding="utf-8") as fh:
+    requirements = fh.read().splitlines()
+
+setup(
+    name="sparrow-parse",
+    version="0.3.2",
+    author="Andrej Baranovskij",
+    author_email="andrejus.baranovskis@gmail.com",
+    description="Sparrow Parse is a Python package for parsing and extracting information from documents.",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/katanaml/sparrow/tree/main/sparrow-data/parse",
+    project_urls={
+        "Homepage": "https://github.com/katanaml/sparrow/tree/main/sparrow-data/parse",
+        "Repository": "https://github.com/katanaml/sparrow",
+    },
+    classifiers=[
+        "Operating System :: OS Independent",
+        "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+        "Topic :: Software Development",
+        "Programming Language :: Python :: 3.10",
+    ],
+    entry_points={
+        'console_scripts': [
+            'sparrow-parse=sparrow_parse:main',
+        ],
+    },
+    keywords="llm, rag, vision",
+    packages=find_packages(),
+    python_requires='>=3.10',
+    install_requires=requirements,
+)

sparrow-parse-0.3.2/sparrow_parse/__init__.py

@@ -0,0 +1 @@
+__version__ = '0.3.2'

{sparrow_parse-0.3.0 → sparrow-parse-0.3.2}/sparrow_parse/extractor/pdf_optimizer.py

@@ -1,4 +1,4 @@
-import PyPDF2
+import pypdf
 from pdf2image import convert_from_path
 import os
 import tempfile
@@ -17,12 +17,12 @@ class PDFOptimizer(object):
         if not convert_to_images:
             # Open the PDF file
             with open(file_path, 'rb') as pdf_file:
-                reader = PyPDF2.PdfReader(pdf_file)
+                reader = pypdf.PdfReader(pdf_file)
                 number_of_pages = len(reader.pages)
 
                 # Split the PDF into separate files per page
                 for page_num in range(number_of_pages):
-                    writer = PyPDF2.PdfWriter()
+                    writer = pypdf.PdfWriter()
                     writer.add_page(reader.pages[page_num])
 
                     output_filename = os.path.join(temp_dir, f'page_{page_num + 1}.pdf')

sparrow-parse-0.3.2/sparrow_parse.egg-info/PKG-INFO

@@ -0,0 +1,189 @@
+Metadata-Version: 2.1
+Name: sparrow-parse
+Version: 0.3.2
+Summary: Sparrow Parse is a Python package for parsing and extracting information from documents.
+Home-page: https://github.com/katanaml/sparrow/tree/main/sparrow-data/parse
+Author: Andrej Baranovskij
+Author-email: andrejus.baranovskis@gmail.com
+License: UNKNOWN
+Project-URL: Homepage, https://github.com/katanaml/sparrow/tree/main/sparrow-data/parse
+Project-URL: Repository, https://github.com/katanaml/sparrow
+Keywords: llm,rag,vision
+Platform: UNKNOWN
+Classifier: Operating System :: OS Independent
+Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
+Classifier: Topic :: Software Development
+Classifier: Programming Language :: Python :: 3.10
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# Sparrow Parse
+
+## Description
+
+This module implements Sparrow Parse [library](https://pypi.org/project/sparrow-parse/) with helpful methods for data pre-processing, parsing and extracting information.
+
+## Install
+
+```
+pip install sparrow-parse
+```
+
+## Pre-processing
+
+### Unstructured
+
+```
+from sparrow_parse.extractor.unstructured_processor import UnstructuredProcessor
+
+processor = UnstructuredProcessor()
+
+content, table_content = processor.extract_data(
+        file_path,  # file to process
+        strategy,  # data processing strategy supported by unstructured
+        model_name,  # model supported by unstructured
+        options,  # table extraction into HTML format
+        local,  # True if running from CLI, or False if running from FastAPI
+        debug)  # Debug
+```
+
+Example:
+
+*file_path* - `/Users/andrejb/infra/shared/katana-git/sparrow/sparrow-ml/llm/data/invoice_1.pdf`
+
+*strategy* - `hi_res`
+
+*model_name* - `yolox`
+
+*options* - `['tables', 'unstructured']`
+
+*local* - `True`
+
+*debug* - `True`
+
+### Markdown
+
+```
+from sparrow_parse.extractor.markdown_processor import MarkdownProcessor
+
+processor = MarkdownProcessor()
+
+content, table_content = processor.extract_data(
+        file_path,  # file to process
+        options,  # table extraction into HTML format
+        local,  # True if running from CLI, or False if running from FastAPI
+        debug)  # Debug
+```
+
+Example:
+
+*file_path* - `/Users/andrejb/infra/shared/katana-git/sparrow/sparrow-ml/llm/data/invoice_1.pdf`
+
+*options* - `['tables', 'markdown']`
+
+*local* - `True`
+
+*debug* - `True`
+
+## Parsing and extraction
+
+```
+from sparrow_parse.extractor.html_extractor import HTMLExtractor
+
+extractor = HTMLExtractor()
+
+answer, targets_unprocessed = extractor.read_data(
+        target_columns,  # list of table columns data to fetch
+        data, # list of HTML tables
+        column_keywords,  # list of valid column names, can be empty. Useful to filter junk content
+        group_by_rows,  # JSON result grouping
+        update_targets,  # Set to true, if page contains multiple tables with the same columns
+        local,  # True if running from CLI, or False if running from FastAPI
+        debug)  # Debug
+
+```
+
+Example:
+
+*target_columns* - `['description', 'qty', 'net_price', 'net_worth', 'vat', 'gross_worth']`
+
+*data* - `list of HTML tables`
+
+*column_keywords* - `None`
+
+*group_by_rows* - `True`
+
+*update_targets* - `True`
+
+*local* - `True`
+
+*debug* - `True`
+
+## PDF optimization
+
+```
+from sparrow_parse.extractor.pdf_optimizer import PDFOptimizer
+
+pdf_optimizer = PDFOptimizer()
+
+num_pages, output_files, temp_dir = pdf_optimizer.split_pdf_to_pages(file_path,
+                                                                     output_directory,
+                                                                     convert_to_images)
+
+```
+
+Example:
+
+*file_path* - `/Users/andrejb/infra/shared/katana-git/sparrow/sparrow-ml/llm/data/invoice_1.pdf`
+
+*output_directory* - set to not `None`, for debug purposes only
+
+*convert_to_images* - default `False`, to split into PDF files
+
+## Library build
+
+Create Python virtual environment
+
+```
+python -m venv .env_sparrow_parse
+```
+
+Install Python libraries
+
+```
+pip install -r requirements.txt
+```
+
+Build package
+
+```
+pip install setuptools wheel
+python setup.py sdist bdist_wheel
+```
+
+Upload to PyPI
+
+```
+pip install twine
+twine upload dist/*
+```
+
+## Commercial usage
+
+Sparrow is available under the GPL 3.0 license, promoting freedom to use, modify, and distribute the software while ensuring any modifications remain open source under the same license. This aligns with our commitment to supporting the open-source community and fostering collaboration.
+
+Additionally, we recognize the diverse needs of organizations, including small to medium-sized enterprises (SMEs). Therefore, Sparrow is also offered for free commercial use to organizations with gross revenue below $5 million USD in the past 12 months, enabling them to leverage Sparrow without the financial burden often associated with high-quality software solutions.
+
+For businesses that exceed this revenue threshold or require usage terms not accommodated by the GPL 3.0 license—such as integrating Sparrow into proprietary software without the obligation to disclose source code modifications—we offer dual licensing options. Dual licensing allows Sparrow to be used under a separate proprietary license, offering greater flexibility for commercial applications and proprietary integrations. This model supports both the project's sustainability and the business's needs for confidentiality and customization.
+
+If your organization is seeking to utilize Sparrow under a proprietary license, or if you are interested in custom workflows, consulting services, or dedicated support and maintenance options, please contact us at abaranovskis@redsamuraiconsulting.com. We're here to provide tailored solutions that meet your unique requirements, ensuring you can maximize the benefits of Sparrow for your projects and workflows.
+
+## Author
+
+[Katana ML](https://katanaml.io), [Andrej Baranovskij](https://github.com/abaranovskis-redsamurai)
+
+## License
+
+Licensed under the GPL 3.0. Copyright 2020-2024 Katana ML, Andrej Baranovskij. [Copy of the license](https://github.com/katanaml/sparrow/blob/main/LICENSE).
+
+

sparrow-parse-0.3.2/sparrow_parse.egg-info/SOURCES.txt

@@ -0,0 +1,17 @@
+README.md
+setup.py
+sparrow_parse/__init__.py
+sparrow_parse/__main__.py
+sparrow_parse/temp.py
+sparrow_parse.egg-info/PKG-INFO
+sparrow_parse.egg-info/SOURCES.txt
+sparrow_parse.egg-info/dependency_links.txt
+sparrow_parse.egg-info/entry_points.txt
+sparrow_parse.egg-info/requires.txt
+sparrow_parse.egg-info/top_level.txt
+sparrow_parse/extractor/__init__.py
+sparrow_parse/extractor/extractor_helper.py
+sparrow_parse/extractor/html_extractor.py
+sparrow_parse/extractor/markdown_processor.py
+sparrow_parse/extractor/pdf_optimizer.py
+sparrow_parse/extractor/unstructured_processor.py

sparrow-parse-0.3.2/sparrow_parse.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

sparrow-parse-0.3.2/sparrow_parse.egg-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+sparrow-parse = sparrow_parse:main
+

sparrow-parse-0.3.2/sparrow_parse.egg-info/requires.txt

@@ -0,0 +1,9 @@
+torch==2.2.2
+unstructured[all-docs]==0.14.5
+unstructured-inference==0.7.33
+rich
+pymupdf4llm==0.0.9
+transformers==4.41.2
+sentence-transformers==3.0.1
+numpy==1.26.4
+pypdf==4.3.0

sparrow-parse-0.3.2/sparrow_parse.egg-info/top_level.txt

@@ -0,0 +1 @@
+sparrow_parse

sparrow_parse-0.3.0/pyproject.toml

@@ -1,41 +0,0 @@
-[tool.poetry]
-name = "sparrow-parse"
-version = "0.3.0"
-description = "Sparrow Parse is a Python package for parsing and extracting information from documents."
-authors = ["Andrej Baranovskij <andrejus.baranovskis@gmail.com>"]
-license = "GPL-3.0"
-readme = "README.md"
-homepage = "https://github.com/katanaml/sparrow/tree/main/sparrow-data/parse"
-repository = "https://github.com/katanaml/sparrow"
-keywords = ["llm", "rag", "vision"]
-classifiers = [
-    "Operating System :: OS Independent",
-    "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
-    "Topic :: Software Development",
-    "Programming Language :: Python :: 3.10",
-]
-include = [
-    "LICENSE",
-]
-
-
-[tool.poetry.dependencies]
-python = ">=3.9,<3.12"
-torch = {version = "2.2.2", source = "pypi"}
-unstructured = {version = "0.14.5", extras = ["all-docs"]}
-unstructured-inference = "0.7.33"
-rich = "^13.7.1"
-pymupdf4llm = "0.0.6"
-transformers = "4.41.2"
-sentence-transformers = "3.0.1"
-numpy = "1.26.4"
-PyPDF2 = "3.0.1"
-
-
-[tool.poetry.scripts]
-sparrow-parse = 'sparrow_parse:main'
-
-
-[build-system]
-requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"

sparrow_parse-0.3.0/sparrow_parse/__init__.py

@@ -1 +0,0 @@
-__version__ = '0.3.0'

sparrow_parse-0.3.0/sparrow_parse/data/invoice_1_table.txt

@@ -1,9 +0,0 @@
-[
-    '<table><thead><th>No.</th><th>Description</th><th>Qty</th><th>UM</th><th>Net price</th><th>Net worth</th><th>VAT [%]</th><th>Gross worth</th></thead><tr><td></td><td>Wine Glasses Goblets Pair Clear
-Glass</td><td>5,00</td><td>eacn</td><td>12,00</td><td>60,00</td><td>10%</td><td>66,00</td></tr><tr><td></td><td>With Hooks Stemware Storage Multiple Uses Iron Wine Rack Hanging
-Glass</td><td>4,00</td><td>eacn</td><td>28,08</td><td>112,32</td><td>10%</td><td>123,55</td></tr><tr><td></td><td>Replacement Corkscrew Parts Spiral Worm Wine Opener Bottle
-Houdini</td><td>1,00</td><td>eacn</td><td>7,50</td><td>7,50</td><td>10%</td><td>8,25</td></tr><tr><td></td><td>HOME ESSENTIALS GRADIENT STEMLESS WINE GLASSES SET OF 4 20 FL OZ (591 ml)
-NEW</td><td>1,00</td><td>eacn</td><td>12,99</td><td>12,99</td><td>10%</td><td>14,29</td></tr></table>',
-    '<table><thead><th>VAT</th><th>[%]</th><th>Net worth</th><th>VAT</th><th>Gross worth</th></thead><tr><td></td><td>10%</td><td>192,81</td><td>19,28</td><td>212,09</td></tr><tr><td colspan="2">Total</td><td>$ 192,81</td><td>$
-19,28</td><td>$ 212,09</td></tr></table>'
-]