docling 0.1.0__py3-none-any.whl

docling/utils/utils.py

@@ -0,0 +1,41 @@
+import hashlib
+from io import BytesIO
+from itertools import islice
+from pathlib import Path
+from typing import List, Union
+
+
+def chunkify(iterator, chunk_size):
+    """Yield successive chunks of chunk_size from the iterable."""
+    if isinstance(iterator, List):
+        iterator = iter(iterator)
+    for first in iterator:  # Take the first element from the iterator
+        yield [first] + list(islice(iterator, chunk_size - 1))
+
+
+def create_file_hash(path_or_stream: Union[BytesIO, Path]) -> str:
+    """Create a stable page_hash of the path_or_stream of a file"""
+
+    block_size = 65536
+    hasher = hashlib.sha256()
+
+    def _hash_buf(binary_stream):
+        buf = binary_stream.read(block_size)  # read and page_hash in chunks
+        while len(buf) > 0:
+            hasher.update(buf)
+            buf = binary_stream.read(block_size)
+
+    if isinstance(path_or_stream, Path):
+        with path_or_stream.open("rb") as afile:
+            _hash_buf(afile)
+    elif isinstance(path_or_stream, BytesIO):
+        _hash_buf(path_or_stream)
+
+    return hasher.hexdigest()
+
+
+def create_hash(string: str):
+    hasher = hashlib.sha256()
+    hasher.update(string.encode("utf-8"))
+
+    return hasher.hexdigest()

docling-0.1.0.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) [year] [fullname]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

docling-0.1.0.dist-info/METADATA

@@ -0,0 +1,130 @@
+Metadata-Version: 2.1
+Name: docling
+Version: 0.1.0
+Summary: Docling PDF conversion package
+License: MIT
+Keywords: docling,convert,document,pdf,layout model,segmentation,table structure,table former
+Author: Christoph Auer
+Author-email: cau@zurich.ibm.com
+Requires-Python: >=3.11,<4.0
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Dist: deepsearch-glm (>=0.18.4,<1)
+Requires-Dist: deepsearch-toolkit (>=0.47.0,<1)
+Requires-Dist: docling-core (>=0.2.0,<0.3.0)
+Requires-Dist: docling-ibm-models (>=0.2.0,<0.3.0)
+Requires-Dist: filetype (>=1.2.0,<2.0.0)
+Requires-Dist: huggingface_hub (>=0.23,<1)
+Requires-Dist: pydantic (>=2.0.0,<3.0.0)
+Requires-Dist: pydantic-settings (>=2.3.0,<3.0.0)
+Requires-Dist: pypdfium2 (>=4.30.0,<5.0.0)
+Description-Content-Type: text/markdown
+
+<p align="center">
+  <a href="https://github.com/ds4sd/docling"> <img loading="lazy" alt="Docling" src="logo.png" width="150" /> </a>
+</p>
+
+# Docling
+
+Dockling bundles PDF document conversion to JSON and Markdown in an easy, self-contained package.
+
+## Features
+* ⚡ Converts any PDF document to JSON or Markdown format, stable and lightning fast
+* 📑 Understands detailed page layout, reading order and recovers table structures
+* 📝 Extracts metadata from the document, such as title, authors, references and language
+* 🔍 Optionally applies OCR (use with scanned PDFs)
+
+## Setup
+
+You need Python 3.11 and poetry. Install poetry from [here](https://python-poetry.org/docs/#installing-with-the-official-installer).
+
+Once you have `poetry` installed, create an environment and install the package:
+
+```bash
+poetry env use $(which python3.11)
+poetry shell
+poetry install
+```
+
+**Notes**:
+* Works on macOS and Linux environments. Windows platforms are currently not tested.
+
+
+## Usage
+
+For basic usage, see the [convert.py](examples/convert.py) example module. Run with:
+
+```
+python examples/convert.py
+```
+The output of the above command will be written to `./scratch`.
+
+### Enable or disable pipeline features
+
+You can control if table structure recognition or OCR should be performed by arguments passed to `DocumentConverter` 
+```python
+doc_converter = DocumentConverter(
+    artifacts_path=artifacts_path,
+    pipeline_options=PipelineOptions(do_table_structure=False, # Controls if table structure is recovered. 
+                                     do_ocr=True), # Controls if OCR is applied (ignores programmatic content)
+)
+```
+
+### Impose limits on the document size
+
+You can limit the file size and number of pages which should be allowed to process per document.
+```python
+paths = [Path("./test/data/2206.01062.pdf")]
+
+input = DocumentConversionInput.from_paths(
+    paths, limits=DocumentLimits(max_num_pages=100, max_file_size=20971520)
+)
+```
+
+### Convert from binary PDF streams 
+
+You can convert PDFs from a binary stream instead of from the filesystem as follows:
+```python
+buf = BytesIO(your_binary_stream)
+docs = [DocumentStream(filename="my_doc.pdf", stream=buf)]
+input = DocumentConversionInput.from_streams(docs)
+converted_docs = doc_converter.convert(input)
+```
+### Limit resource usage
+
+You can limit the CPU threads used by `docling` by setting the environment variable `OMP_NUM_THREADS` accordingly. The default setting is using 4 CPU threads.
+
+
+## Contributing
+
+Please read [Contributing to Docling](./CONTRIBUTING.md) for details.
+
+
+## References
+
+If you use `Docling` in your projects, please consider citing the following:
+
+```bib
+@software{Docling,
+author = {Deep Search Team},
+month = {7},
+title = {{Docling}},
+url = {https://github.com/DS4SD/docling},
+version = {main},
+year = {2024}
+}
+```
+
+## License
+
+The `Docling` codebase is under MIT license.
+For individual model usage, please refer to the model licenses found in the original packages.
+

docling-0.1.0.dist-info/RECORD

@@ -0,0 +1,25 @@
+docling/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+docling/backend/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+docling/backend/abstract_backend.py,sha256=dINr8oTax9Fq31Y1AR0CGWNZtAHN5aqB_M7TAPkJNVQ,1122
+docling/backend/pypdfium2_backend.py,sha256=sJMoActFyc3qdKB6RFly3auHXuXM4noQAG0ypUlj26o,7647
+docling/datamodel/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+docling/datamodel/base_models.py,sha256=UUbimBhbRQSIM5pTsfQmSNi6NoNM5U03Vc0htHDkFZY,6132
+docling/datamodel/document.py,sha256=JIp4TRl9NrVLXwNU-9llkbrFGUKly9B2pwzXaH0GEsE,12615
+docling/datamodel/settings.py,sha256=t5g6wrEJnPa9gBzMMl8ppgBRUYz-8xgopEtfMS0ZH28,733
+docling/document_converter.py,sha256=MZw23oPlRmRi1ggzoD1PukUnqo-6boO3RZB06dZ5Xt0,7305
+docling/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+docling/models/ds_glm_model.py,sha256=wmb--2JKFQby-kvidw6PyM8wURPXYPQ_Z_eKKCBAdYQ,3192
+docling/models/easyocr_model.py,sha256=NaHVs8IN0eW9KB076E2Kae1s-bq74_4IMWueze9QqtE,2290
+docling/models/layout_model.py,sha256=4AfPFiu6pXc8wIQ1sQlEZnHRt7SnBmfzDdctiRveOWw,10944
+docling/models/page_assemble_model.py,sha256=jhjQt0NOkVi-dWBaovJ2KsBim5FF6e47y21uZ8EWfBg,5906
+docling/models/table_structure_model.py,sha256=uvkK2NPvltk9-zScbORUA05JbvymkGX6Dfsal4wLwsI,4103
+docling/pipeline/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+docling/pipeline/base_model_pipeline.py,sha256=ozHdJak0yQAxQf7pQN_C480vI35A2e5KL5Qq1xSkq5c,560
+docling/pipeline/standard_model_pipeline.py,sha256=pDbgVO0oOJry7Q-3KYdMuaypXCQOdoVikR80veizo9o,1489
+docling/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+docling/utils/layout_utils.py,sha256=FOFbL0hKzUoWXdZaeUvEtFqKv0IkPifIr4sdGW4suKs,31804
+docling/utils/utils.py,sha256=llhXSbIDNZ1MHOwBEfLHBAoJIAYI7QlPIonlI1jLUJ0,1208
+docling-0.1.0.dist-info/LICENSE,sha256=ACwmltkrXIz5VsEQcrqljq-fat6ZXAMepjXGoe40KtE,1069
+docling-0.1.0.dist-info/METADATA,sha256=TYMDWosLJnMmi8cDMck1L0uBiCwXtRBTeM6oBFAl8Ls,4228
+docling-0.1.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+docling-0.1.0.dist-info/RECORD,,

docling-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 1.9.0
+Root-Is-Purelib: true
+Tag: py3-none-any