docling 1.19.1__py3-none-any.whl → 2.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- docling/backend/abstract_backend.py +32 -37
- docling/backend/docling_parse_backend.py +16 -12
- docling/backend/docling_parse_v2_backend.py +240 -0
- docling/backend/html_backend.py +425 -0
- docling/backend/mspowerpoint_backend.py +375 -0
- docling/backend/msword_backend.py +509 -0
- docling/backend/pdf_backend.py +78 -0
- docling/backend/pypdfium2_backend.py +15 -10
- docling/cli/main.py +61 -60
- docling/datamodel/base_models.py +73 -193
- docling/datamodel/document.py +379 -324
- docling/datamodel/pipeline_options.py +16 -0
- docling/datamodel/settings.py +1 -0
- docling/document_converter.py +215 -252
- docling/models/base_model.py +25 -0
- docling/models/base_ocr_model.py +19 -6
- docling/models/ds_glm_model.py +220 -22
- docling/models/easyocr_model.py +45 -40
- docling/models/layout_model.py +130 -114
- docling/models/page_assemble_model.py +119 -95
- docling/models/page_preprocessing_model.py +61 -0
- docling/models/table_structure_model.py +122 -111
- docling/models/tesseract_ocr_cli_model.py +63 -56
- docling/models/tesseract_ocr_model.py +58 -50
- docling/pipeline/base_pipeline.py +190 -0
- docling/pipeline/simple_pipeline.py +59 -0
- docling/pipeline/standard_pdf_pipeline.py +198 -0
- docling/utils/export.py +4 -3
- docling/utils/layout_utils.py +17 -11
- docling-2.1.0.dist-info/METADATA +149 -0
- docling-2.1.0.dist-info/RECORD +42 -0
- docling/pipeline/base_model_pipeline.py +0 -18
- docling/pipeline/standard_model_pipeline.py +0 -66
- docling-1.19.1.dist-info/METADATA +0 -380
- docling-1.19.1.dist-info/RECORD +0 -34
- {docling-1.19.1.dist-info → docling-2.1.0.dist-info}/LICENSE +0 -0
- {docling-1.19.1.dist-info → docling-2.1.0.dist-info}/WHEEL +0 -0
- {docling-1.19.1.dist-info → docling-2.1.0.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: docling
|
|
3
|
+
Version: 2.1.0
|
|
4
|
+
Summary: Docling PDF conversion package
|
|
5
|
+
Home-page: https://github.com/DS4SD/docling
|
|
6
|
+
License: MIT
|
|
7
|
+
Keywords: docling,convert,document,pdf,layout model,segmentation,table structure,table former
|
|
8
|
+
Author: Christoph Auer
|
|
9
|
+
Author-email: cau@zurich.ibm.com
|
|
10
|
+
Requires-Python: >=3.10,<4.0
|
|
11
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Operating System :: MacOS :: MacOS X
|
|
16
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
+
Provides-Extra: tesserocr
|
|
23
|
+
Requires-Dist: beautifulsoup4 (>=4.12.3,<5.0.0)
|
|
24
|
+
Requires-Dist: certifi (>=2024.7.4)
|
|
25
|
+
Requires-Dist: deepsearch-glm (>=0.25.0,<0.26.0)
|
|
26
|
+
Requires-Dist: docling-core (>=2.0.0,<3.0.0)
|
|
27
|
+
Requires-Dist: docling-ibm-models (>=2.0.1,<3.0.0)
|
|
28
|
+
Requires-Dist: docling-parse (>=1.6.0,<2.0.0)
|
|
29
|
+
Requires-Dist: easyocr (>=1.7,<2.0)
|
|
30
|
+
Requires-Dist: filetype (>=1.2.0,<2.0.0)
|
|
31
|
+
Requires-Dist: huggingface_hub (>=0.23,<1)
|
|
32
|
+
Requires-Dist: pandas (>=2.1.4,<3.0.0)
|
|
33
|
+
Requires-Dist: pyarrow (>=16.1.0,<17.0.0)
|
|
34
|
+
Requires-Dist: pydantic (>=2.0.0,<3.0.0)
|
|
35
|
+
Requires-Dist: pydantic-settings (>=2.3.0,<3.0.0)
|
|
36
|
+
Requires-Dist: pypdfium2 (>=4.30.0,<5.0.0)
|
|
37
|
+
Requires-Dist: python-docx (>=1.1.2,<2.0.0)
|
|
38
|
+
Requires-Dist: python-pptx (>=1.0.2,<2.0.0)
|
|
39
|
+
Requires-Dist: requests (>=2.32.3,<3.0.0)
|
|
40
|
+
Requires-Dist: rtree (>=1.3.0,<2.0.0)
|
|
41
|
+
Requires-Dist: scipy (>=1.14.1,<2.0.0)
|
|
42
|
+
Requires-Dist: tesserocr (>=2.7.1,<3.0.0) ; extra == "tesserocr"
|
|
43
|
+
Requires-Dist: torch (>=2.2.2,<2.3.0) ; sys_platform == "darwin" and platform_machine == "x86_64"
|
|
44
|
+
Requires-Dist: torch (>=2.2.2,<3.0.0) ; sys_platform != "darwin" or platform_machine != "x86_64"
|
|
45
|
+
Requires-Dist: torchvision (>=0,<1) ; sys_platform != "darwin" or platform_machine != "x86_64"
|
|
46
|
+
Requires-Dist: torchvision (>=0.17.2,<0.18.0) ; sys_platform == "darwin" and platform_machine == "x86_64"
|
|
47
|
+
Requires-Dist: typer (>=0.12.5,<0.13.0)
|
|
48
|
+
Project-URL: Repository, https://github.com/DS4SD/docling
|
|
49
|
+
Description-Content-Type: text/markdown
|
|
50
|
+
|
|
51
|
+
<p align="center">
|
|
52
|
+
<a href="https://github.com/ds4sd/docling">
|
|
53
|
+
<img loading="lazy" alt="Docling" src="docs/assets/docling_processing.png" width="100%"/>
|
|
54
|
+
</a>
|
|
55
|
+
</p>
|
|
56
|
+
|
|
57
|
+
# Docling
|
|
58
|
+
|
|
59
|
+
[](https://arxiv.org/abs/2408.09869)
|
|
60
|
+
[](https://ds4sd.github.io/docling/)
|
|
61
|
+
[](https://pypi.org/project/docling/)
|
|
62
|
+
|
|
63
|
+
[](https://python-poetry.org/)
|
|
64
|
+
[](https://github.com/psf/black)
|
|
65
|
+
[](https://pycqa.github.io/isort/)
|
|
66
|
+
[](https://pydantic.dev)
|
|
67
|
+
[](https://github.com/pre-commit/pre-commit)
|
|
68
|
+
[](https://opensource.org/licenses/MIT)
|
|
69
|
+
|
|
70
|
+
Docling parses documents and exports them to the desired format with ease and speed.
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
## Features
|
|
74
|
+
|
|
75
|
+
* 🗂️ Multi-format support for input (PDF, DOCX etc.) & output (Markdown, JSON etc.)
|
|
76
|
+
* 📑 Advanced PDF document understanding incl. page layout, reading order & table structures
|
|
77
|
+
* 📝 Metadata extraction, including title, authors, references & language
|
|
78
|
+
* 🤖 Seamless LlamaIndex 🦙 & LangChain 🦜🔗 integration for powerful RAG / QA applications
|
|
79
|
+
* 🔍 OCR support for scanned PDFs
|
|
80
|
+
* 💻 Simple and convenient CLI
|
|
81
|
+
|
|
82
|
+
Explore the [documentation](https://ds4sd.github.io/docling/) to discover plenty examples and unlock the full power of Docling!
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
## Installation
|
|
86
|
+
|
|
87
|
+
To use Docling, simply install `docling` from your package manager, e.g. pip:
|
|
88
|
+
```bash
|
|
89
|
+
pip install docling
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
Works on macOS, Linux and Windows environments. Both x86_64 and arm64 architectures.
|
|
93
|
+
|
|
94
|
+
More [detailed installation instructions](https://ds4sd.github.io/docling/installation/) are available in the docs.
|
|
95
|
+
|
|
96
|
+
## Getting started
|
|
97
|
+
|
|
98
|
+
To convert individual documents, use `convert()`, for example:
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
from docling.document_converter import DocumentConverter
|
|
102
|
+
|
|
103
|
+
source = "https://arxiv.org/pdf/2408.09869" # PDF path or URL
|
|
104
|
+
converter = DocumentConverter()
|
|
105
|
+
result = converter.convert(source)
|
|
106
|
+
print(result.document.export_to_markdown()) # output: "## Docling Technical Report[...]"
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
Check out [Getting started](https://ds4sd.github.io/docling/).
|
|
111
|
+
You will find lots of tuning options to leverage all the advanced capabilities.
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
## Get help and support
|
|
115
|
+
|
|
116
|
+
Please feel free to connect with us using the [discussion section](https://github.com/DS4SD/docling/discussions).
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
## Technical report
|
|
120
|
+
|
|
121
|
+
For more details on Docling's inner workings, check out the [Docling Technical Report](https://arxiv.org/abs/2408.09869).
|
|
122
|
+
|
|
123
|
+
## Contributing
|
|
124
|
+
|
|
125
|
+
Please read [Contributing to Docling](https://github.com/DS4SD/docling/blob/main/CONTRIBUTING.md) for details.
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
## References
|
|
129
|
+
|
|
130
|
+
If you use Docling in your projects, please consider citing the following:
|
|
131
|
+
|
|
132
|
+
```bib
|
|
133
|
+
@techreport{Docling,
|
|
134
|
+
author = {Deep Search Team},
|
|
135
|
+
month = {8},
|
|
136
|
+
title = {Docling Technical Report},
|
|
137
|
+
url = {https://arxiv.org/abs/2408.09869},
|
|
138
|
+
eprint = {2408.09869},
|
|
139
|
+
doi = {10.48550/arXiv.2408.09869},
|
|
140
|
+
version = {1.0.0},
|
|
141
|
+
year = {2024}
|
|
142
|
+
}
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
## License
|
|
146
|
+
|
|
147
|
+
The Docling codebase is under MIT license.
|
|
148
|
+
For individual model usage, please refer to the model licenses found in the original packages.
|
|
149
|
+
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
docling/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
docling/backend/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
+
docling/backend/abstract_backend.py,sha256=8Lh1gf1P9AnzlwB989OVBgLmokTpfI0LxYRfuvYTqoo,1646
|
|
4
|
+
docling/backend/docling_parse_backend.py,sha256=UgBpopZIP5YkhwhybiqDnqVsSqv9DAAPFkafhfL0pPo,7623
|
|
5
|
+
docling/backend/docling_parse_v2_backend.py,sha256=VY7MsiyqjN3Vl0UkyezriiVJMLbLRrQVuKjWaTgIUwY,8336
|
|
6
|
+
docling/backend/html_backend.py,sha256=MlhEXaA0tgX_tLuQLnkex43gsKqpqHWnbkssxY4n_kc,14753
|
|
7
|
+
docling/backend/mspowerpoint_backend.py,sha256=2UYfMMeWwgDtvIKQELCA-bYv5Z-rGvbMiBNcidNL_uE,14332
|
|
8
|
+
docling/backend/msword_backend.py,sha256=4SDqZAZxLr6VV50OU3MRBAV8SwZMCyJCUbNVMVUpitc,17659
|
|
9
|
+
docling/backend/pdf_backend.py,sha256=unnw7QiRE1VXg6Pj-eYrtnFGrp5SSYiI324OlFxyv6c,2050
|
|
10
|
+
docling/backend/pypdfium2_backend.py,sha256=MJX6fQqwK3r967fyAAs-RA_YIkeQvhgsLkQAgaBTgaE,8995
|
|
11
|
+
docling/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
12
|
+
docling/cli/main.py,sha256=NRVGz0z-3EBwYNMJGVnLtDBcfOeutaUyYdkM0ymRnGA,8008
|
|
13
|
+
docling/datamodel/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
14
|
+
docling/datamodel/base_models.py,sha256=Ha-DoRZoksjHSZHWqUSiQ79MTBEfY5ur8U_LVtyBRYU,5153
|
|
15
|
+
docling/datamodel/document.py,sha256=FZQyJtHSeGBrZwFf-GGXDu-Dyp4iIl7VbVnTupmlUqk,19532
|
|
16
|
+
docling/datamodel/pipeline_options.py,sha256=WNjluKC-Ww63ifkGMHwws8zIDHnOS1z5Hw7_j3S0qao,2446
|
|
17
|
+
docling/datamodel/settings.py,sha256=KBFVeQviR1hoCFjA1ZwuLuQ6EAAYR7saIa6EUYiOkHI,767
|
|
18
|
+
docling/document_converter.py,sha256=S_t9hs2uZfXC38LC0hTaAihrSJIrCvnTiuY5SvUccgk,9587
|
|
19
|
+
docling/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
20
|
+
docling/models/base_model.py,sha256=wSBGAIAbLqrqP_SMtkzXMuyFvvzjVU6iCqgSNnGIR4Y,603
|
|
21
|
+
docling/models/base_ocr_model.py,sha256=SYelQRValiUo6M_p_9-J7CqNIOFO-EkK58j90SMsKQY,5028
|
|
22
|
+
docling/models/ds_glm_model.py,sha256=vJLngchZonqFzGWbUr2izFSXk9DloPDhAfN2c3nkzNU,11254
|
|
23
|
+
docling/models/easyocr_model.py,sha256=YfvdodjZ20WuOfouQXJmDyQL78QDOqWYsWSs2zSxWFc,3327
|
|
24
|
+
docling/models/layout_model.py,sha256=zd2ULW3U6v9OJl4TnjWFEY6Q2O-lBfrIqtvrnDzF7HU,12596
|
|
25
|
+
docling/models/page_assemble_model.py,sha256=LOKHho-r-RpeIVh8CpJ9tid_QIp5um3ukcrucZsyUlY,6645
|
|
26
|
+
docling/models/page_preprocessing_model.py,sha256=cfhUIlGAGaX1RxILi69ZEV9Kmhhd3Y0XaSlQnGo18o4,1964
|
|
27
|
+
docling/models/table_structure_model.py,sha256=YWSZKOz56gvicjTzVgSE-8Z_hI3NcRD5EN0yOUoM-_g,6979
|
|
28
|
+
docling/models/tesseract_ocr_cli_model.py,sha256=fKc05V73ibMvAeuA4PForhYNtunpT5rR0k_xHZsew-E,5980
|
|
29
|
+
docling/models/tesseract_ocr_model.py,sha256=v6td0vq8NogePuRTJRZhKF0DtZXITj70r9rKJKO5u9k,4984
|
|
30
|
+
docling/pipeline/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
31
|
+
docling/pipeline/base_pipeline.py,sha256=7DTzVvM_jVHCxyY-BuuGRhmUsD_sgX4DD00oBFJWdB8,6723
|
|
32
|
+
docling/pipeline/simple_pipeline.py,sha256=pxce0-3He5Lqa-xXT-7h173XVOSMZiMHl6HOfAJmQ7o,2162
|
|
33
|
+
docling/pipeline/standard_pdf_pipeline.py,sha256=AVNSxGc6kPmBPDLWDc9eI8fryc25eOtiIVrOyVhZMZM,7527
|
|
34
|
+
docling/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
35
|
+
docling/utils/export.py,sha256=KyGF1BVDHPFfHVXZc8vegsWlFfOgGPP2YckWpTadyI8,4694
|
|
36
|
+
docling/utils/layout_utils.py,sha256=vlN0rc8i0ayRGn3WnaG-pdmqEL00KKGl2zez3Gj-hrk,32074
|
|
37
|
+
docling/utils/utils.py,sha256=llhXSbIDNZ1MHOwBEfLHBAoJIAYI7QlPIonlI1jLUJ0,1208
|
|
38
|
+
docling-2.1.0.dist-info/LICENSE,sha256=mBb7ErEcM8VS9OhiGHnQ2kk75HwPhr54W1Oiz3965MY,1088
|
|
39
|
+
docling-2.1.0.dist-info/METADATA,sha256=SorLD4OMK1dU3bX5eqnw5GHqPrPwdhQ7JfYvOyajE20,6109
|
|
40
|
+
docling-2.1.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
41
|
+
docling-2.1.0.dist-info/entry_points.txt,sha256=VOSzV77znM52dz5ysaDuJ0ijl1cnfrh1ZPg8od5OcTs,48
|
|
42
|
+
docling-2.1.0.dist-info/RECORD,,
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
from pathlib import Path
|
|
2
|
-
from typing import Callable, Iterable, List
|
|
3
|
-
|
|
4
|
-
from docling.datamodel.base_models import Page
|
|
5
|
-
from docling.datamodel.pipeline_options import PipelineOptions
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
class BaseModelPipeline:
|
|
9
|
-
def __init__(self, artifacts_path: Path, pipeline_options: PipelineOptions):
|
|
10
|
-
self.model_pipe: List[Callable] = []
|
|
11
|
-
self.artifacts_path = artifacts_path
|
|
12
|
-
self.pipeline_options = pipeline_options
|
|
13
|
-
|
|
14
|
-
def apply(self, page_batch: Iterable[Page]) -> Iterable[Page]:
|
|
15
|
-
for model in self.model_pipe:
|
|
16
|
-
page_batch = model(page_batch)
|
|
17
|
-
|
|
18
|
-
yield from page_batch
|
|
@@ -1,66 +0,0 @@
|
|
|
1
|
-
from pathlib import Path
|
|
2
|
-
|
|
3
|
-
from docling.datamodel.pipeline_options import (
|
|
4
|
-
EasyOcrOptions,
|
|
5
|
-
PipelineOptions,
|
|
6
|
-
TesseractCliOcrOptions,
|
|
7
|
-
TesseractOcrOptions,
|
|
8
|
-
)
|
|
9
|
-
from docling.models.base_ocr_model import BaseOcrModel
|
|
10
|
-
from docling.models.easyocr_model import EasyOcrModel
|
|
11
|
-
from docling.models.layout_model import LayoutModel
|
|
12
|
-
from docling.models.table_structure_model import TableStructureModel
|
|
13
|
-
from docling.models.tesseract_ocr_cli_model import TesseractOcrCliModel
|
|
14
|
-
from docling.models.tesseract_ocr_model import TesseractOcrModel
|
|
15
|
-
from docling.pipeline.base_model_pipeline import BaseModelPipeline
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
class StandardModelPipeline(BaseModelPipeline):
|
|
19
|
-
_layout_model_path = "model_artifacts/layout/beehive_v0.0.5_pt"
|
|
20
|
-
_table_model_path = "model_artifacts/tableformer"
|
|
21
|
-
|
|
22
|
-
def __init__(self, artifacts_path: Path, pipeline_options: PipelineOptions):
|
|
23
|
-
super().__init__(artifacts_path, pipeline_options)
|
|
24
|
-
|
|
25
|
-
ocr_model: BaseOcrModel
|
|
26
|
-
if isinstance(pipeline_options.ocr_options, EasyOcrOptions):
|
|
27
|
-
ocr_model = EasyOcrModel(
|
|
28
|
-
enabled=pipeline_options.do_ocr,
|
|
29
|
-
options=pipeline_options.ocr_options,
|
|
30
|
-
)
|
|
31
|
-
elif isinstance(pipeline_options.ocr_options, TesseractCliOcrOptions):
|
|
32
|
-
ocr_model = TesseractOcrCliModel(
|
|
33
|
-
enabled=pipeline_options.do_ocr,
|
|
34
|
-
options=pipeline_options.ocr_options,
|
|
35
|
-
)
|
|
36
|
-
elif isinstance(pipeline_options.ocr_options, TesseractOcrOptions):
|
|
37
|
-
ocr_model = TesseractOcrModel(
|
|
38
|
-
enabled=pipeline_options.do_ocr,
|
|
39
|
-
options=pipeline_options.ocr_options,
|
|
40
|
-
)
|
|
41
|
-
else:
|
|
42
|
-
raise RuntimeError(
|
|
43
|
-
f"The specified OCR kind is not supported: {pipeline_options.ocr_options.kind}."
|
|
44
|
-
)
|
|
45
|
-
|
|
46
|
-
self.model_pipe = [
|
|
47
|
-
# OCR
|
|
48
|
-
ocr_model,
|
|
49
|
-
# Layout
|
|
50
|
-
LayoutModel(
|
|
51
|
-
config={
|
|
52
|
-
"artifacts_path": artifacts_path
|
|
53
|
-
/ StandardModelPipeline._layout_model_path
|
|
54
|
-
}
|
|
55
|
-
),
|
|
56
|
-
# Table structure
|
|
57
|
-
TableStructureModel(
|
|
58
|
-
config={
|
|
59
|
-
"artifacts_path": artifacts_path
|
|
60
|
-
/ StandardModelPipeline._table_model_path,
|
|
61
|
-
"enabled": pipeline_options.do_table_structure,
|
|
62
|
-
"mode": pipeline_options.table_structure_options.mode,
|
|
63
|
-
"do_cell_matching": pipeline_options.table_structure_options.do_cell_matching,
|
|
64
|
-
}
|
|
65
|
-
),
|
|
66
|
-
]
|
|
@@ -1,380 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.1
|
|
2
|
-
Name: docling
|
|
3
|
-
Version: 1.19.1
|
|
4
|
-
Summary: Docling PDF conversion package
|
|
5
|
-
Home-page: https://github.com/DS4SD/docling
|
|
6
|
-
License: MIT
|
|
7
|
-
Keywords: docling,convert,document,pdf,layout model,segmentation,table structure,table former
|
|
8
|
-
Author: Christoph Auer
|
|
9
|
-
Author-email: cau@zurich.ibm.com
|
|
10
|
-
Requires-Python: >=3.10,<4.0
|
|
11
|
-
Classifier: Development Status :: 5 - Production/Stable
|
|
12
|
-
Classifier: Intended Audience :: Developers
|
|
13
|
-
Classifier: Intended Audience :: Science/Research
|
|
14
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
-
Classifier: Operating System :: MacOS :: MacOS X
|
|
16
|
-
Classifier: Operating System :: POSIX :: Linux
|
|
17
|
-
Classifier: Programming Language :: Python :: 3
|
|
18
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
-
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
-
Provides-Extra: tesserocr
|
|
23
|
-
Requires-Dist: certifi (>=2024.7.4)
|
|
24
|
-
Requires-Dist: deepsearch-glm (>=0.22.0,<0.23.0)
|
|
25
|
-
Requires-Dist: docling-core (>=1.7.1,<2.0.0)
|
|
26
|
-
Requires-Dist: docling-ibm-models (>=2.0.0,<3.0.0)
|
|
27
|
-
Requires-Dist: docling-parse (>=1.4.1,<2.0.0)
|
|
28
|
-
Requires-Dist: easyocr (>=1.7,<2.0)
|
|
29
|
-
Requires-Dist: filetype (>=1.2.0,<2.0.0)
|
|
30
|
-
Requires-Dist: huggingface_hub (>=0.23,<1)
|
|
31
|
-
Requires-Dist: pandas (>=2.1.4,<3.0.0)
|
|
32
|
-
Requires-Dist: pyarrow (>=16.1.0,<17.0.0)
|
|
33
|
-
Requires-Dist: pydantic (>=2.0.0,<3.0.0)
|
|
34
|
-
Requires-Dist: pydantic-settings (>=2.3.0,<3.0.0)
|
|
35
|
-
Requires-Dist: pypdfium2 (>=4.30.0,<5.0.0)
|
|
36
|
-
Requires-Dist: requests (>=2.32.3,<3.0.0)
|
|
37
|
-
Requires-Dist: rtree (>=1.3.0,<2.0.0)
|
|
38
|
-
Requires-Dist: scipy (>=1.14.1,<2.0.0)
|
|
39
|
-
Requires-Dist: tesserocr (>=2.7.1,<3.0.0) ; extra == "tesserocr"
|
|
40
|
-
Requires-Dist: torch (>=2.2.2,<2.3.0) ; sys_platform == "darwin" and platform_machine == "x86_64"
|
|
41
|
-
Requires-Dist: torch (>=2.2.2,<3.0.0) ; sys_platform != "darwin" or platform_machine != "x86_64"
|
|
42
|
-
Requires-Dist: torchvision (>=0,<1) ; sys_platform != "darwin" or platform_machine != "x86_64"
|
|
43
|
-
Requires-Dist: torchvision (>=0.17.2,<0.18.0) ; sys_platform == "darwin" and platform_machine == "x86_64"
|
|
44
|
-
Requires-Dist: typer (>=0.12.5,<0.13.0)
|
|
45
|
-
Project-URL: Repository, https://github.com/DS4SD/docling
|
|
46
|
-
Description-Content-Type: text/markdown
|
|
47
|
-
|
|
48
|
-
<p align="center">
|
|
49
|
-
<a href="https://github.com/ds4sd/docling">
|
|
50
|
-
<img loading="lazy" alt="Docling" src="https://github.com/DS4SD/docling/raw/main/logo.png" width="150" />
|
|
51
|
-
</a>
|
|
52
|
-
</p>
|
|
53
|
-
|
|
54
|
-
# Docling
|
|
55
|
-
|
|
56
|
-
[](https://arxiv.org/abs/2408.09869)
|
|
57
|
-
[](https://pypi.org/project/docling/)
|
|
58
|
-
|
|
59
|
-
[](https://python-poetry.org/)
|
|
60
|
-
[](https://github.com/psf/black)
|
|
61
|
-
[](https://pycqa.github.io/isort/)
|
|
62
|
-
[](https://pydantic.dev)
|
|
63
|
-
[](https://github.com/pre-commit/pre-commit)
|
|
64
|
-
[](https://opensource.org/licenses/MIT)
|
|
65
|
-
|
|
66
|
-
Docling bundles PDF document conversion to JSON and Markdown in an easy, self-contained package.
|
|
67
|
-
|
|
68
|
-
## Features
|
|
69
|
-
* ⚡ Converts any PDF document to JSON or Markdown format, stable and lightning fast
|
|
70
|
-
* 📑 Understands detailed page layout, reading order and recovers table structures
|
|
71
|
-
* 📝 Extracts metadata from the document, such as title, authors, references and language
|
|
72
|
-
* 🔍 Includes OCR support for scanned PDFs
|
|
73
|
-
* 🤖 Integrates easily with LLM app / RAG frameworks like 🦙 LlamaIndex and 🦜🔗 LangChain
|
|
74
|
-
* 💻 Provides a simple and convenient CLI
|
|
75
|
-
|
|
76
|
-
## Installation
|
|
77
|
-
|
|
78
|
-
To use Docling, simply install `docling` from your package manager, e.g. pip:
|
|
79
|
-
```bash
|
|
80
|
-
pip install docling
|
|
81
|
-
```
|
|
82
|
-
|
|
83
|
-
Works on macOS, Linux and Windows environments. Both x86_64 and arm64 architectures.
|
|
84
|
-
|
|
85
|
-
<details>
|
|
86
|
-
<summary><b>Alternative PyTorch distributions</b></summary>
|
|
87
|
-
|
|
88
|
-
The Docling models depend on the [PyTorch](https://pytorch.org/) library.
|
|
89
|
-
Depending on your architecture, you might want to use a different distribution of `torch`.
|
|
90
|
-
For example, you might want support for different accelerator or for a cpu-only version.
|
|
91
|
-
All the different ways for installing `torch` are listed on their website <https://pytorch.org/>.
|
|
92
|
-
|
|
93
|
-
One common situation is the installation on Linux systems with cpu-only support.
|
|
94
|
-
In this case, we suggest the installation of Docling with the following options
|
|
95
|
-
|
|
96
|
-
```bash
|
|
97
|
-
# Example for installing on the Linux cpu-only version
|
|
98
|
-
pip install docling --extra-index-url https://download.pytorch.org/whl/cpu
|
|
99
|
-
```
|
|
100
|
-
</details>
|
|
101
|
-
|
|
102
|
-
<details>
|
|
103
|
-
<summary><b>Alternative OCR engines</b></summary>
|
|
104
|
-
|
|
105
|
-
Docling supports multiple OCR engines for processing scanned documents. The current version provides
|
|
106
|
-
the following engines.
|
|
107
|
-
|
|
108
|
-
| Engine | Installation | Usage |
|
|
109
|
-
| ------ | ------------ | ----- |
|
|
110
|
-
| [EasyOCR](https://github.com/JaidedAI/EasyOCR) | Default in Docling or via `pip install easyocr`. | `EasyOcrOptions` |
|
|
111
|
-
| Tesseract | System dependency. See description for Tesseract and Tesserocr below. | `TesseractOcrOptions` |
|
|
112
|
-
| Tesseract CLI | System dependency. See description below. | `TesseractCliOcrOptions` |
|
|
113
|
-
|
|
114
|
-
The Docling `DocumentConverter` allows to choose the OCR engine with the `ocr_options` settings. For example
|
|
115
|
-
|
|
116
|
-
```python
|
|
117
|
-
from docling.datamodel.base_models import ConversionStatus, PipelineOptions
|
|
118
|
-
from docling.datamodel.pipeline_options import PipelineOptions, EasyOcrOptions, TesseractOcrOptions
|
|
119
|
-
from docling.document_converter import DocumentConverter
|
|
120
|
-
|
|
121
|
-
pipeline_options = PipelineOptions()
|
|
122
|
-
pipeline_options.do_ocr = True
|
|
123
|
-
pipeline_options.ocr_options = TesseractOcrOptions() # Use Tesseract
|
|
124
|
-
|
|
125
|
-
doc_converter = DocumentConverter(
|
|
126
|
-
pipeline_options=pipeline_options,
|
|
127
|
-
)
|
|
128
|
-
```
|
|
129
|
-
|
|
130
|
-
#### Tesseract installation
|
|
131
|
-
|
|
132
|
-
[Tesseract](https://github.com/tesseract-ocr/tesseract) is a popular OCR engine which is available
|
|
133
|
-
on most operating systems. For using this engine with Docling, Tesseract must be installed on your
|
|
134
|
-
system, using the packaging tool of your choice. Below we provide example commands.
|
|
135
|
-
After installing Tesseract you are expected to provide the path to its language files using the
|
|
136
|
-
`TESSDATA_PREFIX` environment variable (note that it must terminate with a slash `/`).
|
|
137
|
-
|
|
138
|
-
For macOS, we reccomend using [Homebrew](https://brew.sh/).
|
|
139
|
-
|
|
140
|
-
```console
|
|
141
|
-
brew install tesseract leptonica pkg-config
|
|
142
|
-
TESSDATA_PREFIX=/opt/homebrew/share/tessdata/
|
|
143
|
-
echo "Set TESSDATA_PREFIX=${TESSDATA_PREFIX}"
|
|
144
|
-
```
|
|
145
|
-
|
|
146
|
-
For Debian-based systems.
|
|
147
|
-
|
|
148
|
-
```console
|
|
149
|
-
apt-get install tesseract-ocr tesseract-ocr-eng libtesseract-dev libleptonica-dev pkg-config
|
|
150
|
-
TESSDATA_PREFIX=$(dpkg -L tesseract-ocr-eng | grep tessdata$)
|
|
151
|
-
echo "Set TESSDATA_PREFIX=${TESSDATA_PREFIX}"
|
|
152
|
-
```
|
|
153
|
-
|
|
154
|
-
For RHEL systems.
|
|
155
|
-
|
|
156
|
-
```console
|
|
157
|
-
dnf install tesseract tesseract-devel tesseract-langpack-eng leptonica-devel
|
|
158
|
-
TESSDATA_PREFIX=/usr/share/tesseract/tessdata/
|
|
159
|
-
echo "Set TESSDATA_PREFIX=${TESSDATA_PREFIX}"
|
|
160
|
-
```
|
|
161
|
-
|
|
162
|
-
#### Linking to Tesseract
|
|
163
|
-
The most efficient usage of the Tesseract library is via linking. Docling is using
|
|
164
|
-
the [Tesserocr](https://github.com/sirfz/tesserocr) package for this.
|
|
165
|
-
|
|
166
|
-
If you get into installation issues of Tesserocr, we suggest using the following
|
|
167
|
-
installation options:
|
|
168
|
-
|
|
169
|
-
```console
|
|
170
|
-
pip uninstall tesserocr
|
|
171
|
-
pip install --no-binary :all: tesserocr
|
|
172
|
-
```
|
|
173
|
-
</details>
|
|
174
|
-
|
|
175
|
-
<details>
|
|
176
|
-
<summary><b>Docling development setup</b></summary>
|
|
177
|
-
|
|
178
|
-
To develop for Docling (features, bugfixes etc.), install as follows from your local clone's root dir:
|
|
179
|
-
```bash
|
|
180
|
-
poetry install --all-extras
|
|
181
|
-
```
|
|
182
|
-
</details>
|
|
183
|
-
|
|
184
|
-
## Getting started
|
|
185
|
-
|
|
186
|
-
### Convert a single document
|
|
187
|
-
|
|
188
|
-
To convert invidual PDF documents, use `convert_single()`, for example:
|
|
189
|
-
```python
|
|
190
|
-
from docling.document_converter import DocumentConverter
|
|
191
|
-
|
|
192
|
-
source = "https://arxiv.org/pdf/2408.09869" # PDF path or URL
|
|
193
|
-
converter = DocumentConverter()
|
|
194
|
-
result = converter.convert_single(source)
|
|
195
|
-
print(result.render_as_markdown()) # output: "## Docling Technical Report[...]"
|
|
196
|
-
print(result.render_as_doctags()) # output: "<document><title><page_1><loc_20>..."
|
|
197
|
-
```
|
|
198
|
-
|
|
199
|
-
### Convert a batch of documents
|
|
200
|
-
|
|
201
|
-
For an example of batch-converting documents, see [batch_convert.py](https://github.com/DS4SD/docling/blob/main/examples/batch_convert.py).
|
|
202
|
-
|
|
203
|
-
From a local repo clone, you can run it with:
|
|
204
|
-
|
|
205
|
-
```
|
|
206
|
-
python examples/batch_convert.py
|
|
207
|
-
```
|
|
208
|
-
The output of the above command will be written to `./scratch`.
|
|
209
|
-
|
|
210
|
-
### CLI
|
|
211
|
-
|
|
212
|
-
You can also use Docling directly from your command line to convert individual files —be it local or by URL— or whole directories.
|
|
213
|
-
|
|
214
|
-
A simple example would look like this:
|
|
215
|
-
```console
|
|
216
|
-
docling https://arxiv.org/pdf/2206.01062
|
|
217
|
-
```
|
|
218
|
-
|
|
219
|
-
To see all available options (export formats etc.) run `docling --help`.
|
|
220
|
-
|
|
221
|
-
<details>
|
|
222
|
-
<summary><b>CLI reference</b></summary>
|
|
223
|
-
|
|
224
|
-
Here are the available options as of this writing (for an up-to-date listing, run `docling --help`):
|
|
225
|
-
|
|
226
|
-
```console
|
|
227
|
-
$ docling --help
|
|
228
|
-
|
|
229
|
-
Usage: docling [OPTIONS] source
|
|
230
|
-
|
|
231
|
-
╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
|
|
232
|
-
│ * input_sources source PDF files to convert. Can be local file / directory paths or URL. [default: None] [required] │
|
|
233
|
-
╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
234
|
-
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
|
|
235
|
-
│ --json --no-json If enabled the document is exported as JSON. [default: no-json] │
|
|
236
|
-
│ --md --no-md If enabled the document is exported as Markdown. [default: md] │
|
|
237
|
-
│ --txt --no-txt If enabled the document is exported as Text. [default: no-txt] │
|
|
238
|
-
│ --doctags --no-doctags If enabled the document is exported as Doc Tags. [default: no-doctags] │
|
|
239
|
-
│ --ocr --no-ocr If enabled, the bitmap content will be processed using OCR. [default: ocr] │
|
|
240
|
-
│ --backend [pypdfium2|docling] The PDF backend to use. [default: docling] │
|
|
241
|
-
│ --output PATH Output directory where results are saved. [default: .] │
|
|
242
|
-
│ --version Show version information. │
|
|
243
|
-
│ --help Show this message and exit. │
|
|
244
|
-
╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
245
|
-
```
|
|
246
|
-
</details>
|
|
247
|
-
|
|
248
|
-
### RAG
|
|
249
|
-
Check out the following examples showcasing RAG using Docling with standard LLM application frameworks:
|
|
250
|
-
- [Basic RAG pipeline with 🦙 LlamaIndex](https://github.com/DS4SD/docling/tree/main/examples/rag_llamaindex.ipynb)
|
|
251
|
-
- [Basic RAG pipeline with 🦜🔗 LangChain](https://github.com/DS4SD/docling/tree/main/examples/rag_langchain.ipynb)
|
|
252
|
-
|
|
253
|
-
## Advanced features
|
|
254
|
-
|
|
255
|
-
### Adjust pipeline features
|
|
256
|
-
|
|
257
|
-
The example file [custom_convert.py](https://github.com/DS4SD/docling/blob/main/examples/custom_convert.py) contains multiple ways
|
|
258
|
-
one can adjust the conversion pipeline and features.
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
#### Control pipeline options
|
|
262
|
-
|
|
263
|
-
You can control if table structure recognition or OCR should be performed by arguments passed to `DocumentConverter`:
|
|
264
|
-
```python
|
|
265
|
-
doc_converter = DocumentConverter(
|
|
266
|
-
artifacts_path=artifacts_path,
|
|
267
|
-
pipeline_options=PipelineOptions(
|
|
268
|
-
do_table_structure=False, # controls if table structure is recovered
|
|
269
|
-
do_ocr=True, # controls if OCR is applied (ignores programmatic content)
|
|
270
|
-
),
|
|
271
|
-
)
|
|
272
|
-
```
|
|
273
|
-
|
|
274
|
-
#### Control table extraction options
|
|
275
|
-
|
|
276
|
-
You can control if table structure recognition should map the recognized structure back to PDF cells (default) or use text cells from the structure prediction itself.
|
|
277
|
-
This can improve output quality if you find that multiple columns in extracted tables are erroneously merged into one.
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
```python
|
|
281
|
-
from docling.datamodel.pipeline_options import PipelineOptions
|
|
282
|
-
|
|
283
|
-
pipeline_options = PipelineOptions(do_table_structure=True)
|
|
284
|
-
pipeline_options.table_structure_options.do_cell_matching = False # uses text cells predicted from table structure model
|
|
285
|
-
|
|
286
|
-
doc_converter = DocumentConverter(
|
|
287
|
-
artifacts_path=artifacts_path,
|
|
288
|
-
pipeline_options=pipeline_options,
|
|
289
|
-
)
|
|
290
|
-
```
|
|
291
|
-
|
|
292
|
-
Since docling 1.16.0: You can control which TableFormer mode you want to use. Choose between `TableFormerMode.FAST` (default) and `TableFormerMode.ACCURATE` (better, but slower) to receive better quality with difficult table structures.
|
|
293
|
-
|
|
294
|
-
```python
|
|
295
|
-
from docling.datamodel.pipeline_options import PipelineOptions, TableFormerMode
|
|
296
|
-
|
|
297
|
-
pipeline_options = PipelineOptions(do_table_structure=True)
|
|
298
|
-
pipeline_options.table_structure_options.mode = TableFormerMode.ACCURATE # use more accurate TableFormer model
|
|
299
|
-
|
|
300
|
-
doc_converter = DocumentConverter(
|
|
301
|
-
artifacts_path=artifacts_path,
|
|
302
|
-
pipeline_options=pipeline_options,
|
|
303
|
-
)
|
|
304
|
-
```
|
|
305
|
-
|
|
306
|
-
### Impose limits on the document size
|
|
307
|
-
|
|
308
|
-
You can limit the file size and number of pages which should be allowed to process per document:
|
|
309
|
-
```python
|
|
310
|
-
conv_input = DocumentConversionInput.from_paths(
|
|
311
|
-
paths=[Path("./test/data/2206.01062.pdf")],
|
|
312
|
-
limits=DocumentLimits(max_num_pages=100, max_file_size=20971520)
|
|
313
|
-
)
|
|
314
|
-
```
|
|
315
|
-
|
|
316
|
-
### Convert from binary PDF streams
|
|
317
|
-
|
|
318
|
-
You can convert PDFs from a binary stream instead of from the filesystem as follows:
|
|
319
|
-
```python
|
|
320
|
-
buf = BytesIO(your_binary_stream)
|
|
321
|
-
docs = [DocumentStream(filename="my_doc.pdf", stream=buf)]
|
|
322
|
-
conv_input = DocumentConversionInput.from_streams(docs)
|
|
323
|
-
results = doc_converter.convert(conv_input)
|
|
324
|
-
```
|
|
325
|
-
### Limit resource usage
|
|
326
|
-
|
|
327
|
-
You can limit the CPU threads used by Docling by setting the environment variable `OMP_NUM_THREADS` accordingly. The default setting is using 4 CPU threads.
|
|
328
|
-
|
|
329
|
-
### Chunking
|
|
330
|
-
|
|
331
|
-
You can perform a hierarchy-aware chunking of a Docling document as follows:
|
|
332
|
-
|
|
333
|
-
```python
|
|
334
|
-
from docling.document_converter import DocumentConverter
|
|
335
|
-
from docling_core.transforms.chunker import HierarchicalChunker
|
|
336
|
-
|
|
337
|
-
doc = DocumentConverter().convert_single("https://arxiv.org/pdf/2206.01062").output
|
|
338
|
-
chunks = list(HierarchicalChunker().chunk(doc))
|
|
339
|
-
print(chunks[0])
|
|
340
|
-
# ChunkWithMetadata(
|
|
341
|
-
# path='#/main-text/1',
|
|
342
|
-
# text='DocLayNet: A Large Human-Annotated Dataset [...]',
|
|
343
|
-
# page=1,
|
|
344
|
-
# bbox=[107.30, 672.38, 505.19, 709.08],
|
|
345
|
-
# [...]
|
|
346
|
-
# )
|
|
347
|
-
```
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
## Technical report
|
|
351
|
-
|
|
352
|
-
For more details on Docling's inner workings, check out the [Docling Technical Report](https://arxiv.org/abs/2408.09869).
|
|
353
|
-
|
|
354
|
-
## Contributing
|
|
355
|
-
|
|
356
|
-
Please read [Contributing to Docling](https://github.com/DS4SD/docling/blob/main/CONTRIBUTING.md) for details.
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
## References
|
|
360
|
-
|
|
361
|
-
If you use Docling in your projects, please consider citing the following:
|
|
362
|
-
|
|
363
|
-
```bib
|
|
364
|
-
@techreport{Docling,
|
|
365
|
-
author = {Deep Search Team},
|
|
366
|
-
month = {8},
|
|
367
|
-
title = {Docling Technical Report},
|
|
368
|
-
url = {https://arxiv.org/abs/2408.09869},
|
|
369
|
-
eprint = {2408.09869},
|
|
370
|
-
doi = {10.48550/arXiv.2408.09869},
|
|
371
|
-
version = {1.0.0},
|
|
372
|
-
year = {2024}
|
|
373
|
-
}
|
|
374
|
-
```
|
|
375
|
-
|
|
376
|
-
## License
|
|
377
|
-
|
|
378
|
-
The Docling codebase is under MIT license.
|
|
379
|
-
For individual model usage, please refer to the model licenses found in the original packages.
|
|
380
|
-
|