docler 2.1.0__tar.gz → 2.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {docler-2.1.0 → docler-2.2.0}/PKG-INFO +29 -42
- {docler-2.1.0 → docler-2.2.0}/README.md +24 -37
- docler-2.2.0/pyproject.toml +338 -0
- docler-2.1.0/pyproject.toml → docler-2.2.0/pyproject.toml.orig +34 -5
- {docler-2.1.0 → docler-2.2.0}/src/docler/__main__.py +136 -133
- {docler-2.1.0 → docler-2.2.0}/src/docler/annotators/__init__.py +1 -1
- {docler-2.1.0 → docler-2.2.0}/src/docler/annotators/base.py +36 -36
- {docler-2.1.0 → docler-2.2.0}/src/docler/chunkers/ai_chunker/chunker.py +1 -1
- {docler-2.1.0 → docler-2.2.0}/src/docler/chunkers/token_chunker/chunker.py +1 -1
- {docler-2.1.0 → docler-2.2.0}/src/docler/common_types.py +49 -49
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/azure_provider/__init__.py +5 -5
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/azure_provider/provider.py +1 -1
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/datalab_provider/__init__.py +5 -5
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/datalab_provider/provider.py +1 -1
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/dir_converter.py +179 -179
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/docling_provider/__init__.py +5 -5
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/docling_provider/provider.py +2 -2
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/docling_provider/utils.py +82 -82
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/docling_remote_provider/provider.py +1 -1
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/exceptions.py +9 -9
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/llamaparse_provider/__init__.py +5 -5
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/llamaparse_provider/provider.py +29 -8
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/llm_provider/__init__.py +5 -5
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/llm_provider/provider.py +2 -3
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/marker_provider/__init__.py +5 -5
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/marker_provider/provider.py +1 -1
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/markitdown_provider/__init__.py +5 -5
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/mistral_provider/__init__.py +5 -5
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/mistral_provider/provider.py +3 -3
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/mistral_provider/utils.py +1 -1
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/upstage_provider/__init__.py +5 -5
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/upstage_provider/provider.py +4 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/diffs.py +90 -90
- {docler-2.1.0 → docler-2.2.0}/src/docler/mime_types.py +174 -174
- {docler-2.1.0 → docler-2.2.0}/src/docler/process_runner.py +276 -276
- {docler-2.1.0 → docler-2.2.0}/src/docler/processors/base.py +35 -35
- {docler-2.1.0 → docler-2.2.0}/src/docler/provider.py +66 -66
- {docler-2.1.0 → docler-2.2.0}/src/docler_api/__init__.py +3 -3
- {docler-2.1.0 → docler-2.2.0}/src/docler_config/__init__.py +91 -91
- {docler-2.1.0 → docler-2.2.0}/src/docler_config/annotator_configs.py +117 -117
- {docler-2.1.0 → docler-2.2.0}/src/docler_config/chunker_configs.py +146 -146
- {docler-2.1.0 → docler-2.2.0}/src/docler_config/converter_configs.py +17 -8
- {docler-2.1.0 → docler-2.2.0}/src/docler_config/processor_configs.py +79 -79
- {docler-2.1.0 → docler-2.2.0}/src/docler_config/provider.py +44 -44
- {docler-2.1.0 → docler-2.2.0}/src/docler_streamlit/__init__.py +6 -6
- {docler-2.1.0 → docler-2.2.0}/src/docler_streamlit/ocr_app.py +56 -56
- docler-2.2.0/src/docler_streamlit/py.typed +0 -0
- {docler-2.1.0 → docler-2.2.0}/LICENSE +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/__init__.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/annotators/ai_document_annotator.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/annotators/ai_image_annotator.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/chunkers/__init__.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/chunkers/ai_chunker/__init__.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/chunkers/ai_chunker/models.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/chunkers/ai_chunker/utils.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/chunkers/base.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/chunkers/llamaindex_chunker/__init__.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/chunkers/llamaindex_chunker/chunker.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/chunkers/markdown_chunker/__init__.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/chunkers/markdown_chunker/chunker.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/chunkers/markdown_chunker/utils.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/chunkers/token_chunker/__init__.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/__init__.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/aggregated_converter.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/azure_provider/utils.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/base.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/datalab_provider/utils.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/docling_remote_provider/__init__.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/docling_remote_provider/utils.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/llamaparse_provider/utils.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/markitdown_provider/provider.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/converters/registry.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/log.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/models.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/pdf_utils.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/processors/__init__.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/processors/ai_processor.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/py.typed +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/resources/pdf_sample.pdf +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/resources/samplepptx.pptx +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/resources/test.docx +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/resources/test.jpg +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/resources/test.pptx +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/resources/test.xls +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/resources/test.xlsx +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler/utils.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler_api/README.md +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler_api/main.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler_api/py.typed +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler_api/routes.py +0 -0
- {docler-2.1.0/src/docler_mcp → docler-2.2.0/src/docler_config}/py.typed +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler_mcp/__init__.py +0 -0
- {docler-2.1.0/src/docler_streamlit → docler-2.2.0/src/docler_mcp}/py.typed +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler_mcp/server.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler_streamlit/chunk_app.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler_streamlit/chunkers.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler_streamlit/converters.py +0 -0
- {docler-2.1.0 → docler-2.2.0}/src/docler_streamlit/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: docler
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.2.0
|
|
4
4
|
Summary: Abstractions & Tools for OCR / document processing
|
|
5
5
|
Keywords:
|
|
6
6
|
Author: Philipp Temminghoff
|
|
@@ -29,18 +29,18 @@ Requires-Dist: pypdf
|
|
|
29
29
|
Requires-Dist: pytest-docker>=3.2.1
|
|
30
30
|
Requires-Dist: schemez>=0.0.1
|
|
31
31
|
Requires-Dist: upathtools>=0.4.3
|
|
32
|
-
Requires-Dist: docler[
|
|
33
|
-
Requires-Dist: agentpool
|
|
32
|
+
Requires-Dist: docler[azure,diffs,docling,llama-parse,markitdown,mcp,mistralai,openai,server] ; extra == 'all'
|
|
33
|
+
Requires-Dist: agentpool>=2.10.0 ; extra == 'all'
|
|
34
34
|
Requires-Dist: azure-ai-documentintelligence ; extra == 'azure'
|
|
35
35
|
Requires-Dist: diff-match-patch ; extra == 'diffs'
|
|
36
36
|
Requires-Dist: docling[ocrmac,vlm,rapidocr] ; python_full_version < '3.14' and extra == 'docling'
|
|
37
37
|
Requires-Dist: docler[mistralai,diffs,llama-parse,openai,azure] ; extra == 'light'
|
|
38
|
-
Requires-Dist: agentpool
|
|
38
|
+
Requires-Dist: agentpool>=2.10.0 ; extra == 'light'
|
|
39
39
|
Requires-Dist: llama-parse ; python_full_version < '3.14' and extra == 'llama-parse'
|
|
40
40
|
Requires-Dist: marker-pdf ; python_full_version < '3.14' and extra == 'marker'
|
|
41
41
|
Requires-Dist: markitdown[all] ; python_full_version < '3.14' and extra == 'markitdown'
|
|
42
42
|
Requires-Dist: fastmcp ; extra == 'mcp'
|
|
43
|
-
Requires-Dist: mistralai ; extra == 'mistralai'
|
|
43
|
+
Requires-Dist: mistralai>=3 ; extra == 'mistralai'
|
|
44
44
|
Requires-Dist: olmocr ; python_full_version < '3.14' and extra == 'olmocr'
|
|
45
45
|
Requires-Dist: accelerate ; python_full_version < '3.14' and extra == 'olmocr'
|
|
46
46
|
Requires-Dist: openai ; extra == 'openai'
|
|
@@ -118,6 +118,7 @@ A unified Python library for document conversion and OCR that provides a consist
|
|
|
118
118
|
import asyncio
|
|
119
119
|
from docler import MistralConverter
|
|
120
120
|
|
|
121
|
+
|
|
121
122
|
async def main():
|
|
122
123
|
# Use the aggregated converter for automatic provider selection
|
|
123
124
|
converter = MistralConverter()
|
|
@@ -130,6 +131,7 @@ async def main():
|
|
|
130
131
|
print(f"Images: {len(result.images)} extracted")
|
|
131
132
|
print(f"Pages: {result.page_count}")
|
|
132
133
|
|
|
134
|
+
|
|
133
135
|
asyncio.run(main())
|
|
134
136
|
```
|
|
135
137
|
|
|
@@ -142,11 +144,7 @@ asyncio.run(main())
|
|
|
142
144
|
```python
|
|
143
145
|
from docler import AzureConverter
|
|
144
146
|
|
|
145
|
-
converter = AzureConverter(
|
|
146
|
-
endpoint="your-endpoint",
|
|
147
|
-
api_key="your-key",
|
|
148
|
-
model="prebuilt-layout"
|
|
149
|
-
)
|
|
147
|
+
converter = AzureConverter(endpoint="your-endpoint", api_key="your-key", model="prebuilt-layout")
|
|
150
148
|
```
|
|
151
149
|
|
|
152
150
|
#### Mistral OCR
|
|
@@ -154,10 +152,7 @@ converter = AzureConverter(
|
|
|
154
152
|
```python
|
|
155
153
|
from docler import MistralConverter
|
|
156
154
|
|
|
157
|
-
converter = MistralConverter(
|
|
158
|
-
api_key="your-key",
|
|
159
|
-
languages=["en", "fr", "de"]
|
|
160
|
-
)
|
|
155
|
+
converter = MistralConverter(api_key="your-key", languages=["en", "fr", "de"])
|
|
161
156
|
```
|
|
162
157
|
|
|
163
158
|
#### LlamaParse
|
|
@@ -165,10 +160,7 @@ converter = MistralConverter(
|
|
|
165
160
|
```python
|
|
166
161
|
from docler import LlamaParseConverter
|
|
167
162
|
|
|
168
|
-
converter = LlamaParseConverter(
|
|
169
|
-
api_key="your-key",
|
|
170
|
-
adaptive_long_table=True
|
|
171
|
-
)
|
|
163
|
+
converter = LlamaParseConverter(api_key="your-key", adaptive_long_table=True)
|
|
172
164
|
```
|
|
173
165
|
|
|
174
166
|
#### Upstage Document AI
|
|
@@ -176,10 +168,7 @@ converter = LlamaParseConverter(
|
|
|
176
168
|
```python
|
|
177
169
|
from docler import UpstageConverter
|
|
178
170
|
|
|
179
|
-
converter = UpstageConverter(
|
|
180
|
-
api_key="your-key",
|
|
181
|
-
chart_recognition=True
|
|
182
|
-
)
|
|
171
|
+
converter = UpstageConverter(api_key="your-key", chart_recognition=True)
|
|
183
172
|
```
|
|
184
173
|
|
|
185
174
|
#### DataLab
|
|
@@ -189,7 +178,7 @@ from docler import DataLabConverter
|
|
|
189
178
|
|
|
190
179
|
converter = DataLabConverter(
|
|
191
180
|
api_key="your-key",
|
|
192
|
-
use_llm=False # Enable for higher accuracy
|
|
181
|
+
use_llm=False, # Enable for higher accuracy
|
|
193
182
|
)
|
|
194
183
|
```
|
|
195
184
|
|
|
@@ -203,7 +192,7 @@ from docler import MarkerConverter
|
|
|
203
192
|
converter = MarkerConverter(
|
|
204
193
|
dpi=192,
|
|
205
194
|
use_llm=True, # Requires local LLM setup
|
|
206
|
-
llm_provider="ollama"
|
|
195
|
+
llm_provider="ollama",
|
|
207
196
|
)
|
|
208
197
|
```
|
|
209
198
|
|
|
@@ -212,10 +201,7 @@ converter = MarkerConverter(
|
|
|
212
201
|
```python
|
|
213
202
|
from docler import DoclingConverter
|
|
214
203
|
|
|
215
|
-
converter = DoclingConverter(
|
|
216
|
-
ocr_engine="easy_ocr",
|
|
217
|
-
image_scale=2.0
|
|
218
|
-
)
|
|
204
|
+
converter = DoclingConverter(ocr_engine="rapid_ocr", image_scale=2.0)
|
|
219
205
|
```
|
|
220
206
|
|
|
221
207
|
#### Docling Remote
|
|
@@ -223,10 +209,7 @@ converter = DoclingConverter(
|
|
|
223
209
|
```python
|
|
224
210
|
from docler import DoclingRemoteConverter
|
|
225
211
|
|
|
226
|
-
converter = DoclingRemoteConverter(
|
|
227
|
-
endpoint="http://localhost:5001",
|
|
228
|
-
pdf_backend="dlparse_v4"
|
|
229
|
-
)
|
|
212
|
+
converter = DoclingRemoteConverter(endpoint="http://localhost:5001", pdf_backend="dlparse_v4")
|
|
230
213
|
```
|
|
231
214
|
|
|
232
215
|
#### MarkItDown (Microsoft)
|
|
@@ -246,7 +229,7 @@ from docler import LLMConverter
|
|
|
246
229
|
|
|
247
230
|
converter = LLMConverter(
|
|
248
231
|
model="gpt-4o", # or claude-3-5-sonnet, etc.
|
|
249
|
-
system_prompt="Extract text preserving formatting..."
|
|
232
|
+
system_prompt="Extract text preserving formatting...",
|
|
250
233
|
)
|
|
251
234
|
```
|
|
252
235
|
|
|
@@ -315,13 +298,13 @@ All converters return a standardized `Document` object with:
|
|
|
315
298
|
|
|
316
299
|
```python
|
|
317
300
|
class Document:
|
|
318
|
-
content: str
|
|
319
|
-
images: list[Image]
|
|
320
|
-
title: str
|
|
321
|
-
source_path: str
|
|
322
|
-
mime_type: str
|
|
323
|
-
metadata: dict
|
|
324
|
-
page_count: int
|
|
301
|
+
content: str # Extracted text in markdown format
|
|
302
|
+
images: list[Image] # Extracted images with metadata
|
|
303
|
+
title: str # Document title
|
|
304
|
+
source_path: str # Original file path
|
|
305
|
+
mime_type: str # File MIME type
|
|
306
|
+
metadata: dict # Provider-specific metadata
|
|
307
|
+
page_count: int # Number of pages processed
|
|
325
308
|
```
|
|
326
309
|
|
|
327
310
|
The markdown content includes standardized metadata for page breaks and structure:
|
|
@@ -344,11 +327,15 @@ pip install docler
|
|
|
344
327
|
|
|
345
328
|
# With specific provider dependencies
|
|
346
329
|
pip install docler[azure] # Azure Document Intelligence
|
|
347
|
-
pip install docler[
|
|
330
|
+
pip install docler[mistralai] # Mistral OCR
|
|
348
331
|
pip install docler[marker] # Marker PDF processing
|
|
349
|
-
pip install docler[all] # All providers
|
|
332
|
+
pip install docler[all] # All providers except Marker
|
|
350
333
|
```
|
|
351
334
|
|
|
335
|
+
Marker cannot be installed together with `docler[all]`/`docler[light]`: marker-pdf
|
|
336
|
+
requires `openai<3`, while the AI features (agentpool) require `openai>=3.31`.
|
|
337
|
+
Install `docler[marker]` in a separate environment.
|
|
338
|
+
|
|
352
339
|
## Environment Variables
|
|
353
340
|
|
|
354
341
|
Configure API keys via environment variables:
|
|
@@ -44,6 +44,7 @@ A unified Python library for document conversion and OCR that provides a consist
|
|
|
44
44
|
import asyncio
|
|
45
45
|
from docler import MistralConverter
|
|
46
46
|
|
|
47
|
+
|
|
47
48
|
async def main():
|
|
48
49
|
# Use the aggregated converter for automatic provider selection
|
|
49
50
|
converter = MistralConverter()
|
|
@@ -56,6 +57,7 @@ async def main():
|
|
|
56
57
|
print(f"Images: {len(result.images)} extracted")
|
|
57
58
|
print(f"Pages: {result.page_count}")
|
|
58
59
|
|
|
60
|
+
|
|
59
61
|
asyncio.run(main())
|
|
60
62
|
```
|
|
61
63
|
|
|
@@ -68,11 +70,7 @@ asyncio.run(main())
|
|
|
68
70
|
```python
|
|
69
71
|
from docler import AzureConverter
|
|
70
72
|
|
|
71
|
-
converter = AzureConverter(
|
|
72
|
-
endpoint="your-endpoint",
|
|
73
|
-
api_key="your-key",
|
|
74
|
-
model="prebuilt-layout"
|
|
75
|
-
)
|
|
73
|
+
converter = AzureConverter(endpoint="your-endpoint", api_key="your-key", model="prebuilt-layout")
|
|
76
74
|
```
|
|
77
75
|
|
|
78
76
|
#### Mistral OCR
|
|
@@ -80,10 +78,7 @@ converter = AzureConverter(
|
|
|
80
78
|
```python
|
|
81
79
|
from docler import MistralConverter
|
|
82
80
|
|
|
83
|
-
converter = MistralConverter(
|
|
84
|
-
api_key="your-key",
|
|
85
|
-
languages=["en", "fr", "de"]
|
|
86
|
-
)
|
|
81
|
+
converter = MistralConverter(api_key="your-key", languages=["en", "fr", "de"])
|
|
87
82
|
```
|
|
88
83
|
|
|
89
84
|
#### LlamaParse
|
|
@@ -91,10 +86,7 @@ converter = MistralConverter(
|
|
|
91
86
|
```python
|
|
92
87
|
from docler import LlamaParseConverter
|
|
93
88
|
|
|
94
|
-
converter = LlamaParseConverter(
|
|
95
|
-
api_key="your-key",
|
|
96
|
-
adaptive_long_table=True
|
|
97
|
-
)
|
|
89
|
+
converter = LlamaParseConverter(api_key="your-key", adaptive_long_table=True)
|
|
98
90
|
```
|
|
99
91
|
|
|
100
92
|
#### Upstage Document AI
|
|
@@ -102,10 +94,7 @@ converter = LlamaParseConverter(
|
|
|
102
94
|
```python
|
|
103
95
|
from docler import UpstageConverter
|
|
104
96
|
|
|
105
|
-
converter = UpstageConverter(
|
|
106
|
-
api_key="your-key",
|
|
107
|
-
chart_recognition=True
|
|
108
|
-
)
|
|
97
|
+
converter = UpstageConverter(api_key="your-key", chart_recognition=True)
|
|
109
98
|
```
|
|
110
99
|
|
|
111
100
|
#### DataLab
|
|
@@ -115,7 +104,7 @@ from docler import DataLabConverter
|
|
|
115
104
|
|
|
116
105
|
converter = DataLabConverter(
|
|
117
106
|
api_key="your-key",
|
|
118
|
-
use_llm=False # Enable for higher accuracy
|
|
107
|
+
use_llm=False, # Enable for higher accuracy
|
|
119
108
|
)
|
|
120
109
|
```
|
|
121
110
|
|
|
@@ -129,7 +118,7 @@ from docler import MarkerConverter
|
|
|
129
118
|
converter = MarkerConverter(
|
|
130
119
|
dpi=192,
|
|
131
120
|
use_llm=True, # Requires local LLM setup
|
|
132
|
-
llm_provider="ollama"
|
|
121
|
+
llm_provider="ollama",
|
|
133
122
|
)
|
|
134
123
|
```
|
|
135
124
|
|
|
@@ -138,10 +127,7 @@ converter = MarkerConverter(
|
|
|
138
127
|
```python
|
|
139
128
|
from docler import DoclingConverter
|
|
140
129
|
|
|
141
|
-
converter = DoclingConverter(
|
|
142
|
-
ocr_engine="easy_ocr",
|
|
143
|
-
image_scale=2.0
|
|
144
|
-
)
|
|
130
|
+
converter = DoclingConverter(ocr_engine="rapid_ocr", image_scale=2.0)
|
|
145
131
|
```
|
|
146
132
|
|
|
147
133
|
#### Docling Remote
|
|
@@ -149,10 +135,7 @@ converter = DoclingConverter(
|
|
|
149
135
|
```python
|
|
150
136
|
from docler import DoclingRemoteConverter
|
|
151
137
|
|
|
152
|
-
converter = DoclingRemoteConverter(
|
|
153
|
-
endpoint="http://localhost:5001",
|
|
154
|
-
pdf_backend="dlparse_v4"
|
|
155
|
-
)
|
|
138
|
+
converter = DoclingRemoteConverter(endpoint="http://localhost:5001", pdf_backend="dlparse_v4")
|
|
156
139
|
```
|
|
157
140
|
|
|
158
141
|
#### MarkItDown (Microsoft)
|
|
@@ -172,7 +155,7 @@ from docler import LLMConverter
|
|
|
172
155
|
|
|
173
156
|
converter = LLMConverter(
|
|
174
157
|
model="gpt-4o", # or claude-3-5-sonnet, etc.
|
|
175
|
-
system_prompt="Extract text preserving formatting..."
|
|
158
|
+
system_prompt="Extract text preserving formatting...",
|
|
176
159
|
)
|
|
177
160
|
```
|
|
178
161
|
|
|
@@ -241,13 +224,13 @@ All converters return a standardized `Document` object with:
|
|
|
241
224
|
|
|
242
225
|
```python
|
|
243
226
|
class Document:
|
|
244
|
-
content: str
|
|
245
|
-
images: list[Image]
|
|
246
|
-
title: str
|
|
247
|
-
source_path: str
|
|
248
|
-
mime_type: str
|
|
249
|
-
metadata: dict
|
|
250
|
-
page_count: int
|
|
227
|
+
content: str # Extracted text in markdown format
|
|
228
|
+
images: list[Image] # Extracted images with metadata
|
|
229
|
+
title: str # Document title
|
|
230
|
+
source_path: str # Original file path
|
|
231
|
+
mime_type: str # File MIME type
|
|
232
|
+
metadata: dict # Provider-specific metadata
|
|
233
|
+
page_count: int # Number of pages processed
|
|
251
234
|
```
|
|
252
235
|
|
|
253
236
|
The markdown content includes standardized metadata for page breaks and structure:
|
|
@@ -270,11 +253,15 @@ pip install docler
|
|
|
270
253
|
|
|
271
254
|
# With specific provider dependencies
|
|
272
255
|
pip install docler[azure] # Azure Document Intelligence
|
|
273
|
-
pip install docler[
|
|
256
|
+
pip install docler[mistralai] # Mistral OCR
|
|
274
257
|
pip install docler[marker] # Marker PDF processing
|
|
275
|
-
pip install docler[all] # All providers
|
|
258
|
+
pip install docler[all] # All providers except Marker
|
|
276
259
|
```
|
|
277
260
|
|
|
261
|
+
Marker cannot be installed together with `docler[all]`/`docler[light]`: marker-pdf
|
|
262
|
+
requires `openai<3`, while the AI features (agentpool) require `openai>=3.31`.
|
|
263
|
+
Install `docler[marker]` in a separate environment.
|
|
264
|
+
|
|
278
265
|
## Environment Variables
|
|
279
266
|
|
|
280
267
|
Configure API keys via environment variables:
|
|
@@ -0,0 +1,338 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "docler"
|
|
3
|
+
version = "2.2.0"
|
|
4
|
+
description = "Abstractions & Tools for OCR / document processing"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.13"
|
|
7
|
+
license = "MIT"
|
|
8
|
+
license-files = ["LICENSE"]
|
|
9
|
+
keywords = []
|
|
10
|
+
classifiers = [
|
|
11
|
+
"Development Status :: 4 - Beta",
|
|
12
|
+
"Framework :: Pydantic",
|
|
13
|
+
"Framework :: Pydantic :: 2",
|
|
14
|
+
"Intended Audience :: Developers",
|
|
15
|
+
"Operating System :: OS Independent",
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
18
|
+
"Programming Language :: Python :: 3.12",
|
|
19
|
+
"Programming Language :: Python :: 3.13",
|
|
20
|
+
"Programming Language :: Python :: 3.14",
|
|
21
|
+
"Topic :: Documentation",
|
|
22
|
+
"Topic :: Software Development",
|
|
23
|
+
"Topic :: Utilities",
|
|
24
|
+
"Typing :: Typed",
|
|
25
|
+
]
|
|
26
|
+
dependencies = [
|
|
27
|
+
"anyenv>=0.4.1",
|
|
28
|
+
"mkdown>=0.12.1",
|
|
29
|
+
"pydantic",
|
|
30
|
+
"pydantic-settings>=2.8.1",
|
|
31
|
+
"pypdf",
|
|
32
|
+
"pytest-docker>=3.2.1",
|
|
33
|
+
"schemez>=0.0.1",
|
|
34
|
+
"upathtools>=0.4.3",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
[[project.authors]]
|
|
38
|
+
name = "Philipp Temminghoff"
|
|
39
|
+
email = "philipptemminghoff@googlemail.com"
|
|
40
|
+
|
|
41
|
+
[project.urls]
|
|
42
|
+
"Code coverage" = "https://app.codecov.io/gh/phil65/docler"
|
|
43
|
+
Discussions = "https://github.com/phil65/docler/discussions"
|
|
44
|
+
Documentation = "https://phil65.github.io/docler/"
|
|
45
|
+
Issues = "https://github.com/phil65/docler/issues"
|
|
46
|
+
Source = "https://github.com/phil65/docler"
|
|
47
|
+
|
|
48
|
+
[project.scripts]
|
|
49
|
+
docler = "docler.__main__:cli"
|
|
50
|
+
|
|
51
|
+
[project.optional-dependencies]
|
|
52
|
+
all = [
|
|
53
|
+
"docler[azure,diffs,docling,llama-parse,markitdown,mcp,mistralai,openai,server]",
|
|
54
|
+
"agentpool>=2.10.0",
|
|
55
|
+
]
|
|
56
|
+
azure = ["azure-ai-documentintelligence"]
|
|
57
|
+
diffs = ["diff-match-patch"]
|
|
58
|
+
docling = ["docling[ocrmac,vlm,rapidocr]; python_version < '3.14'"]
|
|
59
|
+
light = [
|
|
60
|
+
"docler[mistralai,diffs,llama-parse,openai,azure]",
|
|
61
|
+
"agentpool>=2.10.0",
|
|
62
|
+
]
|
|
63
|
+
llama-parse = ["llama-parse; python_version < '3.14'"]
|
|
64
|
+
marker = ["marker-pdf; python_version < '3.14'"]
|
|
65
|
+
markitdown = ["markitdown[all]; python_version < '3.14'"]
|
|
66
|
+
mcp = ["fastmcp"]
|
|
67
|
+
mistralai = ["mistralai>=3"]
|
|
68
|
+
olmocr = [
|
|
69
|
+
"olmocr; python_version < '3.14'",
|
|
70
|
+
"accelerate; python_version < '3.14'",
|
|
71
|
+
]
|
|
72
|
+
openai = ["openai"]
|
|
73
|
+
server = [
|
|
74
|
+
"uvicorn",
|
|
75
|
+
"fastapi",
|
|
76
|
+
]
|
|
77
|
+
streamlit = [
|
|
78
|
+
"streamlit; python_version < '3.14'",
|
|
79
|
+
"streambricks; python_version < '3.14'",
|
|
80
|
+
]
|
|
81
|
+
token-chunking = ["tokonomics"]
|
|
82
|
+
|
|
83
|
+
[dependency-groups]
|
|
84
|
+
dev = [
|
|
85
|
+
"devtools",
|
|
86
|
+
"pyreadline3",
|
|
87
|
+
"pytest",
|
|
88
|
+
"pytest-asyncio",
|
|
89
|
+
"pytest-cov",
|
|
90
|
+
"syrupy>=4.9.1",
|
|
91
|
+
]
|
|
92
|
+
benchmark = ["pyinstrument"]
|
|
93
|
+
docs = [
|
|
94
|
+
"mkdocs-material",
|
|
95
|
+
"mkdocs-mknodes",
|
|
96
|
+
]
|
|
97
|
+
lint = [
|
|
98
|
+
"mypy; python_version >= '3.14'",
|
|
99
|
+
"mypy[faster-cache]; python_version < '3.14'",
|
|
100
|
+
"ruff",
|
|
101
|
+
]
|
|
102
|
+
|
|
103
|
+
[build-system]
|
|
104
|
+
requires = ["uv_build>=0.8.22"]
|
|
105
|
+
build-backend = "uv_build"
|
|
106
|
+
|
|
107
|
+
[tool.coverage.report]
|
|
108
|
+
exclude_lines = [
|
|
109
|
+
"pragma: no cover",
|
|
110
|
+
"if TYPE_CHECKING:",
|
|
111
|
+
"@overload",
|
|
112
|
+
"except ImportError",
|
|
113
|
+
'if __name__ == "__main__":',
|
|
114
|
+
]
|
|
115
|
+
|
|
116
|
+
[tool.mknodes]
|
|
117
|
+
allowed-commit-types = [
|
|
118
|
+
"fix",
|
|
119
|
+
"feat",
|
|
120
|
+
"refactor",
|
|
121
|
+
"docs",
|
|
122
|
+
"test",
|
|
123
|
+
"build",
|
|
124
|
+
"chore",
|
|
125
|
+
]
|
|
126
|
+
docstring-style = "google"
|
|
127
|
+
|
|
128
|
+
[tool.mypy]
|
|
129
|
+
python_version = "3.13"
|
|
130
|
+
disable_error_code = ["import"]
|
|
131
|
+
pretty = true
|
|
132
|
+
strict = true
|
|
133
|
+
check_untyped_defs = true
|
|
134
|
+
exclude = [
|
|
135
|
+
"venv/",
|
|
136
|
+
".venv/",
|
|
137
|
+
"tests/",
|
|
138
|
+
]
|
|
139
|
+
plugins = ["pydantic.mypy"]
|
|
140
|
+
|
|
141
|
+
[tool.pyright]
|
|
142
|
+
venvPath = "."
|
|
143
|
+
venv = ".venv"
|
|
144
|
+
pythonVersion = "3.13"
|
|
145
|
+
pythonPlatform = "All"
|
|
146
|
+
typeCheckingMode = "basic"
|
|
147
|
+
deprecateTypingAliases = true
|
|
148
|
+
reportMissingTypeStubs = false
|
|
149
|
+
reportUnusedCallResult = false
|
|
150
|
+
reportUnknownVariableType = false
|
|
151
|
+
reportAny = false
|
|
152
|
+
reportImplicitOverride = false
|
|
153
|
+
reportUnusedFunction = false
|
|
154
|
+
reportImplicitStringConcatenation = false
|
|
155
|
+
reportIgnoreCommentWithoutRule = false
|
|
156
|
+
reportUnannotatedClassAttribute = false
|
|
157
|
+
reportSelfClsParameterName = false
|
|
158
|
+
reportPrivateImportUsage = false
|
|
159
|
+
|
|
160
|
+
[tool.pytest]
|
|
161
|
+
log_cli = true
|
|
162
|
+
log_date_format = "%Y-%m-%d %H:%M:%S"
|
|
163
|
+
log_format = "%(asctime)s %(levelname)s %(message)s"
|
|
164
|
+
minversion = "9.0"
|
|
165
|
+
testpaths = ["tests/"]
|
|
166
|
+
asyncio_mode = "auto"
|
|
167
|
+
|
|
168
|
+
[tool.ruff]
|
|
169
|
+
line-length = 100
|
|
170
|
+
extend-exclude = ["docs"]
|
|
171
|
+
target-version = "py313"
|
|
172
|
+
|
|
173
|
+
[tool.ruff.lint]
|
|
174
|
+
future-annotations = true
|
|
175
|
+
select = [
|
|
176
|
+
"A",
|
|
177
|
+
"B",
|
|
178
|
+
"BLE",
|
|
179
|
+
"C",
|
|
180
|
+
"C4",
|
|
181
|
+
"COM",
|
|
182
|
+
"D",
|
|
183
|
+
"E",
|
|
184
|
+
"EM",
|
|
185
|
+
"EXE",
|
|
186
|
+
"F",
|
|
187
|
+
"FA",
|
|
188
|
+
"FLY",
|
|
189
|
+
"G",
|
|
190
|
+
"I",
|
|
191
|
+
"ICN",
|
|
192
|
+
"INP",
|
|
193
|
+
"INT",
|
|
194
|
+
"ISC",
|
|
195
|
+
"N",
|
|
196
|
+
"PERF",
|
|
197
|
+
"PIE",
|
|
198
|
+
"PLE",
|
|
199
|
+
"PLC",
|
|
200
|
+
"PLR",
|
|
201
|
+
"PT",
|
|
202
|
+
"PTH",
|
|
203
|
+
"PYI",
|
|
204
|
+
"Q",
|
|
205
|
+
"RET",
|
|
206
|
+
"RSE",
|
|
207
|
+
"RUF",
|
|
208
|
+
"SIM",
|
|
209
|
+
"SLF",
|
|
210
|
+
"SLOT",
|
|
211
|
+
"T10",
|
|
212
|
+
"TC",
|
|
213
|
+
"TID",
|
|
214
|
+
"TRY",
|
|
215
|
+
"UP",
|
|
216
|
+
"W",
|
|
217
|
+
"YTT",
|
|
218
|
+
]
|
|
219
|
+
ignore = [
|
|
220
|
+
"C408",
|
|
221
|
+
"B905",
|
|
222
|
+
"C901",
|
|
223
|
+
"COM812",
|
|
224
|
+
"D100",
|
|
225
|
+
"D101",
|
|
226
|
+
"D102",
|
|
227
|
+
"D103",
|
|
228
|
+
"D104",
|
|
229
|
+
"D105",
|
|
230
|
+
"D106",
|
|
231
|
+
"D107",
|
|
232
|
+
"D203",
|
|
233
|
+
"D204",
|
|
234
|
+
"D213",
|
|
235
|
+
"D215",
|
|
236
|
+
"D400",
|
|
237
|
+
"D401",
|
|
238
|
+
"D404",
|
|
239
|
+
"D406",
|
|
240
|
+
"D407",
|
|
241
|
+
"D408",
|
|
242
|
+
"D409",
|
|
243
|
+
"D413",
|
|
244
|
+
"EM101",
|
|
245
|
+
"EM102",
|
|
246
|
+
"ISC001",
|
|
247
|
+
"PLC0415",
|
|
248
|
+
"PLR0912",
|
|
249
|
+
"PLR0913",
|
|
250
|
+
"PLR0915",
|
|
251
|
+
"PLR0917",
|
|
252
|
+
"SLF001",
|
|
253
|
+
"TRY003",
|
|
254
|
+
"TC006",
|
|
255
|
+
]
|
|
256
|
+
|
|
257
|
+
[tool.ruff.lint.flake8-quotes]
|
|
258
|
+
docstring-quotes = "double"
|
|
259
|
+
|
|
260
|
+
[tool.ruff.lint.flake8-type-checking]
|
|
261
|
+
runtime-evaluated-base-classes = [
|
|
262
|
+
"pydantic.BaseModel",
|
|
263
|
+
"sqlalchemy.orm.DeclarativeBase",
|
|
264
|
+
"schemez.Schema",
|
|
265
|
+
"sqlmodel.SQLModel",
|
|
266
|
+
]
|
|
267
|
+
runtime-evaluated-decorators = [
|
|
268
|
+
"pydantic.validate_call",
|
|
269
|
+
"fastapi.FastAPI.get",
|
|
270
|
+
"fastapi.FastAPI.post",
|
|
271
|
+
"fastmcp.FastMCP.tool",
|
|
272
|
+
"typer.Typer.command",
|
|
273
|
+
]
|
|
274
|
+
|
|
275
|
+
[tool.ruff.lint.isort]
|
|
276
|
+
lines-after-imports = 2
|
|
277
|
+
force-sort-within-sections = true
|
|
278
|
+
combine-as-imports = true
|
|
279
|
+
|
|
280
|
+
[tool.ruff.lint.mccabe]
|
|
281
|
+
max-complexity = 15
|
|
282
|
+
|
|
283
|
+
[tool.ruff.lint.per-file-ignores]
|
|
284
|
+
"__init__.py" = [
|
|
285
|
+
"E402",
|
|
286
|
+
"I001",
|
|
287
|
+
]
|
|
288
|
+
"scripts/*" = ["INP001"]
|
|
289
|
+
"*tests/*" = ["D100"]
|
|
290
|
+
|
|
291
|
+
[tool.ruff.format]
|
|
292
|
+
preview = true
|
|
293
|
+
|
|
294
|
+
[tool.ty.environment]
|
|
295
|
+
python-version = "3.13"
|
|
296
|
+
python-platform = "all"
|
|
297
|
+
|
|
298
|
+
[tool.uv]
|
|
299
|
+
default-groups = [
|
|
300
|
+
"dev",
|
|
301
|
+
"lint",
|
|
302
|
+
"docs",
|
|
303
|
+
]
|
|
304
|
+
conflicts = [
|
|
305
|
+
[
|
|
306
|
+
{ extra = "marker" },
|
|
307
|
+
{ extra = "all" },
|
|
308
|
+
],
|
|
309
|
+
[
|
|
310
|
+
{ extra = "marker" },
|
|
311
|
+
{ extra = "light" },
|
|
312
|
+
],
|
|
313
|
+
]
|
|
314
|
+
|
|
315
|
+
[tool.uv.build-backend]
|
|
316
|
+
module-name = [
|
|
317
|
+
"docler",
|
|
318
|
+
"docler_config",
|
|
319
|
+
"docler_streamlit",
|
|
320
|
+
"docler_api",
|
|
321
|
+
"docler_mcp",
|
|
322
|
+
]
|
|
323
|
+
wheel-exclude = [
|
|
324
|
+
".mypy_cache/**",
|
|
325
|
+
"__pycache__/**",
|
|
326
|
+
"*.pyc",
|
|
327
|
+
"*.pyo",
|
|
328
|
+
".pytest_cache/**",
|
|
329
|
+
".ruff_cache/**",
|
|
330
|
+
]
|
|
331
|
+
source-exclude = [
|
|
332
|
+
".mypy_cache/**",
|
|
333
|
+
"__pycache__/**",
|
|
334
|
+
"*.pyc",
|
|
335
|
+
"*.pyo",
|
|
336
|
+
".pytest_cache/**",
|
|
337
|
+
".ruff_cache/**",
|
|
338
|
+
]
|