natural-pdf 0.1.7__py3-none-any.whl → 0.1.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- natural_pdf/__init__.py +3 -0
- natural_pdf/analyzers/layout/base.py +1 -5
- natural_pdf/analyzers/layout/gemini.py +61 -51
- natural_pdf/analyzers/layout/layout_analyzer.py +40 -11
- natural_pdf/analyzers/layout/layout_manager.py +26 -84
- natural_pdf/analyzers/layout/layout_options.py +7 -0
- natural_pdf/analyzers/layout/pdfplumber_table_finder.py +142 -0
- natural_pdf/analyzers/layout/surya.py +46 -123
- natural_pdf/analyzers/layout/tatr.py +51 -4
- natural_pdf/analyzers/text_structure.py +3 -5
- natural_pdf/analyzers/utils.py +3 -3
- natural_pdf/classification/manager.py +422 -0
- natural_pdf/classification/mixin.py +163 -0
- natural_pdf/classification/results.py +80 -0
- natural_pdf/collections/mixins.py +111 -0
- natural_pdf/collections/pdf_collection.py +434 -15
- natural_pdf/core/element_manager.py +83 -0
- natural_pdf/core/highlighting_service.py +13 -22
- natural_pdf/core/page.py +578 -93
- natural_pdf/core/pdf.py +912 -460
- natural_pdf/elements/base.py +134 -40
- natural_pdf/elements/collections.py +712 -109
- natural_pdf/elements/region.py +722 -69
- natural_pdf/elements/text.py +4 -1
- natural_pdf/export/mixin.py +137 -0
- natural_pdf/exporters/base.py +3 -3
- natural_pdf/exporters/paddleocr.py +5 -4
- natural_pdf/extraction/manager.py +135 -0
- natural_pdf/extraction/mixin.py +279 -0
- natural_pdf/extraction/result.py +23 -0
- natural_pdf/ocr/__init__.py +5 -5
- natural_pdf/ocr/engine_doctr.py +346 -0
- natural_pdf/ocr/engine_easyocr.py +6 -3
- natural_pdf/ocr/ocr_factory.py +24 -4
- natural_pdf/ocr/ocr_manager.py +122 -26
- natural_pdf/ocr/ocr_options.py +94 -11
- natural_pdf/ocr/utils.py +19 -6
- natural_pdf/qa/document_qa.py +0 -4
- natural_pdf/search/__init__.py +20 -34
- natural_pdf/search/haystack_search_service.py +309 -265
- natural_pdf/search/haystack_utils.py +99 -75
- natural_pdf/search/search_service_protocol.py +11 -12
- natural_pdf/selectors/parser.py +431 -230
- natural_pdf/utils/debug.py +3 -3
- natural_pdf/utils/identifiers.py +1 -1
- natural_pdf/utils/locks.py +8 -0
- natural_pdf/utils/packaging.py +8 -6
- natural_pdf/utils/text_extraction.py +60 -1
- natural_pdf/utils/tqdm_utils.py +51 -0
- natural_pdf/utils/visualization.py +18 -0
- natural_pdf/widgets/viewer.py +4 -25
- {natural_pdf-0.1.7.dist-info → natural_pdf-0.1.9.dist-info}/METADATA +17 -3
- natural_pdf-0.1.9.dist-info/RECORD +80 -0
- {natural_pdf-0.1.7.dist-info → natural_pdf-0.1.9.dist-info}/WHEEL +1 -1
- {natural_pdf-0.1.7.dist-info → natural_pdf-0.1.9.dist-info}/top_level.txt +0 -2
- docs/api/index.md +0 -386
- docs/assets/favicon.png +0 -3
- docs/assets/favicon.svg +0 -3
- docs/assets/javascripts/custom.js +0 -17
- docs/assets/logo.svg +0 -3
- docs/assets/sample-screen.png +0 -0
- docs/assets/social-preview.png +0 -17
- docs/assets/social-preview.svg +0 -17
- docs/assets/stylesheets/custom.css +0 -65
- docs/document-qa/index.ipynb +0 -435
- docs/document-qa/index.md +0 -79
- docs/element-selection/index.ipynb +0 -915
- docs/element-selection/index.md +0 -229
- docs/finetuning/index.md +0 -176
- docs/index.md +0 -170
- docs/installation/index.md +0 -69
- docs/interactive-widget/index.ipynb +0 -962
- docs/interactive-widget/index.md +0 -12
- docs/layout-analysis/index.ipynb +0 -818
- docs/layout-analysis/index.md +0 -185
- docs/ocr/index.md +0 -209
- docs/pdf-navigation/index.ipynb +0 -314
- docs/pdf-navigation/index.md +0 -97
- docs/regions/index.ipynb +0 -816
- docs/regions/index.md +0 -294
- docs/tables/index.ipynb +0 -658
- docs/tables/index.md +0 -144
- docs/text-analysis/index.ipynb +0 -370
- docs/text-analysis/index.md +0 -105
- docs/text-extraction/index.ipynb +0 -1478
- docs/text-extraction/index.md +0 -292
- docs/tutorials/01-loading-and-extraction.ipynb +0 -194
- docs/tutorials/01-loading-and-extraction.md +0 -95
- docs/tutorials/02-finding-elements.ipynb +0 -340
- docs/tutorials/02-finding-elements.md +0 -149
- docs/tutorials/03-extracting-blocks.ipynb +0 -147
- docs/tutorials/03-extracting-blocks.md +0 -48
- docs/tutorials/04-table-extraction.ipynb +0 -114
- docs/tutorials/04-table-extraction.md +0 -50
- docs/tutorials/05-excluding-content.ipynb +0 -270
- docs/tutorials/05-excluding-content.md +0 -109
- docs/tutorials/06-document-qa.ipynb +0 -332
- docs/tutorials/06-document-qa.md +0 -91
- docs/tutorials/07-layout-analysis.ipynb +0 -288
- docs/tutorials/07-layout-analysis.md +0 -66
- docs/tutorials/07-working-with-regions.ipynb +0 -413
- docs/tutorials/07-working-with-regions.md +0 -151
- docs/tutorials/08-spatial-navigation.ipynb +0 -508
- docs/tutorials/08-spatial-navigation.md +0 -190
- docs/tutorials/09-section-extraction.ipynb +0 -2434
- docs/tutorials/09-section-extraction.md +0 -256
- docs/tutorials/10-form-field-extraction.ipynb +0 -512
- docs/tutorials/10-form-field-extraction.md +0 -201
- docs/tutorials/11-enhanced-table-processing.ipynb +0 -54
- docs/tutorials/11-enhanced-table-processing.md +0 -9
- docs/tutorials/12-ocr-integration.ipynb +0 -604
- docs/tutorials/12-ocr-integration.md +0 -175
- docs/tutorials/13-semantic-search.ipynb +0 -1328
- docs/tutorials/13-semantic-search.md +0 -77
- docs/visual-debugging/index.ipynb +0 -2970
- docs/visual-debugging/index.md +0 -157
- docs/visual-debugging/region.png +0 -0
- natural_pdf/templates/finetune/fine_tune_paddleocr.md +0 -415
- natural_pdf/templates/spa/css/style.css +0 -334
- natural_pdf/templates/spa/index.html +0 -31
- natural_pdf/templates/spa/js/app.js +0 -472
- natural_pdf/templates/spa/words.txt +0 -235976
- natural_pdf/widgets/frontend/viewer.js +0 -88
- natural_pdf-0.1.7.dist-info/RECORD +0 -145
- notebooks/Examples.ipynb +0 -1293
- pdfs/.gitkeep +0 -0
- pdfs/01-practice.pdf +0 -543
- pdfs/0500000US42001.pdf +0 -0
- pdfs/0500000US42007.pdf +0 -0
- pdfs/2014 Statistics.pdf +0 -0
- pdfs/2019 Statistics.pdf +0 -0
- pdfs/Atlanta_Public_Schools_GA_sample.pdf +0 -0
- pdfs/needs-ocr.pdf +0 -0
- {natural_pdf-0.1.7.dist-info → natural_pdf-0.1.9.dist-info}/licenses/LICENSE +0 -0
@@ -1,604 +0,0 @@
|
|
1
|
-
{
|
2
|
-
"cells": [
|
3
|
-
{
|
4
|
-
"cell_type": "markdown",
|
5
|
-
"id": "8b02fa9e",
|
6
|
-
"metadata": {},
|
7
|
-
"source": [
|
8
|
-
"# OCR Integration for Scanned Documents\n",
|
9
|
-
"\n",
|
10
|
-
"Optical Character Recognition (OCR) allows you to extract text from scanned documents where the text isn't embedded in the PDF. This tutorial demonstrates how to work with scanned documents."
|
11
|
-
]
|
12
|
-
},
|
13
|
-
{
|
14
|
-
"cell_type": "code",
|
15
|
-
"execution_count": 1,
|
16
|
-
"id": "bde55ac1",
|
17
|
-
"metadata": {
|
18
|
-
"execution": {
|
19
|
-
"iopub.execute_input": "2025-04-21T21:32:06.104226Z",
|
20
|
-
"iopub.status.busy": "2025-04-21T21:32:06.104019Z",
|
21
|
-
"iopub.status.idle": "2025-04-21T21:32:06.108232Z",
|
22
|
-
"shell.execute_reply": "2025-04-21T21:32:06.107754Z"
|
23
|
-
}
|
24
|
-
},
|
25
|
-
"outputs": [],
|
26
|
-
"source": [
|
27
|
-
"#%pip install \"natural-pdf[all]\""
|
28
|
-
]
|
29
|
-
},
|
30
|
-
{
|
31
|
-
"cell_type": "code",
|
32
|
-
"execution_count": 2,
|
33
|
-
"id": "5c624a53",
|
34
|
-
"metadata": {
|
35
|
-
"execution": {
|
36
|
-
"iopub.execute_input": "2025-04-21T21:32:06.110125Z",
|
37
|
-
"iopub.status.busy": "2025-04-21T21:32:06.109925Z",
|
38
|
-
"iopub.status.idle": "2025-04-21T21:32:14.008764Z",
|
39
|
-
"shell.execute_reply": "2025-04-21T21:32:14.008268Z"
|
40
|
-
}
|
41
|
-
},
|
42
|
-
"outputs": [
|
43
|
-
{
|
44
|
-
"data": {
|
45
|
-
"text/plain": [
|
46
|
-
"'Without OCR: 0 characters extracted'"
|
47
|
-
]
|
48
|
-
},
|
49
|
-
"execution_count": 2,
|
50
|
-
"metadata": {},
|
51
|
-
"output_type": "execute_result"
|
52
|
-
}
|
53
|
-
],
|
54
|
-
"source": [
|
55
|
-
"from natural_pdf import PDF\n",
|
56
|
-
"\n",
|
57
|
-
"# Load a PDF\n",
|
58
|
-
"pdf = PDF(\"https://github.com/jsoma/natural-pdf/raw/refs/heads/main/pdfs/needs-ocr.pdf\")\n",
|
59
|
-
"page = pdf.pages[0]\n",
|
60
|
-
"\n",
|
61
|
-
"# Try extracting text without OCR\n",
|
62
|
-
"text_without_ocr = page.extract_text()\n",
|
63
|
-
"f\"Without OCR: {len(text_without_ocr)} characters extracted\""
|
64
|
-
]
|
65
|
-
},
|
66
|
-
{
|
67
|
-
"cell_type": "markdown",
|
68
|
-
"id": "461a5090",
|
69
|
-
"metadata": {},
|
70
|
-
"source": [
|
71
|
-
"## Finding Text Elements with OCR"
|
72
|
-
]
|
73
|
-
},
|
74
|
-
{
|
75
|
-
"cell_type": "code",
|
76
|
-
"execution_count": 3,
|
77
|
-
"id": "895e3c2c",
|
78
|
-
"metadata": {
|
79
|
-
"execution": {
|
80
|
-
"iopub.execute_input": "2025-04-21T21:32:14.010745Z",
|
81
|
-
"iopub.status.busy": "2025-04-21T21:32:14.010324Z",
|
82
|
-
"iopub.status.idle": "2025-04-21T21:32:28.416856Z",
|
83
|
-
"shell.execute_reply": "2025-04-21T21:32:28.416360Z"
|
84
|
-
}
|
85
|
-
},
|
86
|
-
"outputs": [
|
87
|
-
{
|
88
|
-
"name": "stderr",
|
89
|
-
"output_type": "stream",
|
90
|
-
"text": [
|
91
|
-
"\u001b[2m2025-04-21T21:32:14.064078Z\u001b[0m [\u001b[33m\u001b[1mwarning \u001b[0m] \u001b[1mUsing CPU. Note: This module is much faster with a GPU.\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m71\u001b[0m \u001b[36mmodule\u001b[0m=\u001b[35measyocr.easyocr\u001b[0m\n"
|
92
|
-
]
|
93
|
-
},
|
94
|
-
{
|
95
|
-
"name": "stderr",
|
96
|
-
"output_type": "stream",
|
97
|
-
"text": [
|
98
|
-
"[2025-04-21 17:32:14,064] [ WARNING] easyocr.py:71 - Using CPU. Note: This module is much faster with a GPU.\n"
|
99
|
-
]
|
100
|
-
},
|
101
|
-
{
|
102
|
-
"data": {
|
103
|
-
"text/plain": [
|
104
|
-
"<ElementCollection[TextElement](count=47)>"
|
105
|
-
]
|
106
|
-
},
|
107
|
-
"execution_count": 3,
|
108
|
-
"metadata": {},
|
109
|
-
"output_type": "execute_result"
|
110
|
-
}
|
111
|
-
],
|
112
|
-
"source": [
|
113
|
-
"# Convert text-as-image to text elements\n",
|
114
|
-
"page.apply_ocr()\n",
|
115
|
-
"\n",
|
116
|
-
"# Select all text pieces on the page\n",
|
117
|
-
"text_elements = page.find_all('text')\n",
|
118
|
-
"f\"Found {len(text_elements)} text elements\"\n",
|
119
|
-
"\n",
|
120
|
-
"# Visualize the elements\n",
|
121
|
-
"text_elements.highlight()"
|
122
|
-
]
|
123
|
-
},
|
124
|
-
{
|
125
|
-
"cell_type": "markdown",
|
126
|
-
"id": "36051d57",
|
127
|
-
"metadata": {},
|
128
|
-
"source": [
|
129
|
-
"## OCR Configuration Options"
|
130
|
-
]
|
131
|
-
},
|
132
|
-
{
|
133
|
-
"cell_type": "code",
|
134
|
-
"execution_count": 4,
|
135
|
-
"id": "d4461746",
|
136
|
-
"metadata": {
|
137
|
-
"execution": {
|
138
|
-
"iopub.execute_input": "2025-04-21T21:32:28.418763Z",
|
139
|
-
"iopub.status.busy": "2025-04-21T21:32:28.418565Z",
|
140
|
-
"iopub.status.idle": "2025-04-21T21:32:28.423024Z",
|
141
|
-
"shell.execute_reply": "2025-04-21T21:32:28.422671Z"
|
142
|
-
}
|
143
|
-
},
|
144
|
-
"outputs": [
|
145
|
-
{
|
146
|
-
"data": {
|
147
|
-
"text/plain": [
|
148
|
-
"' \\n \\n ...'"
|
149
|
-
]
|
150
|
-
},
|
151
|
-
"execution_count": 4,
|
152
|
-
"metadata": {},
|
153
|
-
"output_type": "execute_result"
|
154
|
-
}
|
155
|
-
],
|
156
|
-
"source": [
|
157
|
-
"# Set OCR configuration for better results\n",
|
158
|
-
"page.ocr_config = {\n",
|
159
|
-
" 'language': 'eng', # English\n",
|
160
|
-
" 'dpi': 300, # Higher resolution\n",
|
161
|
-
"}\n",
|
162
|
-
"\n",
|
163
|
-
"# Extract text with the improved configuration\n",
|
164
|
-
"improved_text = page.extract_text()\n",
|
165
|
-
"\n",
|
166
|
-
"# Preview the text\n",
|
167
|
-
"improved_text[:200] + \"...\" if len(improved_text) > 200 else improved_text"
|
168
|
-
]
|
169
|
-
},
|
170
|
-
{
|
171
|
-
"cell_type": "markdown",
|
172
|
-
"id": "d5a96ac7",
|
173
|
-
"metadata": {},
|
174
|
-
"source": [
|
175
|
-
"## Working with Multi-language Documents"
|
176
|
-
]
|
177
|
-
},
|
178
|
-
{
|
179
|
-
"cell_type": "code",
|
180
|
-
"execution_count": 5,
|
181
|
-
"id": "9fa156f5",
|
182
|
-
"metadata": {
|
183
|
-
"execution": {
|
184
|
-
"iopub.execute_input": "2025-04-21T21:32:28.424374Z",
|
185
|
-
"iopub.status.busy": "2025-04-21T21:32:28.424235Z",
|
186
|
-
"iopub.status.idle": "2025-04-21T21:32:28.428114Z",
|
187
|
-
"shell.execute_reply": "2025-04-21T21:32:28.427816Z"
|
188
|
-
}
|
189
|
-
},
|
190
|
-
"outputs": [
|
191
|
-
{
|
192
|
-
"data": {
|
193
|
-
"text/plain": [
|
194
|
-
"' \\n \\n '"
|
195
|
-
]
|
196
|
-
},
|
197
|
-
"execution_count": 5,
|
198
|
-
"metadata": {},
|
199
|
-
"output_type": "execute_result"
|
200
|
-
}
|
201
|
-
],
|
202
|
-
"source": [
|
203
|
-
"# Configure for multiple languages\n",
|
204
|
-
"page.ocr_config = {\n",
|
205
|
-
" 'language': 'eng+fra+deu', # English, French, German\n",
|
206
|
-
" 'dpi': 300\n",
|
207
|
-
"}\n",
|
208
|
-
"\n",
|
209
|
-
"# Extract text with multi-language support\n",
|
210
|
-
"multilang_text = page.extract_text()\n",
|
211
|
-
"multilang_text[:200]"
|
212
|
-
]
|
213
|
-
},
|
214
|
-
{
|
215
|
-
"cell_type": "markdown",
|
216
|
-
"id": "d3ccf43f",
|
217
|
-
"metadata": {},
|
218
|
-
"source": [
|
219
|
-
"## Extracting Tables from Scanned Documents"
|
220
|
-
]
|
221
|
-
},
|
222
|
-
{
|
223
|
-
"cell_type": "code",
|
224
|
-
"execution_count": 6,
|
225
|
-
"id": "ee7a7e7d",
|
226
|
-
"metadata": {
|
227
|
-
"execution": {
|
228
|
-
"iopub.execute_input": "2025-04-21T21:32:28.429414Z",
|
229
|
-
"iopub.status.busy": "2025-04-21T21:32:28.429283Z",
|
230
|
-
"iopub.status.idle": "2025-04-21T21:32:30.754086Z",
|
231
|
-
"shell.execute_reply": "2025-04-21T21:32:30.753700Z"
|
232
|
-
}
|
233
|
-
},
|
234
|
-
"outputs": [
|
235
|
-
{
|
236
|
-
"name": "stderr",
|
237
|
-
"output_type": "stream",
|
238
|
-
"text": [
|
239
|
-
"\u001b[2m2025-04-21T21:32:28.446098Z\u001b[0m [\u001b[33m\u001b[1mwarning \u001b[0m] \u001b[1mGOOGLE_API_KEY environment variable not set. Gemini detector (via OpenAI lib) will not be available.\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m72\u001b[0m \u001b[36mmodule\u001b[0m=\u001b[35mnatural_pdf.analyzers.layout.gemini\u001b[0m\n"
|
240
|
-
]
|
241
|
-
},
|
242
|
-
{
|
243
|
-
"name": "stderr",
|
244
|
-
"output_type": "stream",
|
245
|
-
"text": [
|
246
|
-
"[2025-04-21 17:32:28,446] [ WARNING] gemini.py:72 - GOOGLE_API_KEY environment variable not set. Gemini detector (via OpenAI lib) will not be available.\n"
|
247
|
-
]
|
248
|
-
},
|
249
|
-
{
|
250
|
-
"name": "stderr",
|
251
|
-
"output_type": "stream",
|
252
|
-
"text": [
|
253
|
-
"\u001b[2m2025-04-21T21:32:28.446834Z\u001b[0m [\u001b[33m\u001b[1mwarning \u001b[0m] \u001b[1mGOOGLE_API_KEY environment variable not set. Gemini detector (via OpenAI lib) will not be available.\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m72\u001b[0m \u001b[36mmodule\u001b[0m=\u001b[35mnatural_pdf.analyzers.layout.gemini\u001b[0m\n"
|
254
|
-
]
|
255
|
-
},
|
256
|
-
{
|
257
|
-
"name": "stderr",
|
258
|
-
"output_type": "stream",
|
259
|
-
"text": [
|
260
|
-
"[2025-04-21 17:32:28,446] [ WARNING] gemini.py:72 - GOOGLE_API_KEY environment variable not set. Gemini detector (via OpenAI lib) will not be available.\n"
|
261
|
-
]
|
262
|
-
},
|
263
|
-
{
|
264
|
-
"name": "stdout",
|
265
|
-
"output_type": "stream",
|
266
|
-
"text": [
|
267
|
-
"\n"
|
268
|
-
]
|
269
|
-
},
|
270
|
-
{
|
271
|
-
"name": "stdout",
|
272
|
-
"output_type": "stream",
|
273
|
-
"text": [
|
274
|
-
"image 1/1 /var/folders/25/h3prywj14qb0mlkl2s8bxq5m0000gn/T/tmpjbbxsx1v/temp_layout_image.png: 1024x800 2 titles, 2 plain texts, 3 abandons, 1 table, 1940.4ms\n"
|
275
|
-
]
|
276
|
-
},
|
277
|
-
{
|
278
|
-
"name": "stdout",
|
279
|
-
"output_type": "stream",
|
280
|
-
"text": [
|
281
|
-
"Speed: 5.4ms preprocess, 1940.4ms inference, 1.0ms postprocess per image at shape (1, 3, 1024, 800)\n"
|
282
|
-
]
|
283
|
-
}
|
284
|
-
],
|
285
|
-
"source": [
|
286
|
-
"# Enable OCR and analyze the document layout\n",
|
287
|
-
"page.use_ocr = True\n",
|
288
|
-
"page.analyze_layout()\n",
|
289
|
-
"\n",
|
290
|
-
"# Find table regions\n",
|
291
|
-
"table_regions = page.find_all('region[type=table]')\n",
|
292
|
-
"\n",
|
293
|
-
"# Visualize any detected tables\n",
|
294
|
-
"table_regions.highlight()\n",
|
295
|
-
"\n",
|
296
|
-
"# Extract the first table if found\n",
|
297
|
-
"if table_regions:\n",
|
298
|
-
" table_data = table_regions[0].extract_table()\n",
|
299
|
-
" table_data\n",
|
300
|
-
"else:\n",
|
301
|
-
" \"No tables found in the document\""
|
302
|
-
]
|
303
|
-
},
|
304
|
-
{
|
305
|
-
"cell_type": "markdown",
|
306
|
-
"id": "6a3c701e",
|
307
|
-
"metadata": {},
|
308
|
-
"source": [
|
309
|
-
"## Finding Form Fields in Scanned Documents"
|
310
|
-
]
|
311
|
-
},
|
312
|
-
{
|
313
|
-
"cell_type": "code",
|
314
|
-
"execution_count": 7,
|
315
|
-
"id": "7180badd",
|
316
|
-
"metadata": {
|
317
|
-
"execution": {
|
318
|
-
"iopub.execute_input": "2025-04-21T21:32:30.755960Z",
|
319
|
-
"iopub.status.busy": "2025-04-21T21:32:30.755766Z",
|
320
|
-
"iopub.status.idle": "2025-04-21T21:32:30.762760Z",
|
321
|
-
"shell.execute_reply": "2025-04-21T21:32:30.762434Z"
|
322
|
-
}
|
323
|
-
},
|
324
|
-
"outputs": [
|
325
|
-
{
|
326
|
-
"data": {
|
327
|
-
"text/plain": [
|
328
|
-
"{\"Site: Durham's Meatpacking Chicago, IIl.\": 'Jungle Health and Satety Inspection Service\\nINS-UPONSINCLAIR \\n \\n \\n \\n \\n \\nSummary: Worst of any, however; were the fertilizer men, and those who served in the cooking rooms\\nThese people could not be shown to the visitor for the odor of a fertilizer man would scare any\\nvisitor at a hundred yards, and as for the other men, who worked in tank rooms full of steam, and in\\nsome of which there were open vats near the level of the floor; their peculiar trouble was that they fell\\ninlo the vats; and when they were fished out; there was never enough of them left to be worth\\nwould be overlooked for days, till all but the bones of them had gone out\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\nLevel \\nUnsanitary Working Conditions Critical\\nInadequate Protective Equipment: Serious\\n \\nSerious \\nFailure to Properly Storc Hazardous Materials_ Critical\\nSafety Measures_ Serious \\nInadequate Ventilation Systems Serious\\n \\nInsufficient Employee Training for Safe Work Practices Serious\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\nJungle Health and Salety Irspection Service',\n",
|
329
|
-
" 'Date: February 3, 1905': \"Jungle Health and Satety Inspection Service\\n INS-UPONSINCLAIR \\n \\nSite: Durham's Meatpacking Chicago, IIl.\\n \\n \\n \\nSummary: Worst of any, however; were the fertilizer men, and those who served in the cooking rooms\\nThese people could not be shown to the visitor for the odor of a fertilizer man would scare any\\nvisitor at a hundred yards, and as for the other men, who worked in tank rooms full of steam, and in\\nsome of which there were open vats near the level of the floor; their peculiar trouble was that they fell\\ninlo the vats; and when they were fished out; there was never enough of them left to be worth\\ntheywould be overlooked for days, till all but the bones of them had gone out\\nto thc world as Durham's Purc Lcaf Lard!\\n \\n \\n \\n \\n \\n \\n \\n \\n \\nDescription \\n \\nUnsanitary Working Conditions\\nInadequate Protective Equipment:\\nIneffective Injury Prevention _\\n \\nFailure to Properly Storc Hazardous Materials_\\nLack of AdequateFireSafety Measures_\\nInadequate Ventilation Systems\\n \\nInsufficient Employee Training for Safe Work Practices\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\nJungle Health and Salety Irspection Service\",\n",
|
330
|
-
" 'Violation Count': \"Site: Durham's Meatpacking Chicago, IIl.\\nDate: February 3, 1905 \\n \\n \\nSummary: Worst of any, however; were the fertilizer men, and those who served in the cooking rooms\\nThese people could not be shown to the visitor for the odor of a fertilizer man would scare any\\nvisitor at a hundred yards, and as for the other men, who worked in tank rooms full of steam, and in\\nsome of which there were open vats near the level of the floor; their peculiar trouble was that they fell\\ninlo the vats; and when they were fished out; there was never enough of them left to be worth\\nsometimestheywould be overlooked for days, till all but the bones of them had gone out\\nto thc world as Durham's Purc Lcaf Lard!\\n \\n \\n \\n \\n \\n \\n \\n \\n \\nDescription \\n \\nUnsanitary Working Conditions\\nInadequate Protective Equipment:\\nIneffective Injury Prevention _\\n \\nFailure to Properly Storc Hazardous Materials_\\nLack of AdequateFireSafety Measures_\\nInadequate Ventilation Systems\\n \\nInsufficient Employee Training for Safe Work Practices\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\nJungle Health and Salety Irspection Service\",\n",
|
331
|
-
" 'Summary: Worst of any, however; were the fertilizer men, and those who served in the cooking rooms': 'Red (ZGB tuple] \\n \\nJungle Health and Satety Inspection Service\\n \\n \\n \\n \\n \\n \\n \\nordinary \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\nRepeat?',\n",
|
332
|
-
" 'Inadequate Protective Equipment': 'Jungle Health and Satety Inspection Service\\nINS-UPONSINCLAIR \\n \\n \\n \\n \\n \\nSummary: Worst of any, however; were the fertilizer men, and those who served in the cooking rooms\\nThese people could not be shown to the visitor for the odor of a fertilizer man would scare anyordinary\\nvisitor at a hundred yards, and as for the other men, who worked in tank rooms full of steam, and in\\nsome of which there were open vats near the level of the floor; their peculiar trouble was that they fell\\ninlo the vats; and when they were fished out; there was never enough of them left to be worth\\nwould be overlooked for days, till all but the bones of them had gone out\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\nLevel \\nCritical \\nSerious \\n \\nSerious \\nFailure to Properly Storc Hazardous Materials_ Critical\\nSafety Measures_ Serious \\nSerious \\n \\nInsufficient Employee Training for Safe Work Practices Serious\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\nJungle Health and Salety Irspection Service'}"
|
333
|
-
]
|
334
|
-
},
|
335
|
-
"execution_count": 7,
|
336
|
-
"metadata": {},
|
337
|
-
"output_type": "execute_result"
|
338
|
-
}
|
339
|
-
],
|
340
|
-
"source": [
|
341
|
-
"# Look for potential form labels (containing a colon)\n",
|
342
|
-
"labels = page.find_all('text:contains(\":\")') \n",
|
343
|
-
"\n",
|
344
|
-
"# Visualize the labels\n",
|
345
|
-
"labels.highlight()\n",
|
346
|
-
"\n",
|
347
|
-
"# Extract form data by looking to the right of each label\n",
|
348
|
-
"form_data = {}\n",
|
349
|
-
"for label in labels:\n",
|
350
|
-
" # Clean the label text\n",
|
351
|
-
" field_name = label.text.strip().rstrip(':')\n",
|
352
|
-
" \n",
|
353
|
-
" # Find the value to the right\n",
|
354
|
-
" value_element = label.right(width=200)\n",
|
355
|
-
" value = value_element.extract_text().strip()\n",
|
356
|
-
" \n",
|
357
|
-
" # Add to our dictionary\n",
|
358
|
-
" form_data[field_name] = value\n",
|
359
|
-
"\n",
|
360
|
-
"# Display the extracted data\n",
|
361
|
-
"form_data"
|
362
|
-
]
|
363
|
-
},
|
364
|
-
{
|
365
|
-
"cell_type": "markdown",
|
366
|
-
"id": "5495e93c",
|
367
|
-
"metadata": {},
|
368
|
-
"source": [
|
369
|
-
"## Combining OCR with Layout Analysis"
|
370
|
-
]
|
371
|
-
},
|
372
|
-
{
|
373
|
-
"cell_type": "code",
|
374
|
-
"execution_count": 8,
|
375
|
-
"id": "20b489df",
|
376
|
-
"metadata": {
|
377
|
-
"execution": {
|
378
|
-
"iopub.execute_input": "2025-04-21T21:32:30.764203Z",
|
379
|
-
"iopub.status.busy": "2025-04-21T21:32:30.764045Z",
|
380
|
-
"iopub.status.idle": "2025-04-21T21:32:32.790129Z",
|
381
|
-
"shell.execute_reply": "2025-04-21T21:32:32.789771Z"
|
382
|
-
}
|
383
|
-
},
|
384
|
-
"outputs": [
|
385
|
-
{
|
386
|
-
"name": "stderr",
|
387
|
-
"output_type": "stream",
|
388
|
-
"text": [
|
389
|
-
"\u001b[2m2025-04-21T21:32:30.782293Z\u001b[0m [\u001b[33m\u001b[1mwarning \u001b[0m] \u001b[1mGOOGLE_API_KEY environment variable not set. Gemini detector (via OpenAI lib) will not be available.\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m72\u001b[0m \u001b[36mmodule\u001b[0m=\u001b[35mnatural_pdf.analyzers.layout.gemini\u001b[0m\n"
|
390
|
-
]
|
391
|
-
},
|
392
|
-
{
|
393
|
-
"name": "stderr",
|
394
|
-
"output_type": "stream",
|
395
|
-
"text": [
|
396
|
-
"[2025-04-21 17:32:30,782] [ WARNING] gemini.py:72 - GOOGLE_API_KEY environment variable not set. Gemini detector (via OpenAI lib) will not be available.\n"
|
397
|
-
]
|
398
|
-
},
|
399
|
-
{
|
400
|
-
"name": "stderr",
|
401
|
-
"output_type": "stream",
|
402
|
-
"text": [
|
403
|
-
"\u001b[2m2025-04-21T21:32:30.783192Z\u001b[0m [\u001b[33m\u001b[1mwarning \u001b[0m] \u001b[1mGOOGLE_API_KEY environment variable not set. Gemini detector (via OpenAI lib) will not be available.\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m72\u001b[0m \u001b[36mmodule\u001b[0m=\u001b[35mnatural_pdf.analyzers.layout.gemini\u001b[0m\n"
|
404
|
-
]
|
405
|
-
},
|
406
|
-
{
|
407
|
-
"name": "stderr",
|
408
|
-
"output_type": "stream",
|
409
|
-
"text": [
|
410
|
-
"[2025-04-21 17:32:30,783] [ WARNING] gemini.py:72 - GOOGLE_API_KEY environment variable not set. Gemini detector (via OpenAI lib) will not be available.\n"
|
411
|
-
]
|
412
|
-
},
|
413
|
-
{
|
414
|
-
"name": "stdout",
|
415
|
-
"output_type": "stream",
|
416
|
-
"text": [
|
417
|
-
"\n"
|
418
|
-
]
|
419
|
-
},
|
420
|
-
{
|
421
|
-
"name": "stdout",
|
422
|
-
"output_type": "stream",
|
423
|
-
"text": [
|
424
|
-
"image 1/1 /var/folders/25/h3prywj14qb0mlkl2s8bxq5m0000gn/T/tmprtsl29ey/temp_layout_image.png: 1024x800 2 titles, 2 plain texts, 3 abandons, 1 table, 1925.6ms\n"
|
425
|
-
]
|
426
|
-
},
|
427
|
-
{
|
428
|
-
"name": "stdout",
|
429
|
-
"output_type": "stream",
|
430
|
-
"text": [
|
431
|
-
"Speed: 4.7ms preprocess, 1925.6ms inference, 1.2ms postprocess per image at shape (1, 3, 1024, 800)\n"
|
432
|
-
]
|
433
|
-
},
|
434
|
-
{
|
435
|
-
"data": {
|
436
|
-
"text/plain": [
|
437
|
-
"[]"
|
438
|
-
]
|
439
|
-
},
|
440
|
-
"execution_count": 8,
|
441
|
-
"metadata": {},
|
442
|
-
"output_type": "execute_result"
|
443
|
-
}
|
444
|
-
],
|
445
|
-
"source": [
|
446
|
-
"# Apply OCR and analyze layout\n",
|
447
|
-
"page.use_ocr = True\n",
|
448
|
-
"page.analyze_layout()\n",
|
449
|
-
"\n",
|
450
|
-
"# Find document structure elements\n",
|
451
|
-
"headings = page.find_all('region[type=heading]')\n",
|
452
|
-
"paragraphs = page.find_all('region[type=paragraph]')\n",
|
453
|
-
"\n",
|
454
|
-
"# Visualize the structure\n",
|
455
|
-
"headings.highlight(color=\"red\", label=\"Headings\")\n",
|
456
|
-
"paragraphs.highlight(color=\"blue\", label=\"Paragraphs\")\n",
|
457
|
-
"\n",
|
458
|
-
"# Create a simple document outline\n",
|
459
|
-
"document_outline = []\n",
|
460
|
-
"for heading in headings:\n",
|
461
|
-
" heading_text = heading.extract_text()\n",
|
462
|
-
" document_outline.append(heading_text)\n",
|
463
|
-
"\n",
|
464
|
-
"document_outline"
|
465
|
-
]
|
466
|
-
},
|
467
|
-
{
|
468
|
-
"cell_type": "markdown",
|
469
|
-
"id": "320bdfc4",
|
470
|
-
"metadata": {},
|
471
|
-
"source": [
|
472
|
-
"## Working with Multiple Pages"
|
473
|
-
]
|
474
|
-
},
|
475
|
-
{
|
476
|
-
"cell_type": "code",
|
477
|
-
"execution_count": 9,
|
478
|
-
"id": "9421a04d",
|
479
|
-
"metadata": {
|
480
|
-
"execution": {
|
481
|
-
"iopub.execute_input": "2025-04-21T21:32:32.791525Z",
|
482
|
-
"iopub.status.busy": "2025-04-21T21:32:32.791398Z",
|
483
|
-
"iopub.status.idle": "2025-04-21T21:32:32.796295Z",
|
484
|
-
"shell.execute_reply": "2025-04-21T21:32:32.795973Z"
|
485
|
-
}
|
486
|
-
},
|
487
|
-
"outputs": [
|
488
|
-
{
|
489
|
-
"data": {
|
490
|
-
"text/plain": [
|
491
|
-
"['Page 1: \\n ...']"
|
492
|
-
]
|
493
|
-
},
|
494
|
-
"execution_count": 9,
|
495
|
-
"metadata": {},
|
496
|
-
"output_type": "execute_result"
|
497
|
-
}
|
498
|
-
],
|
499
|
-
"source": [
|
500
|
-
"# Process all pages in the document\n",
|
501
|
-
"all_text = []\n",
|
502
|
-
"\n",
|
503
|
-
"for i, page in enumerate(pdf.pages):\n",
|
504
|
-
" # Enable OCR for each page\n",
|
505
|
-
" page.use_ocr = True\n",
|
506
|
-
" \n",
|
507
|
-
" # Extract text\n",
|
508
|
-
" page_text = page.extract_text()\n",
|
509
|
-
" \n",
|
510
|
-
" # Add to our collection with page number\n",
|
511
|
-
" all_text.append(f\"Page {i+1}: {page_text[:100]}...\")\n",
|
512
|
-
"\n",
|
513
|
-
"# Show the first few pages\n",
|
514
|
-
"all_text"
|
515
|
-
]
|
516
|
-
},
|
517
|
-
{
|
518
|
-
"cell_type": "markdown",
|
519
|
-
"id": "d69c14d1",
|
520
|
-
"metadata": {},
|
521
|
-
"source": [
|
522
|
-
"## Saving PDFs with Searchable Text\n",
|
523
|
-
"\n",
|
524
|
-
"After applying OCR to a PDF, you can save a new version of the PDF where the recognized text is embedded as an invisible layer. This makes the text searchable and copyable in standard PDF viewers.\n",
|
525
|
-
"\n",
|
526
|
-
"Use the `save_searchable()` method on the `PDF` object:"
|
527
|
-
]
|
528
|
-
},
|
529
|
-
{
|
530
|
-
"cell_type": "code",
|
531
|
-
"execution_count": 10,
|
532
|
-
"id": "e84f8946",
|
533
|
-
"metadata": {
|
534
|
-
"execution": {
|
535
|
-
"iopub.execute_input": "2025-04-21T21:32:32.797789Z",
|
536
|
-
"iopub.status.busy": "2025-04-21T21:32:32.797610Z",
|
537
|
-
"iopub.status.idle": "2025-04-21T21:32:49.165749Z",
|
538
|
-
"shell.execute_reply": "2025-04-21T21:32:49.165293Z"
|
539
|
-
}
|
540
|
-
},
|
541
|
-
"outputs": [
|
542
|
-
{
|
543
|
-
"name": "stderr",
|
544
|
-
"output_type": "stream",
|
545
|
-
"text": [
|
546
|
-
"\u001b[2m2025-04-21T21:32:32.910436Z\u001b[0m [\u001b[33m\u001b[1mwarning \u001b[0m] \u001b[1mUsing CPU. Note: This module is much faster with a GPU.\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m71\u001b[0m \u001b[36mmodule\u001b[0m=\u001b[35measyocr.easyocr\u001b[0m\n"
|
547
|
-
]
|
548
|
-
},
|
549
|
-
{
|
550
|
-
"name": "stderr",
|
551
|
-
"output_type": "stream",
|
552
|
-
"text": [
|
553
|
-
"[2025-04-21 17:32:32,910] [ WARNING] easyocr.py:71 - Using CPU. Note: This module is much faster with a GPU.\n"
|
554
|
-
]
|
555
|
-
}
|
556
|
-
],
|
557
|
-
"source": [
|
558
|
-
"from natural_pdf import PDF\n",
|
559
|
-
"\n",
|
560
|
-
"input_pdf_path = \"https://github.com/jsoma/natural-pdf/raw/refs/heads/main/pdfs/needs-ocr.pdf\"\n",
|
561
|
-
"\n",
|
562
|
-
"pdf = PDF(input_pdf_path)\n",
|
563
|
-
"pdf.apply_ocr() \n",
|
564
|
-
"\n",
|
565
|
-
"pdf.save_searchable(\"needs-ocr-searchable.pdf\")"
|
566
|
-
]
|
567
|
-
},
|
568
|
-
{
|
569
|
-
"cell_type": "markdown",
|
570
|
-
"id": "cd0b43ed",
|
571
|
-
"metadata": {},
|
572
|
-
"source": [
|
573
|
-
"This creates `needs-ocr-searchable.pdf`, which looks identical to the original but now has a text layer corresponding to the OCR results. You can adjust the rendering resolution used during saving with the `dpi` parameter (default is 300).\n",
|
574
|
-
"\n",
|
575
|
-
"OCR integration enables you to work with scanned documents, historical archives, and image-based PDFs that don't have embedded text. By combining OCR with natural-pdf's layout analysis capabilities, you can turn any document into structured, searchable data. "
|
576
|
-
]
|
577
|
-
}
|
578
|
-
],
|
579
|
-
"metadata": {
|
580
|
-
"jupytext": {
|
581
|
-
"cell_metadata_filter": "-all",
|
582
|
-
"main_language": "python",
|
583
|
-
"notebook_metadata_filter": "-all",
|
584
|
-
"text_representation": {
|
585
|
-
"extension": ".md",
|
586
|
-
"format_name": "markdown"
|
587
|
-
}
|
588
|
-
},
|
589
|
-
"language_info": {
|
590
|
-
"codemirror_mode": {
|
591
|
-
"name": "ipython",
|
592
|
-
"version": 3
|
593
|
-
},
|
594
|
-
"file_extension": ".py",
|
595
|
-
"mimetype": "text/x-python",
|
596
|
-
"name": "python",
|
597
|
-
"nbconvert_exporter": "python",
|
598
|
-
"pygments_lexer": "ipython3",
|
599
|
-
"version": "3.10.13"
|
600
|
-
}
|
601
|
-
},
|
602
|
-
"nbformat": 4,
|
603
|
-
"nbformat_minor": 5
|
604
|
-
}
|