structx 0.6.4__tar.gz → 0.6.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {structx-0.6.4/structx.egg-info → structx-0.6.5}/PKG-INFO +2 -2
- {structx-0.6.4 → structx-0.6.5}/README.md +1 -1
- {structx-0.6.4 → structx-0.6.5}/pyproject.toml +2 -2
- {structx-0.6.4 → structx-0.6.5}/structx/__init__.py +1 -1
- {structx-0.6.4 → structx-0.6.5}/structx/measurement.py +171 -73
- {structx-0.6.4 → structx-0.6.5}/structx/utils/measurement_ocr.py +48 -2
- {structx-0.6.4 → structx-0.6.5/structx.egg-info}/PKG-INFO +2 -2
- {structx-0.6.4 → structx-0.6.5}/tests/test_measurement.py +138 -0
- {structx-0.6.4 → structx-0.6.5}/LICENSE +0 -0
- {structx-0.6.4 → structx-0.6.5}/setup.cfg +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx/core/__init__.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx/core/config.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx/core/exceptions.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx/core/input.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx/core/models.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx/core/type_system.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx/extraction/__init__.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx/extraction/core/__init__.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx/extraction/core/llm_core.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx/extraction/engines/__init__.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx/extraction/engines/extraction_engine.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx/extraction/extractor.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx/extraction/generator.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx/extraction/processors/__init__.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx/extraction/processors/batch_processor.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx/extraction/processors/content_analyzer.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx/extraction/processors/input_processor.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx/extraction/processors/model_operations.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx/extraction/result_manager.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx/schema.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx/utils/__init__.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx/utils/file_reader.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx/utils/helpers.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx/utils/prompts.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx/utils/types.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx/utils/usage.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx.egg-info/SOURCES.txt +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx.egg-info/dependency_links.txt +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx.egg-info/requires.txt +0 -0
- {structx-0.6.4 → structx-0.6.5}/structx.egg-info/top_level.txt +0 -0
- {structx-0.6.4 → structx-0.6.5}/tests/test_config_and_model_utils.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/tests/test_content_analyzer.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/tests/test_document_conversion_integration.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/tests/test_extraction_engine.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/tests/test_extractor_async.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/tests/test_extractor_flow.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/tests/test_file_reader.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/tests/test_input_and_batch_processors.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/tests/test_live_llm.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/tests/test_llm_configuration.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/tests/test_result_manager_and_helpers.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/tests/test_schema_contract.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/tests/test_type_system_and_planning.py +0 -0
- {structx-0.6.4 → structx-0.6.5}/tests/test_usage.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: structx
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.5
|
|
4
4
|
Summary: Structured data extraction from text using LLMs and dynamic model generation
|
|
5
5
|
Author-email: blacksuan19 <py@aolabs.dev>
|
|
6
6
|
Classifier: Development Status :: 5 - Production/Stable
|
|
@@ -70,7 +70,7 @@ Advanced structured data extraction from any document using LLMs with multimodal
|
|
|
70
70
|
support.
|
|
71
71
|
|
|
72
72
|
[](https://structx.aolabs.dev "Documentation")
|
|
73
|
-
[](https://pypi.org/project/structx "Package")
|
|
74
74
|
[](# "Build with GitHub Actions")
|
|
75
75
|
|
|
76
76
|
See the [project roadmap](docs/ROADMAP.md) for planned portable business rules
|
|
@@ -4,7 +4,7 @@ Advanced structured data extraction from any document using LLMs with multimodal
|
|
|
4
4
|
support.
|
|
5
5
|
|
|
6
6
|
[](https://structx.aolabs.dev "Documentation")
|
|
7
|
-
[](https://pypi.org/project/structx "Package")
|
|
8
8
|
[](# "Build with GitHub Actions")
|
|
9
9
|
|
|
10
10
|
See the [project roadmap](docs/ROADMAP.md) for planned portable business rules
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "structx"
|
|
7
|
-
version = "0.6.
|
|
7
|
+
version = "0.6.5"
|
|
8
8
|
description = "Structured data extraction from text using LLMs and dynamic model generation"
|
|
9
9
|
authors = [{ name = "blacksuan19", email = "py@aolabs.dev" }]
|
|
10
10
|
license-files = ["LICENSE"]
|
|
@@ -106,7 +106,7 @@ torchvision = [
|
|
|
106
106
|
]
|
|
107
107
|
|
|
108
108
|
[tool.bumpver]
|
|
109
|
-
current_version = "0.6.
|
|
109
|
+
current_version = "0.6.5"
|
|
110
110
|
version_pattern = "MAJOR.MINOR.PATCH"
|
|
111
111
|
commit_message = "chore(release): bump version {old_version} -> {new_version}"
|
|
112
112
|
commit = true
|
|
@@ -21,6 +21,7 @@ Example:
|
|
|
21
21
|
"""
|
|
22
22
|
|
|
23
23
|
import threading
|
|
24
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
24
25
|
from dataclasses import dataclass, field
|
|
25
26
|
from math import ceil, sqrt
|
|
26
27
|
from pathlib import Path
|
|
@@ -118,6 +119,17 @@ class DocumentMeasurement:
|
|
|
118
119
|
return tuple(codes)
|
|
119
120
|
|
|
120
121
|
|
|
122
|
+
@dataclass
|
|
123
|
+
class _PagePlan:
|
|
124
|
+
"""Work-in-progress state for one page, used between rendering and OCR."""
|
|
125
|
+
|
|
126
|
+
page_number: int
|
|
127
|
+
native_text: str = ""
|
|
128
|
+
native_failed: bool = False
|
|
129
|
+
image: Any = None
|
|
130
|
+
measurement: Optional[PageMeasurement] = None
|
|
131
|
+
|
|
132
|
+
|
|
121
133
|
def _positive_int(value: Any, name: str) -> int:
|
|
122
134
|
"""Validate a positive integer setting."""
|
|
123
135
|
if isinstance(value, bool) or not isinstance(value, int) or value < 1:
|
|
@@ -167,6 +179,11 @@ class DocumentMeasurer:
|
|
|
167
179
|
render_dpi: Resolution used when rendering a page for OCR.
|
|
168
180
|
max_render_pixels: Upper bound on rendered pixels per page. Larger
|
|
169
181
|
pages are rendered at a reduced scale rather than skipped.
|
|
182
|
+
ocr_workers: Pages recognized at once. The default of 1 keeps OCR
|
|
183
|
+
serial. Higher values give each worker thread its own engine and
|
|
184
|
+
bound how many page images are held in memory at once.
|
|
185
|
+
ocr_engine_params: Parameters for the built-in OCR engine. Ignored when
|
|
186
|
+
``ocr_reader`` is supplied.
|
|
170
187
|
"""
|
|
171
188
|
|
|
172
189
|
def __init__(
|
|
@@ -174,6 +191,8 @@ class DocumentMeasurer:
|
|
|
174
191
|
*,
|
|
175
192
|
ocr_mode: str = "never",
|
|
176
193
|
ocr_reader: Optional[Callable[[Any], str]] = None,
|
|
194
|
+
ocr_workers: int = 1,
|
|
195
|
+
ocr_engine_params: Optional[dict] = None,
|
|
177
196
|
max_pages: int = MEASUREMENT_PAGE_LIMIT,
|
|
178
197
|
max_input_bytes: int = MEASUREMENT_BYTES_LIMIT,
|
|
179
198
|
render_dpi: float = MEASUREMENT_RENDER_DPI,
|
|
@@ -183,17 +202,27 @@ class DocumentMeasurer:
|
|
|
183
202
|
raise ConfigurationError(f"ocr_mode must be one of {', '.join(OCR_MODES)}")
|
|
184
203
|
if ocr_reader is not None and not callable(ocr_reader):
|
|
185
204
|
raise ConfigurationError("ocr_reader must be callable")
|
|
205
|
+
if ocr_engine_params is not None and not isinstance(ocr_engine_params, dict):
|
|
206
|
+
raise ConfigurationError("ocr_engine_params must be a dict")
|
|
186
207
|
|
|
187
208
|
self.ocr_mode = ocr_mode
|
|
209
|
+
self.ocr_workers = _positive_int(ocr_workers, "ocr_workers")
|
|
210
|
+
self.ocr_engine_params = dict(ocr_engine_params or {})
|
|
188
211
|
self.max_pages = _positive_int(max_pages, "max_pages")
|
|
189
212
|
self.max_input_bytes = _positive_int(max_input_bytes, "max_input_bytes")
|
|
190
213
|
self.render_dpi = _positive_real(render_dpi, "render_dpi")
|
|
191
214
|
self.max_render_pixels = _positive_int(max_render_pixels, "max_render_pixels")
|
|
192
215
|
self._ocr_reader = ocr_reader
|
|
216
|
+
self._ocr_pool: Optional[ThreadPoolExecutor] = None
|
|
217
|
+
self._reader_lock = threading.Lock()
|
|
193
218
|
|
|
194
219
|
def measure(self, file_path: Union[str, Path]) -> DocumentMeasurement:
|
|
195
220
|
"""Measure one PDF document.
|
|
196
221
|
|
|
222
|
+
PDF reading and rendering are serialized because the PDF backend is not
|
|
223
|
+
thread-safe. OCR runs outside that lock, so a slow scanned document does
|
|
224
|
+
not block other documents measured at the same time.
|
|
225
|
+
|
|
197
226
|
Args:
|
|
198
227
|
file_path: Path to an existing PDF file.
|
|
199
228
|
|
|
@@ -207,8 +236,7 @@ class DocumentMeasurer:
|
|
|
207
236
|
"""
|
|
208
237
|
path = self._validated_path(file_path)
|
|
209
238
|
pdfium = _import_pdfium()
|
|
210
|
-
|
|
211
|
-
pages = self._measure_pages(pdfium, path)
|
|
239
|
+
pages = self._measure_pages(pdfium, path)
|
|
212
240
|
return DocumentMeasurement(
|
|
213
241
|
pages=tuple(pages),
|
|
214
242
|
ocr_mode=self.ocr_mode,
|
|
@@ -243,108 +271,178 @@ class DocumentMeasurer:
|
|
|
243
271
|
return path
|
|
244
272
|
|
|
245
273
|
def _measure_pages(self, pdfium: Any, path: Path) -> list:
|
|
246
|
-
"""
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
try:
|
|
253
|
-
if _is_encrypted(pdfium, document):
|
|
254
|
-
raise FileError(f"Encrypted PDFs cannot be measured: {path}")
|
|
255
|
-
|
|
256
|
-
page_count = len(document)
|
|
257
|
-
if page_count < 1:
|
|
258
|
-
raise FileError(f"The PDF contains no pages: {path}")
|
|
259
|
-
if page_count > self.max_pages:
|
|
274
|
+
"""Measure every page in order, in batches bounded by the worker count."""
|
|
275
|
+
with _PDFIUM_LOCK:
|
|
276
|
+
try:
|
|
277
|
+
document = pdfium.PdfDocument(path)
|
|
278
|
+
except Exception as error:
|
|
260
279
|
raise FileError(
|
|
261
|
-
f"
|
|
262
|
-
)
|
|
263
|
-
|
|
280
|
+
f"Could not open PDF for measurement: {path}"
|
|
281
|
+
) from error
|
|
282
|
+
try:
|
|
283
|
+
if _is_encrypted(pdfium, document):
|
|
284
|
+
raise FileError(f"Encrypted PDFs cannot be measured: {path}")
|
|
285
|
+
page_count = len(document)
|
|
286
|
+
if page_count < 1:
|
|
287
|
+
raise FileError(f"The PDF contains no pages: {path}")
|
|
288
|
+
if page_count > self.max_pages:
|
|
289
|
+
raise FileError(
|
|
290
|
+
f"PDF exceeds the {self.max_pages} page measurement "
|
|
291
|
+
f"limit: {path}"
|
|
292
|
+
)
|
|
293
|
+
except FileError:
|
|
294
|
+
document.close()
|
|
295
|
+
raise
|
|
296
|
+
|
|
297
|
+
measurements: list = []
|
|
298
|
+
try:
|
|
299
|
+
# Only the pages of one batch are rendered at a time, so a long
|
|
300
|
+
# document does not hold every page image in memory at once.
|
|
301
|
+
for start in range(0, page_count, self.ocr_workers):
|
|
302
|
+
end = min(start + self.ocr_workers, page_count)
|
|
303
|
+
with _PDFIUM_LOCK:
|
|
304
|
+
plans = [self._prepare_page(document, i) for i in range(start, end)]
|
|
305
|
+
measurements.extend(self._recognize_batch(plans))
|
|
264
306
|
finally:
|
|
265
|
-
|
|
307
|
+
with _PDFIUM_LOCK:
|
|
308
|
+
document.close()
|
|
309
|
+
return measurements
|
|
310
|
+
|
|
311
|
+
def _prepare_page(self, document: Any, index: int) -> _PagePlan:
|
|
312
|
+
"""Read a page's text layer and render it only when OCR is needed.
|
|
266
313
|
|
|
267
|
-
|
|
268
|
-
"""
|
|
314
|
+
This runs under the PDF lock. Recognition itself happens afterwards.
|
|
315
|
+
"""
|
|
269
316
|
page_number = index + 1
|
|
270
317
|
try:
|
|
271
318
|
page = document[index]
|
|
272
319
|
except Exception:
|
|
273
|
-
return
|
|
274
|
-
page_number,
|
|
320
|
+
return _PagePlan(
|
|
321
|
+
page_number,
|
|
322
|
+
measurement=PageMeasurement(
|
|
323
|
+
page_number, 0, "none", "partial", "page_load_failed"
|
|
324
|
+
),
|
|
275
325
|
)
|
|
276
326
|
|
|
277
327
|
try:
|
|
278
328
|
native_text, native_failed = _native_text(page)
|
|
329
|
+
plan = _PagePlan(
|
|
330
|
+
page_number, native_text=native_text, native_failed=native_failed
|
|
331
|
+
)
|
|
279
332
|
|
|
280
|
-
if self.ocr_mode == "
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
if native_failed:
|
|
284
|
-
|
|
285
|
-
return self._ocr_page(page, page_number, "", True)
|
|
286
|
-
return PageMeasurement(
|
|
287
|
-
page_number, 0, "none", "partial", "native_text_failed"
|
|
288
|
-
)
|
|
289
|
-
|
|
290
|
-
if native_text.strip():
|
|
291
|
-
return PageMeasurement(
|
|
333
|
+
if self.ocr_mode == "never":
|
|
334
|
+
plan.measurement = self._without_ocr(plan)
|
|
335
|
+
return plan
|
|
336
|
+
if self.ocr_mode == "auto" and not native_failed and native_text.strip():
|
|
337
|
+
plan.measurement = PageMeasurement(
|
|
292
338
|
page_number, len(native_text), "native", "complete"
|
|
293
339
|
)
|
|
294
|
-
|
|
295
|
-
if self.ocr_mode == "auto":
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
340
|
+
return plan
|
|
341
|
+
if self.ocr_mode == "auto" and native_failed:
|
|
342
|
+
# A failed text layer says nothing about the page, so the
|
|
343
|
+
# native text is not reused as a fallback count.
|
|
344
|
+
plan.native_text = ""
|
|
345
|
+
|
|
346
|
+
try:
|
|
347
|
+
plan.image = self._rendered_image(page)
|
|
348
|
+
except Exception:
|
|
349
|
+
plan.measurement = self._ocr_unavailable(plan, "page_render_failed")
|
|
350
|
+
return plan
|
|
301
351
|
finally:
|
|
302
352
|
page.close()
|
|
303
353
|
|
|
304
|
-
def
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
page_number: int,
|
|
308
|
-
native_text: str,
|
|
309
|
-
native_failed: bool,
|
|
310
|
-
) -> PageMeasurement:
|
|
311
|
-
"""Measure a page with OCR, falling back to any usable native count."""
|
|
312
|
-
text, error_code = self._recognized_text(page)
|
|
313
|
-
if error_code is None:
|
|
314
|
-
return PageMeasurement(page_number, len(text), "ocr", "complete")
|
|
315
|
-
if not native_failed and native_text.strip():
|
|
354
|
+
def _without_ocr(self, plan: _PagePlan) -> PageMeasurement:
|
|
355
|
+
"""Resolve a page that will not be sent to OCR."""
|
|
356
|
+
if plan.native_failed:
|
|
316
357
|
return PageMeasurement(
|
|
317
|
-
page_number,
|
|
358
|
+
plan.page_number, 0, "none", "partial", "native_text_failed"
|
|
318
359
|
)
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
360
|
+
if plan.native_text.strip():
|
|
361
|
+
return PageMeasurement(
|
|
362
|
+
plan.page_number, len(plan.native_text), "native", "complete"
|
|
363
|
+
)
|
|
364
|
+
# Without OCR, an empty text layer cannot be distinguished from a
|
|
365
|
+
# scanned page, so the page is reported as incomplete.
|
|
366
|
+
return PageMeasurement(plan.page_number, 0, "none", "partial", "ocr_skipped")
|
|
367
|
+
|
|
368
|
+
def _recognize_batch(self, plans: list) -> list:
|
|
369
|
+
"""Recognize the rendered pages of one batch, in page order."""
|
|
370
|
+
pending = [plan for plan in plans if plan.measurement is None]
|
|
371
|
+
if pending:
|
|
372
|
+
if len(pending) > 1 and self.ocr_workers > 1:
|
|
373
|
+
# Created before fan-out so workers share one reader object.
|
|
374
|
+
self._resolve_reader()
|
|
375
|
+
results = list(self._pool().map(self._recognized_text, pending))
|
|
376
|
+
else:
|
|
377
|
+
results = [self._recognized_text(plan) for plan in pending]
|
|
378
|
+
for plan, (text, error_code) in zip(pending, results):
|
|
379
|
+
plan.measurement = self._from_recognition(plan, text, error_code)
|
|
380
|
+
return [plan.measurement for plan in plans]
|
|
381
|
+
|
|
382
|
+
def _recognized_text(self, plan: _PagePlan) -> Tuple[str, Optional[str]]:
|
|
383
|
+
"""Recognize one rendered page, reporting bounded failures."""
|
|
323
384
|
reader = self._resolve_reader()
|
|
324
|
-
if reader is None:
|
|
325
|
-
return "", "ocr_unavailable"
|
|
326
|
-
|
|
327
385
|
try:
|
|
328
|
-
|
|
329
|
-
except Exception:
|
|
330
|
-
return "", "page_render_failed"
|
|
331
|
-
|
|
332
|
-
try:
|
|
333
|
-
text = reader(image)
|
|
386
|
+
text = reader(plan.image)
|
|
334
387
|
except OcrEngineUnavailableError:
|
|
335
388
|
return "", "ocr_unavailable"
|
|
336
389
|
except Exception:
|
|
337
390
|
return "", "ocr_failed"
|
|
391
|
+
finally:
|
|
392
|
+
plan.image = None
|
|
338
393
|
|
|
339
394
|
if not isinstance(text, str):
|
|
340
395
|
return "", "ocr_failed"
|
|
341
396
|
return _normalized(text), None
|
|
342
397
|
|
|
343
|
-
def
|
|
398
|
+
def _from_recognition(
|
|
399
|
+
self, plan: _PagePlan, text: str, error_code: Optional[str]
|
|
400
|
+
) -> PageMeasurement:
|
|
401
|
+
"""Build a page result, falling back to any usable native count."""
|
|
402
|
+
if error_code is None:
|
|
403
|
+
return PageMeasurement(plan.page_number, len(text), "ocr", "complete")
|
|
404
|
+
return self._ocr_unavailable(plan, error_code)
|
|
405
|
+
|
|
406
|
+
def _ocr_unavailable(self, plan: _PagePlan, error_code: str) -> PageMeasurement:
|
|
407
|
+
"""Report a page OCR could not measure, keeping a usable native count."""
|
|
408
|
+
if not plan.native_failed and plan.native_text.strip():
|
|
409
|
+
return PageMeasurement(
|
|
410
|
+
plan.page_number,
|
|
411
|
+
len(plan.native_text),
|
|
412
|
+
"native",
|
|
413
|
+
"partial",
|
|
414
|
+
error_code,
|
|
415
|
+
)
|
|
416
|
+
return PageMeasurement(plan.page_number, 0, "none", "partial", error_code)
|
|
417
|
+
|
|
418
|
+
def _resolve_reader(self) -> Callable[[Any], str]:
|
|
344
419
|
"""Return the configured reader, creating the built-in one on demand."""
|
|
345
|
-
|
|
346
|
-
self._ocr_reader
|
|
347
|
-
|
|
420
|
+
with self._reader_lock:
|
|
421
|
+
if self._ocr_reader is None:
|
|
422
|
+
self._ocr_reader = create_default_ocr_reader(
|
|
423
|
+
workers=self.ocr_workers, **self.ocr_engine_params
|
|
424
|
+
)
|
|
425
|
+
return self._ocr_reader
|
|
426
|
+
|
|
427
|
+
def _pool(self) -> ThreadPoolExecutor:
|
|
428
|
+
"""Return this measurer's OCR worker pool, creating it on demand."""
|
|
429
|
+
with self._reader_lock:
|
|
430
|
+
if self._ocr_pool is None:
|
|
431
|
+
self._ocr_pool = ThreadPoolExecutor(
|
|
432
|
+
max_workers=self.ocr_workers,
|
|
433
|
+
thread_name_prefix="structx-measure-ocr",
|
|
434
|
+
)
|
|
435
|
+
return self._ocr_pool
|
|
436
|
+
|
|
437
|
+
def close(self) -> None:
|
|
438
|
+
"""Release worker threads held for parallel OCR.
|
|
439
|
+
|
|
440
|
+
Measuring again after closing recreates them.
|
|
441
|
+
"""
|
|
442
|
+
with self._reader_lock:
|
|
443
|
+
pool, self._ocr_pool = self._ocr_pool, None
|
|
444
|
+
if pool is not None:
|
|
445
|
+
pool.shutdown(wait=True)
|
|
348
446
|
|
|
349
447
|
def _rendered_image(self, page: Any) -> Any:
|
|
350
448
|
"""Render a page to pixels within the configured pixel budget."""
|
|
@@ -17,6 +17,18 @@ class OcrEngineUnavailableError(StructXError):
|
|
|
17
17
|
pass
|
|
18
18
|
|
|
19
19
|
|
|
20
|
+
PARALLEL_ENGINE_PARAMS: Dict[str, Any] = {
|
|
21
|
+
"params": {"EngineConfig.onnxruntime.intra_op_num_threads": 1}
|
|
22
|
+
}
|
|
23
|
+
"""Engine parameters used when several reader threads run at once.
|
|
24
|
+
|
|
25
|
+
ONNX Runtime already spreads a single recognition call across CPU cores, so
|
|
26
|
+
parallel workers only help when each engine is limited to one inference thread.
|
|
27
|
+
This applies to the built-in onnxruntime engine; pass explicit engine parameters
|
|
28
|
+
for other backends.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
|
|
20
32
|
def _joined_text(result: Any) -> str:
|
|
21
33
|
"""Join recognized text lines from a RapidOCR result."""
|
|
22
34
|
lines = getattr(result, "txts", None)
|
|
@@ -80,13 +92,47 @@ class RapidOcrReader:
|
|
|
80
92
|
return _joined_text(engine(image))
|
|
81
93
|
|
|
82
94
|
|
|
83
|
-
|
|
95
|
+
class ThreadLocalRapidOcrReader:
|
|
96
|
+
"""Reader that gives every worker thread its own OCR engine.
|
|
97
|
+
|
|
98
|
+
``RapidOCR`` updates instance state on each call, so one engine cannot be
|
|
99
|
+
shared across threads. This reader creates an engine per thread instead,
|
|
100
|
+
which is what makes page-level parallel OCR safe.
|
|
101
|
+
|
|
102
|
+
Attributes:
|
|
103
|
+
engine_params: Parameters forwarded to ``RapidOCR`` on creation.
|
|
104
|
+
"""
|
|
105
|
+
|
|
106
|
+
def __init__(self, **engine_params: Any) -> None:
|
|
107
|
+
self.engine_params: Dict[str, Any] = dict(engine_params)
|
|
108
|
+
self._local = threading.local()
|
|
109
|
+
|
|
110
|
+
def __call__(self, image: Any) -> str:
|
|
111
|
+
"""Recognize text using this thread's engine, creating it on demand."""
|
|
112
|
+
reader = getattr(self._local, "reader", None)
|
|
113
|
+
if reader is None:
|
|
114
|
+
reader = RapidOcrReader(**self.engine_params)
|
|
115
|
+
self._local.reader = reader
|
|
116
|
+
return reader(image)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def create_default_ocr_reader(
|
|
120
|
+
*, workers: int = 1, **engine_params: Any
|
|
121
|
+
) -> Callable[[Any], str]:
|
|
84
122
|
"""Create the built-in RapidOCR reader without initializing the engine.
|
|
85
123
|
|
|
86
124
|
Args:
|
|
125
|
+
workers: Number of threads that may call the reader at once. Values
|
|
126
|
+
above one create one engine per thread and, unless the caller
|
|
127
|
+
overrides engine parameters, limit each engine to a single
|
|
128
|
+
inference thread so workers do not oversubscribe the CPU.
|
|
87
129
|
**engine_params: Optional parameters forwarded to ``RapidOCR``.
|
|
88
130
|
|
|
89
131
|
Returns:
|
|
90
132
|
A callable that accepts rendered page pixels and returns text.
|
|
91
133
|
"""
|
|
92
|
-
|
|
134
|
+
if workers <= 1:
|
|
135
|
+
return RapidOcrReader(**engine_params)
|
|
136
|
+
if not engine_params:
|
|
137
|
+
engine_params = dict(PARALLEL_ENGINE_PARAMS)
|
|
138
|
+
return ThreadLocalRapidOcrReader(**engine_params)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: structx
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.5
|
|
4
4
|
Summary: Structured data extraction from text using LLMs and dynamic model generation
|
|
5
5
|
Author-email: blacksuan19 <py@aolabs.dev>
|
|
6
6
|
Classifier: Development Status :: 5 - Production/Stable
|
|
@@ -70,7 +70,7 @@ Advanced structured data extraction from any document using LLMs with multimodal
|
|
|
70
70
|
support.
|
|
71
71
|
|
|
72
72
|
[](https://structx.aolabs.dev "Documentation")
|
|
73
|
-
[](https://pypi.org/project/structx "Package")
|
|
74
74
|
[](# "Build with GitHub Actions")
|
|
75
75
|
|
|
76
76
|
See the [project roadmap](docs/ROADMAP.md) for planned portable business rules
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import sys
|
|
2
|
+
import threading
|
|
2
3
|
import types
|
|
3
4
|
from pathlib import Path
|
|
4
5
|
|
|
@@ -7,8 +8,10 @@ import pytest
|
|
|
7
8
|
from structx.core.exceptions import ConfigurationError, FileError
|
|
8
9
|
from structx.measurement import DocumentMeasurer, DocumentMeasurement
|
|
9
10
|
from structx.utils.measurement_ocr import (
|
|
11
|
+
PARALLEL_ENGINE_PARAMS,
|
|
10
12
|
OcrEngineUnavailableError,
|
|
11
13
|
RapidOcrReader,
|
|
14
|
+
ThreadLocalRapidOcrReader,
|
|
12
15
|
create_default_ocr_reader,
|
|
13
16
|
)
|
|
14
17
|
|
|
@@ -266,6 +269,9 @@ def test_extraction_input_preparation_is_unaffected_by_measurement(native_pdf):
|
|
|
266
269
|
lambda: DocumentMeasurer(ocr_mode="sometimes"),
|
|
267
270
|
lambda: DocumentMeasurer(ocr_reader="not callable"),
|
|
268
271
|
lambda: DocumentMeasurer(max_pages=0),
|
|
272
|
+
lambda: DocumentMeasurer(ocr_workers=0),
|
|
273
|
+
lambda: DocumentMeasurer(ocr_workers=True),
|
|
274
|
+
lambda: DocumentMeasurer(ocr_engine_params="params"),
|
|
269
275
|
lambda: DocumentMeasurer(max_pages=True),
|
|
270
276
|
lambda: DocumentMeasurer(max_input_bytes=-1),
|
|
271
277
|
lambda: DocumentMeasurer(render_dpi=0),
|
|
@@ -427,6 +433,138 @@ def _import_failing_on(missing_name):
|
|
|
427
433
|
return guarded_import
|
|
428
434
|
|
|
429
435
|
|
|
436
|
+
def test_parallel_workers_recognize_pages_concurrently_and_in_order(tmp_path):
|
|
437
|
+
scanned = build_pdf(tmp_path / "scan.pdf", [None] * 4)
|
|
438
|
+
barrier = threading.Barrier(2, timeout=10)
|
|
439
|
+
worker_threads = set()
|
|
440
|
+
|
|
441
|
+
def reader(image):
|
|
442
|
+
worker_threads.add(threading.current_thread().name)
|
|
443
|
+
barrier.wait()
|
|
444
|
+
return "page text"
|
|
445
|
+
|
|
446
|
+
measurer = DocumentMeasurer(ocr_mode="auto", ocr_reader=reader, ocr_workers=2)
|
|
447
|
+
try:
|
|
448
|
+
measurement = measurer.measure(scanned)
|
|
449
|
+
finally:
|
|
450
|
+
measurer.close()
|
|
451
|
+
|
|
452
|
+
assert [page.page_number for page in measurement.pages] == [1, 2, 3, 4]
|
|
453
|
+
assert all(page.method == "ocr" for page in measurement.pages)
|
|
454
|
+
assert measurement.character_count == 4 * len("page text")
|
|
455
|
+
assert len(worker_threads) >= 2
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
def test_only_one_batch_of_pages_is_rendered_at_a_time(tmp_path):
|
|
459
|
+
scanned = build_pdf(tmp_path / "scan.pdf", [None] * 6)
|
|
460
|
+
in_flight = 0
|
|
461
|
+
peak = 0
|
|
462
|
+
guard = threading.Lock()
|
|
463
|
+
ready = threading.Barrier(2, timeout=10)
|
|
464
|
+
|
|
465
|
+
def reader(image):
|
|
466
|
+
nonlocal in_flight, peak
|
|
467
|
+
with guard:
|
|
468
|
+
in_flight += 1
|
|
469
|
+
peak = max(peak, in_flight)
|
|
470
|
+
ready.wait()
|
|
471
|
+
with guard:
|
|
472
|
+
in_flight -= 1
|
|
473
|
+
return "text"
|
|
474
|
+
|
|
475
|
+
measurer = DocumentMeasurer(ocr_mode="auto", ocr_reader=reader, ocr_workers=2)
|
|
476
|
+
try:
|
|
477
|
+
measurement = measurer.measure(scanned)
|
|
478
|
+
finally:
|
|
479
|
+
measurer.close()
|
|
480
|
+
|
|
481
|
+
assert measurement.page_count == 6
|
|
482
|
+
assert peak <= 2
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
def test_ocr_does_not_hold_the_pdf_lock(tmp_path, native_pdf):
|
|
486
|
+
scanned = build_pdf(tmp_path / "scan.pdf", [None])
|
|
487
|
+
recognizing = threading.Event()
|
|
488
|
+
release = threading.Event()
|
|
489
|
+
|
|
490
|
+
def blocking_reader(image):
|
|
491
|
+
recognizing.set()
|
|
492
|
+
assert release.wait(10)
|
|
493
|
+
return "slow page"
|
|
494
|
+
|
|
495
|
+
slow = DocumentMeasurer(ocr_mode="auto", ocr_reader=blocking_reader)
|
|
496
|
+
worker = threading.Thread(target=slow.measure, args=(scanned,))
|
|
497
|
+
worker.start()
|
|
498
|
+
try:
|
|
499
|
+
assert recognizing.wait(10)
|
|
500
|
+
# Another document must still be measurable while OCR is running.
|
|
501
|
+
quick = DocumentMeasurer().measure(native_pdf)
|
|
502
|
+
assert quick.status == "complete"
|
|
503
|
+
finally:
|
|
504
|
+
release.set()
|
|
505
|
+
worker.join(10)
|
|
506
|
+
assert not worker.is_alive()
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def test_serial_measurement_does_not_create_worker_threads(blank_pdf):
|
|
510
|
+
measurer = DocumentMeasurer(ocr_mode="auto", ocr_reader=RecordingReader())
|
|
511
|
+
measurer.measure(blank_pdf)
|
|
512
|
+
|
|
513
|
+
assert measurer._ocr_pool is None
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
def test_measuring_again_after_close_still_works(tmp_path):
|
|
517
|
+
scanned = build_pdf(tmp_path / "scan.pdf", [None, None])
|
|
518
|
+
measurer = DocumentMeasurer(
|
|
519
|
+
ocr_mode="auto", ocr_reader=RecordingReader(), ocr_workers=2
|
|
520
|
+
)
|
|
521
|
+
try:
|
|
522
|
+
first = measurer.measure(scanned)
|
|
523
|
+
measurer.close()
|
|
524
|
+
second = measurer.measure(scanned)
|
|
525
|
+
finally:
|
|
526
|
+
measurer.close()
|
|
527
|
+
|
|
528
|
+
assert first == second
|
|
529
|
+
assert first.status == "complete"
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
def test_parallel_default_reader_uses_one_engine_per_thread(monkeypatch):
|
|
533
|
+
calls = install_fake_rapidocr(monkeypatch)
|
|
534
|
+
reader = create_default_ocr_reader(workers=2)
|
|
535
|
+
assert isinstance(reader, ThreadLocalRapidOcrReader)
|
|
536
|
+
assert reader.engine_params == PARALLEL_ENGINE_PARAMS
|
|
537
|
+
|
|
538
|
+
results = []
|
|
539
|
+
threads = [
|
|
540
|
+
threading.Thread(target=lambda: results.append(reader("image")))
|
|
541
|
+
for _ in range(2)
|
|
542
|
+
]
|
|
543
|
+
for thread in threads:
|
|
544
|
+
thread.start()
|
|
545
|
+
for thread in threads:
|
|
546
|
+
thread.join(10)
|
|
547
|
+
|
|
548
|
+
assert results == ["first line\nsecond line"] * 2
|
|
549
|
+
assert calls["init"] == 2
|
|
550
|
+
assert calls["kwargs"] == PARALLEL_ENGINE_PARAMS
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
def test_serial_default_reader_keeps_engine_defaults(monkeypatch):
|
|
554
|
+
install_fake_rapidocr(monkeypatch)
|
|
555
|
+
reader = create_default_ocr_reader(workers=1)
|
|
556
|
+
|
|
557
|
+
assert isinstance(reader, RapidOcrReader)
|
|
558
|
+
assert reader.engine_params == {}
|
|
559
|
+
|
|
560
|
+
|
|
561
|
+
def test_explicit_engine_parameters_survive_parallel_readers(monkeypatch):
|
|
562
|
+
calls = install_fake_rapidocr(monkeypatch)
|
|
563
|
+
create_default_ocr_reader(workers=4, params={"Rec.model_type": "tiny"})("image")
|
|
564
|
+
|
|
565
|
+
assert calls["kwargs"] == {"params": {"Rec.model_type": "tiny"}}
|
|
566
|
+
|
|
567
|
+
|
|
430
568
|
@pytest.mark.integration
|
|
431
569
|
def test_real_ocr_measures_a_rasterized_page(tmp_path, native_pdf):
|
|
432
570
|
scanned = tmp_path / "scanned.pdf"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|