structx 0.6.4__tar.gz → 0.6.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. {structx-0.6.4/structx.egg-info → structx-0.6.5}/PKG-INFO +2 -2
  2. {structx-0.6.4 → structx-0.6.5}/README.md +1 -1
  3. {structx-0.6.4 → structx-0.6.5}/pyproject.toml +2 -2
  4. {structx-0.6.4 → structx-0.6.5}/structx/__init__.py +1 -1
  5. {structx-0.6.4 → structx-0.6.5}/structx/measurement.py +171 -73
  6. {structx-0.6.4 → structx-0.6.5}/structx/utils/measurement_ocr.py +48 -2
  7. {structx-0.6.4 → structx-0.6.5/structx.egg-info}/PKG-INFO +2 -2
  8. {structx-0.6.4 → structx-0.6.5}/tests/test_measurement.py +138 -0
  9. {structx-0.6.4 → structx-0.6.5}/LICENSE +0 -0
  10. {structx-0.6.4 → structx-0.6.5}/setup.cfg +0 -0
  11. {structx-0.6.4 → structx-0.6.5}/structx/core/__init__.py +0 -0
  12. {structx-0.6.4 → structx-0.6.5}/structx/core/config.py +0 -0
  13. {structx-0.6.4 → structx-0.6.5}/structx/core/exceptions.py +0 -0
  14. {structx-0.6.4 → structx-0.6.5}/structx/core/input.py +0 -0
  15. {structx-0.6.4 → structx-0.6.5}/structx/core/models.py +0 -0
  16. {structx-0.6.4 → structx-0.6.5}/structx/core/type_system.py +0 -0
  17. {structx-0.6.4 → structx-0.6.5}/structx/extraction/__init__.py +0 -0
  18. {structx-0.6.4 → structx-0.6.5}/structx/extraction/core/__init__.py +0 -0
  19. {structx-0.6.4 → structx-0.6.5}/structx/extraction/core/llm_core.py +0 -0
  20. {structx-0.6.4 → structx-0.6.5}/structx/extraction/engines/__init__.py +0 -0
  21. {structx-0.6.4 → structx-0.6.5}/structx/extraction/engines/extraction_engine.py +0 -0
  22. {structx-0.6.4 → structx-0.6.5}/structx/extraction/extractor.py +0 -0
  23. {structx-0.6.4 → structx-0.6.5}/structx/extraction/generator.py +0 -0
  24. {structx-0.6.4 → structx-0.6.5}/structx/extraction/processors/__init__.py +0 -0
  25. {structx-0.6.4 → structx-0.6.5}/structx/extraction/processors/batch_processor.py +0 -0
  26. {structx-0.6.4 → structx-0.6.5}/structx/extraction/processors/content_analyzer.py +0 -0
  27. {structx-0.6.4 → structx-0.6.5}/structx/extraction/processors/input_processor.py +0 -0
  28. {structx-0.6.4 → structx-0.6.5}/structx/extraction/processors/model_operations.py +0 -0
  29. {structx-0.6.4 → structx-0.6.5}/structx/extraction/result_manager.py +0 -0
  30. {structx-0.6.4 → structx-0.6.5}/structx/schema.py +0 -0
  31. {structx-0.6.4 → structx-0.6.5}/structx/utils/__init__.py +0 -0
  32. {structx-0.6.4 → structx-0.6.5}/structx/utils/file_reader.py +0 -0
  33. {structx-0.6.4 → structx-0.6.5}/structx/utils/helpers.py +0 -0
  34. {structx-0.6.4 → structx-0.6.5}/structx/utils/prompts.py +0 -0
  35. {structx-0.6.4 → structx-0.6.5}/structx/utils/types.py +0 -0
  36. {structx-0.6.4 → structx-0.6.5}/structx/utils/usage.py +0 -0
  37. {structx-0.6.4 → structx-0.6.5}/structx.egg-info/SOURCES.txt +0 -0
  38. {structx-0.6.4 → structx-0.6.5}/structx.egg-info/dependency_links.txt +0 -0
  39. {structx-0.6.4 → structx-0.6.5}/structx.egg-info/requires.txt +0 -0
  40. {structx-0.6.4 → structx-0.6.5}/structx.egg-info/top_level.txt +0 -0
  41. {structx-0.6.4 → structx-0.6.5}/tests/test_config_and_model_utils.py +0 -0
  42. {structx-0.6.4 → structx-0.6.5}/tests/test_content_analyzer.py +0 -0
  43. {structx-0.6.4 → structx-0.6.5}/tests/test_document_conversion_integration.py +0 -0
  44. {structx-0.6.4 → structx-0.6.5}/tests/test_extraction_engine.py +0 -0
  45. {structx-0.6.4 → structx-0.6.5}/tests/test_extractor_async.py +0 -0
  46. {structx-0.6.4 → structx-0.6.5}/tests/test_extractor_flow.py +0 -0
  47. {structx-0.6.4 → structx-0.6.5}/tests/test_file_reader.py +0 -0
  48. {structx-0.6.4 → structx-0.6.5}/tests/test_input_and_batch_processors.py +0 -0
  49. {structx-0.6.4 → structx-0.6.5}/tests/test_live_llm.py +0 -0
  50. {structx-0.6.4 → structx-0.6.5}/tests/test_llm_configuration.py +0 -0
  51. {structx-0.6.4 → structx-0.6.5}/tests/test_result_manager_and_helpers.py +0 -0
  52. {structx-0.6.4 → structx-0.6.5}/tests/test_schema_contract.py +0 -0
  53. {structx-0.6.4 → structx-0.6.5}/tests/test_type_system_and_planning.py +0 -0
  54. {structx-0.6.4 → structx-0.6.5}/tests/test_usage.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: structx
3
- Version: 0.6.4
3
+ Version: 0.6.5
4
4
  Summary: Structured data extraction from text using LLMs and dynamic model generation
5
5
  Author-email: blacksuan19 <py@aolabs.dev>
6
6
  Classifier: Development Status :: 5 - Production/Stable
@@ -70,7 +70,7 @@ Advanced structured data extraction from any document using LLMs with multimodal
70
70
  support.
71
71
 
72
72
  [![Documentation](https://img.shields.io/badge/docs-mkdocs-blue.svg?style=for-the-badge)](https://structx.aolabs.dev "Documentation")
73
- [![PyPI](https://img.shields.io/badge/PyPi-0.6.4-blue?style=for-the-badge)](https://pypi.org/project/structx "Package")
73
+ [![PyPI](https://img.shields.io/badge/PyPi-0.6.5-blue?style=for-the-badge)](https://pypi.org/project/structx "Package")
74
74
  [![GitHub Actions](https://img.shields.io/badge/github%20actions-%232671E5.svg?style=for-the-badge&logo=githubactions&logoColor=white)](# "Build with GitHub Actions")
75
75
 
76
76
  See the [project roadmap](docs/ROADMAP.md) for planned portable business rules
@@ -4,7 +4,7 @@ Advanced structured data extraction from any document using LLMs with multimodal
4
4
  support.
5
5
 
6
6
  [![Documentation](https://img.shields.io/badge/docs-mkdocs-blue.svg?style=for-the-badge)](https://structx.aolabs.dev "Documentation")
7
- [![PyPI](https://img.shields.io/badge/PyPi-0.6.4-blue?style=for-the-badge)](https://pypi.org/project/structx "Package")
7
+ [![PyPI](https://img.shields.io/badge/PyPi-0.6.5-blue?style=for-the-badge)](https://pypi.org/project/structx "Package")
8
8
  [![GitHub Actions](https://img.shields.io/badge/github%20actions-%232671E5.svg?style=for-the-badge&logo=githubactions&logoColor=white)](# "Build with GitHub Actions")
9
9
 
10
10
  See the [project roadmap](docs/ROADMAP.md) for planned portable business rules
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "structx"
7
- version = "0.6.4"
7
+ version = "0.6.5"
8
8
  description = "Structured data extraction from text using LLMs and dynamic model generation"
9
9
  authors = [{ name = "blacksuan19", email = "py@aolabs.dev" }]
10
10
  license-files = ["LICENSE"]
@@ -106,7 +106,7 @@ torchvision = [
106
106
  ]
107
107
 
108
108
  [tool.bumpver]
109
- current_version = "0.6.4"
109
+ current_version = "0.6.5"
110
110
  version_pattern = "MAJOR.MINOR.PATCH"
111
111
  commit_message = "chore(release): bump version {old_version} -> {new_version}"
112
112
  commit = true
@@ -22,7 +22,7 @@ from structx.schema import (
22
22
  model_to_extraction_request,
23
23
  )
24
24
 
25
- __version__ = "0.6.4"
25
+ __version__ = "0.6.5"
26
26
  __all__ = [
27
27
  "Extractor",
28
28
  "ExtractionConfig",
@@ -21,6 +21,7 @@ Example:
21
21
  """
22
22
 
23
23
  import threading
24
+ from concurrent.futures import ThreadPoolExecutor
24
25
  from dataclasses import dataclass, field
25
26
  from math import ceil, sqrt
26
27
  from pathlib import Path
@@ -118,6 +119,17 @@ class DocumentMeasurement:
118
119
  return tuple(codes)
119
120
 
120
121
 
122
+ @dataclass
123
+ class _PagePlan:
124
+ """Work-in-progress state for one page, used between rendering and OCR."""
125
+
126
+ page_number: int
127
+ native_text: str = ""
128
+ native_failed: bool = False
129
+ image: Any = None
130
+ measurement: Optional[PageMeasurement] = None
131
+
132
+
121
133
  def _positive_int(value: Any, name: str) -> int:
122
134
  """Validate a positive integer setting."""
123
135
  if isinstance(value, bool) or not isinstance(value, int) or value < 1:
@@ -167,6 +179,11 @@ class DocumentMeasurer:
167
179
  render_dpi: Resolution used when rendering a page for OCR.
168
180
  max_render_pixels: Upper bound on rendered pixels per page. Larger
169
181
  pages are rendered at a reduced scale rather than skipped.
182
+ ocr_workers: Pages recognized at once. The default of 1 keeps OCR
183
+ serial. Higher values give each worker thread its own engine and
184
+ bound how many page images are held in memory at once.
185
+ ocr_engine_params: Parameters for the built-in OCR engine. Ignored when
186
+ ``ocr_reader`` is supplied.
170
187
  """
171
188
 
172
189
  def __init__(
@@ -174,6 +191,8 @@ class DocumentMeasurer:
174
191
  *,
175
192
  ocr_mode: str = "never",
176
193
  ocr_reader: Optional[Callable[[Any], str]] = None,
194
+ ocr_workers: int = 1,
195
+ ocr_engine_params: Optional[dict] = None,
177
196
  max_pages: int = MEASUREMENT_PAGE_LIMIT,
178
197
  max_input_bytes: int = MEASUREMENT_BYTES_LIMIT,
179
198
  render_dpi: float = MEASUREMENT_RENDER_DPI,
@@ -183,17 +202,27 @@ class DocumentMeasurer:
183
202
  raise ConfigurationError(f"ocr_mode must be one of {', '.join(OCR_MODES)}")
184
203
  if ocr_reader is not None and not callable(ocr_reader):
185
204
  raise ConfigurationError("ocr_reader must be callable")
205
+ if ocr_engine_params is not None and not isinstance(ocr_engine_params, dict):
206
+ raise ConfigurationError("ocr_engine_params must be a dict")
186
207
 
187
208
  self.ocr_mode = ocr_mode
209
+ self.ocr_workers = _positive_int(ocr_workers, "ocr_workers")
210
+ self.ocr_engine_params = dict(ocr_engine_params or {})
188
211
  self.max_pages = _positive_int(max_pages, "max_pages")
189
212
  self.max_input_bytes = _positive_int(max_input_bytes, "max_input_bytes")
190
213
  self.render_dpi = _positive_real(render_dpi, "render_dpi")
191
214
  self.max_render_pixels = _positive_int(max_render_pixels, "max_render_pixels")
192
215
  self._ocr_reader = ocr_reader
216
+ self._ocr_pool: Optional[ThreadPoolExecutor] = None
217
+ self._reader_lock = threading.Lock()
193
218
 
194
219
  def measure(self, file_path: Union[str, Path]) -> DocumentMeasurement:
195
220
  """Measure one PDF document.
196
221
 
222
+ PDF reading and rendering are serialized because the PDF backend is not
223
+ thread-safe. OCR runs outside that lock, so a slow scanned document does
224
+ not block other documents measured at the same time.
225
+
197
226
  Args:
198
227
  file_path: Path to an existing PDF file.
199
228
 
@@ -207,8 +236,7 @@ class DocumentMeasurer:
207
236
  """
208
237
  path = self._validated_path(file_path)
209
238
  pdfium = _import_pdfium()
210
- with _PDFIUM_LOCK:
211
- pages = self._measure_pages(pdfium, path)
239
+ pages = self._measure_pages(pdfium, path)
212
240
  return DocumentMeasurement(
213
241
  pages=tuple(pages),
214
242
  ocr_mode=self.ocr_mode,
@@ -243,108 +271,178 @@ class DocumentMeasurer:
243
271
  return path
244
272
 
245
273
  def _measure_pages(self, pdfium: Any, path: Path) -> list:
246
- """Open the document once and measure every page in order."""
247
- try:
248
- document = pdfium.PdfDocument(path)
249
- except Exception as error:
250
- raise FileError(f"Could not open PDF for measurement: {path}") from error
251
-
252
- try:
253
- if _is_encrypted(pdfium, document):
254
- raise FileError(f"Encrypted PDFs cannot be measured: {path}")
255
-
256
- page_count = len(document)
257
- if page_count < 1:
258
- raise FileError(f"The PDF contains no pages: {path}")
259
- if page_count > self.max_pages:
274
+ """Measure every page in order, in batches bounded by the worker count."""
275
+ with _PDFIUM_LOCK:
276
+ try:
277
+ document = pdfium.PdfDocument(path)
278
+ except Exception as error:
260
279
  raise FileError(
261
- f"PDF exceeds the {self.max_pages} page measurement limit: {path}"
262
- )
263
- return [self._measure_page(document, index) for index in range(page_count)]
280
+ f"Could not open PDF for measurement: {path}"
281
+ ) from error
282
+ try:
283
+ if _is_encrypted(pdfium, document):
284
+ raise FileError(f"Encrypted PDFs cannot be measured: {path}")
285
+ page_count = len(document)
286
+ if page_count < 1:
287
+ raise FileError(f"The PDF contains no pages: {path}")
288
+ if page_count > self.max_pages:
289
+ raise FileError(
290
+ f"PDF exceeds the {self.max_pages} page measurement "
291
+ f"limit: {path}"
292
+ )
293
+ except FileError:
294
+ document.close()
295
+ raise
296
+
297
+ measurements: list = []
298
+ try:
299
+ # Only the pages of one batch are rendered at a time, so a long
300
+ # document does not hold every page image in memory at once.
301
+ for start in range(0, page_count, self.ocr_workers):
302
+ end = min(start + self.ocr_workers, page_count)
303
+ with _PDFIUM_LOCK:
304
+ plans = [self._prepare_page(document, i) for i in range(start, end)]
305
+ measurements.extend(self._recognize_batch(plans))
264
306
  finally:
265
- document.close()
307
+ with _PDFIUM_LOCK:
308
+ document.close()
309
+ return measurements
310
+
311
+ def _prepare_page(self, document: Any, index: int) -> _PagePlan:
312
+ """Read a page's text layer and render it only when OCR is needed.
266
313
 
267
- def _measure_page(self, document: Any, index: int) -> PageMeasurement:
268
- """Measure one page without letting a page failure end the document."""
314
+ This runs under the PDF lock. Recognition itself happens afterwards.
315
+ """
269
316
  page_number = index + 1
270
317
  try:
271
318
  page = document[index]
272
319
  except Exception:
273
- return PageMeasurement(
274
- page_number, 0, "none", "partial", "page_load_failed"
320
+ return _PagePlan(
321
+ page_number,
322
+ measurement=PageMeasurement(
323
+ page_number, 0, "none", "partial", "page_load_failed"
324
+ ),
275
325
  )
276
326
 
277
327
  try:
278
328
  native_text, native_failed = _native_text(page)
329
+ plan = _PagePlan(
330
+ page_number, native_text=native_text, native_failed=native_failed
331
+ )
279
332
 
280
- if self.ocr_mode == "always":
281
- return self._ocr_page(page, page_number, native_text, native_failed)
282
-
283
- if native_failed:
284
- if self.ocr_mode == "auto":
285
- return self._ocr_page(page, page_number, "", True)
286
- return PageMeasurement(
287
- page_number, 0, "none", "partial", "native_text_failed"
288
- )
289
-
290
- if native_text.strip():
291
- return PageMeasurement(
333
+ if self.ocr_mode == "never":
334
+ plan.measurement = self._without_ocr(plan)
335
+ return plan
336
+ if self.ocr_mode == "auto" and not native_failed and native_text.strip():
337
+ plan.measurement = PageMeasurement(
292
338
  page_number, len(native_text), "native", "complete"
293
339
  )
294
-
295
- if self.ocr_mode == "auto":
296
- return self._ocr_page(page, page_number, "", False)
297
-
298
- # Without OCR, an empty text layer cannot be distinguished from a
299
- # scanned page, so the page is reported as incomplete.
300
- return PageMeasurement(page_number, 0, "none", "partial", "ocr_skipped")
340
+ return plan
341
+ if self.ocr_mode == "auto" and native_failed:
342
+ # A failed text layer says nothing about the page, so the
343
+ # native text is not reused as a fallback count.
344
+ plan.native_text = ""
345
+
346
+ try:
347
+ plan.image = self._rendered_image(page)
348
+ except Exception:
349
+ plan.measurement = self._ocr_unavailable(plan, "page_render_failed")
350
+ return plan
301
351
  finally:
302
352
  page.close()
303
353
 
304
- def _ocr_page(
305
- self,
306
- page: Any,
307
- page_number: int,
308
- native_text: str,
309
- native_failed: bool,
310
- ) -> PageMeasurement:
311
- """Measure a page with OCR, falling back to any usable native count."""
312
- text, error_code = self._recognized_text(page)
313
- if error_code is None:
314
- return PageMeasurement(page_number, len(text), "ocr", "complete")
315
- if not native_failed and native_text.strip():
354
+ def _without_ocr(self, plan: _PagePlan) -> PageMeasurement:
355
+ """Resolve a page that will not be sent to OCR."""
356
+ if plan.native_failed:
316
357
  return PageMeasurement(
317
- page_number, len(native_text), "native", "partial", error_code
358
+ plan.page_number, 0, "none", "partial", "native_text_failed"
318
359
  )
319
- return PageMeasurement(page_number, 0, "none", "partial", error_code)
320
-
321
- def _recognized_text(self, page: Any) -> Tuple[str, Optional[str]]:
322
- """Render one page and recognize its text, reporting bounded failures."""
360
+ if plan.native_text.strip():
361
+ return PageMeasurement(
362
+ plan.page_number, len(plan.native_text), "native", "complete"
363
+ )
364
+ # Without OCR, an empty text layer cannot be distinguished from a
365
+ # scanned page, so the page is reported as incomplete.
366
+ return PageMeasurement(plan.page_number, 0, "none", "partial", "ocr_skipped")
367
+
368
+ def _recognize_batch(self, plans: list) -> list:
369
+ """Recognize the rendered pages of one batch, in page order."""
370
+ pending = [plan for plan in plans if plan.measurement is None]
371
+ if pending:
372
+ if len(pending) > 1 and self.ocr_workers > 1:
373
+ # Created before fan-out so workers share one reader object.
374
+ self._resolve_reader()
375
+ results = list(self._pool().map(self._recognized_text, pending))
376
+ else:
377
+ results = [self._recognized_text(plan) for plan in pending]
378
+ for plan, (text, error_code) in zip(pending, results):
379
+ plan.measurement = self._from_recognition(plan, text, error_code)
380
+ return [plan.measurement for plan in plans]
381
+
382
+ def _recognized_text(self, plan: _PagePlan) -> Tuple[str, Optional[str]]:
383
+ """Recognize one rendered page, reporting bounded failures."""
323
384
  reader = self._resolve_reader()
324
- if reader is None:
325
- return "", "ocr_unavailable"
326
-
327
385
  try:
328
- image = self._rendered_image(page)
329
- except Exception:
330
- return "", "page_render_failed"
331
-
332
- try:
333
- text = reader(image)
386
+ text = reader(plan.image)
334
387
  except OcrEngineUnavailableError:
335
388
  return "", "ocr_unavailable"
336
389
  except Exception:
337
390
  return "", "ocr_failed"
391
+ finally:
392
+ plan.image = None
338
393
 
339
394
  if not isinstance(text, str):
340
395
  return "", "ocr_failed"
341
396
  return _normalized(text), None
342
397
 
343
- def _resolve_reader(self) -> Optional[Callable[[Any], str]]:
398
+ def _from_recognition(
399
+ self, plan: _PagePlan, text: str, error_code: Optional[str]
400
+ ) -> PageMeasurement:
401
+ """Build a page result, falling back to any usable native count."""
402
+ if error_code is None:
403
+ return PageMeasurement(plan.page_number, len(text), "ocr", "complete")
404
+ return self._ocr_unavailable(plan, error_code)
405
+
406
+ def _ocr_unavailable(self, plan: _PagePlan, error_code: str) -> PageMeasurement:
407
+ """Report a page OCR could not measure, keeping a usable native count."""
408
+ if not plan.native_failed and plan.native_text.strip():
409
+ return PageMeasurement(
410
+ plan.page_number,
411
+ len(plan.native_text),
412
+ "native",
413
+ "partial",
414
+ error_code,
415
+ )
416
+ return PageMeasurement(plan.page_number, 0, "none", "partial", error_code)
417
+
418
+ def _resolve_reader(self) -> Callable[[Any], str]:
344
419
  """Return the configured reader, creating the built-in one on demand."""
345
- if self._ocr_reader is None:
346
- self._ocr_reader = create_default_ocr_reader()
347
- return self._ocr_reader
420
+ with self._reader_lock:
421
+ if self._ocr_reader is None:
422
+ self._ocr_reader = create_default_ocr_reader(
423
+ workers=self.ocr_workers, **self.ocr_engine_params
424
+ )
425
+ return self._ocr_reader
426
+
427
+ def _pool(self) -> ThreadPoolExecutor:
428
+ """Return this measurer's OCR worker pool, creating it on demand."""
429
+ with self._reader_lock:
430
+ if self._ocr_pool is None:
431
+ self._ocr_pool = ThreadPoolExecutor(
432
+ max_workers=self.ocr_workers,
433
+ thread_name_prefix="structx-measure-ocr",
434
+ )
435
+ return self._ocr_pool
436
+
437
+ def close(self) -> None:
438
+ """Release worker threads held for parallel OCR.
439
+
440
+ Measuring again after closing recreates them.
441
+ """
442
+ with self._reader_lock:
443
+ pool, self._ocr_pool = self._ocr_pool, None
444
+ if pool is not None:
445
+ pool.shutdown(wait=True)
348
446
 
349
447
  def _rendered_image(self, page: Any) -> Any:
350
448
  """Render a page to pixels within the configured pixel budget."""
@@ -17,6 +17,18 @@ class OcrEngineUnavailableError(StructXError):
17
17
  pass
18
18
 
19
19
 
20
+ PARALLEL_ENGINE_PARAMS: Dict[str, Any] = {
21
+ "params": {"EngineConfig.onnxruntime.intra_op_num_threads": 1}
22
+ }
23
+ """Engine parameters used when several reader threads run at once.
24
+
25
+ ONNX Runtime already spreads a single recognition call across CPU cores, so
26
+ parallel workers only help when each engine is limited to one inference thread.
27
+ This applies to the built-in onnxruntime engine; pass explicit engine parameters
28
+ for other backends.
29
+ """
30
+
31
+
20
32
  def _joined_text(result: Any) -> str:
21
33
  """Join recognized text lines from a RapidOCR result."""
22
34
  lines = getattr(result, "txts", None)
@@ -80,13 +92,47 @@ class RapidOcrReader:
80
92
  return _joined_text(engine(image))
81
93
 
82
94
 
83
- def create_default_ocr_reader(**engine_params: Any) -> Callable[[Any], str]:
95
+ class ThreadLocalRapidOcrReader:
96
+ """Reader that gives every worker thread its own OCR engine.
97
+
98
+ ``RapidOCR`` updates instance state on each call, so one engine cannot be
99
+ shared across threads. This reader creates an engine per thread instead,
100
+ which is what makes page-level parallel OCR safe.
101
+
102
+ Attributes:
103
+ engine_params: Parameters forwarded to ``RapidOCR`` on creation.
104
+ """
105
+
106
+ def __init__(self, **engine_params: Any) -> None:
107
+ self.engine_params: Dict[str, Any] = dict(engine_params)
108
+ self._local = threading.local()
109
+
110
+ def __call__(self, image: Any) -> str:
111
+ """Recognize text using this thread's engine, creating it on demand."""
112
+ reader = getattr(self._local, "reader", None)
113
+ if reader is None:
114
+ reader = RapidOcrReader(**self.engine_params)
115
+ self._local.reader = reader
116
+ return reader(image)
117
+
118
+
119
+ def create_default_ocr_reader(
120
+ *, workers: int = 1, **engine_params: Any
121
+ ) -> Callable[[Any], str]:
84
122
  """Create the built-in RapidOCR reader without initializing the engine.
85
123
 
86
124
  Args:
125
+ workers: Number of threads that may call the reader at once. Values
126
+ above one create one engine per thread and, unless the caller
127
+ overrides engine parameters, limit each engine to a single
128
+ inference thread so workers do not oversubscribe the CPU.
87
129
  **engine_params: Optional parameters forwarded to ``RapidOCR``.
88
130
 
89
131
  Returns:
90
132
  A callable that accepts rendered page pixels and returns text.
91
133
  """
92
- return RapidOcrReader(**engine_params)
134
+ if workers <= 1:
135
+ return RapidOcrReader(**engine_params)
136
+ if not engine_params:
137
+ engine_params = dict(PARALLEL_ENGINE_PARAMS)
138
+ return ThreadLocalRapidOcrReader(**engine_params)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: structx
3
- Version: 0.6.4
3
+ Version: 0.6.5
4
4
  Summary: Structured data extraction from text using LLMs and dynamic model generation
5
5
  Author-email: blacksuan19 <py@aolabs.dev>
6
6
  Classifier: Development Status :: 5 - Production/Stable
@@ -70,7 +70,7 @@ Advanced structured data extraction from any document using LLMs with multimodal
70
70
  support.
71
71
 
72
72
  [![Documentation](https://img.shields.io/badge/docs-mkdocs-blue.svg?style=for-the-badge)](https://structx.aolabs.dev "Documentation")
73
- [![PyPI](https://img.shields.io/badge/PyPi-0.6.4-blue?style=for-the-badge)](https://pypi.org/project/structx "Package")
73
+ [![PyPI](https://img.shields.io/badge/PyPi-0.6.5-blue?style=for-the-badge)](https://pypi.org/project/structx "Package")
74
74
  [![GitHub Actions](https://img.shields.io/badge/github%20actions-%232671E5.svg?style=for-the-badge&logo=githubactions&logoColor=white)](# "Build with GitHub Actions")
75
75
 
76
76
  See the [project roadmap](docs/ROADMAP.md) for planned portable business rules
@@ -1,4 +1,5 @@
1
1
  import sys
2
+ import threading
2
3
  import types
3
4
  from pathlib import Path
4
5
 
@@ -7,8 +8,10 @@ import pytest
7
8
  from structx.core.exceptions import ConfigurationError, FileError
8
9
  from structx.measurement import DocumentMeasurer, DocumentMeasurement
9
10
  from structx.utils.measurement_ocr import (
11
+ PARALLEL_ENGINE_PARAMS,
10
12
  OcrEngineUnavailableError,
11
13
  RapidOcrReader,
14
+ ThreadLocalRapidOcrReader,
12
15
  create_default_ocr_reader,
13
16
  )
14
17
 
@@ -266,6 +269,9 @@ def test_extraction_input_preparation_is_unaffected_by_measurement(native_pdf):
266
269
  lambda: DocumentMeasurer(ocr_mode="sometimes"),
267
270
  lambda: DocumentMeasurer(ocr_reader="not callable"),
268
271
  lambda: DocumentMeasurer(max_pages=0),
272
+ lambda: DocumentMeasurer(ocr_workers=0),
273
+ lambda: DocumentMeasurer(ocr_workers=True),
274
+ lambda: DocumentMeasurer(ocr_engine_params="params"),
269
275
  lambda: DocumentMeasurer(max_pages=True),
270
276
  lambda: DocumentMeasurer(max_input_bytes=-1),
271
277
  lambda: DocumentMeasurer(render_dpi=0),
@@ -427,6 +433,138 @@ def _import_failing_on(missing_name):
427
433
  return guarded_import
428
434
 
429
435
 
436
+ def test_parallel_workers_recognize_pages_concurrently_and_in_order(tmp_path):
437
+ scanned = build_pdf(tmp_path / "scan.pdf", [None] * 4)
438
+ barrier = threading.Barrier(2, timeout=10)
439
+ worker_threads = set()
440
+
441
+ def reader(image):
442
+ worker_threads.add(threading.current_thread().name)
443
+ barrier.wait()
444
+ return "page text"
445
+
446
+ measurer = DocumentMeasurer(ocr_mode="auto", ocr_reader=reader, ocr_workers=2)
447
+ try:
448
+ measurement = measurer.measure(scanned)
449
+ finally:
450
+ measurer.close()
451
+
452
+ assert [page.page_number for page in measurement.pages] == [1, 2, 3, 4]
453
+ assert all(page.method == "ocr" for page in measurement.pages)
454
+ assert measurement.character_count == 4 * len("page text")
455
+ assert len(worker_threads) >= 2
456
+
457
+
458
+ def test_only_one_batch_of_pages_is_rendered_at_a_time(tmp_path):
459
+ scanned = build_pdf(tmp_path / "scan.pdf", [None] * 6)
460
+ in_flight = 0
461
+ peak = 0
462
+ guard = threading.Lock()
463
+ ready = threading.Barrier(2, timeout=10)
464
+
465
+ def reader(image):
466
+ nonlocal in_flight, peak
467
+ with guard:
468
+ in_flight += 1
469
+ peak = max(peak, in_flight)
470
+ ready.wait()
471
+ with guard:
472
+ in_flight -= 1
473
+ return "text"
474
+
475
+ measurer = DocumentMeasurer(ocr_mode="auto", ocr_reader=reader, ocr_workers=2)
476
+ try:
477
+ measurement = measurer.measure(scanned)
478
+ finally:
479
+ measurer.close()
480
+
481
+ assert measurement.page_count == 6
482
+ assert peak <= 2
483
+
484
+
485
+ def test_ocr_does_not_hold_the_pdf_lock(tmp_path, native_pdf):
486
+ scanned = build_pdf(tmp_path / "scan.pdf", [None])
487
+ recognizing = threading.Event()
488
+ release = threading.Event()
489
+
490
+ def blocking_reader(image):
491
+ recognizing.set()
492
+ assert release.wait(10)
493
+ return "slow page"
494
+
495
+ slow = DocumentMeasurer(ocr_mode="auto", ocr_reader=blocking_reader)
496
+ worker = threading.Thread(target=slow.measure, args=(scanned,))
497
+ worker.start()
498
+ try:
499
+ assert recognizing.wait(10)
500
+ # Another document must still be measurable while OCR is running.
501
+ quick = DocumentMeasurer().measure(native_pdf)
502
+ assert quick.status == "complete"
503
+ finally:
504
+ release.set()
505
+ worker.join(10)
506
+ assert not worker.is_alive()
507
+
508
+
509
+ def test_serial_measurement_does_not_create_worker_threads(blank_pdf):
510
+ measurer = DocumentMeasurer(ocr_mode="auto", ocr_reader=RecordingReader())
511
+ measurer.measure(blank_pdf)
512
+
513
+ assert measurer._ocr_pool is None
514
+
515
+
516
+ def test_measuring_again_after_close_still_works(tmp_path):
517
+ scanned = build_pdf(tmp_path / "scan.pdf", [None, None])
518
+ measurer = DocumentMeasurer(
519
+ ocr_mode="auto", ocr_reader=RecordingReader(), ocr_workers=2
520
+ )
521
+ try:
522
+ first = measurer.measure(scanned)
523
+ measurer.close()
524
+ second = measurer.measure(scanned)
525
+ finally:
526
+ measurer.close()
527
+
528
+ assert first == second
529
+ assert first.status == "complete"
530
+
531
+
532
+ def test_parallel_default_reader_uses_one_engine_per_thread(monkeypatch):
533
+ calls = install_fake_rapidocr(monkeypatch)
534
+ reader = create_default_ocr_reader(workers=2)
535
+ assert isinstance(reader, ThreadLocalRapidOcrReader)
536
+ assert reader.engine_params == PARALLEL_ENGINE_PARAMS
537
+
538
+ results = []
539
+ threads = [
540
+ threading.Thread(target=lambda: results.append(reader("image")))
541
+ for _ in range(2)
542
+ ]
543
+ for thread in threads:
544
+ thread.start()
545
+ for thread in threads:
546
+ thread.join(10)
547
+
548
+ assert results == ["first line\nsecond line"] * 2
549
+ assert calls["init"] == 2
550
+ assert calls["kwargs"] == PARALLEL_ENGINE_PARAMS
551
+
552
+
553
+ def test_serial_default_reader_keeps_engine_defaults(monkeypatch):
554
+ install_fake_rapidocr(monkeypatch)
555
+ reader = create_default_ocr_reader(workers=1)
556
+
557
+ assert isinstance(reader, RapidOcrReader)
558
+ assert reader.engine_params == {}
559
+
560
+
561
+ def test_explicit_engine_parameters_survive_parallel_readers(monkeypatch):
562
+ calls = install_fake_rapidocr(monkeypatch)
563
+ create_default_ocr_reader(workers=4, params={"Rec.model_type": "tiny"})("image")
564
+
565
+ assert calls["kwargs"] == {"params": {"Rec.model_type": "tiny"}}
566
+
567
+
430
568
  @pytest.mark.integration
431
569
  def test_real_ocr_measures_a_rasterized_page(tmp_path, native_pdf):
432
570
  scanned = tmp_path / "scanned.pdf"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes