extract-python 0.5.4__tar.gz → 0.5.7__tar.gz

{extract_python-0.5.4 → extract_python-0.5.7}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: extract-python
-Version: 0.5.4
+Version: 0.5.7
 Summary: Structured content extraction
 Project-URL: Homepage, https://github.com/ICIJ/extract-python
 Project-URL: Repository, https://github.com/ICIJ/extract-python
 Project-URL: Issues, https://github.com/ICIJ/extract-python/issues
 Author-email: Clément Doumouro <cdoumouro@icij.org>
 Requires-Python: <3.14,>=3.11
-Requires-Dist: extract-core~=0.1
+Requires-Dist: extract-core~=0.5.5
 Requires-Dist: icij-common~=0.8.2
 Provides-Extra: benches
 Requires-Dist: html2image~=2.0.7; extra == 'benches'

{extract_python-0.5.4 → extract_python-0.5.7}/extract_python/docling_.py

@@ -1,16 +1,20 @@
+import asyncio
 import shutil
 import tempfile
 from collections.abc import AsyncGenerator, Iterable, Iterator
 from pathlib import Path
+from typing import Any, Self
 
 from docling.datamodel.base_models import InputFormat
 from docling.datamodel.document import ConversionResult
-from docling.document_converter import DocumentConverter
+from docling.datamodel.pipeline_options import PipelineOptions
+from docling.document_converter import DocumentConverter, FormatOption
 
 # TODO: this is long to load improve it
 from docling_core.types.doc import ImageRefMode
 from docling_core.types.io import DocumentStream
 from extract_core import (
+    BaseModel,
     DoclingFormatOption,
     DoclingPipelineConfig,
     Error,
@@ -23,7 +27,10 @@ from extract_core import (
     Result,
     Status,
 )
+from icij_common.pydantic_utils import merge_configs
 from icij_common.registrable import FromConfig
+from pydantic import ConfigDict, field_serializer
+from pydantic_core.core_schema import SerializerFunctionWrapHandler
 
 from .constants import ARTIFACTS, DEFAULT_MD_PAGE_SEP
 from .utils import chdir, map_and_preserve, path_to_artifacts_dirname
@@ -49,7 +56,13 @@ class DoclingPipeline(Pipeline):
     ) -> AsyncGenerator[Result, None]:
         docs, path_or_streams = map_and_preserve(_to_docling, docs)
         outputs = self._converter.convert_all(path_or_streams, raises_on_error=False)
-        for doc, res in zip(docs, outputs, strict=True):
+
+        sentinel = object()
+        while True:
+            res = await asyncio.to_thread(next, outputs, sentinel)
+            if res is sentinel:
+                return
+            doc = next(docs)
             yield _to_result(res, doc, output_format, output_path=output_path)
 
     @classmethod
@@ -128,3 +141,52 @@ def _to_markdown_doc(
         shutil.move(tmp_dir, md_dir)
     pages = PageIndexes.from_page_end_indices(end_indices)
     return MarkdownDoc(path=Path(md_dir_name), pages=pages)
+
+
+class SerializableFormatOptions(DoclingFormatOption):
+    # Utility class to serialize Python format options into a JSON which can be
+    # correctly deserialized into a docling FormatOption
+    # via DoclingFormatOption.to_docling
+    model_config = merge_configs(
+        BaseModel.model_config, ConfigDict(polymorphic_serialization=True)
+    )
+
+    pipeline_options: PipelineOptions | None = None
+
+    @classmethod
+    def from_docling(cls, format_opts: FormatOption) -> Self:
+        return cls(
+            pipeline_cls=format_opts.pipeline_cls.__name__,
+            pipeline_options=format_opts.pipeline_options,
+            backend=format_opts.backend.__name__,
+            backend_options=format_opts.backend_options,
+        )
+
+    @field_serializer("pipeline_options", mode="wrap")
+    def _serialize_pipeline_opts(
+        self, v: PipelineOptions | None, handler: SerializerFunctionWrapHandler
+    ) -> Any:
+        if v is None:
+            return handler(v)
+        serialized = handler(v)
+        picture_desc_opts = getattr(v, "picture_description_options", None)
+        if picture_desc_opts is not None:
+            if "picture_description_options" not in serialized:
+                serialized["picture_description_options"] = dict()
+            serialized["picture_description_options"]["kind"] = picture_desc_opts.kind
+        ocr_opts = getattr(v, "ocr_options", None)
+        if ocr_opts is not None:
+            if "ocr_options" not in serialized:
+                serialized["ocr_options"] = dict()
+            serialized["ocr_options"]["kind"] = ocr_opts.kind
+        layout_opts = getattr(v, "layout_options", None)
+        if layout_opts is not None:
+            if "layout_options" not in serialized:
+                serialized["layout_options"] = dict()
+            serialized["layout_opts"]["kind"] = layout_opts.kind
+        table_structure_opts = getattr(v, "table_structure_options", None)
+        if table_structure_opts is not None:
+            if "table_structure_options" not in serialized:
+                serialized["table_structure_options"] = dict()
+            serialized["table_structure_options"]["kind"] = table_structure_opts.kind
+        return serialized

{extract_python-0.5.4 → extract_python-0.5.7}/extract_python/marker_.py

@@ -1,3 +1,4 @@
+import asyncio
 import gc
 from collections.abc import AsyncGenerator, Iterable
 from copy import deepcopy
@@ -86,7 +87,7 @@ class MarkerPipeline(Pipeline):
             renderer=renderer,
         )
         for doc in docs:
-            yield _process_doc(doc, converter, output_format, output_path)
+            yield await _process_doc(doc, converter, output_format, output_path)
 
     @classmethod
     def _from_config(cls, config: MarkerPipelineConfig) -> Self:
@@ -94,7 +95,7 @@ class MarkerPipeline(Pipeline):
 
 
 @report_recoverable_errors(_MARKER_CONVERSION_ERRORS)
-def _process_doc(
+async def _process_doc(
     doc: InputDoc,
     converter: "PdfConverter",
     output_format: OutputFormat,
@@ -102,7 +103,7 @@ def _process_doc(
 ) -> Result:
     from marker.output import text_from_rendered  # noqa: PLC0415
 
-    rendered = converter(str(doc.path))
+    rendered = await asyncio.to_thread(converter, str(doc.path))
     content, _, images = text_from_rendered(rendered)
     match output_format:
         case OutputFormat.MARKDOWN:

{extract_python-0.5.4 → extract_python-0.5.7}/pyproject.toml

@@ -9,7 +9,7 @@ readme = "README.md"
 requires-python = ">=3.11,<3.14"
 dependencies = [
   "icij-common~=0.8.2",
-  "extract-core~=0.1",
+  "extract-core~=0.5.5",
 ]
 
 [project.optional-dependencies]

{extract_python-0.5.4 → extract_python-0.5.7}/uv.lock

@@ -871,12 +871,14 @@ requires-dist = [
 [package.metadata.requires-dev]
 dev = [
     { name = "docling", specifier = "~=2.96" },
-    { name = "huggingface-hub", specifier = "~=0.36.2" },
+    { name = "docling-slim", extras = ["models-vlm-inline"], specifier = "~=2.96" },
     { name = "mpmath", specifier = "~=1.3.0" },
     { name = "pytest", specifier = "~=8.3.5" },
     { name = "pytest-asyncio", specifier = "~=0.25.3" },
     { name = "ruff", specifier = "==0.15.2" },
     { name = "sympy", specifier = "~=1.14.0" },
+    { name = "torch", specifier = "==2.12.0", index = "https://download.pytorch.org/whl/cpu" },
+    { name = "torchvision", specifier = "==0.27.0", index = "https://download.pytorch.org/whl/cpu" },
 ]
 
 [[package]]