docling-jobkit 1.7.1__py3-none-any.whl → 1.8.1__py3-none-any.whl

docling_jobkit/convert/manager.py

@@ -31,7 +31,12 @@ from docling.datamodel.pipeline_options import (
     VlmPipelineOptions,
 )
 from docling.datamodel.pipeline_options_vlm_model import ApiVlmOptions, InlineVlmOptions
-from docling.document_converter import DocumentConverter, FormatOption, PdfFormatOption
+from docling.document_converter import (
+    DocumentConverter,
+    FormatOption,
+    ImageFormatOption,
+    PdfFormatOption,
+)
 from docling.models.factories import get_ocr_factory
 from docling.pipeline.vlm_pipeline import VlmPipeline
 from docling_core.types.doc import ImageRefMode
@@ -50,6 +55,13 @@ class DoclingConverterManagerConfig(BaseModel):
     max_num_pages: int = sys.maxsize
     max_file_size: int = sys.maxsize
 
+    # Threading pipeline
+    queue_max_size: Optional[int] = None
+    ocr_batch_size: Optional[int] = None
+    layout_batch_size: Optional[int] = None
+    table_batch_size: Optional[int] = None
+    batch_polling_interval_seconds: Optional[float] = None
+
 
 # Custom serializer for PdfFormatOption
 # (model_dump_json does not work with some classes)
@@ -61,12 +73,28 @@ def _hash_pdf_format_option(pdf_format_option: PdfFormatOption) -> bytes:
         data["pipeline_options"] = pdf_format_option.pipeline_options.model_dump(
             serialize_as_any=True, mode="json"
         )
+        data["pipeline_options_type"] = (
+            f"{pdf_format_option.pipeline_options.__class__.__module__}."
+            f"{pdf_format_option.pipeline_options.__class__.__qualname__}"
+        )
+    else:
+        data["pipeline_options_type"] = None
 
     # Replace `pipeline_cls` with a string representation
-    data["pipeline_cls"] = repr(data["pipeline_cls"])
+    pipeline_cls = pdf_format_option.pipeline_cls
+    data["pipeline_cls"] = (
+        f"{pipeline_cls.__module__}.{pipeline_cls.__qualname__}"
+        if pipeline_cls is not None
+        else "None"
+    )
 
     # Replace `backend` with a string representation
-    data["backend"] = repr(data["backend"])
+    backend = pdf_format_option.backend
+    data["backend"] = (
+        f"{backend.__module__}.{backend.__qualname__}"
+        if backend is not None
+        else "None"
+    )
 
     # Serialize the dictionary to JSON with sorted keys to have consistent hashes
     serialized_data = json.dumps(data, sort_keys=True)
@@ -114,9 +142,19 @@ class DoclingConverterManager:
         @lru_cache(maxsize=cache_size)
         def _get_converter_from_hash(options_hash: bytes) -> DocumentConverter:
             pdf_format_option = self._options_map[options_hash]
+            image_format_option: FormatOption = pdf_format_option
+            if isinstance(pdf_format_option.pipeline_cls, type) and issubclass(
+                pdf_format_option.pipeline_cls, VlmPipeline
+            ):
+                image_format_option = ImageFormatOption(
+                    pipeline_cls=pdf_format_option.pipeline_cls,
+                    pipeline_options=pdf_format_option.pipeline_options,
+                    backend_options=pdf_format_option.backend_options,
+                )
+
             format_options: dict[InputFormat, FormatOption] = {
                 InputFormat.PDF: pdf_format_option,
-                InputFormat.IMAGE: pdf_format_option,
+                InputFormat.IMAGE: image_format_option,
             }
 
             return DocumentConverter(format_options=format_options)
@@ -202,6 +240,17 @@ class DoclingConverterManager:
             request.picture_description_area_threshold
         )
 
+        # Forward the definition of the following attributes, if they are not none
+        for attr in (
+            "queue_max_size",
+            "ocr_batch_size",
+            "layout_batch_size",
+            "table_batch_size",
+            "batch_polling_interval_seconds",
+        ):
+            if value := getattr(self.config, attr):
+                setattr(pipeline_options, attr, value)
+
         return pipeline_options
 
     def _parse_backend(
@@ -264,6 +313,27 @@ class DoclingConverterManager:
                 request.vlm_pipeline_model_api.model_dump()
             )
 
+        pipeline_options.do_picture_classification = request.do_picture_classification
+        pipeline_options.do_picture_description = request.do_picture_description
+
+        if request.picture_description_local is not None:
+            pipeline_options.picture_description_options = (
+                PictureDescriptionVlmOptions.model_validate(
+                    request.picture_description_local.model_dump()
+                )
+            )
+
+        if request.picture_description_api is not None:
+            pipeline_options.picture_description_options = (
+                PictureDescriptionApiOptions.model_validate(
+                    request.picture_description_api.model_dump()
+                )
+            )
+
+        pipeline_options.picture_description_options.picture_area_threshold = (
+            request.picture_description_area_threshold
+        )
+
         return pipeline_options
 
     # Computes the PDF pipeline options and returns the PdfFormatOption and its hash

{docling_jobkit-1.7.1.dist-info → docling_jobkit-1.8.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docling-jobkit
-Version: 1.7.1
+Version: 1.8.1
 Summary: Running a distributed job processing documents with Docling.
 Project-URL: Homepage, https://github.com/docling-project/docling-jobkit
 Project-URL: Repository, https://github.com/docling-project/docling-jobkit
@@ -25,7 +25,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
 Requires-Dist: boto3~=1.35
-Requires-Dist: docling~=2.56
+Requires-Dist: docling~=2.60
 Requires-Dist: fastparquet~=2024.11
 Requires-Dist: httpx~=0.28
 Requires-Dist: pandas~=2.2
@@ -45,7 +45,7 @@ Provides-Extra: rq
 Requires-Dist: msgpack~=1.1; extra == 'rq'
 Requires-Dist: rq~=2.4; extra == 'rq'
 Provides-Extra: vlm
-Requires-Dist: docling[vlm]~=2.53; extra == 'vlm'
+Requires-Dist: docling[vlm]~=2.60; extra == 'vlm'
 Description-Content-Type: text/markdown
 
 # Docling Jobkit

{docling_jobkit-1.7.1.dist-info → docling_jobkit-1.8.1.dist-info}/RECORD

@@ -16,7 +16,7 @@ docling_jobkit/connectors/target_processor.py,sha256=2iIJE7Ip_-1dxJGt02_ALwDC2BP
 docling_jobkit/connectors/target_processor_factory.py,sha256=b_Q3L_mlvfQlZG7A2cskzf6-LzQ1G_seGd2vLT51b5o,688
 docling_jobkit/convert/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 docling_jobkit/convert/chunking.py,sha256=jFl7g8rGFmIBV_-0rxfyp6970N_0dqLehLgDSHKOM-o,11933
-docling_jobkit/convert/manager.py,sha256=Ir7lHzDPi-NdlYtBbTqb-PcKVM4kefLU9xU7TXB_7jM,13341
+docling_jobkit/convert/manager.py,sha256=M6kB5hFzQpcDCrdQRXxImdadbYJuwoLel7fTDeXWRtw,15949
 docling_jobkit/convert/results.py,sha256=vQvOuXIdlmPskHwUJlXX2zyJSb2k20ip5TfzuyPH5mU,9053
 docling_jobkit/convert/results_processor.py,sha256=TtiN6hqcUriEYMsEiyAutrgpMIz78D4pf-1HtiSjrXQ,16558
 docling_jobkit/datamodel/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -51,8 +51,8 @@ docling_jobkit/orchestrators/rq/orchestrator.py,sha256=MFINSpXb5tQh0PKk3xK10eZ8c
 docling_jobkit/orchestrators/rq/worker.py,sha256=P9rXhH9k814sNPGgjY24CATwQSzL6Hfd5Th5d4I3ejs,6591
 docling_jobkit/ray_job/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 docling_jobkit/ray_job/main.py,sha256=o52gLtEdGqyxa9XcVmwN55bCSVsIUq8zRm-FjgRlYN8,13465
-docling_jobkit-1.7.1.dist-info/METADATA,sha256=9IBxO3tGluaxQnTG8lyEwMYr8tbSn-oq3gedyoq438E,8105
-docling_jobkit-1.7.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-docling_jobkit-1.7.1.dist-info/entry_points.txt,sha256=QWmq6d0B14If8Zshc7pRnBs6zO1e9vhEnrMHOgBfzj8,121
-docling_jobkit-1.7.1.dist-info/licenses/LICENSE,sha256=mBb7ErEcM8VS9OhiGHnQ2kk75HwPhr54W1Oiz3965MY,1088
-docling_jobkit-1.7.1.dist-info/RECORD,,
+docling_jobkit-1.8.1.dist-info/METADATA,sha256=i1VO-KS2Ub3H8mbnZi4rGJpp_noOrrZ1MOq6XTz874g,8105
+docling_jobkit-1.8.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+docling_jobkit-1.8.1.dist-info/entry_points.txt,sha256=QWmq6d0B14If8Zshc7pRnBs6zO1e9vhEnrMHOgBfzj8,121
+docling_jobkit-1.8.1.dist-info/licenses/LICENSE,sha256=mBb7ErEcM8VS9OhiGHnQ2kk75HwPhr54W1Oiz3965MY,1088
+docling_jobkit-1.8.1.dist-info/RECORD,,

{docling_jobkit-1.7.1.dist-info → docling_jobkit-1.8.1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.27.0
+Generator: hatchling 1.28.0
 Root-Is-Purelib: true
 Tag: py3-none-any