docling-jobkit 1.8.0__py3-none-any.whl → 1.9.0__py3-none-any.whl

docling_jobkit/orchestrators/local/orchestrator.py

@@ -1,4 +1,5 @@
 import asyncio
+import gc
 import logging
 import tempfile
 import uuid
@@ -10,6 +11,7 @@ from pydantic import BaseModel
 
 from docling.datamodel.base_models import InputFormat
 
+from docling_jobkit.convert.chunking import DocumentChunkerManager
 from docling_jobkit.convert.manager import DoclingConverterManager
 from docling_jobkit.datamodel.chunking import BaseChunkerOptions, ChunkingExportOptions
 from docling_jobkit.datamodel.convert import ConvertDocumentsOptions
@@ -41,6 +43,8 @@ class LocalOrchestrator(BaseOrchestrator):
         self.task_queue: asyncio.Queue[str] = asyncio.Queue()
         self.queue_list: list[str] = []
         self.cm = converter_manager
+        self.chunker_manager = DocumentChunkerManager()
+        self.worker_cms: list[DoclingConverterManager] = []
         self._task_results: dict[str, DoclingTaskResult] = {}
         self.scratch_dir = self.config.scratch_dir or Path(
             tempfile.mkdtemp(prefix="docling_")
@@ -129,6 +133,10 @@ class LocalOrchestrator(BaseOrchestrator):
 
     async def clear_converters(self):
         self.cm.clear_cache()
+        self.chunker_manager.clear_cache()
+        for cm in self.worker_cms:
+            cm.clear_cache()
+        gc.collect()
 
     async def check_connection(self):
         pass

docling_jobkit/orchestrators/local/worker.py

@@ -34,11 +34,11 @@ class AsyncLocalWorker:
 
     async def loop(self):
         _log.debug(f"Starting loop for worker {self.worker_id}")
-        cm = (
-            self.orchestrator.cm
-            if self.use_shared_manager
-            else DoclingConverterManager(self.orchestrator.cm.config)
-        )
+        if self.use_shared_manager:
+            cm = self.orchestrator.cm
+        else:
+            cm = DoclingConverterManager(self.orchestrator.cm.config)
+            self.orchestrator.worker_cms.append(cm)
         while True:
             task_id: str = await self.orchestrator.task_queue.get()
             self.orchestrator.queue_list.remove(task_id)
@@ -94,6 +94,7 @@ class AsyncLocalWorker:
                             task=task,
                             conv_results=conv_results,
                             work_dir=workdir,
+                            chunker_manager=self.orchestrator.chunker_manager,
                         )
 
                     return processed_results

docling_jobkit/orchestrators/rq/orchestrator.py

@@ -226,12 +226,22 @@ class RQOrchestrator(BaseOrchestrator):
 
     async def delete_task(self, task_id: str):
         _log.info(f"Deleting result of task {task_id=}")
+
+        # Delete the result data from Redis if it exists
         if task_id in self._task_result_keys:
             await self._async_redis_conn.delete(self._task_result_keys[task_id])
             del self._task_result_keys[task_id]
-            # TODO: consider also deleting the task
-            # job = Job.fetch(task_id, connection=self._redis_conn)
-            # job.delete()
+
+        # Delete the RQ job itself to free up Redis memory
+        # This includes the job metadata and result stream
+        try:
+            job = Job.fetch(task_id, connection=self._redis_conn)
+            job.delete()
+            _log.debug(f"Deleted RQ job {task_id=}")
+        except Exception as e:
+            # Job may not exist or already be deleted - this is not an error
+            _log.debug(f"Could not delete RQ job {task_id=}: {e}")
+
         await super().delete_task(task_id)
 
     async def warm_up_caches(self):

docling_jobkit/orchestrators/rq/worker.py

@@ -69,6 +69,9 @@ class CustomRQWorker(SimpleWorker):
         self.conversion_manager = DoclingConverterManager(cm_config)
         self.scratch_dir = scratch_dir
 
+        if "default_result_ttl" not in kwargs:
+            kwargs["default_result_ttl"] = self.orchestrator_config.results_ttl
+
         # Call parent class constructor
         super().__init__(*args, **kwargs)
 

docling_jobkit/ray_job/main.py

@@ -1,3 +1,8 @@
+import sys
+
+if sys.version_info >= (3, 14):
+    raise ImportError("ray support is not yet available for Python 3.14.")
+
 import argparse
 import json
 import os
@@ -16,7 +21,11 @@ from ray._raylet import ObjectRefGenerator  # type: ignore
 from docling.backend.docling_parse_backend import DoclingParseDocumentBackend
 from docling.datamodel.base_models import ConversionStatus, InputFormat
 from docling.datamodel.document import ConversionResult
-from docling.datamodel.pipeline_options import PdfPipelineOptions, TableFormerMode
+from docling.datamodel.pipeline_options import (
+    PdfPipelineOptions,
+    TableFormerMode,
+    TableStructureOptions,
+)
 from docling.document_converter import DocumentConverter, PdfFormatOption
 
 # Load credentials
@@ -224,8 +233,8 @@ class DoclingConvert:
         pipeline_options = PdfPipelineOptions()
         pipeline_options.do_ocr = do_ocr
         pipeline_options.do_table_structure = do_table_structure
-        pipeline_options.table_structure_options.mode = TableFormerMode(
-            table_structure_mode
+        pipeline_options.table_structure_options = TableStructureOptions(
+            mode=TableFormerMode(table_structure_mode)
         )
         pipeline_options.generate_page_images = generate_page_images
 

{docling_jobkit-1.8.0.dist-info → docling_jobkit-1.9.0.dist-info}/METADATA

@@ -1,8 +1,9 @@
 Metadata-Version: 2.4
 Name: docling-jobkit
-Version: 1.8.0
+Version: 1.9.0
 Summary: Running a distributed job processing documents with Docling.
 Project-URL: Homepage, https://github.com/docling-project/docling-jobkit
+Project-URL: Documentation, https://docling-project.github.io/docling/usage/jobkit/
 Project-URL: Repository, https://github.com/docling-project/docling-jobkit
 Project-URL: Issues, https://github.com/docling-project/docling-jobkit/issues
 Project-URL: Changelog, https://github.com/docling-project/docling-jobkit/blob/main/CHANGELOG.md
@@ -18,6 +19,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Classifier: Topic :: Software Development :: Build Tools
@@ -26,21 +28,18 @@ Classifier: Typing :: Typed
 Requires-Python: >=3.10
 Requires-Dist: boto3~=1.35
 Requires-Dist: docling~=2.60
-Requires-Dist: fastparquet~=2024.11
-Requires-Dist: httpx~=0.28
+Requires-Dist: httpx<1,>=0.28
 Requires-Dist: pandas~=2.2
-Requires-Dist: pyarrow~=19.0
 Requires-Dist: pydantic-settings~=2.4
 Requires-Dist: pydantic~=2.10
-Requires-Dist: typer>=0.12.5
-Requires-Dist: typer~=0.12
+Requires-Dist: typer<1,>=0.12.5
 Provides-Extra: gdrive
 Requires-Dist: google-api-python-client>=2.183.0; extra == 'gdrive'
 Requires-Dist: google-auth-oauthlib>=1.2.2; extra == 'gdrive'
 Provides-Extra: kfp
 Requires-Dist: kfp[kubernetes]>=2.10.0; extra == 'kfp'
 Provides-Extra: ray
-Requires-Dist: ray~=2.30; extra == 'ray'
+Requires-Dist: ray~=2.30; (python_version < '3.14') and extra == 'ray'
 Provides-Extra: rq
 Requires-Dist: msgpack~=1.1; extra == 'rq'
 Requires-Dist: rq~=2.4; extra == 'rq'
@@ -55,6 +54,77 @@ Running a distributed job processing documents with Docling.
 
 ## How to use it
 
+### Local Multiprocessing CLI
+
+The `docling-jobkit-multiproc` CLI enables parallel batch processing of documents using Python's multiprocessing. Each batch of documents is processed in a separate subprocess, allowing efficient parallel processing on a single machine.
+
+#### Usage
+
+```bash
+# Basic usage with default settings (batch_size=10, num_processes=CPU count)
+docling-jobkit-multiproc config.yaml
+
+# Custom batch size and number of processes
+docling-jobkit-multiproc config.yaml --batch-size 20 --num-processes 4
+
+# With model artifacts
+docling-jobkit-multiproc config.yaml --artifacts-path /path/to/models
+
+# Quiet mode (suppress progress bar)
+docling-jobkit-multiproc config.yaml --quiet
+
+# Full options
+docling-jobkit-multiproc config.yaml \
+  --batch-size 30 \
+  --num-processes 8 \
+  --artifacts-path /path/to/models \
+  --enable-remote-services \
+  --allow-external-plugins
+```
+
+#### Configuration
+
+The configuration file format is the same as `docling-jobkit-local`. See example configurations:
+- S3 source/target: `dev/configs/run_multiproc_s3_example.yaml`
+- Local path source/target: `dev/configs/run_local_folder_example.yaml`
+
+**Note:** Only S3, Google Drive, and local_path sources support batch processing. File and HTTP sources do not support chunking.
+
+#### CLI Options
+
+- `--batch-size, -b`: Number of documents to process in each batch (default: 10)
+- `--num-processes, -n`: Number of parallel processes (default: CPU count)
+- `--artifacts-path`: Path to model artifacts directory
+- `--enable-remote-services`: Enable models connecting to remote services
+- `--allow-external-plugins`: Enable loading modules from third-party plugins
+- `--quiet, -q`: Suppress progress bar and detailed output
+
+### Local Sequential CLI
+
+The `docling-jobkit-local` CLI processes documents sequentially in a single process.
+
+```bash
+docling-jobkit-local config.yaml
+```
+
+### Using Local Path Sources and Targets
+
+Both CLIs support local file system sources and targets. Example configuration:
+
+```yaml
+sources:
+  - kind: local_path
+    path: ./input_documents/
+    recursive: true  # optional, default true
+    pattern: "*.pdf"  # optional glob pattern
+
+target:
+  kind: local_path
+  path: ./output_documents/
+```
+
+See `dev/configs/run_local_folder_example.yaml` for a complete example.
+
 ## Kubeflow pipeline with Docling Jobkit
 
 ### Using Kubeflow pipeline web dashboard UI

{docling_jobkit-1.8.0.dist-info → docling_jobkit-1.9.0.dist-info}/RECORD

@@ -1,22 +1,25 @@
 docling_jobkit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 docling_jobkit/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 docling_jobkit/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-docling_jobkit/cli/local.py,sha256=EzI8TXxzZDdxtbeT0plpHKAOoK7gbR6K_IhI2KN0N8c,3665
+docling_jobkit/cli/local.py,sha256=DZZvzwcxjsm-q7AD2EsjpP-B7nCw1RBY9yXNc4ifeiY,3789
+docling_jobkit/cli/multiproc.py,sha256=X8M16Alv_eQ7QPB0u9NHpSPtI6nMlRzSbD_0hcRTwMw,16633
 docling_jobkit/connectors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-docling_jobkit/connectors/google_drive_helper.py,sha256=8e0loUWxbhA7oaZc_Dapb-vRIu6WMn18bZDKVN77YGA,10489
-docling_jobkit/connectors/google_drive_source_processor.py,sha256=LxpuJXPiZZQeLNXOg0gyT8nZBA-nZHzfdSbZITEcQEc,1384
+docling_jobkit/connectors/google_drive_helper.py,sha256=YDYLhLD-0Kb8QhRwi35PSbFu8Lc-CBhI9zWE5qyPPv0,10554
+docling_jobkit/connectors/google_drive_source_processor.py,sha256=0bl7mEpF-wRMmJMyi6e2mcDNrqgBxWQzU4rwi81ER8I,2384
 docling_jobkit/connectors/google_drive_target_processor.py,sha256=sp54RIsH5DEfW54LaOGkCoEUJvYP3BYGcOo955aQs10,1884
-docling_jobkit/connectors/http_source_processor.py,sha256=F5Z8IzqY9aWCZ0Ak2HCgKHD4tbk51dY4FDNc4I3bBNw,797
+docling_jobkit/connectors/http_source_processor.py,sha256=i8gYWdPGE19cMoPAPwKSKe46fCm1B1NAQ64YJKyvgI8,1654
+docling_jobkit/connectors/local_path_source_processor.py,sha256=3R-w8HEZAZGPDn_vlkpdegdK4fiIoBWlsjaV2_NbXpg,4063
+docling_jobkit/connectors/local_path_target_processor.py,sha256=DhGCMLqMV7PuV_3jp7DL2Y9Zp1eNbs8onDAadfwz5XY,3173
 docling_jobkit/connectors/s3_helper.py,sha256=YIDytDkuEWIeTNRa-k0LsDG3V-tG31mvbAT3JyC1iF4,6320
-docling_jobkit/connectors/s3_source_processor.py,sha256=gLsYLksLGvn6pLID4RH0NN1w6nuy8HZVTnmmM6NfF9k,1365
+docling_jobkit/connectors/s3_source_processor.py,sha256=TZ1X0Lfj4eMM_QpOKz9KYF6HSbxDZ8XjtKqPqbxOLJM,2353
 docling_jobkit/connectors/s3_target_processor.py,sha256=CaX109JhZ8nFlYWcz3Sy0_Kl-pSD6Y9oz5SOPsFe9I4,1998
-docling_jobkit/connectors/source_processor.py,sha256=BbbsePSPIkyda73Ezwn29HnZSuerSJ7GrSPyPOOJFpo,1244
-docling_jobkit/connectors/source_processor_factory.py,sha256=CLanqD5PwvDqHQ6WP5ivhsXA5xMx-8PFnm8RtmVHvlI,932
+docling_jobkit/connectors/source_processor.py,sha256=HW3u6x_Btl4219LNdSPJhBi-xCGbh6yMf4vA1H7iXz4,2726
+docling_jobkit/connectors/source_processor_factory.py,sha256=FF93NMHPNMvOTdmahVk-tOdDGLJF93j6EVCAMT-YaCw,1155
 docling_jobkit/connectors/target_processor.py,sha256=2iIJE7Ip_-1dxJGt02_ALwDC2BPBOk0AomYI6EOFTCA,1447
-docling_jobkit/connectors/target_processor_factory.py,sha256=b_Q3L_mlvfQlZG7A2cskzf6-LzQ1G_seGd2vLT51b5o,688
+docling_jobkit/connectors/target_processor_factory.py,sha256=A-ZuwQiuYK6HVNHTZomIVBZ3tC9D6femv3-7PSNQkEA,901
 docling_jobkit/convert/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-docling_jobkit/convert/chunking.py,sha256=jFl7g8rGFmIBV_-0rxfyp6970N_0dqLehLgDSHKOM-o,11933
-docling_jobkit/convert/manager.py,sha256=oU_TiYANGQEGFR1mRVF7salwdiTYXulxAIQHqRr3UQ0,13990
+docling_jobkit/convert/chunking.py,sha256=vvdrvg66JB8Snwnbo7yzYXox6b2rRFDS9R4UyTXvq58,12014
+docling_jobkit/convert/manager.py,sha256=SPsPGf600vXL1H0tcXyQ7L7CCJlQcAz9gsetX4XWSPo,15938
 docling_jobkit/convert/results.py,sha256=vQvOuXIdlmPskHwUJlXX2zyJSb2k20ip5TfzuyPH5mU,9053
 docling_jobkit/convert/results_processor.py,sha256=TtiN6hqcUriEYMsEiyAutrgpMIz78D4pf-1HtiSjrXQ,16558
 docling_jobkit/datamodel/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -29,8 +32,8 @@ docling_jobkit/datamodel/result.py,sha256=WaasDSc_aAyuN3wfGlWqt1ZksC_5tIDHcafgeV
 docling_jobkit/datamodel/s3_coords.py,sha256=9MrrtQaLJRBOdqi7bzuT3XVSbXJAdKjPlch0IAXDnfc,1260
 docling_jobkit/datamodel/task.py,sha256=llcL3G6FM5ktkFSib8WFTuO5c_-KbJHUWugsD9RitbA,3163
 docling_jobkit/datamodel/task_meta.py,sha256=QB_u9_TZEZlXYFlmmCgsFnsuMjzaO6QNVsnREmDu6hc,389
-docling_jobkit/datamodel/task_sources.py,sha256=_aw4ymzC0lkjBRzz3UqrtM76Wu35_nNL704-jY2J0E8,733
-docling_jobkit/datamodel/task_targets.py,sha256=P7eEwpalak5E8TiTrYA7Oq9y38qs0KQR06eU0WtROfY,749
+docling_jobkit/datamodel/task_sources.py,sha256=HQVsy6d_qcClL-I9fjo1U9n_EGY2JJVZe91pmILUw9Q,2291
+docling_jobkit/datamodel/task_targets.py,sha256=j31iiZWAkw0viYF7vFuwP7yyorTZEcnlpVzE6RgJ_Y8,1388
 docling_jobkit/kfp_pipeline/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 docling_jobkit/kfp_pipeline/docling-s3in-s3out.yaml,sha256=tarhfU24S7eTnuTKSycCqbzLanJCfXb4EHmu_D3oYq8,15207
 docling_jobkit/kfp_pipeline/docling_s3in_s3out.py,sha256=DbIPQetoBIs9dX-JZszimjTTEEIWwiQOlLyhSt29Ybs,9818
@@ -44,15 +47,15 @@ docling_jobkit/orchestrators/kfp/kfp_pipeline.py,sha256=oglqcFAW2JS9G4Vyfby76T7a
 docling_jobkit/orchestrators/kfp/notify.py,sha256=uG9c18LJn9T0RuvRZtllWL497Uhv2Qe_whglB_ta8XY,883
 docling_jobkit/orchestrators/kfp/orchestrator.py,sha256=FkNCb9cscE4vckSYd9sA4_1SmnRJBepV738JeOwygcs,11653
 docling_jobkit/orchestrators/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-docling_jobkit/orchestrators/local/orchestrator.py,sha256=usG5nLEra3GqhfERnw6Pff0aNLWWSqE9T2TOU2jjiUI,4600
-docling_jobkit/orchestrators/local/worker.py,sha256=KB-lB4kV2eyWzluGcJCmRq9b3XHzo4i3w1C-VLlrTVU,5696
+docling_jobkit/orchestrators/local/orchestrator.py,sha256=swMw3a-Lm4a13poLV2JE33uF_EeBDyM3VZ71Dhbt-_o,4921
+docling_jobkit/orchestrators/local/worker.py,sha256=v4YNAZsSIcnNzsLDXot_3jdiy_lfIlH4h6E1hLRixS0,5818
 docling_jobkit/orchestrators/rq/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-docling_jobkit/orchestrators/rq/orchestrator.py,sha256=MFINSpXb5tQh0PKk3xK10eZ8cNc3gNflfeqLx5r9ueo,8896
-docling_jobkit/orchestrators/rq/worker.py,sha256=P9rXhH9k814sNPGgjY24CATwQSzL6Hfd5Th5d4I3ejs,6591
+docling_jobkit/orchestrators/rq/orchestrator.py,sha256=sP0EzjhKST8R6tPW3cVyaAzGWNDd-7AdaWhWMUi7QY8,9259
+docling_jobkit/orchestrators/rq/worker.py,sha256=tSGQCMFgHZTUqmIVkhVq8bYFdVl0eyhcm7NT22vmsHk,6719
 docling_jobkit/ray_job/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-docling_jobkit/ray_job/main.py,sha256=o52gLtEdGqyxa9XcVmwN55bCSVsIUq8zRm-FjgRlYN8,13465
-docling_jobkit-1.8.0.dist-info/METADATA,sha256=PjO4LuYeZpP03bpJwhmM9b6GQkPtidTGoAi59BCw_Lk,8105
-docling_jobkit-1.8.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-docling_jobkit-1.8.0.dist-info/entry_points.txt,sha256=QWmq6d0B14If8Zshc7pRnBs6zO1e9vhEnrMHOgBfzj8,121
-docling_jobkit-1.8.0.dist-info/licenses/LICENSE,sha256=mBb7ErEcM8VS9OhiGHnQ2kk75HwPhr54W1Oiz3965MY,1088
-docling_jobkit-1.8.0.dist-info/RECORD,,
+docling_jobkit/ray_job/main.py,sha256=6VyAsn9wk3v09qH4uQb4u1YnesX_-1DJrEg2MkDXy2k,13648
+docling_jobkit-1.9.0.dist-info/METADATA,sha256=RkesOBVu3Ue73C37am3HArxmYTLjzIhwaCjlJM5LqUs,10475
+docling_jobkit-1.9.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+docling_jobkit-1.9.0.dist-info/entry_points.txt,sha256=-tTX7hZPMCPZ2zVSUhI2BTPFglsds_A6PbCpPR7gUVM,181
+docling_jobkit-1.9.0.dist-info/licenses/LICENSE,sha256=mBb7ErEcM8VS9OhiGHnQ2kk75HwPhr54W1Oiz3965MY,1088
+docling_jobkit-1.9.0.dist-info/RECORD,,

{docling_jobkit-1.8.0.dist-info → docling_jobkit-1.9.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.27.0
+Generator: hatchling 1.28.0
 Root-Is-Purelib: true
 Tag: py3-none-any

{docling_jobkit-1.8.0.dist-info → docling_jobkit-1.9.0.dist-info}/entry_points.txt

@@ -1,3 +1,4 @@
 [console_scripts]
 docling-jobkit-local = docling_jobkit.cli.local:app
+docling-jobkit-multiproc = docling_jobkit.cli.multiproc:app
 docling-ray-job = docling_jobkit.ray_job.main:main