docling-jobkit 1.8.0__tar.gz → 1.9.0__tar.gz

{docling_jobkit-1.8.0 → docling_jobkit-1.9.0}/PKG-INFO

@@ -1,8 +1,9 @@
 Metadata-Version: 2.4
 Name: docling-jobkit
-Version: 1.8.0
+Version: 1.9.0
 Summary: Running a distributed job processing documents with Docling.
 Project-URL: Homepage, https://github.com/docling-project/docling-jobkit
+Project-URL: Documentation, https://docling-project.github.io/docling/usage/jobkit/
 Project-URL: Repository, https://github.com/docling-project/docling-jobkit
 Project-URL: Issues, https://github.com/docling-project/docling-jobkit/issues
 Project-URL: Changelog, https://github.com/docling-project/docling-jobkit/blob/main/CHANGELOG.md
@@ -18,6 +19,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Classifier: Topic :: Software Development :: Build Tools
@@ -26,21 +28,18 @@ Classifier: Typing :: Typed
 Requires-Python: >=3.10
 Requires-Dist: boto3~=1.35
 Requires-Dist: docling~=2.60
-Requires-Dist: fastparquet~=2024.11
-Requires-Dist: httpx~=0.28
+Requires-Dist: httpx<1,>=0.28
 Requires-Dist: pandas~=2.2
-Requires-Dist: pyarrow~=19.0
 Requires-Dist: pydantic-settings~=2.4
 Requires-Dist: pydantic~=2.10
-Requires-Dist: typer>=0.12.5
-Requires-Dist: typer~=0.12
+Requires-Dist: typer<1,>=0.12.5
 Provides-Extra: gdrive
 Requires-Dist: google-api-python-client>=2.183.0; extra == 'gdrive'
 Requires-Dist: google-auth-oauthlib>=1.2.2; extra == 'gdrive'
 Provides-Extra: kfp
 Requires-Dist: kfp[kubernetes]>=2.10.0; extra == 'kfp'
 Provides-Extra: ray
-Requires-Dist: ray~=2.30; extra == 'ray'
+Requires-Dist: ray~=2.30; (python_version < '3.14') and extra == 'ray'
 Provides-Extra: rq
 Requires-Dist: msgpack~=1.1; extra == 'rq'
 Requires-Dist: rq~=2.4; extra == 'rq'
@@ -55,6 +54,77 @@ Running a distributed job processing documents with Docling.
 
 ## How to use it
 
+### Local Multiprocessing CLI
+
+The `docling-jobkit-multiproc` CLI enables parallel batch processing of documents using Python's multiprocessing. Each batch of documents is processed in a separate subprocess, allowing efficient parallel processing on a single machine.
+
+#### Usage
+
+```bash
+# Basic usage with default settings (batch_size=10, num_processes=CPU count)
+docling-jobkit-multiproc config.yaml
+
+# Custom batch size and number of processes
+docling-jobkit-multiproc config.yaml --batch-size 20 --num-processes 4
+
+# With model artifacts
+docling-jobkit-multiproc config.yaml --artifacts-path /path/to/models
+
+# Quiet mode (suppress progress bar)
+docling-jobkit-multiproc config.yaml --quiet
+
+# Full options
+docling-jobkit-multiproc config.yaml \
+  --batch-size 30 \
+  --num-processes 8 \
+  --artifacts-path /path/to/models \
+  --enable-remote-services \
+  --allow-external-plugins
+```
+
+#### Configuration
+
+The configuration file format is the same as `docling-jobkit-local`. See example configurations:
+- S3 source/target: `dev/configs/run_multiproc_s3_example.yaml`
+- Local path source/target: `dev/configs/run_local_folder_example.yaml`
+
+**Note:** Only S3, Google Drive, and local_path sources support batch processing. File and HTTP sources do not support chunking.
+
+#### CLI Options
+
+- `--batch-size, -b`: Number of documents to process in each batch (default: 10)
+- `--num-processes, -n`: Number of parallel processes (default: CPU count)
+- `--artifacts-path`: Path to model artifacts directory
+- `--enable-remote-services`: Enable models connecting to remote services
+- `--allow-external-plugins`: Enable loading modules from third-party plugins
+- `--quiet, -q`: Suppress progress bar and detailed output
+
+### Local Sequential CLI
+
+The `docling-jobkit-local` CLI processes documents sequentially in a single process.
+
+```bash
+docling-jobkit-local config.yaml
+```
+
+### Using Local Path Sources and Targets
+
+Both CLIs support local file system sources and targets. Example configuration:
+
+```yaml
+sources:
+  - kind: local_path
+    path: ./input_documents/
+    recursive: true  # optional, default true
+    pattern: "*.pdf"  # optional glob pattern
+
+target:
+  kind: local_path
+  path: ./output_documents/
+```
+
+See `dev/configs/run_local_folder_example.yaml` for a complete example.
+
 ## Kubeflow pipeline with Docling Jobkit
 
 ### Using Kubeflow pipeline web dashboard UI

{docling_jobkit-1.8.0 → docling_jobkit-1.9.0}/README.md

@@ -5,6 +5,77 @@ Running a distributed job processing documents with Docling.
 
 ## How to use it
 
+### Local Multiprocessing CLI
+
+The `docling-jobkit-multiproc` CLI enables parallel batch processing of documents using Python's multiprocessing. Each batch of documents is processed in a separate subprocess, allowing efficient parallel processing on a single machine.
+
+#### Usage
+
+```bash
+# Basic usage with default settings (batch_size=10, num_processes=CPU count)
+docling-jobkit-multiproc config.yaml
+
+# Custom batch size and number of processes
+docling-jobkit-multiproc config.yaml --batch-size 20 --num-processes 4
+
+# With model artifacts
+docling-jobkit-multiproc config.yaml --artifacts-path /path/to/models
+
+# Quiet mode (suppress progress bar)
+docling-jobkit-multiproc config.yaml --quiet
+
+# Full options
+docling-jobkit-multiproc config.yaml \
+  --batch-size 30 \
+  --num-processes 8 \
+  --artifacts-path /path/to/models \
+  --enable-remote-services \
+  --allow-external-plugins
+```
+
+#### Configuration
+
+The configuration file format is the same as `docling-jobkit-local`. See example configurations:
+- S3 source/target: `dev/configs/run_multiproc_s3_example.yaml`
+- Local path source/target: `dev/configs/run_local_folder_example.yaml`
+
+**Note:** Only S3, Google Drive, and local_path sources support batch processing. File and HTTP sources do not support chunking.
+
+#### CLI Options
+
+- `--batch-size, -b`: Number of documents to process in each batch (default: 10)
+- `--num-processes, -n`: Number of parallel processes (default: CPU count)
+- `--artifacts-path`: Path to model artifacts directory
+- `--enable-remote-services`: Enable models connecting to remote services
+- `--allow-external-plugins`: Enable loading modules from third-party plugins
+- `--quiet, -q`: Suppress progress bar and detailed output
+
+### Local Sequential CLI
+
+The `docling-jobkit-local` CLI processes documents sequentially in a single process.
+
+```bash
+docling-jobkit-local config.yaml
+```
+
+### Using Local Path Sources and Targets
+
+Both CLIs support local file system sources and targets. Example configuration:
+
+```yaml
+sources:
+  - kind: local_path
+    path: ./input_documents/
+    recursive: true  # optional, default true
+    pattern: "*.pdf"  # optional glob pattern
+
+target:
+  kind: local_path
+  path: ./output_documents/
+```
+
+See `dev/configs/run_local_folder_example.yaml` for a complete example.
+
 ## Kubeflow pipeline with Docling Jobkit
 
 ### Using Kubeflow pipeline web dashboard UI

{docling_jobkit-1.8.0 → docling_jobkit-1.9.0}/docling_jobkit/cli/local.py

@@ -18,9 +18,15 @@ from docling_jobkit.datamodel.task_sources import (
     TaskFileSource,
     TaskGoogleDriveSource,
     TaskHttpSource,
+    TaskLocalPathSource,
     TaskS3Source,
 )
-from docling_jobkit.datamodel.task_targets import GoogleDriveTarget, S3Target, ZipTarget
+from docling_jobkit.datamodel.task_targets import (
+    GoogleDriveTarget,
+    LocalPathTarget,
+    S3Target,
+    ZipTarget,
+)
 
 console = Console()
 err_console = Console(stderr=True)
@@ -34,12 +40,17 @@ app = typer.Typer(
 )
 
 JobTaskSource = Annotated[
-    TaskFileSource | TaskHttpSource | TaskS3Source | TaskGoogleDriveSource,
+    TaskFileSource
+    | TaskHttpSource
+    | TaskLocalPathSource
+    | TaskS3Source
+    | TaskGoogleDriveSource,
     Field(discriminator="kind"),
 ]
 
 JobTaskTarget = Annotated[
-    ZipTarget | S3Target | GoogleDriveTarget, Field(discriminator="kind")
+    ZipTarget | LocalPathTarget | S3Target | GoogleDriveTarget,
+    Field(discriminator="kind"),
 ]