nv-ingest-client 2025.10.14.dev20251014__tar.gz → 2025.10.15.dev20251015__tar.gz

{nv_ingest_client-2025.10.14.dev20251014/src/nv_ingest_client.egg-info → nv_ingest_client-2025.10.15.dev20251015}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest-client
-Version: 2025.10.14.dev20251014
+Version: 2025.10.15.dev20251015
 Summary: Python client for the nv-ingest service
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License

{nv_ingest_client-2025.10.14.dev20251014 → nv_ingest_client-2025.10.15.dev20251015}/src/nv_ingest_client/client/client.py

@@ -36,7 +36,11 @@ from nv_ingest_client.primitives.tasks import TaskType
 from nv_ingest_client.primitives.tasks import is_valid_task_type
 from nv_ingest_client.primitives.tasks import task_factory
 from nv_ingest_client.util.processing import handle_future_result, IngestJobFailure
-from nv_ingest_client.util.util import create_job_specs_for_batch, check_ingest_result
+from nv_ingest_client.util.util import (
+    create_job_specs_for_batch,
+    check_ingest_result,
+    apply_pdf_split_config_to_job_specs,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -688,11 +692,12 @@ class NvIngestClient:
         message_client_port : int, optional
             Port of the REST/message service. Defaults to 7670.
         message_client_kwargs : dict, optional
-            Extra keyword arguments passed to the client allocator.
+            Extra keyword arguments passed to the client allocator. For RestClient,
+            can include 'api_version' (e.g., 'v1' or 'v2'). Defaults to 'v1'.
         msg_counter_id : str, optional
             Identifier for message counting. Defaults to "nv-ingest-message-id".
         worker_pool_size : int, optional
-            Number of workers in the thread pool. Defaults to 1.
+            Number of workers in the thread pool. Defaults to 8.
 
         Returns
         -------
@@ -1707,7 +1712,9 @@ class NvIngestClient:
 
         return results
 
-    def create_jobs_for_batch(self, files_batch: List[str], tasks: Dict[str, Any]) -> List[str]:
+    def create_jobs_for_batch(
+        self, files_batch: List[str], tasks: Dict[str, Any], pdf_split_page_count: int = None
+    ) -> List[str]:
         """
         Create and submit job specifications (JobSpecs) for a batch of files, returning the job IDs.
         This function takes a batch of files, processes each file to extract its content and type,
@@ -1723,6 +1730,9 @@ class NvIngestClient:
             A dictionary of tasks to be added to each job. The keys represent task names, and the
             values represent task specifications or configurations. Standard tasks include "split",
             "extract", "store", "caption", "dedup", "filter", "embed".
+        pdf_split_page_count : int, optional
+            Number of pages per PDF chunk for splitting (1-128). If provided, this will be added
+            to the job spec's extended_options for PDF files.
 
         Returns
         -------
@@ -1769,6 +1779,10 @@ class NvIngestClient:
 
         job_specs = create_job_specs_for_batch(files_batch)
 
+        # Apply PDF split config if provided
+        if pdf_split_page_count is not None:
+            apply_pdf_split_config_to_job_specs(job_specs, pdf_split_page_count)
+
         job_ids = []
         for job_spec in job_specs:
             logger.debug(f"Tasks: {tasks.keys()}")

{nv_ingest_client-2025.10.14.dev20251014 → nv_ingest_client-2025.10.15.dev20251015}/src/nv_ingest_client/client/ingest_job_handler.py

@@ -45,6 +45,7 @@ class IngestJobHandler:
         show_progress: bool = True,
         show_telemetry: bool = False,
         job_queue_id: str = "ingest_task_queue",
+        pdf_split_page_count: int = None,
     ) -> None:
         self.client = client
         self.files = files
@@ -56,6 +57,7 @@ class IngestJobHandler:
         self.show_progress = show_progress
         self.show_telemetry = show_telemetry
         self.job_queue_id = job_queue_id
+        self.pdf_split_page_count = pdf_split_page_count
         self._pbar = None
         # Internal state used across iterations
         self._retry_job_ids: List[str] = []
@@ -144,7 +146,9 @@ class IngestJobHandler:
             new_job_count: int = min(self.batch_size - cur_job_count, len(self.files) - self._processed)
             batch_files: List[str] = self.files[self._processed : self._processed + new_job_count]
 
-            new_job_indices: List[str] = self.client.create_jobs_for_batch(batch_files, self.tasks)
+            new_job_indices: List[str] = self.client.create_jobs_for_batch(
+                batch_files, self.tasks, pdf_split_page_count=self.pdf_split_page_count
+            )
             if len(new_job_indices) != new_job_count:
                 missing_jobs: int = new_job_count - len(new_job_indices)
                 error_msg: str = (
@@ -304,6 +308,7 @@ class IngestJobHandler:
         trace_ids: Dict[str, str] = defaultdict(list)  # type: ignore
         failed_jobs: List[str] = []
         retry_counts: Dict[str, int] = defaultdict(int)
+        pages_per_sec: float = None
 
         start_time_ns: int = time.time_ns()
         self._init_progress_bar(total_files)

{nv_ingest_client-2025.10.14.dev20251014 → nv_ingest_client-2025.10.15.dev20251015}/src/nv_ingest_client/client/interface.py

@@ -54,7 +54,7 @@ from nv_ingest_client.primitives.tasks import StoreEmbedTask
 from nv_ingest_client.primitives.tasks import UDFTask
 from nv_ingest_client.util.processing import check_schema
 from nv_ingest_client.util.system import ensure_directory_with_permissions
-from nv_ingest_client.util.util import filter_function_kwargs
+from nv_ingest_client.util.util import filter_function_kwargs, apply_pdf_split_config_to_job_specs
 from nv_ingest_client.util.vdb import VDB, get_vdb_op_cls
 from tqdm import tqdm
 
@@ -1237,6 +1237,44 @@ class Ingestor:
 
         return self
 
+    @ensure_job_specs
+    def pdf_split_config(self, pages_per_chunk: int = 32) -> "Ingestor":
+        """
+        Configure PDF splitting behavior for V2 API.
+
+        Parameters
+        ----------
+        pages_per_chunk : int, optional
+            Number of pages per PDF chunk (default: 32)
+            Server enforces boundaries: min=1, max=128
+
+        Returns
+        -------
+        Ingestor
+            Self for method chaining
+
+        Notes
+        -----
+        - Only affects V2 API endpoints with PDF splitting support
+        - Server will clamp values outside [1, 128] range
+        - Smaller chunks = more parallelism but more overhead
+        - Larger chunks = less overhead but reduced concurrency
+        """
+        MIN_PAGES = 1
+        MAX_PAGES = 128
+
+        # Warn if value will be clamped by server
+        if pages_per_chunk < MIN_PAGES:
+            logger.warning(f"pages_per_chunk={pages_per_chunk} is below minimum. Server will clamp to {MIN_PAGES}.")
+        elif pages_per_chunk > MAX_PAGES:
+            logger.warning(f"pages_per_chunk={pages_per_chunk} exceeds maximum. Server will clamp to {MAX_PAGES}.")
+
+        # Flatten all job specs and apply PDF config using shared utility
+        all_job_specs = [spec for job_specs in self._job_specs._file_type_to_job_spec.values() for spec in job_specs]
+        apply_pdf_split_config_to_job_specs(all_job_specs, pages_per_chunk)
+
+        return self
+
     def _count_job_states(self, job_states: set[JobStateEnum]) -> int:
         """
         Counts the jobs in specified states.

{nv_ingest_client-2025.10.14.dev20251014 → nv_ingest_client-2025.10.15.dev20251015}/src/nv_ingest_client/nv_ingest_cli.py

@@ -74,6 +74,12 @@ logger = logging.getLogger(__name__)
 @click.option("--client_host", default="localhost", help="DNS name or URL for the endpoint.")
 @click.option("--client_port", default=7670, type=int, help="Port for the client endpoint.")
 @click.option("--client_kwargs", help="Additional arguments to pass to the client.", default="{}")
+@click.option(
+    "--api_version",
+    default="v1",
+    type=click.Choice(["v1", "v2"], case_sensitive=False),
+    help="API version to use (v1 or v2). V2 required for PDF split page count feature.",
+)
 @click.option(
     "--client_type",
     default="rest",
@@ -119,6 +125,8 @@ Example:
   --task 'extract:{"document_type":"docx", "extract_text":true, "extract_images":true}'
   --task 'embed'
   --task 'caption:{}'
+  --pdf_split_page_count 64  # Configure PDF splitting (requires --api_version v2)
+  --api_version v2           # Use V2 API for PDF splitting support
 
 \b
 Tasks and Options:
@@ -207,6 +215,12 @@ for locating portions of the system that might be bottlenecks for the overall ru
 )
 @click.option("--zipkin_host", default="localhost", help="DNS name or Zipkin API.")
 @click.option("--zipkin_port", default=9411, type=int, help="Port for the Zipkin trace API")
+@click.option(
+    "--pdf_split_page_count",
+    default=None,
+    type=int,
+    help="Number of pages per PDF chunk for splitting. Allows per-request tuning of PDF split size in v2 api.",
+)
 @click.option("--version", is_flag=True, help="Show version.")
 @click.pass_context
 def main(
@@ -215,6 +229,7 @@ def main(
     client_host: str,
     client_kwargs: str,
     client_port: int,
+    api_version: str,
     client_type: str,
     concurrency_n: int,
     dataset: str,
@@ -228,6 +243,7 @@ def main(
     collect_profiling_traces: bool,
     zipkin_host: str,
     zipkin_port: int,
+    pdf_split_page_count: int,
     task: [str],
     version: [bool],
 ):
@@ -268,6 +284,10 @@ def main(
                 _client_kwargs_obj = json.loads(client_kwargs)
             except Exception:
                 _client_kwargs_obj = {"raw": client_kwargs}
+
+            # Merge api_version into client_kwargs
+            _client_kwargs_obj["api_version"] = api_version
+
             _sanitized_client_kwargs = sanitize_for_logging(_client_kwargs_obj)
             logging.debug(
                 f"Creating message client: {client_host} and port: {client_port} -> "
@@ -285,7 +305,7 @@ def main(
                 message_client_allocator=client_allocator,
                 message_client_hostname=client_host,
                 message_client_port=client_port,
-                message_client_kwargs=json.loads(client_kwargs),
+                message_client_kwargs=_client_kwargs_obj,
                 worker_pool_size=concurrency_n,
             )
 
@@ -300,6 +320,7 @@ def main(
                 save_images_separately=save_images_separately,
                 show_progress=True,
                 show_telemetry=True,
+                pdf_split_page_count=pdf_split_page_count,
             )
             (total_files, trace_times, pages_processed, trace_ids) = handler.run()
 

{nv_ingest_client-2025.10.14.dev20251014 → nv_ingest_client-2025.10.15.dev20251015}/src/nv_ingest_client/primitives/jobs/job_spec.py

@@ -110,6 +110,7 @@ class JobSpec:
             "job_id": str(self._job_id),
             "tasks": [task.to_dict() for task in self._tasks],
             "tracing_options": self._extended_options.get("tracing_options", {}),
+            "pdf_config": self._extended_options.get("pdf_config", {}),
         }
 
     @property

{nv_ingest_client-2025.10.14.dev20251014 → nv_ingest_client-2025.10.15.dev20251015}/src/nv_ingest_client/util/document_analysis.py

@@ -20,7 +20,7 @@ logger = logging.getLogger(__name__)
 
 
 def analyze_document_chunks(
-    results: Union[List[List[Dict[str, Any]]], List[Dict[str, Any]]]
+    results: Union[List[List[Dict[str, Any]]], List[Dict[str, Any]]],
 ) -> Dict[str, Dict[str, Dict[str, int]]]:
     """
     Analyze ingestor results to count elements by type and page for each document.

{nv_ingest_client-2025.10.14.dev20251014 → nv_ingest_client-2025.10.15.dev20251015}/src/nv_ingest_client/util/util.py

@@ -350,6 +350,32 @@ def create_job_specs_for_batch(files_batch: List[str]) -> List[JobSpec]:
     return job_specs
 
 
+def apply_pdf_split_config_to_job_specs(job_specs: List[JobSpec], pages_per_chunk: int) -> None:
+    """
+    Apply PDF split configuration to a list of JobSpec objects.
+
+    Modifies job specs in-place by adding pdf_config to extended_options for PDF files only.
+
+    Parameters
+    ----------
+    job_specs : List[JobSpec]
+        List of job specifications to potentially modify
+    pages_per_chunk : int
+        Number of pages per PDF chunk (will be stored as-is; server performs clamping)
+
+    Notes
+    -----
+    - Only modifies job specs with document_type == "pdf" (case-insensitive)
+    - Modifies job specs in-place
+    - Safe to call on mixed document types (only PDFs are affected)
+    """
+    for job_spec in job_specs:
+        if job_spec.document_type.lower() == "pdf":
+            if "pdf_config" not in job_spec._extended_options:
+                job_spec._extended_options["pdf_config"] = {}
+            job_spec._extended_options["pdf_config"]["split_page_count"] = pages_per_chunk
+
+
 def filter_function_kwargs(func, **kwargs):
     """
     Filters and returns keyword arguments that match the parameters of a given function.

{nv_ingest_client-2025.10.14.dev20251014 → nv_ingest_client-2025.10.15.dev20251015}/src/nv_ingest_client/util/vdb/milvus.py

@@ -917,7 +917,9 @@ def wait_for_index(collection_name: str, num_elements: int, client: MilvusClient
                     break
                 # check if indexed_rows is staying the same, too many times means something is wrong
                 if new_indexed_rows == indexed_rows:
-                    pos_movement = -1
+                    pos_movement -= 1
+                else:
+                    pos_movement = 10
                 # if pos_movement is 0, raise an error, means the rows are not getting indexed as expected
                 if pos_movement == 0:
                     raise ValueError("Rows are not getting indexed as expected")
@@ -1046,9 +1048,10 @@ def write_to_nvingest_collection(
             client,
             collection_name,
         )
-        # Make sure all rows are indexed, decided not to wrap in a timeout because we dont
-        # know how long this should take, it is num_elements dependent.
-        wait_for_index(collection_name, num_elements, client)
+        if not local_index:
+            # Make sure all rows are indexed, decided not to wrap in a timeout because we dont
+            # know how long this should take, it is num_elements dependent.
+            wait_for_index(collection_name, num_elements, client)
     else:
         minio_client = Minio(minio_endpoint, access_key=access_key, secret_key=secret_key, secure=False)
         bucket_name = bucket_name if bucket_name else ClientConfigSchema().minio_bucket_name

{nv_ingest_client-2025.10.14.dev20251014 → nv_ingest_client-2025.10.15.dev20251015/src/nv_ingest_client.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest-client
-Version: 2025.10.14.dev20251014
+Version: 2025.10.15.dev20251015
 Summary: Python client for the nv-ingest service
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License