nv-ingest-api 2025.10.27.dev20251027__py3-none-any.whl → 2025.10.29.dev20251029__py3-none-any.whl

nv_ingest_api/util/dataloader/dataloader.py

@@ -254,22 +254,29 @@ else:
         file = None
         try:
             for file in paths:
+                if thread_stop.is_set():
+                    return
                 if isinstance(file, tuple):
                     video_file, audio_file = file
+                    if thread_stop.is_set():
+                        return
                     with open(video_file, "rb") as f:
                         video = f.read()
+                    if thread_stop.is_set():
+                        return
                     with open(audio_file, "rb") as f:
                         audio = f.read()
                     queue.put((video, audio))
                 else:
-                    if thread_stop:
+                    if thread_stop.is_set():
                         return
                     with open(file, "rb") as f:
                         queue.put(f.read())
         except Exception as e:
             logging.error(f"Error processing file {file}: {e}")
             queue.put(RuntimeError(f"Error processing file {file}: {e}"))
-        queue.put(StopIteration)
+        finally:
+            queue.put(StopIteration)
 
     class DataLoader:
         """
@@ -290,7 +297,7 @@ else:
         ):
             interface = interface if interface else MediaInterface()
             self.thread = None
-            self.thread_stop = False
+            self.thread_stop = threading.Event()
             self.queue = queue.Queue(size)
             self.path = Path(path)
             self.output_dir = output_dir
@@ -323,16 +330,20 @@ else:
             Reset itertor by stopping the thread and clearing the queue.
             """
             if self.thread:
-                self.thread_stop = True
+                self.thread_stop.set()
                 self.thread.join()
-            self.thread_stop = False
-            while self.queue.qsize() != 0:
-                with self.queue.mutex:
-                    self.queue.queue.clear()
+                self.thread = None
+            try:
+                while True:
+                    self.queue.get_nowait()
+            except Exception:
+                pass
+            finally:
+                self.thread_stop.clear()
 
         def __iter__(self):
             self.stop()
-            self.thread_stop = False
+            self.thread_stop.clear()
             self.thread = threading.Thread(
                 target=load_data,
                 args=(

nv_ingest_api/util/multi_processing/mp_pool_singleton.py

@@ -5,8 +5,9 @@
 
 import logging
 import math
-import multiprocessing as mp
 import os
+import sys
+import multiprocessing as mp
 from threading import Lock
 from typing import Any, Callable, Optional
 
@@ -103,7 +104,12 @@ class ProcessWorkerPoolSingleton:
             The total number of worker processes to start.
         """
         self._total_workers = total_max_workers
-        self._context: mp.context.ForkContext = mp.get_context("fork")
+
+        start_method = "fork"
+        if sys.platform.lower() == "darwin":
+            start_method = "spawn"
+        self._context: mp.context.ForkContext = mp.get_context(start_method)
+
         # Bounded task queue: maximum tasks queued = 2 * total_max_workers.
         self._task_queue: mp.Queue = self._context.Queue(maxsize=2 * total_max_workers)
         self._next_task_id: int = 0

{nv_ingest_api-2025.10.27.dev20251027.dist-info → nv_ingest_api-2025.10.29.dev20251029.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest-api
-Version: 2025.10.27.dev20251027
+Version: 2025.10.29.dev20251029
 Summary: Python module with core document ingestion functions.
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License

{nv_ingest_api-2025.10.27.dev20251027.dist-info → nv_ingest_api-2025.10.29.dev20251029.dist-info}/RECORD

@@ -116,7 +116,7 @@ nv_ingest_api/util/converters/dftools.py,sha256=FjHjazIeiUd1LdFwWuummJmraqZe1a90
 nv_ingest_api/util/converters/formats.py,sha256=L11FtormO2SeHSebbwsGE_uuCv6Jk0D3VvVW2avU0vI,2258
 nv_ingest_api/util/converters/type_mappings.py,sha256=5TVXRyU6BlQvFOdqknEuQw3ss4PXeCvSUynJnjvgQpA,1102
 nv_ingest_api/util/dataloader/__init__.py,sha256=B6ybDORMI9IzXGdhM7w_agcVj1BNYgAlcfTA0lG5jng,308
-nv_ingest_api/util/dataloader/dataloader.py,sha256=r_TU-RfdYerl3k3jRsGIVByxejwz-UQDuallx5-YAGM,14790
+nv_ingest_api/util/dataloader/dataloader.py,sha256=1SG0cHKo7X_eBRTVMJ9EFJOfpqe37QqfEoMxeAWxkEU,15124
 nv_ingest_api/util/detectors/__init__.py,sha256=HIHfzSig66GT0Uk8qsGBm_f13fKYcPtItBicRUWOOVA,183
 nv_ingest_api/util/detectors/language.py,sha256=TvzcESYY0bn0U4aLY6GjB4VaCWA6XrXxAGZbVzHTMuE,965
 nv_ingest_api/util/exception_handlers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -147,7 +147,7 @@ nv_ingest_api/util/message_brokers/simple_message_broker/simple_client.py,sha256
 nv_ingest_api/util/metadata/__init__.py,sha256=HIHfzSig66GT0Uk8qsGBm_f13fKYcPtItBicRUWOOVA,183
 nv_ingest_api/util/metadata/aggregators.py,sha256=YYdvJ1E04eGFZKKHUxXoH6mzLg8nor9Smvnv0qzqK5w,15988
 nv_ingest_api/util/multi_processing/__init__.py,sha256=4fojP8Rp_5Hu1YAkqGylqTyEZ-HBVVEunn5Z9I99swA,242
-nv_ingest_api/util/multi_processing/mp_pool_singleton.py,sha256=dTfP82DgGPaXEJH3jywTO8rNlLZUniD4FFzwv84_giE,7372
+nv_ingest_api/util/multi_processing/mp_pool_singleton.py,sha256=34O7I8Lin5GvO_zNZGbsqEGkDvIbqy_0Eh3ejoPNDVE,7501
 nv_ingest_api/util/nim/__init__.py,sha256=No45pMstom1Jo0EENT6VEFkZn3YmTha7lYaBZU7xtHk,2116
 nv_ingest_api/util/pdf/__init__.py,sha256=uLsBITo_XfgbwpzqXUm1IYX6XlZrTfx6T1cIhdILwG8,140
 nv_ingest_api/util/pdf/pdfium.py,sha256=1aPCnPKXHWnncYoMO8HllYjrhODSXIeRBIsSLDevpYs,15667
@@ -165,10 +165,10 @@ nv_ingest_api/util/string_processing/configuration.py,sha256=2HS08msccuPCT0fn_jf
 nv_ingest_api/util/string_processing/yaml.py,sha256=4Zdmc4474lUZn6kznqaNTlQJwsmRnnJQZ-DvAWLu-zo,2678
 nv_ingest_api/util/system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nv_ingest_api/util/system/hardware_info.py,sha256=1UFM8XE6M3pgQcpbVsCsqDQ7Dj-zzptL-XRE-DEu9UA,27213
-nv_ingest_api-2025.10.27.dev20251027.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+nv_ingest_api-2025.10.29.dev20251029.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
 udfs/__init__.py,sha256=pXFqPgXIUqHDfj7SAR1Q19tt8KwGv_iMvhHyziz4AYM,205
 udfs/llm_summarizer_udf.py,sha256=lH5c5NHoT-5ecHC3og_40u1Ujta8SpsKU4X0e4wzbMU,7314
-nv_ingest_api-2025.10.27.dev20251027.dist-info/METADATA,sha256=fQdSW5uS_1SsaCF8ddbueLFilG6l51Cg2HaZ076Pj0M,14106
-nv_ingest_api-2025.10.27.dev20251027.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nv_ingest_api-2025.10.27.dev20251027.dist-info/top_level.txt,sha256=I1lseG9FF0CH93SPx4kFblsxFuv190cfzaas_CLNIiw,19
-nv_ingest_api-2025.10.27.dev20251027.dist-info/RECORD,,
+nv_ingest_api-2025.10.29.dev20251029.dist-info/METADATA,sha256=0aFHQxZbEF6-N4iGk5aUxQpasZM73Ae49nQFa90k_-Q,14106
+nv_ingest_api-2025.10.29.dev20251029.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nv_ingest_api-2025.10.29.dev20251029.dist-info/top_level.txt,sha256=I1lseG9FF0CH93SPx4kFblsxFuv190cfzaas_CLNIiw,19
+nv_ingest_api-2025.10.29.dev20251029.dist-info/RECORD,,