nv-ingest-client 2025.8.19.dev20250819__py3-none-any.whl → 2025.8.21.dev20250821__py3-none-any.whl

nv_ingest_client/nv_ingest_cli.py

@@ -32,6 +32,7 @@ from nv_ingest_client.client import NvIngestClient
 from nv_ingest_client.util.dataset import get_dataset_files
 from nv_ingest_client.util.dataset import get_dataset_statistics
 from nv_ingest_client.util.system import ensure_directory_with_permissions
+from nv_ingest_api.util.logging.sanitize import sanitize_for_logging
 
 try:
     NV_INGEST_VERSION = version("nv_ingest")
@@ -237,7 +238,9 @@ def main(
 
     try:
         configure_logging(logger, log_level)
-        logging.debug(f"nv-ingest-cli:params:\n{json.dumps(ctx.params, indent=2, default=repr)}")
+        # Sanitize CLI params before logging to avoid leaking secrets
+        _sanitized_params = sanitize_for_logging(dict(ctx.params))
+        logging.debug(f"nv-ingest-cli:params:\n{json.dumps(_sanitized_params, indent=2, default=repr)}")
 
         docs = list(doc)
         if dataset:
@@ -260,7 +263,16 @@ def main(
             logger.info(_msg)
 
         if not dry_run:
-            logging.debug(f"Creating message client: {client_host} and port: {client_port} -> {client_kwargs}")
+            # Sanitize client kwargs (JSON string) before logging
+            try:
+                _client_kwargs_obj = json.loads(client_kwargs)
+            except Exception:
+                _client_kwargs_obj = {"raw": client_kwargs}
+            _sanitized_client_kwargs = sanitize_for_logging(_client_kwargs_obj)
+            logging.debug(
+                f"Creating message client: {client_host} and port: {client_port} -> "
+                f"{json.dumps(_sanitized_client_kwargs, indent=2, default=repr)}"
+            )
 
             if client_type == "rest":
                 client_allocator = RestClient

nv_ingest_client/primitives/tasks/caption.py

@@ -9,8 +9,8 @@
 import logging
 from typing import Dict
 
-from nv_ingest_api.internal.schemas.meta.ingest_job_schema import IngestTaskCaptionSchema
 
+from nv_ingest_api.internal.schemas.meta.ingest_job_schema import IngestTaskCaptionSchema
 from .task_base import Task
 
 logger = logging.getLogger(__name__)

nv_ingest_client/util/dataset.py

@@ -94,8 +94,14 @@ def get_dataset_files(dataset_bytes: BytesIO, shuffle: bool = False) -> list:
         dataset_bytes.seek(0)
         dataset = json.load(dataset_bytes)
         sampled_files = dataset.get("sampled_files", [])
-        if shuffle:
-            random.shuffle(sampled_files)
+        if shuffle and len(sampled_files) > 1:
+            original = list(sampled_files)
+            # Create a shuffled copy without mutating the original list
+            shuffled = random.sample(sampled_files, k=len(sampled_files))
+            # Guard against seeded RNG or accidental identity by forcing a different order
+            if shuffled == original:
+                shuffled = shuffled[1:] + shuffled[:1]
+            return shuffled
         return sampled_files
     except json.JSONDecodeError as err:
         raise ValueError(f"{err}")

nv_ingest_client/util/vdb/milvus.py

@@ -170,10 +170,15 @@ def grab_meta_collection_info(
     embedding_model: str = None,
     embedding_dim: int = None,
     client: MilvusClient = None,
+    milvus_uri: str = None,
+    username: str = None,
+    password: str = None,
 ):
     timestamp = timestamp or ""
     embedding_model = embedding_model or ""
     embedding_dim = embedding_dim or ""
+    if milvus_uri:
+        client = MilvusClient(milvus_uri, token=f"{username}:{password}")
     results = client.query_iterator(
         collection_name=meta_collection_name,
         output_fields=[
@@ -771,7 +776,7 @@ def bulk_insert_milvus(
     """
     minio_client = Minio(minio_endpoint, access_key=access_key, secret_key=secret_key, secure=False)
 
-    connections.connect(uri=milvus_uri, username=username, password=password)
+    connections.connect(uri=milvus_uri, token=f"{username}:{password}")
     t_bulk_start = time.time()
     task_ids = []
     uploaded_files = []

{nv_ingest_client-2025.8.19.dev20250819.dist-info → nv_ingest_client-2025.8.21.dev20250821.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest-client
-Version: 2025.8.19.dev20250819
+Version: 2025.8.21.dev20250821
 Summary: Python client for the nv-ingest service
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License

{nv_ingest_client-2025.8.19.dev20250819.dist-info → nv_ingest_client-2025.8.21.dev20250821.dist-info}/RECORD

@@ -1,5 +1,5 @@
 nv_ingest_client/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
-nv_ingest_client/nv_ingest_cli.py,sha256=uonsSDRSKXqUl5meY2u_FMgUjgDCOkDRPDzCpAIIM6I,12966
+nv_ingest_client/nv_ingest_cli.py,sha256=GG7x_fe423NHQRDmpNcTtNI2P_g1xgg9SQ5JjbdBAIU,13592
 nv_ingest_client/cli/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest_client/cli/util/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest_client/cli/util/click.py,sha256=YjQU1uF148FU5D3ozC2m1kkfOOJxO1U8U552-T8PjU4,20029
@@ -15,7 +15,7 @@ nv_ingest_client/primitives/jobs/job_spec.py,sha256=NYT8K31b6p2v0zbIYugcARqJ8DTH
 nv_ingest_client/primitives/jobs/job_state.py,sha256=CEe_oZr4p_MobauWIyhuNrP8y7AUwxhIGBuO7dN-VOQ,5277
 nv_ingest_client/primitives/tasks/__init__.py,sha256=D8X4XuwCxk4g_sMSpNRL1XsjVE1eACYaUdEjSanSEfU,1130
 nv_ingest_client/primitives/tasks/audio_extraction.py,sha256=KD5VvaRm6PYelfofZq_-83CbOmupgosokZzFERI5wDA,3559
-nv_ingest_client/primitives/tasks/caption.py,sha256=J8sMIYujPb-ysWj1w3TXPSBLCnhHns_z4tZjzhDOQIs,2130
+nv_ingest_client/primitives/tasks/caption.py,sha256=I1nOpfGb1Ts7QsElwfayhw-F_UcYqtesS-HaZzeh4rI,2130
 nv_ingest_client/primitives/tasks/chart_extraction.py,sha256=s5hsljgSXxQMZHGekpAg6OYJ9k3-DHk5NmFpvtKJ6Zs,1493
 nv_ingest_client/primitives/tasks/dedup.py,sha256=qort6p3t6ZJuK_74sfOOLp3vMT3hkB5DAu3467WenyY,1719
 nv_ingest_client/primitives/tasks/embed.py,sha256=I6Irmvm1Qj9oqzDGSgfykCtfz8pz9LNxiXO-t29nXv8,5916
@@ -30,7 +30,7 @@ nv_ingest_client/primitives/tasks/task_factory.py,sha256=x8FXrhlgRYTxM0rLvsUvM8w
 nv_ingest_client/primitives/tasks/udf.py,sha256=5e_WJVgocnK-z0EGCEwPO_zG8WJEhuIsOUTjPmr8REY,12833
 nv_ingest_client/primitives/tasks/vdb_upload.py,sha256=mXOyQJfQfaoN96nntzevd0sKUs60-AHi8lc1jxG3DAw,1765
 nv_ingest_client/util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-nv_ingest_client/util/dataset.py,sha256=b6if_hM15iUJC4rvSHS0cmGBsSuZ3W-NoKDMTulx4b8,3316
+nv_ingest_client/util/dataset.py,sha256=2yDPs47HNj8AOdOAfJL4XVji0BMRJq_NH8CG4s4xT-Q,3701
 nv_ingest_client/util/milvus.py,sha256=MwBix_UBg54i7xONBIwjcqeKSBkqunxBJBK2f0bPMoo,61
 nv_ingest_client/util/process_json_files.py,sha256=YKR-fGT4kM8zO2p8r5tpo5-vvFywkcLuNieozvPWvo0,3785
 nv_ingest_client/util/processing.py,sha256=bAy8it-OUgGFO3pcy6D3ezpyZ6p2DfmoQUGhx3QmVf8,8989
@@ -42,11 +42,11 @@ nv_ingest_client/util/file_processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCe
 nv_ingest_client/util/file_processing/extract.py,sha256=uXEATBYZXjxdymGTNQvvzDD2eHgpuq4PdU6HsMl0Lp0,4662
 nv_ingest_client/util/vdb/__init__.py,sha256=ZmoEzeM9LzwwrVvu_DVUnjRNx-x8ahkNeIrSfSKzbAk,513
 nv_ingest_client/util/vdb/adt_vdb.py,sha256=UubzAMSfyrqqpD-OQErpBs25hC2Mw8zGZ4waenGXPOk,515
-nv_ingest_client/util/vdb/milvus.py,sha256=PC3qXjrdTab2xVS3FZkhj_28T5R9DNaHZ8a7D721Pik,77269
+nv_ingest_client/util/vdb/milvus.py,sha256=5yjn9uZ0fB10RrJml0WdImsfvfcowDtwrPrl_oYnnF0,77436
 nv_ingest_client/util/vdb/opensearch.py,sha256=I4FzF95VWCOkyzhfm-szdfK1Zd9ugUc8AxxpAdEMWGE,7538
-nv_ingest_client-2025.8.19.dev20250819.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-nv_ingest_client-2025.8.19.dev20250819.dist-info/METADATA,sha256=QYd4COuKD4YRil1snCzCeQpUBEuSzpnb5wGVGW__VMk,30737
-nv_ingest_client-2025.8.19.dev20250819.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nv_ingest_client-2025.8.19.dev20250819.dist-info/entry_points.txt,sha256=3uQVZkTZIjO08_bjTV-g0CDF5H1nrP1zWXU9gJOweuI,137
-nv_ingest_client-2025.8.19.dev20250819.dist-info/top_level.txt,sha256=1eMhBFD3SiWmpXnod2LM66C1HrSLSk96ninZi5XX-cE,17
-nv_ingest_client-2025.8.19.dev20250819.dist-info/RECORD,,
+nv_ingest_client-2025.8.21.dev20250821.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+nv_ingest_client-2025.8.21.dev20250821.dist-info/METADATA,sha256=NHS_9MLL0jBbpb1bvSD2b0jF2jrmicXTbJep_gd2v2U,30737
+nv_ingest_client-2025.8.21.dev20250821.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nv_ingest_client-2025.8.21.dev20250821.dist-info/entry_points.txt,sha256=3uQVZkTZIjO08_bjTV-g0CDF5H1nrP1zWXU9gJOweuI,137
+nv_ingest_client-2025.8.21.dev20250821.dist-info/top_level.txt,sha256=1eMhBFD3SiWmpXnod2LM66C1HrSLSk96ninZi5XX-cE,17
+nv_ingest_client-2025.8.21.dev20250821.dist-info/RECORD,,