nv-ingest-client 2025.8.18.dev20250818__py3-none-any.whl → 2025.8.20.dev20250820__py3-none-any.whl

nv_ingest_client/util/dataset.py

@@ -94,8 +94,14 @@ def get_dataset_files(dataset_bytes: BytesIO, shuffle: bool = False) -> list:
         dataset_bytes.seek(0)
         dataset = json.load(dataset_bytes)
         sampled_files = dataset.get("sampled_files", [])
-        if shuffle:
-            random.shuffle(sampled_files)
+        if shuffle and len(sampled_files) > 1:
+            original = list(sampled_files)
+            # Create a shuffled copy without mutating the original list
+            shuffled = random.sample(sampled_files, k=len(sampled_files))
+            # Guard against seeded RNG or accidental identity by forcing a different order
+            if shuffled == original:
+                shuffled = shuffled[1:] + shuffled[:1]
+            return shuffled
         return sampled_files
     except json.JSONDecodeError as err:
         raise ValueError(f"{err}")

nv_ingest_client/util/vdb/milvus.py

@@ -84,11 +84,10 @@ def create_nvingest_meta_schema():
 
 def create_meta_collection(
     schema: CollectionSchema,
-    milvus_uri: str = "http://localhost:19530",
     collection_name: str = "meta",
     recreate=False,
+    client: MilvusClient = None,
 ):
-    client = MilvusClient(milvus_uri)
     if client.has_collection(collection_name) and not recreate:
         # already exists, dont erase and recreate
         return
@@ -106,7 +105,6 @@ def create_meta_collection(
 def write_meta_collection(
     collection_name: str,
     fields: List[str],
-    milvus_uri: str = "http://localhost:19530",
     creation_timestamp: str = None,
     dense_index: str = None,
     dense_dim: int = None,
@@ -114,6 +112,7 @@ def write_meta_collection(
     embedding_model: str = None,
     sparse_model: str = None,
     meta_collection_name: str = "meta",
+    client: MilvusClient = None,
 ):
     client_config = ClientConfigSchema()
     data = {
@@ -132,14 +131,12 @@ def write_meta_collection(
         },
         "user_fields": [field.name for field in fields],
     }
-    client = MilvusClient(milvus_uri)
     client.insert(collection_name=meta_collection_name, data=data)
 
 
 def log_new_meta_collection(
     collection_name: str,
     fields: List[str],
-    milvus_uri: str = "http://localhost:19530",
     creation_timestamp: str = None,
     dense_index: str = None,
     dense_dim: int = None,
@@ -148,13 +145,13 @@ def log_new_meta_collection(
     sparse_model: str = None,
     meta_collection_name: str = "meta",
     recreate: bool = False,
+    client: MilvusClient = None,
 ):
     schema = create_nvingest_meta_schema()
-    create_meta_collection(schema, milvus_uri, recreate=recreate)
+    create_meta_collection(schema, client=client, recreate=recreate)
     write_meta_collection(
         collection_name,
         fields=fields,
-        milvus_uri=milvus_uri,
         creation_timestamp=creation_timestamp,
         dense_index=dense_index,
         dense_dim=dense_dim,
@@ -162,6 +159,7 @@ def log_new_meta_collection(
         embedding_model=embedding_model,
         sparse_model=sparse_model,
         meta_collection_name=meta_collection_name,
+        client=client,
     )
 
 
@@ -171,12 +169,16 @@ def grab_meta_collection_info(
     timestamp: str = None,
     embedding_model: str = None,
     embedding_dim: int = None,
-    milvus_uri: str = "http://localhost:19530",
+    client: MilvusClient = None,
+    milvus_uri: str = None,
+    username: str = None,
+    password: str = None,
 ):
     timestamp = timestamp or ""
     embedding_model = embedding_model or ""
     embedding_dim = embedding_dim or ""
-    client = MilvusClient(milvus_uri)
+    if milvus_uri:
+        client = MilvusClient(milvus_uri, token=f"{username}:{password}")
     results = client.query_iterator(
         collection_name=meta_collection_name,
         output_fields=[
@@ -404,6 +406,8 @@ def create_nvingest_collection(
     gpu_search: bool = False,
     dense_dim: int = 2048,
     recreate_meta: bool = False,
+    username: str = None,
+    password: str = None,
 ) -> CollectionSchema:
     """
     Creates a milvus collection with an nv-ingest compatible schema under
@@ -413,9 +417,7 @@ def create_nvingest_collection(
     ----------
     collection_name : str
         Name of the collection to be created.
-    milvus_uri : str,
-        Milvus address with http(s) preffix and port. Can also be a file path, to activate
-        milvus-lite.
+
     sparse : bool, optional
         When set to true, this adds a Sparse index to the IndexParams, usually activated for
         hybrid search.
@@ -426,6 +428,11 @@ def create_nvingest_collection(
         If true, creates a GPU_CAGRA index for dense embeddings.
     dense_dim : int, optional
         Sets the dimension size for the dense embedding in the milvus schema.
+    username : str, optional
+        Milvus username.
+    password : str, optional
+        Milvus password.
+
 
     Returns
     -------
@@ -435,7 +442,7 @@ def create_nvingest_collection(
     """
     local_index = False
     if urlparse(milvus_uri).scheme:
-        connections.connect(uri=milvus_uri)
+        connections.connect(uri=milvus_uri, token=f"{username}:{password}")
         server_version = utility.get_server_version()
         if "lite" in server_version:
             gpu_index = False
@@ -444,7 +451,7 @@ def create_nvingest_collection(
         if milvus_uri.endswith(".db"):
             local_index = True
 
-    client = MilvusClient(milvus_uri)
+    client = MilvusClient(milvus_uri, token=f"{username}:{password}")
     schema = create_nvingest_schema(dense_dim=dense_dim, sparse=sparse, local_index=local_index)
     index_params = create_nvingest_index_params(
         sparse=sparse,
@@ -457,11 +464,11 @@ def create_nvingest_collection(
     log_new_meta_collection(
         collection_name,
         fields=schema.fields,
-        milvus_uri=milvus_uri,
         dense_index=str(d_idx),
         dense_dim=dense_dim,
         sparse_index=str(s_idx),
         recreate=recreate_meta,
+        client=client,
     )
     return schema
 
@@ -744,6 +751,8 @@ def bulk_insert_milvus(
     access_key: str = "minioadmin",
     secret_key: str = "minioadmin",
     bucket_name: str = "nv-ingest",
+    username: str = None,
+    password: str = None,
 ):
     """
     This function initialize the bulk ingest of all minio uploaded records, and checks for
@@ -760,10 +769,14 @@ def bulk_insert_milvus(
     milvus_uri : str,
         Milvus address with http(s) preffix and port. Can also be a file path, to activate
         milvus-lite.
+    username : str, optional
+        Milvus username.
+    password : str, optional
+        Milvus password.
     """
     minio_client = Minio(minio_endpoint, access_key=access_key, secret_key=secret_key, secure=False)
 
-    connections.connect(uri=milvus_uri)
+    connections.connect(uri=milvus_uri, token=f"{username}:{password}")
     t_bulk_start = time.time()
     task_ids = []
     uploaded_files = []
@@ -913,6 +926,8 @@ def write_to_nvingest_collection(
     meta_source_field=None,
     meta_fields=None,
     stream: bool = False,
+    username: str = None,
+    password: str = None,
     **kwargs,
 ):
     """
@@ -953,9 +968,13 @@ def write_to_nvingest_collection(
         Minio bucket name.
     stream : bool, optional
         When true, the records will be inserted into milvus using the stream insert method.
+    username : str, optional
+        Milvus username.
+    password : str, optional
+        Milvus password.
     """
     local_index = False
-    connections.connect(uri=milvus_uri)
+    connections.connect(uri=milvus_uri, token=f"{username}:{password}")
     if urlparse(milvus_uri).scheme:
         server_version = utility.get_server_version()
         if "lite" in server_version:
@@ -978,7 +997,7 @@ def write_to_nvingest_collection(
     elif local_index and sparse:
         bm25_ef = BM25EmbeddingFunction(build_default_analyzer(language="en"))
         bm25_ef.load(bm25_save_path)
-    client = MilvusClient(milvus_uri)
+    client = MilvusClient(milvus_uri, token=f"{username}:{password}")
     schema = Collection(collection_name).schema
     if isinstance(meta_dataframe, str):
         meta_dataframe = pandas_file_reader(meta_dataframe)
@@ -1038,6 +1057,8 @@ def write_to_nvingest_collection(
             access_key,
             secret_key,
             bucket_name,
+            username=username,
+            password=password,
         )
         # fixes bulk insert lag time https://github.com/milvus-io/milvus/issues/21746
         client.refresh_load(collection_name)
@@ -1222,6 +1243,9 @@ def nvingest_retrieval(
     nv_ranker_max_batch_size: int = 64,
     _filter: str = "",
     ef_param: int = 200,
+    client: MilvusClient = None,
+    username: str = None,
+    password: str = None,
     **kwargs,
 ):
     """
@@ -1268,6 +1292,12 @@ def nvingest_retrieval(
         Max size for the number of candidates to rerank.
     nv_ranker_top_k : int,
         The number of candidates to return after reranking.
+    client : MilvusClient, optional
+        Milvus client instance.
+    username : str, optional
+        Milvus username.
+    password : str, optional
+        Milvus password.
     Returns
     -------
     List
@@ -1289,7 +1319,7 @@ def nvingest_retrieval(
     model_name = model_name if model_name else client_config.embedding_nim_model_name
     local_index = False
     embed_model = NVIDIAEmbedding(base_url=embedding_endpoint, model=model_name, nvidia_api_key=nvidia_api_key)
-    client = MilvusClient(milvus_uri)
+    client = client or MilvusClient(milvus_uri, token=f"{username}:{password}")
     final_top_k = top_k
     if nv_ranker:
         top_k = nv_ranker_top_k
@@ -1345,7 +1375,14 @@ def nvingest_retrieval(
     return results
 
 
-def remove_records(source_name: str, collection_name: str, milvus_uri: str = "http://localhost:19530"):
+def remove_records(
+    source_name: str,
+    collection_name: str,
+    milvus_uri: str = "http://localhost:19530",
+    username: str = None,
+    password: str = None,
+    client: MilvusClient = None,
+):
     """
     This function allows a user to remove chunks associated with an ingested file.
     Supply the full path of the file you would like to remove and this function will
@@ -1360,6 +1397,12 @@ def remove_records(source_name: str, collection_name: str, milvus_uri: str = "ht
     milvus_uri : str,
         Milvus address with http(s) preffix and port. Can also be a file path, to activate
         milvus-lite.
+    client : MilvusClient, optional
+        Milvus client instance.
+    username : str, optional
+        Milvus username.
+    password : str, optional
+        Milvus password.
 
     Returns
     -------
@@ -1367,7 +1410,7 @@ def remove_records(source_name: str, collection_name: str, milvus_uri: str = "ht
         Dictionary with one key, `delete_cnt`. The value represents the number of entities
         removed.
     """
-    client = MilvusClient(milvus_uri)
+    client = client or MilvusClient(milvus_uri, token=f"{username}:{password}")
     result_ids = client.delete(
         collection_name=collection_name,
         filter=f'(source["source_name"] == "{source_name}")',
@@ -1474,6 +1517,9 @@ def pull_all_milvus(
     write_dir: str = None,
     batch_size: int = 1000,
     include_embeddings: bool = False,
+    username: str = None,
+    password: str = None,
+    client: MilvusClient = None,
 ):
     """
     This function takes the input collection name and pulls all the records
@@ -1492,12 +1538,18 @@ def pull_all_milvus(
         The number of records to pull in each batch. Defaults to 1000.
     include_embeddings : bool, optional
         Whether to include the embeddings in the output. Defaults to False.
+    username : str, optional
+        Milvus username.
+    password : str, optional
+        Milvus password.
+    client : MilvusClient, optional
+        Milvus client instance.
     Returns
     -------
     List
         List of records/files with records from the collection.
     """
-    client = MilvusClient(milvus_uri)
+    client = client or MilvusClient(milvus_uri, token=f"{username}:{password}")
     output_fields = ["source", "content_metadata", "text"]
     if include_embeddings:
         output_fields.append("vector")
@@ -1572,6 +1624,9 @@ def embed_index_collection(
     meta_fields: list[str] = None,
     intput_type: str = "passage",
     truncate: str = "END",
+    client: MilvusClient = None,
+    username: str = None,
+    password: str = None,
     **kwargs,
 ):
     """
@@ -1609,6 +1664,12 @@ def embed_index_collection(
         meta_source_field (str, optional): The field in the metadata that serves as the source identifier.
             Defaults to None.
         meta_fields (list[str], optional): A list of metadata fields to include. Defaults to None.
+        client : MilvusClient, optional
+            Milvus client instance.
+        username : str, optional
+            Milvus username.
+        password : str, optional
+            Milvus password.
         **kwargs: Additional keyword arguments for customization.
     """
     client_config = ClientConfigSchema()
@@ -1642,6 +1703,8 @@ def embed_index_collection(
         meta_dataframe=meta_dataframe,
         meta_source_field=meta_source_field,
         meta_fields=meta_fields,
+        username=username,
+        password=password,
         **kwargs,
     )
     # running in parts
@@ -1866,6 +1929,8 @@ class Milvus(VDB):
         meta_fields: list[str] = None,
         stream: bool = False,
         threshold: int = 1000,
+        username: str = None,
+        password: str = None,
         **kwargs,
     ):
         """
@@ -1895,9 +1960,11 @@ class Milvus(VDB):
             meta_source_field (str, optional): The field in the metadata that serves as the source identifier.
                 Defaults to None.
             meta_fields (list[str], optional): A list of metadata fields to include. Defaults to None.
-            **kwargs: Additional keyword arguments for customization.
             stream (bool, optional): When true, the records will be inserted into milvus using the stream
                 insert method.
+            username (str, optional): The username for Milvus authentication. Defaults to None.
+            password (str, optional): The password for Milvus authentication. Defaults to None.
+            **kwargs: Additional keyword arguments for customization.
         """
         kwargs = locals().copy()
         kwargs.pop("self", None)
@@ -1927,6 +1994,8 @@ class Milvus(VDB):
             "gpu_index": self.__dict__.get("gpu_index", True),
             "gpu_search": self.__dict__.get("gpu_search", True),
             "dense_dim": self.__dict__.get("dense_dim", 2048),
+            "username": self.__dict__.get("username", None),
+            "password": self.__dict__.get("password", None),
         }
         return (self.collection_name, conn_dict)
 

{nv_ingest_client-2025.8.18.dev20250818.dist-info → nv_ingest_client-2025.8.20.dev20250820.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest-client
-Version: 2025.8.18.dev20250818
+Version: 2025.8.20.dev20250820
 Summary: Python client for the nv-ingest service
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License

{nv_ingest_client-2025.8.18.dev20250818.dist-info → nv_ingest_client-2025.8.20.dev20250820.dist-info}/RECORD

@@ -30,7 +30,7 @@ nv_ingest_client/primitives/tasks/task_factory.py,sha256=x8FXrhlgRYTxM0rLvsUvM8w
 nv_ingest_client/primitives/tasks/udf.py,sha256=5e_WJVgocnK-z0EGCEwPO_zG8WJEhuIsOUTjPmr8REY,12833
 nv_ingest_client/primitives/tasks/vdb_upload.py,sha256=mXOyQJfQfaoN96nntzevd0sKUs60-AHi8lc1jxG3DAw,1765
 nv_ingest_client/util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-nv_ingest_client/util/dataset.py,sha256=b6if_hM15iUJC4rvSHS0cmGBsSuZ3W-NoKDMTulx4b8,3316
+nv_ingest_client/util/dataset.py,sha256=2yDPs47HNj8AOdOAfJL4XVji0BMRJq_NH8CG4s4xT-Q,3701
 nv_ingest_client/util/milvus.py,sha256=MwBix_UBg54i7xONBIwjcqeKSBkqunxBJBK2f0bPMoo,61
 nv_ingest_client/util/process_json_files.py,sha256=YKR-fGT4kM8zO2p8r5tpo5-vvFywkcLuNieozvPWvo0,3785
 nv_ingest_client/util/processing.py,sha256=bAy8it-OUgGFO3pcy6D3ezpyZ6p2DfmoQUGhx3QmVf8,8989
@@ -42,11 +42,11 @@ nv_ingest_client/util/file_processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCe
 nv_ingest_client/util/file_processing/extract.py,sha256=uXEATBYZXjxdymGTNQvvzDD2eHgpuq4PdU6HsMl0Lp0,4662
 nv_ingest_client/util/vdb/__init__.py,sha256=ZmoEzeM9LzwwrVvu_DVUnjRNx-x8ahkNeIrSfSKzbAk,513
 nv_ingest_client/util/vdb/adt_vdb.py,sha256=UubzAMSfyrqqpD-OQErpBs25hC2Mw8zGZ4waenGXPOk,515
-nv_ingest_client/util/vdb/milvus.py,sha256=SIUiW285lDFUXwJjes_58Y3c4pK51SHFqbn0QEqOmm4,75243
+nv_ingest_client/util/vdb/milvus.py,sha256=5yjn9uZ0fB10RrJml0WdImsfvfcowDtwrPrl_oYnnF0,77436
 nv_ingest_client/util/vdb/opensearch.py,sha256=I4FzF95VWCOkyzhfm-szdfK1Zd9ugUc8AxxpAdEMWGE,7538
-nv_ingest_client-2025.8.18.dev20250818.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-nv_ingest_client-2025.8.18.dev20250818.dist-info/METADATA,sha256=7AoAmMB2B45WV5L1-nHJPTjMQDaO3fExvKWv-5xp6gg,30737
-nv_ingest_client-2025.8.18.dev20250818.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nv_ingest_client-2025.8.18.dev20250818.dist-info/entry_points.txt,sha256=3uQVZkTZIjO08_bjTV-g0CDF5H1nrP1zWXU9gJOweuI,137
-nv_ingest_client-2025.8.18.dev20250818.dist-info/top_level.txt,sha256=1eMhBFD3SiWmpXnod2LM66C1HrSLSk96ninZi5XX-cE,17
-nv_ingest_client-2025.8.18.dev20250818.dist-info/RECORD,,
+nv_ingest_client-2025.8.20.dev20250820.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+nv_ingest_client-2025.8.20.dev20250820.dist-info/METADATA,sha256=54Czy3ATSEasGQ0SwUpgxKw1wErWaryTKkHw9-LYpcE,30737
+nv_ingest_client-2025.8.20.dev20250820.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nv_ingest_client-2025.8.20.dev20250820.dist-info/entry_points.txt,sha256=3uQVZkTZIjO08_bjTV-g0CDF5H1nrP1zWXU9gJOweuI,137
+nv_ingest_client-2025.8.20.dev20250820.dist-info/top_level.txt,sha256=1eMhBFD3SiWmpXnod2LM66C1HrSLSk96ninZi5XX-cE,17
+nv_ingest_client-2025.8.20.dev20250820.dist-info/RECORD,,