nv-ingest-client 2025.8.10.dev20250810__py3-none-any.whl → 2025.8.12.dev20250812__py3-none-any.whl

nv_ingest_client/util/transport.py

@@ -26,7 +26,7 @@ def infer_microservice(
         client = NimClient(
             model_interface=EmbeddingModelInterface(),
             protocol="grpc",
-            endpoints=(embedding_endpoint, embedding_endpoint),
+            endpoints=(embedding_endpoint, None),
             auth_token=nvidia_api_key,
         )
         return client.infer(
@@ -43,7 +43,7 @@ def infer_microservice(
         client = NimClient(
             model_interface=EmbeddingModelInterface(),
             protocol="http",
-            endpoints=(embedding_endpoint, embedding_endpoint),
+            endpoints=(None, embedding_endpoint),
             auth_token=nvidia_api_key,
         )
         return client.infer(data, model_name, input_type=input_type, truncate=truncate, batch_size=batch_size)

nv_ingest_client/util/vdb/milvus.py

@@ -1,23 +1,28 @@
+import ast
+import copy
 import datetime
+import json
 import logging
+import os
 import time
+from functools import partial
+from pathlib import Path
 from typing import Dict
 from typing import List
 from typing import Tuple
 from typing import Union
 from urllib.parse import urlparse
-from pathlib import Path
-import pandas as pd
-from functools import partial
-import json
-import os
-import numpy as np
-import ast
-import copy
 
+import numpy as np
+import pandas as pd
 import requests
+from minio import Minio
+from minio.commonconfig import CopySource
+from minio.deleteobjects import DeleteObject
 from nv_ingest_client.util.process_json_files import ingest_json_results_to_blob
+from nv_ingest_client.util.transport import infer_microservice
 from nv_ingest_client.util.util import ClientConfigSchema
+from nv_ingest_client.util.vdb.adt_vdb import VDB
 from pymilvus import AnnSearchRequest
 from pymilvus import BulkInsertState
 from pymilvus import Collection
@@ -36,13 +41,11 @@ from pymilvus.model.sparse import BM25EmbeddingFunction
 from pymilvus.model.sparse.bm25.tokenizers import build_default_analyzer
 from pymilvus.orm.types import CONSISTENCY_BOUNDED
 from scipy.sparse import csr_array
-from nv_ingest_client.util.transport import infer_microservice
-from nv_ingest_client.util.vdb.adt_vdb import VDB
-
 
 logger = logging.getLogger(__name__)
 
 CONSISTENCY = CONSISTENCY_BOUNDED
+MINIO_DEFAULT_BUCKET_NAME = "a-bucket"
 
 pandas_reader_map = {
     ".json": pd.read_json,
@@ -737,6 +740,10 @@ def bulk_insert_milvus(
     collection_name: str,
     writer: RemoteBulkWriter,
     milvus_uri: str = "http://localhost:19530",
+    minio_endpoint: str = "localhost:9000",
+    access_key: str = "minioadmin",
+    secret_key: str = "minioadmin",
+    bucket_name: str = "nv-ingest",
 ):
     """
     This function initialize the bulk ingest of all minio uploaded records, and checks for
@@ -754,28 +761,49 @@ def bulk_insert_milvus(
         Milvus address with http(s) preffix and port. Can also be a file path, to activate
         milvus-lite.
     """
+    minio_client = Minio(minio_endpoint, access_key=access_key, secret_key=secret_key, secure=False)
 
     connections.connect(uri=milvus_uri)
     t_bulk_start = time.time()
-    files_to_upload = [_file for file_set in writer.batch_files for _file in file_set]
-    task_id = utility.do_bulk_insert(
-        collection_name=collection_name,
-        files=files_to_upload,
-        consistency_level=CONSISTENCY,
-    )
+    task_ids = []
+    uploaded_files = []
+    for files in writer.batch_files:
+        for f in files:
+            # Hack: do_bulk_insert only reads from the default bucket ('a-bucket'),
+            # so we first copy objects from the source bucket into 'a-bucket' before inserting.
+            try:
+                minio_client.copy_object(MINIO_DEFAULT_BUCKET_NAME, f, CopySource(bucket_name, f))
+                uploaded_files.append(f)
+            except Exception as e:
+                logger.error(f"Error copying {f} from {bucket_name} to {MINIO_DEFAULT_BUCKET_NAME}: {e}")
+
+        task_id = utility.do_bulk_insert(
+            collection_name=collection_name,
+            files=files,
+            consistency_level=CONSISTENCY,
+        )
+        task_ids.append(task_id)
     # list_bulk_insert_tasks = utility.list_bulk_insert_tasks(collection_name=collection_name)
-    state = "Pending"
-    while state != "Completed":
-        task = utility.get_bulk_insert_state(task_id=task_id)
-        state = task.state_name
-        if state == "Completed":
-            t_bulk_end = time.time()
-            logger.info("Start time:", task.create_time_str)
-            logger.info("Imported row count:", task.row_count)
-            logger.info(f"Bulk {collection_name} upload took {t_bulk_end - t_bulk_start} s")
-        if task.state == BulkInsertState.ImportFailed:
-            logger.error("Failed reason:", task.failed_reason)
+    while len(task_ids) > 0:
         time.sleep(1)
+        for task_id in task_ids:
+            task = utility.get_bulk_insert_state(task_id=task_id)
+            state = task.state_name
+            if state == "Completed":
+                logger.info(f"Task: {task_id}")
+                logger.info(f"Start time: {task.create_time_str}")
+                logger.info(f"Imported row count: {task.row_count}")
+                task_ids.remove(task_id)
+            if task.state == BulkInsertState.ImportFailed:
+                logger.error(f"Task: {task_id}")
+                logger.error(f"Failed reason: {task.failed_reason}")
+                task_ids.remove(task_id)
+
+    # Cleanup: remove the copied files to undo the temporary workaround before bulk insert.
+    minio_client.remove_objects(MINIO_DEFAULT_BUCKET_NAME, [DeleteObject(f) for f in uploaded_files])
+
+    t_bulk_end = time.time()
+    logger.info(f"Bulk {collection_name} upload took {t_bulk_end - t_bulk_start} s")
 
 
 def create_bm25_model(
@@ -980,6 +1008,10 @@ def write_to_nvingest_collection(
             collection_name,
         )
     else:
+        minio_client = Minio(minio_endpoint, access_key=access_key, secret_key=secret_key, secure=False)
+        if not minio_client.bucket_exists(bucket_name):
+            minio_client.make_bucket(bucket_name)
+
         # Connections parameters to access the remote bucket
         conn = RemoteBulkWriter.S3ConnectParam(
             endpoint=minio_endpoint,  # the default MinIO service started along with Milvus
@@ -998,7 +1030,15 @@ def write_to_nvingest_collection(
             cleaned_records,
             text_writer,
         )
-        bulk_insert_milvus(collection_name, writer, milvus_uri)
+        bulk_insert_milvus(
+            collection_name,
+            writer,
+            milvus_uri,
+            minio_endpoint,
+            access_key,
+            secret_key,
+            bucket_name,
+        )
         # fixes bulk insert lag time https://github.com/milvus-io/milvus/issues/21746
         client.refresh_load(collection_name)
 

{nv_ingest_client-2025.8.10.dev20250810.dist-info → nv_ingest_client-2025.8.12.dev20250812.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest-client
-Version: 2025.8.10.dev20250810
+Version: 2025.8.12.dev20250812
 Summary: Python client for the nv-ingest service
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License

{nv_ingest_client-2025.8.10.dev20250810.dist-info → nv_ingest_client-2025.8.12.dev20250812.dist-info}/RECORD

@@ -37,18 +37,18 @@ nv_ingest_client/util/milvus.py,sha256=MwBix_UBg54i7xONBIwjcqeKSBkqunxBJBK2f0bPM
 nv_ingest_client/util/process_json_files.py,sha256=YKR-fGT4kM8zO2p8r5tpo5-vvFywkcLuNieozvPWvo0,3785
 nv_ingest_client/util/processing.py,sha256=bAy8it-OUgGFO3pcy6D3ezpyZ6p2DfmoQUGhx3QmVf8,8989
 nv_ingest_client/util/system.py,sha256=DVIRLlEWkpqftqxazCuPNdaFSjQiHGMYcHzBufJSRUM,2216
-nv_ingest_client/util/transport.py,sha256=Rzdj9GxYsJVbGuh95H2AoHTMsFj-oZC1TiN9pT5vRPA,1674
+nv_ingest_client/util/transport.py,sha256=G1wrwaJLXf8S2yTuq6ZwG1NNMf4cHfp1igLxGQ87apc,1646
 nv_ingest_client/util/util.py,sha256=6KQkE5vXmBUoxETpjLdtPYm1pNCYCnqwC-Df56ETGQ4,14748
 nv_ingest_client/util/zipkin.py,sha256=p2tMtTVAqrZGxmAxWKE42wkx7U5KywiX5munI7rJt_k,4473
 nv_ingest_client/util/file_processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nv_ingest_client/util/file_processing/extract.py,sha256=uXEATBYZXjxdymGTNQvvzDD2eHgpuq4PdU6HsMl0Lp0,4662
 nv_ingest_client/util/vdb/__init__.py,sha256=ZmoEzeM9LzwwrVvu_DVUnjRNx-x8ahkNeIrSfSKzbAk,513
 nv_ingest_client/util/vdb/adt_vdb.py,sha256=UubzAMSfyrqqpD-OQErpBs25hC2Mw8zGZ4waenGXPOk,515
-nv_ingest_client/util/vdb/milvus.py,sha256=uVgCTjg-Njz9Lq_sURNlZEky4KtdapxPr6a5Ug6vCmo,73511
+nv_ingest_client/util/vdb/milvus.py,sha256=qIX-evj5wIxODi7I6w2Hu9jOupEmLQLeom-eLn4Xon8,75212
 nv_ingest_client/util/vdb/opensearch.py,sha256=I4FzF95VWCOkyzhfm-szdfK1Zd9ugUc8AxxpAdEMWGE,7538
-nv_ingest_client-2025.8.10.dev20250810.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-nv_ingest_client-2025.8.10.dev20250810.dist-info/METADATA,sha256=jqMm_wuxEx1RnNUukpX93lZMHI3L0N5rnfSf6VYet40,30737
-nv_ingest_client-2025.8.10.dev20250810.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nv_ingest_client-2025.8.10.dev20250810.dist-info/entry_points.txt,sha256=3uQVZkTZIjO08_bjTV-g0CDF5H1nrP1zWXU9gJOweuI,137
-nv_ingest_client-2025.8.10.dev20250810.dist-info/top_level.txt,sha256=1eMhBFD3SiWmpXnod2LM66C1HrSLSk96ninZi5XX-cE,17
-nv_ingest_client-2025.8.10.dev20250810.dist-info/RECORD,,
+nv_ingest_client-2025.8.12.dev20250812.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+nv_ingest_client-2025.8.12.dev20250812.dist-info/METADATA,sha256=GYWKUN1-UQpWKxTCmSewl9frvvEvm3DZNtqg5wajKXA,30737
+nv_ingest_client-2025.8.12.dev20250812.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nv_ingest_client-2025.8.12.dev20250812.dist-info/entry_points.txt,sha256=3uQVZkTZIjO08_bjTV-g0CDF5H1nrP1zWXU9gJOweuI,137
+nv_ingest_client-2025.8.12.dev20250812.dist-info/top_level.txt,sha256=1eMhBFD3SiWmpXnod2LM66C1HrSLSk96ninZi5XX-cE,17
+nv_ingest_client-2025.8.12.dev20250812.dist-info/RECORD,,