nv-ingest-client 2025.8.11.dev20250811__py3-none-any.whl → 2025.8.13.dev20250813__py3-none-any.whl

nv_ingest_client/util/vdb/milvus.py

@@ -1,23 +1,28 @@
+import ast
+import copy
 import datetime
+import json
 import logging
+import os
 import time
+from functools import partial
+from pathlib import Path
 from typing import Dict
 from typing import List
 from typing import Tuple
 from typing import Union
 from urllib.parse import urlparse
-from pathlib import Path
-import pandas as pd
-from functools import partial
-import json
-import os
-import numpy as np
-import ast
-import copy
 
+import numpy as np
+import pandas as pd
 import requests
+from minio import Minio
+from minio.commonconfig import CopySource
+from minio.deleteobjects import DeleteObject
 from nv_ingest_client.util.process_json_files import ingest_json_results_to_blob
+from nv_ingest_client.util.transport import infer_microservice
 from nv_ingest_client.util.util import ClientConfigSchema
+from nv_ingest_client.util.vdb.adt_vdb import VDB
 from pymilvus import AnnSearchRequest
 from pymilvus import BulkInsertState
 from pymilvus import Collection
@@ -36,13 +41,11 @@ from pymilvus.model.sparse import BM25EmbeddingFunction
 from pymilvus.model.sparse.bm25.tokenizers import build_default_analyzer
 from pymilvus.orm.types import CONSISTENCY_BOUNDED
 from scipy.sparse import csr_array
-from nv_ingest_client.util.transport import infer_microservice
-from nv_ingest_client.util.vdb.adt_vdb import VDB
-
 
 logger = logging.getLogger(__name__)
 
 CONSISTENCY = CONSISTENCY_BOUNDED
+MINIO_DEFAULT_BUCKET_NAME = "a-bucket"
 
 pandas_reader_map = {
     ".json": pd.read_json,
@@ -737,6 +740,10 @@ def bulk_insert_milvus(
     collection_name: str,
     writer: RemoteBulkWriter,
     milvus_uri: str = "http://localhost:19530",
+    minio_endpoint: str = "localhost:9000",
+    access_key: str = "minioadmin",
+    secret_key: str = "minioadmin",
+    bucket_name: str = "nv-ingest",
 ):
     """
     This function initialize the bulk ingest of all minio uploaded records, and checks for
@@ -754,11 +761,22 @@ def bulk_insert_milvus(
         Milvus address with http(s) preffix and port. Can also be a file path, to activate
         milvus-lite.
     """
+    minio_client = Minio(minio_endpoint, access_key=access_key, secret_key=secret_key, secure=False)
 
     connections.connect(uri=milvus_uri)
     t_bulk_start = time.time()
     task_ids = []
+    uploaded_files = []
     for files in writer.batch_files:
+        for f in files:
+            # Hack: do_bulk_insert only reads from the default bucket ('a-bucket'),
+            # so we first copy objects from the source bucket into 'a-bucket' before inserting.
+            try:
+                minio_client.copy_object(MINIO_DEFAULT_BUCKET_NAME, f, CopySource(bucket_name, f))
+                uploaded_files.append(f)
+            except Exception as e:
+                logger.error(f"Error copying {f} from {bucket_name} to {MINIO_DEFAULT_BUCKET_NAME}: {e}")
+
         task_id = utility.do_bulk_insert(
             collection_name=collection_name,
             files=files,
@@ -780,6 +798,10 @@ def bulk_insert_milvus(
                 logger.error(f"Task: {task_id}")
                 logger.error(f"Failed reason: {task.failed_reason}")
                 task_ids.remove(task_id)
+
+    # Cleanup: remove the copied files to undo the temporary workaround before bulk insert.
+    minio_client.remove_objects(MINIO_DEFAULT_BUCKET_NAME, [DeleteObject(f) for f in uploaded_files])
+
     t_bulk_end = time.time()
     logger.info(f"Bulk {collection_name} upload took {t_bulk_end - t_bulk_start} s")
 
@@ -986,6 +1008,10 @@ def write_to_nvingest_collection(
             collection_name,
         )
     else:
+        minio_client = Minio(minio_endpoint, access_key=access_key, secret_key=secret_key, secure=False)
+        if not minio_client.bucket_exists(bucket_name):
+            minio_client.make_bucket(bucket_name)
+
         # Connections parameters to access the remote bucket
         conn = RemoteBulkWriter.S3ConnectParam(
             endpoint=minio_endpoint,  # the default MinIO service started along with Milvus
@@ -1004,7 +1030,15 @@ def write_to_nvingest_collection(
             cleaned_records,
             text_writer,
         )
-        bulk_insert_milvus(collection_name, writer, milvus_uri)
+        bulk_insert_milvus(
+            collection_name,
+            writer,
+            milvus_uri,
+            minio_endpoint,
+            access_key,
+            secret_key,
+            bucket_name,
+        )
         # fixes bulk insert lag time https://github.com/milvus-io/milvus/issues/21746
         client.refresh_load(collection_name)
 

{nv_ingest_client-2025.8.11.dev20250811.dist-info → nv_ingest_client-2025.8.13.dev20250813.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest-client
-Version: 2025.8.11.dev20250811
+Version: 2025.8.13.dev20250813
 Summary: Python client for the nv-ingest service
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License

{nv_ingest_client-2025.8.11.dev20250811.dist-info → nv_ingest_client-2025.8.13.dev20250813.dist-info}/RECORD

@@ -44,11 +44,11 @@ nv_ingest_client/util/file_processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCe
 nv_ingest_client/util/file_processing/extract.py,sha256=uXEATBYZXjxdymGTNQvvzDD2eHgpuq4PdU6HsMl0Lp0,4662
 nv_ingest_client/util/vdb/__init__.py,sha256=ZmoEzeM9LzwwrVvu_DVUnjRNx-x8ahkNeIrSfSKzbAk,513
 nv_ingest_client/util/vdb/adt_vdb.py,sha256=UubzAMSfyrqqpD-OQErpBs25hC2Mw8zGZ4waenGXPOk,515
-nv_ingest_client/util/vdb/milvus.py,sha256=0WrJg0L8kFrbCJnAlDoRs8qrFX2GLnOrAkxhVB-GJcQ,73727
+nv_ingest_client/util/vdb/milvus.py,sha256=qIX-evj5wIxODi7I6w2Hu9jOupEmLQLeom-eLn4Xon8,75212
 nv_ingest_client/util/vdb/opensearch.py,sha256=I4FzF95VWCOkyzhfm-szdfK1Zd9ugUc8AxxpAdEMWGE,7538
-nv_ingest_client-2025.8.11.dev20250811.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-nv_ingest_client-2025.8.11.dev20250811.dist-info/METADATA,sha256=taoo_JPimeMWMcK-cFUe8pymGiML4Ob_TVqY70sHOWE,30737
-nv_ingest_client-2025.8.11.dev20250811.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nv_ingest_client-2025.8.11.dev20250811.dist-info/entry_points.txt,sha256=3uQVZkTZIjO08_bjTV-g0CDF5H1nrP1zWXU9gJOweuI,137
-nv_ingest_client-2025.8.11.dev20250811.dist-info/top_level.txt,sha256=1eMhBFD3SiWmpXnod2LM66C1HrSLSk96ninZi5XX-cE,17
-nv_ingest_client-2025.8.11.dev20250811.dist-info/RECORD,,
+nv_ingest_client-2025.8.13.dev20250813.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+nv_ingest_client-2025.8.13.dev20250813.dist-info/METADATA,sha256=XAxE0M-2XLRkOhng51l7ImFGj8An7ridxkpMKTMhCw0,30737
+nv_ingest_client-2025.8.13.dev20250813.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nv_ingest_client-2025.8.13.dev20250813.dist-info/entry_points.txt,sha256=3uQVZkTZIjO08_bjTV-g0CDF5H1nrP1zWXU9gJOweuI,137
+nv_ingest_client-2025.8.13.dev20250813.dist-info/top_level.txt,sha256=1eMhBFD3SiWmpXnod2LM66C1HrSLSk96ninZi5XX-cE,17
+nv_ingest_client-2025.8.13.dev20250813.dist-info/RECORD,,