nv-ingest-client 2025.8.11.dev20250811__tar.gz → 2025.8.13.dev20250813__tar.gz

{nv_ingest_client-2025.8.11.dev20250811/src/nv_ingest_client.egg-info → nv_ingest_client-2025.8.13.dev20250813}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest-client
-Version: 2025.8.11.dev20250811
+Version: 2025.8.13.dev20250813
 Summary: Python client for the nv-ingest service
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License

{nv_ingest_client-2025.8.11.dev20250811 → nv_ingest_client-2025.8.13.dev20250813}/src/nv_ingest_client/util/vdb/milvus.py

@@ -1,23 +1,28 @@
+import ast
+import copy
 import datetime
+import json
 import logging
+import os
 import time
+from functools import partial
+from pathlib import Path
 from typing import Dict
 from typing import List
 from typing import Tuple
 from typing import Union
 from urllib.parse import urlparse
-from pathlib import Path
-import pandas as pd
-from functools import partial
-import json
-import os
-import numpy as np
-import ast
-import copy
 
+import numpy as np
+import pandas as pd
 import requests
+from minio import Minio
+from minio.commonconfig import CopySource
+from minio.deleteobjects import DeleteObject
 from nv_ingest_client.util.process_json_files import ingest_json_results_to_blob
+from nv_ingest_client.util.transport import infer_microservice
 from nv_ingest_client.util.util import ClientConfigSchema
+from nv_ingest_client.util.vdb.adt_vdb import VDB
 from pymilvus import AnnSearchRequest
 from pymilvus import BulkInsertState
 from pymilvus import Collection
@@ -36,13 +41,11 @@ from pymilvus.model.sparse import BM25EmbeddingFunction
 from pymilvus.model.sparse.bm25.tokenizers import build_default_analyzer
 from pymilvus.orm.types import CONSISTENCY_BOUNDED
 from scipy.sparse import csr_array
-from nv_ingest_client.util.transport import infer_microservice
-from nv_ingest_client.util.vdb.adt_vdb import VDB
-
 
 logger = logging.getLogger(__name__)
 
 CONSISTENCY = CONSISTENCY_BOUNDED
+MINIO_DEFAULT_BUCKET_NAME = "a-bucket"
 
 pandas_reader_map = {
     ".json": pd.read_json,
@@ -737,6 +740,10 @@ def bulk_insert_milvus(
     collection_name: str,
     writer: RemoteBulkWriter,
     milvus_uri: str = "http://localhost:19530",
+    minio_endpoint: str = "localhost:9000",
+    access_key: str = "minioadmin",
+    secret_key: str = "minioadmin",
+    bucket_name: str = "nv-ingest",
 ):
     """
     This function initialize the bulk ingest of all minio uploaded records, and checks for
@@ -754,11 +761,22 @@ def bulk_insert_milvus(
         Milvus address with http(s) preffix and port. Can also be a file path, to activate
         milvus-lite.
     """
+    minio_client = Minio(minio_endpoint, access_key=access_key, secret_key=secret_key, secure=False)
 
     connections.connect(uri=milvus_uri)
     t_bulk_start = time.time()
     task_ids = []
+    uploaded_files = []
     for files in writer.batch_files:
+        for f in files:
+            # Hack: do_bulk_insert only reads from the default bucket ('a-bucket'),
+            # so we first copy objects from the source bucket into 'a-bucket' before inserting.
+            try:
+                minio_client.copy_object(MINIO_DEFAULT_BUCKET_NAME, f, CopySource(bucket_name, f))
+                uploaded_files.append(f)
+            except Exception as e:
+                logger.error(f"Error copying {f} from {bucket_name} to {MINIO_DEFAULT_BUCKET_NAME}: {e}")
+
         task_id = utility.do_bulk_insert(
             collection_name=collection_name,
             files=files,
@@ -780,6 +798,10 @@ def bulk_insert_milvus(
                 logger.error(f"Task: {task_id}")
                 logger.error(f"Failed reason: {task.failed_reason}")
                 task_ids.remove(task_id)
+
+    # Cleanup: remove the copied files to undo the temporary workaround before bulk insert.
+    minio_client.remove_objects(MINIO_DEFAULT_BUCKET_NAME, [DeleteObject(f) for f in uploaded_files])
+
     t_bulk_end = time.time()
     logger.info(f"Bulk {collection_name} upload took {t_bulk_end - t_bulk_start} s")
 
@@ -986,6 +1008,10 @@ def write_to_nvingest_collection(
             collection_name,
         )
     else:
+        minio_client = Minio(minio_endpoint, access_key=access_key, secret_key=secret_key, secure=False)
+        if not minio_client.bucket_exists(bucket_name):
+            minio_client.make_bucket(bucket_name)
+
         # Connections parameters to access the remote bucket
         conn = RemoteBulkWriter.S3ConnectParam(
             endpoint=minio_endpoint,  # the default MinIO service started along with Milvus
@@ -1004,7 +1030,15 @@ def write_to_nvingest_collection(
             cleaned_records,
             text_writer,
         )
-        bulk_insert_milvus(collection_name, writer, milvus_uri)
+        bulk_insert_milvus(
+            collection_name,
+            writer,
+            milvus_uri,
+            minio_endpoint,
+            access_key,
+            secret_key,
+            bucket_name,
+        )
         # fixes bulk insert lag time https://github.com/milvus-io/milvus/issues/21746
         client.refresh_load(collection_name)
 

{nv_ingest_client-2025.8.11.dev20250811 → nv_ingest_client-2025.8.13.dev20250813/src/nv_ingest_client.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest-client
-Version: 2025.8.11.dev20250811
+Version: 2025.8.13.dev20250813
 Summary: Python client for the nv-ingest service
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License