nv-ingest-client 2025.9.9.dev20250909__tar.gz → 2025.9.11.dev20250911__tar.gz

{nv_ingest_client-2025.9.9.dev20250909/src/nv_ingest_client.egg-info → nv_ingest_client-2025.9.11.dev20250911}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest-client
-Version: 2025.9.9.dev20250909
+Version: 2025.9.11.dev20250911
 Summary: Python client for the nv-ingest service
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License

nv_ingest_client-2025.9.11.dev20250911/src/nv_ingest_client/util/document_analysis.py

@@ -0,0 +1,314 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Utility functions for analyzing document-level chunk composition from nv-ingest results.
+
+This module provides analysis capabilities for understanding the distribution and types
+of extracted content elements across individual documents. It enables customers to
+gain visibility into their document composition for performance optimization and
+capacity planning decisions.
+"""
+
+import logging
+import os
+from collections import defaultdict
+from typing import Any, Dict, List, Union
+
+logger = logging.getLogger(__name__)
+
+
+def analyze_document_chunks(
+    results: Union[List[List[Dict[str, Any]]], List[Dict[str, Any]]]
+) -> Dict[str, Dict[str, Dict[str, int]]]:
+    """
+    Analyze ingestor results to count elements by type and page for each document.
+
+    This function processes results from nv-ingest ingestion and provides a per-document,
+    per-page breakdown of extracted content types, enabling customers to understand document
+    composition and page-level distribution for optimization and planning purposes.
+
+    Parameters
+    ----------
+    results : Union[List[List[Dict[str, Any]]], List[Dict[str, Any]]]
+        Ingestor results from ingestor.ingest() in standard List[List[Dict]] format,
+        or flattened List[Dict] format. Handles both regular lists and
+        LazyLoadedList objects automatically.
+
+    Returns
+    -------
+    Dict[str, Dict[str, Dict[str, int]]]
+        Dictionary mapping document names to page-level element type counts with structure:
+        {
+            "document1.pdf": {
+                "total": {
+                    "text": 7, "charts": 1, "tables": 1,
+                    "unstructured_images": 0, "infographics": 0, "page_images": 0
+                },
+                "1": {
+                    "text": 3, "charts": 1, "tables": 0,
+                    "unstructured_images": 0, "infographics": 0, "page_images": 0
+                },
+                "2": {
+                    "text": 4, "charts": 0, "tables": 1,
+                    "unstructured_images": 0, "infographics": 0, "page_images": 0
+                }
+            },
+            "document2.pdf": {...}
+        }
+
+    Notes
+    -----
+    - Requires purge_results_after_upload=False in vdb_upload() configuration
+    - Automatically handles LazyLoadedList objects from nv-ingest client
+    - Returns zero counts for missing element types
+    - Assumes valid nv-ingest output format with guaranteed metadata structure
+
+    Examples
+    --------
+    >>> from nv_ingest_client.util.document_analysis import analyze_document_chunks
+    >>>
+    >>> # After running ingestion
+    >>> results, failures = ingestor.ingest(show_progress=True, return_failures=True)
+    >>>
+    >>> # Analyze document composition by page
+    >>> breakdown = analyze_document_chunks(results)
+    >>>
+    >>> for doc_name, pages in breakdown.items():
+    ...     total_counts = pages["total"]
+    ...     total_elements = sum(total_counts.values())
+    ...     page_count = len(pages) - 1  # Subtract 1 for "total" key
+    ...     print(f"{doc_name}: {total_elements} elements across {page_count} pages")
+    ...     print(f"  total: {total_elements} elements ({total_counts['text']} text, {total_counts['charts']} charts)")
+    ...     for page_name, counts in pages.items():
+    ...         if page_name != "total":  # Skip total when listing pages
+    ...             page_total = sum(counts.values())
+    ...             print(
+                f"  page {page_name}: {page_total} elements "
+                f"({counts['text']} text, {counts['charts']} charts)"
+            )
+    """
+
+    if not results:
+        logger.warning("No results provided for analysis")
+        return {}
+
+    # Normalize input format to handle both List[List[Dict]] and List[Dict] structures
+    normalized_results = _normalize_results_format(results)
+
+    # Group elements by document name and page number
+    document_page_elements = defaultdict(lambda: defaultdict(list))
+
+    for doc_results in normalized_results:
+        # Handle LazyLoadedList and other iterable types
+        elements = _extract_elements_from_doc(doc_results)
+
+        for element in elements:
+            doc_name = _extract_document_name(element)
+            page_key = _extract_page_key(element)
+            document_page_elements[doc_name][page_key].append(element)
+
+    # Count element types per page within each document and calculate totals
+    document_page_counts = {}
+
+    for doc_name, pages in document_page_elements.items():
+        document_page_counts[doc_name] = {}
+        total_counts = _initialize_element_counts()
+
+        for page_key, elements in pages.items():
+            counts = _initialize_element_counts()
+
+            for element in elements:
+                element_type = _categorize_element(element)
+                counts[element_type] += 1
+                total_counts[element_type] += 1  # Add to document total
+
+            document_page_counts[doc_name][page_key] = counts
+
+        # Add the total counts for this document
+        document_page_counts[doc_name]["total"] = total_counts
+
+    if document_page_counts:
+        total_docs = len(document_page_counts)
+        total_pages = sum(len(pages) - 1 for pages in document_page_counts.values())  # Subtract 1 for "total" key
+        total_elements = sum(sum(page_counts["total"].values()) for page_counts in document_page_counts.values())
+        logger.info(f"Analyzed {total_elements} elements across {total_pages} pages in {total_docs} documents")
+    else:
+        logger.warning("No valid documents found for analysis")
+
+    return document_page_counts
+
+
+def _normalize_results_format(results: Union[List[List[Dict]], List[Dict]]) -> List[List[Dict]]:
+    """
+    Normalize various input formats to consistent List[List[Dict]] structure.
+
+    Parameters
+    ----------
+    results : Union[List[List[Dict]], List[Dict]]
+        Input results in various formats
+
+    Returns
+    -------
+    List[List[Dict]]
+        Normalized results in standard format
+    """
+
+    if not results:
+        return []
+
+    # Handle List[List[Dict]] or List[LazyLoadedList] formats
+    if isinstance(results, list) and len(results) > 0:
+        first_elem = results[0]
+        # Check for list, LazyLoadedList, or any sequence-like object
+        if isinstance(first_elem, list) or (
+            hasattr(first_elem, "__iter__") and hasattr(first_elem, "__len__") and not isinstance(first_elem, dict)
+        ):
+            return results
+
+    # Handle flattened List[Dict] format by grouping elements by document
+    if isinstance(results, list) and len(results) > 0 and isinstance(results[0], dict):
+        doc_groups = defaultdict(list)
+        for element in results:
+            doc_name = _extract_document_name(element)
+            doc_groups[doc_name].append(element)
+
+        return list(doc_groups.values())
+
+    # Fallback for unexpected formats
+    return [[item] for item in results if item]
+
+
+def _extract_elements_from_doc(doc_results) -> List[Dict]:
+    """
+    Extract elements from document results, handling various data types.
+
+    Parameters
+    ----------
+    doc_results : Any
+        Document results which may be a list, LazyLoadedList, or other iterable
+
+    Returns
+    -------
+    List[Dict]
+        List of element dictionaries
+    """
+
+    if isinstance(doc_results, list):
+        return doc_results
+    elif hasattr(doc_results, "__iter__") and hasattr(doc_results, "__len__"):
+        # Handle LazyLoadedList and other sequence-like objects
+        return list(doc_results)
+    else:
+        # Single element case
+        return [doc_results] if doc_results else []
+
+
+def _extract_document_name(element: Dict[str, Any]) -> str:
+    """
+    Extract clean document name from element metadata.
+
+    Parameters
+    ----------
+    element : Dict[str, Any]
+        Element dictionary containing metadata
+
+    Returns
+    -------
+    str
+        Clean document filename (basename of source_id)
+    """
+
+    # nv-ingest guarantees this structure exists
+    source_id = element["metadata"]["source_metadata"]["source_id"]
+    return os.path.basename(source_id)
+
+
+def _extract_page_key(element: Dict[str, Any]) -> str:
+    """
+    Extract page key from element metadata for consistent page naming.
+
+    Parameters
+    ----------
+    element : Dict[str, Any]
+        Element dictionary containing metadata
+
+    Returns
+    -------
+    str
+        Page number as string (e.g., "1", "2", or "unknown")
+    """
+
+    try:
+        page_number = element["metadata"]["content_metadata"]["page_number"]
+        if page_number is not None and page_number >= 0:
+            return str(page_number)
+        else:
+            return "unknown"
+    except (KeyError, TypeError):
+        logger.warning("Missing or invalid page_number in element metadata")
+        return "unknown"
+
+
+def _categorize_element(element: Dict[str, Any]) -> str:
+    """
+    Categorize element by type using document_type and content metadata.
+
+    Parameters
+    ----------
+    element : Dict[str, Any]
+        Element dictionary with document_type and metadata fields
+
+    Returns
+    -------
+    str
+        Element category: "text", "charts", "tables", "unstructured_images",
+        "infographics", or "page_images"
+    """
+
+    doc_type = element["document_type"]
+
+    # Text elements
+    if doc_type == "text":
+        return "text"
+
+    # Structured elements with subtypes
+    elif doc_type == "structured":
+        subtype = element["metadata"]["content_metadata"]["subtype"]
+        if subtype == "chart":
+            return "charts"
+        elif subtype == "table":
+            return "tables"
+        elif subtype == "infographic":
+            return "infographics"
+        elif subtype == "page_image":
+            return "page_images"
+
+    # Image elements (unstructured)
+    elif doc_type == "image":
+        return "unstructured_images"
+
+    # Should not reach here with valid nv-ingest output
+    logger.warning(f"Unexpected element type: {doc_type}")
+    return "text"  # Default to text for safety
+
+
+def _initialize_element_counts() -> Dict[str, int]:
+    """
+    Initialize element counts dictionary with all supported types.
+
+    Returns
+    -------
+    Dict[str, int]
+        Dictionary with zero counts for all element types
+    """
+
+    return {
+        "text": 0,
+        "charts": 0,
+        "tables": 0,
+        "unstructured_images": 0,
+        "infographics": 0,
+        "page_images": 0,
+    }

{nv_ingest_client-2025.9.9.dev20250909 → nv_ingest_client-2025.9.11.dev20250911}/src/nv_ingest_client/util/util.py

@@ -35,6 +35,7 @@ class ClientConfigSchema:
             "https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-3_2-nv-rerankqa-1b-v2/reranking",
         )
         self.nv_ranker_nim_model_name: str = os.getenv("RERANKER_NIM_MODEL_NAME", "nvidia/llama-3.2-nv-rerankqa-1b-v2")
+        self.minio_bucket_name: str = os.getenv("MINIO_BUCKET", "nv-ingest")
 
 
 @unified_exception_handler

{nv_ingest_client-2025.9.9.dev20250909 → nv_ingest_client-2025.9.11.dev20250911}/src/nv_ingest_client/util/vdb/milvus.py

@@ -17,8 +17,6 @@ import numpy as np
 import pandas as pd
 import requests
 from minio import Minio
-from minio.commonconfig import CopySource
-from minio.deleteobjects import DeleteObject
 from nv_ingest_client.util.process_json_files import ingest_json_results_to_blob
 from nv_ingest_client.util.transport import infer_microservice
 from nv_ingest_client.util.util import ClientConfigSchema
@@ -46,7 +44,6 @@ from scipy.sparse import csr_array
 logger = logging.getLogger(__name__)
 
 CONSISTENCY = CONSISTENCY_BOUNDED
-MINIO_DEFAULT_BUCKET_NAME = "a-bucket"
 
 pandas_reader_map = {
     ".json": pd.read_json,
@@ -751,7 +748,7 @@ def bulk_insert_milvus(
     minio_endpoint: str = "localhost:9000",
     access_key: str = "minioadmin",
     secret_key: str = "minioadmin",
-    bucket_name: str = "nv-ingest",
+    bucket_name: str = None,
     username: str = None,
     password: str = None,
 ):
@@ -775,29 +772,16 @@ def bulk_insert_milvus(
     password : str, optional
         Milvus password.
     """
-    minio_client = Minio(minio_endpoint, access_key=access_key, secret_key=secret_key, secure=False)
-
     connections.connect(uri=milvus_uri, token=f"{username}:{password}")
     t_bulk_start = time.time()
     task_ids = []
-    uploaded_files = []
-    for files in writer.batch_files:
-        for f in files:
-            # Hack: do_bulk_insert only reads from the default bucket ('a-bucket'),
-            # so we first copy objects from the source bucket into 'a-bucket' before inserting.
-            try:
-                minio_client.copy_object(MINIO_DEFAULT_BUCKET_NAME, f, CopySource(bucket_name, f))
-                uploaded_files.append(f)
-            except Exception as e:
-                logger.error(f"Error copying {f} from {bucket_name} to {MINIO_DEFAULT_BUCKET_NAME}: {e}")
-
-        task_id = utility.do_bulk_insert(
-            collection_name=collection_name,
-            files=files,
-            consistency_level=CONSISTENCY,
-        )
-        task_ids.append(task_id)
-    # list_bulk_insert_tasks = utility.list_bulk_insert_tasks(collection_name=collection_name)
+
+    task_id = utility.do_bulk_insert(
+        collection_name=collection_name,
+        files=[file for files in writer.batch_files for file in files],
+        consistency_level=CONSISTENCY,
+    )
+
     while len(task_ids) > 0:
         time.sleep(1)
         for task_id in task_ids:
@@ -813,9 +797,6 @@ def bulk_insert_milvus(
                 logger.error(f"Failed reason: {task.failed_reason}")
                 task_ids.remove(task_id)
 
-    # Cleanup: remove the copied files to undo the temporary workaround before bulk insert.
-    minio_client.remove_objects(MINIO_DEFAULT_BUCKET_NAME, [DeleteObject(f) for f in uploaded_files])
-
     t_bulk_end = time.time()
     logger.info(f"Bulk {collection_name} upload took {t_bulk_end - t_bulk_start} s")
 
@@ -903,6 +884,7 @@ def stream_insert_milvus(records, client: MilvusClient, collection_name: str, ba
     for idx in range(0, len(records), batch_size):
         client.insert(collection_name=collection_name, data=records[idx : idx + batch_size])
         count += len(records[idx : idx + batch_size])
+    client.flush(collection_name)
     logger.info(f"streamed {count} records")
 
 
@@ -923,6 +905,9 @@ def wait_for_index(collection_name: str, num_elements: int, client: MilvusClient
             for i in range(20):
                 new_indexed_rows = client.describe_index(collection_name, index_name)["indexed_rows"]
                 time.sleep(1)
+                logger.info(
+                    f"polling for indexed rows, {collection_name}, {index_name} -  {new_indexed_rows} / {num_elements}"
+                )
                 if new_indexed_rows == num_elements:
                     indexed_rows = new_indexed_rows
                     break
@@ -951,7 +936,7 @@ def write_to_nvingest_collection(
     compute_bm25_stats: bool = True,
     access_key: str = "minioadmin",
     secret_key: str = "minioadmin",
-    bucket_name: str = "nv-ingest",
+    bucket_name: str = None,
     threshold: int = 1000,
     meta_dataframe=None,
     meta_source_field=None,
@@ -1062,6 +1047,7 @@ def write_to_nvingest_collection(
         wait_for_index(collection_name, num_elements, client)
     else:
         minio_client = Minio(minio_endpoint, access_key=access_key, secret_key=secret_key, secure=False)
+        bucket_name = bucket_name if bucket_name else ClientConfigSchema().minio_bucket_name
         if not minio_client.bucket_exists(bucket_name):
             minio_client.make_bucket(bucket_name)
 
@@ -1652,7 +1638,7 @@ def embed_index_collection(
     compute_bm25_stats: bool = True,
     access_key: str = "minioadmin",
     secret_key: str = "minioadmin",
-    bucket_name: str = "nv-ingest",
+    bucket_name: str = None,
     meta_dataframe: Union[str, pd.DataFrame] = None,
     meta_source_field: str = None,
     meta_fields: list[str] = None,
@@ -1692,7 +1678,7 @@ def embed_index_collection(
         compute_bm25_stats (bool, optional): Whether to compute BM25 statistics. Defaults to True.
         access_key (str, optional): The access key for MinIO authentication. Defaults to "minioadmin".
         secret_key (str, optional): The secret key for MinIO authentication. Defaults to "minioadmin".
-        bucket_name (str, optional): The name of the MinIO bucket. Defaults to "nv-ingest".
+        bucket_name (str, optional): The name of the MinIO bucket.
         meta_dataframe (Union[str, pd.DataFrame], optional): A metadata DataFrame or the path to a CSV file
             containing metadata. Defaults to None.
         meta_source_field (str, optional): The field in the metadata that serves as the source identifier.
@@ -1808,7 +1794,7 @@ def reindex_collection(
     compute_bm25_stats: bool = True,
     access_key: str = "minioadmin",
     secret_key: str = "minioadmin",
-    bucket_name: str = "nv-ingest",
+    bucket_name: str = None,
     meta_dataframe: Union[str, pd.DataFrame] = None,
     meta_source_field: str = None,
     meta_fields: list[str] = None,
@@ -1849,7 +1835,7 @@ def reindex_collection(
         compute_bm25_stats (bool, optional): Whether to compute BM25 statistics. Defaults to True.
         access_key (str, optional): The access key for MinIO authentication. Defaults to "minioadmin".
         secret_key (str, optional): The secret key for MinIO authentication. Defaults to "minioadmin".
-        bucket_name (str, optional): The name of the MinIO bucket. Defaults to "nv-ingest".
+        bucket_name (str, optional): The name of the MinIO bucket.
         meta_dataframe (Union[str, pd.DataFrame], optional): A metadata DataFrame or the path to a CSV file
             containing metadata. Defaults to None.
         meta_source_field (str, optional): The field in the metadata that serves as the source identifier.
@@ -1957,7 +1943,7 @@ class Milvus(VDB):
         compute_bm25_stats: bool = True,
         access_key: str = "minioadmin",
         secret_key: str = "minioadmin",
-        bucket_name: str = "nv-ingest",
+        bucket_name: str = None,
         meta_dataframe: Union[str, pd.DataFrame] = None,
         meta_source_field: str = None,
         meta_fields: list[str] = None,
@@ -1988,7 +1974,7 @@ class Milvus(VDB):
             compute_bm25_stats (bool, optional): Whether to compute BM25 statistics. Defaults to True.
             access_key (str, optional): The access key for MinIO authentication. Defaults to "minioadmin".
             secret_key (str, optional): The secret key for MinIO authentication. Defaults to "minioadmin".
-            bucket_name (str, optional): The name of the MinIO bucket. Defaults to "nv-ingest".
+            bucket_name (str, optional): The name of the MinIO bucket.
             meta_dataframe (Union[str, pd.DataFrame], optional): A metadata DataFrame or the path to a CSV file
                 containing metadata. Defaults to None.
             meta_source_field (str, optional): The field in the metadata that serves as the source identifier.

{nv_ingest_client-2025.9.9.dev20250909 → nv_ingest_client-2025.9.11.dev20250911/src/nv_ingest_client.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest-client
-Version: 2025.9.9.dev20250909
+Version: 2025.9.11.dev20250911
 Summary: Python client for the nv-ingest service
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License

{nv_ingest_client-2025.9.9.dev20250909 → nv_ingest_client-2025.9.11.dev20250911}/src/nv_ingest_client.egg-info/SOURCES.txt

@@ -42,6 +42,7 @@ src/nv_ingest_client/primitives/tasks/udf.py
 src/nv_ingest_client/primitives/tasks/vdb_upload.py
 src/nv_ingest_client/util/__init__.py
 src/nv_ingest_client/util/dataset.py
+src/nv_ingest_client/util/document_analysis.py
 src/nv_ingest_client/util/image_disk_utils.py
 src/nv_ingest_client/util/milvus.py
 src/nv_ingest_client/util/process_json_files.py