nv-ingest-api 2025.8.20.dev20250820__tar.gz → 2025.8.22.dev20250822__tar.gz

{nv_ingest_api-2025.8.20.dev20250820/src/nv_ingest_api.egg-info → nv_ingest_api-2025.8.22.dev20250822}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest-api
-Version: 2025.8.20.dev20250820
+Version: 2025.8.22.dev20250822
 Summary: Python module with core document ingestion functions.
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License

{nv_ingest_api-2025.8.20.dev20250820 → nv_ingest_api-2025.8.22.dev20250822}/src/nv_ingest_api/interface/__init__.py

@@ -11,6 +11,7 @@ from typing import Dict, Any, Optional, List
 from pydantic import BaseModel
 
 from nv_ingest_api.internal.schemas.extract.extract_pdf_schema import PDFiumConfigSchema, NemoRetrieverParseConfigSchema
+from nv_ingest_api.util.logging.sanitize import sanitize_for_logging
 
 logger = logging.getLogger(__name__)
 
@@ -180,29 +181,31 @@ def extraction_interface_relay_constructor(api_fn, task_keys: Optional[List[str]
             if extractor_schema is None:
                 extractor_schema = {f"{extract_method}_config": extraction_config_dict}
 
-            # Log the task and extractor configurations for debugging
+            # Log the task and extractor configurations for debugging (sanitized)
             logger.debug("\n" + "=" * 80)
             logger.debug(f"DEBUG - API Function: {api_fn.__name__}")
             logger.debug(f"DEBUG - Extract Method: {extract_method}")
             logger.debug("-" * 80)
 
-            # Format the task config as a string and log it
-            task_config_str = pprint.pformat(task_config, width=100, sort_dicts=False)
-            logger.debug(f"DEBUG - Task Config:\n{task_config_str}")
+            # Sanitize and format the task config as a string and log it
+            sanitized_task_config = sanitize_for_logging(task_config)
+            task_config_str = pprint.pformat(sanitized_task_config, width=100, sort_dicts=False)
+            logger.debug(f"DEBUG - Task Config (sanitized):\n{task_config_str}")
             logger.debug("-" * 80)
 
-            # Format the extractor config as a string and log it
+            # Sanitize and format the extractor config as a string and log it
             if hasattr(extractor_schema, "model_dump"):
-                extractor_config_str = pprint.pformat(extractor_schema.model_dump(), width=100, sort_dicts=False)
+                sanitized_extractor_config = sanitize_for_logging(extractor_schema.model_dump())
             else:
-                extractor_config_str = pprint.pformat(extractor_schema, width=100, sort_dicts=False)
+                sanitized_extractor_config = sanitize_for_logging(extractor_schema)
+            extractor_config_str = pprint.pformat(sanitized_extractor_config, width=100, sort_dicts=False)
             logger.debug(f"DEBUG - Extractor Config Type: {type(extractor_schema)}")
-            logger.debug(f"DEBUG - Extractor Config:\n{extractor_config_str}")
+            logger.debug(f"DEBUG - Extractor Config (sanitized):\n{extractor_config_str}")
             logger.debug("=" * 80 + "\n")
 
-            # Call the backend API function.
-            pprint.pprint(task_config)
-            pprint.pprint(extractor_schema)
+            # Call the backend API function. Print sanitized configs for any debug consumers of stdout.
+            pprint.pprint(sanitized_task_config)
+            pprint.pprint(sanitized_extractor_config)
             result = api_fn(ledger, task_config, extractor_schema, execution_trace_log)
 
             # If the result is a tuple, return only the first element

{nv_ingest_api-2025.8.20.dev20250820 → nv_ingest_api-2025.8.22.dev20250822}/src/nv_ingest_api/internal/extract/pdf/engines/pdf_helpers/__init__.py

@@ -11,6 +11,7 @@ from typing import Any
 from typing import Dict
 from typing import List
 from typing import Optional
+from nv_ingest_api.util.logging.sanitize import sanitize_for_logging
 
 import pandas as pd
 from nv_ingest_api.internal.extract.pdf.engines import adobe_extractor
@@ -131,7 +132,7 @@ def _orchestrate_row_extraction(
         method_config = extractor_config[config_key]
     else:
         # If no matching config is found, log a warning but don't fail
-        logger.warning(f"No {config_key} found in extractor_config: {extractor_config}")
+        logger.warning(f"No {config_key} found in extractor_config: {sanitize_for_logging(extractor_config)}")
         method_config = None
 
     # Add the method-specific config to the parameters if available
@@ -141,7 +142,7 @@ def _orchestrate_row_extraction(
 
     # The resulting parameters constitute the complete extractor_config
     extractor_config = params
-    logger.debug(f"Final extractor_config: {extractor_config}")
+    logger.debug(f"Final extractor_config: {sanitize_for_logging(extractor_config)}")
 
     result = _work_extract_pdf(
         pdf_stream=pdf_stream,

{nv_ingest_api-2025.8.20.dev20250820 → nv_ingest_api-2025.8.22.dev20250822}/src/nv_ingest_api/internal/schemas/extract/extract_audio_schema.py

@@ -7,7 +7,7 @@ import logging
 from typing import Optional
 from typing import Tuple
 
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 from pydantic import root_validator
 
 logger = logging.getLogger(__name__)
@@ -42,12 +42,12 @@ class AudioConfigSchema(BaseModel):
         Pydantic config option to forbid extra fields.
     """
 
-    auth_token: Optional[str] = None
+    auth_token: Optional[str] = Field(default=None, repr=False)
     audio_endpoints: Tuple[Optional[str], Optional[str]] = (None, None)
     audio_infer_protocol: Optional[str] = None
     function_id: Optional[str] = None
     use_ssl: Optional[bool] = None
-    ssl_cert: Optional[str] = None
+    ssl_cert: Optional[str] = Field(default=None, repr=False)
     segment_audio: Optional[bool] = None
 
     @root_validator(pre=True)

{nv_ingest_api-2025.8.20.dev20250820 → nv_ingest_api-2025.8.22.dev20250822}/src/nv_ingest_api/internal/schemas/extract/extract_chart_schema.py

@@ -6,7 +6,7 @@ import logging
 from typing import Optional
 from typing import Tuple
 
-from pydantic import field_validator, model_validator, ConfigDict, BaseModel
+from pydantic import field_validator, model_validator, ConfigDict, BaseModel, Field
 
 logger = logging.getLogger(__name__)
 
@@ -44,7 +44,7 @@ class ChartExtractorConfigSchema(BaseModel):
         Pydantic config option to forbid extra fields.
     """
 
-    auth_token: Optional[str] = None
+    auth_token: Optional[str] = Field(default=None, repr=False)
 
     yolox_endpoints: Tuple[Optional[str], Optional[str]] = (None, None)
     yolox_infer_protocol: str = ""

{nv_ingest_api-2025.8.20.dev20250820 → nv_ingest_api-2025.8.22.dev20250822}/src/nv_ingest_api/internal/schemas/extract/extract_docx_schema.py

@@ -7,7 +7,7 @@ import logging
 from typing import Optional
 from typing import Tuple
 
-from pydantic import model_validator, ConfigDict, BaseModel
+from pydantic import model_validator, ConfigDict, BaseModel, Field
 
 logger = logging.getLogger(__name__)
 
@@ -41,7 +41,7 @@ class DocxConfigSchema(BaseModel):
         Pydantic config option to forbid extra fields.
     """
 
-    auth_token: Optional[str] = None
+    auth_token: Optional[str] = Field(default=None, repr=False)
 
     yolox_endpoints: Tuple[Optional[str], Optional[str]] = (None, None)
     yolox_infer_protocol: str = ""

{nv_ingest_api-2025.8.20.dev20250820 → nv_ingest_api-2025.8.22.dev20250822}/src/nv_ingest_api/internal/schemas/extract/extract_image_schema.py

@@ -7,7 +7,7 @@ import logging
 from typing import Optional
 from typing import Tuple
 
-from pydantic import model_validator, ConfigDict, BaseModel
+from pydantic import model_validator, ConfigDict, BaseModel, Field
 
 logger = logging.getLogger(__name__)
 
@@ -41,7 +41,7 @@ class ImageConfigSchema(BaseModel):
         Pydantic config option to forbid extra fields.
     """
 
-    auth_token: Optional[str] = None
+    auth_token: Optional[str] = Field(default=None, repr=False)
 
     yolox_endpoints: Tuple[Optional[str], Optional[str]] = (None, None)
     yolox_infer_protocol: str = ""

{nv_ingest_api-2025.8.20.dev20250820 → nv_ingest_api-2025.8.22.dev20250822}/src/nv_ingest_api/internal/schemas/extract/extract_infographic_schema.py

@@ -6,7 +6,7 @@ import logging
 from typing import Optional
 from typing import Tuple
 
-from pydantic import field_validator, model_validator, ConfigDict, BaseModel
+from pydantic import field_validator, model_validator, ConfigDict, BaseModel, Field
 
 logger = logging.getLogger(__name__)
 
@@ -40,7 +40,7 @@ class InfographicExtractorConfigSchema(BaseModel):
         Pydantic config option to forbid extra fields.
     """
 
-    auth_token: Optional[str] = None
+    auth_token: Optional[str] = Field(default=None, repr=False)
 
     ocr_endpoints: Tuple[Optional[str], Optional[str]] = (None, None)
     ocr_infer_protocol: str = ""

{nv_ingest_api-2025.8.20.dev20250820 → nv_ingest_api-2025.8.22.dev20250822}/src/nv_ingest_api/internal/schemas/extract/extract_pdf_schema.py

@@ -7,7 +7,7 @@ import logging
 from typing import Optional
 from typing import Tuple
 
-from pydantic import model_validator, ConfigDict, BaseModel
+from pydantic import model_validator, ConfigDict, BaseModel, Field
 
 logger = logging.getLogger(__name__)
 
@@ -41,7 +41,7 @@ class PDFiumConfigSchema(BaseModel):
         Pydantic config option to forbid extra fields.
     """
 
-    auth_token: Optional[str] = None
+    auth_token: Optional[str] = Field(default=None, repr=False)
 
     yolox_endpoints: Tuple[Optional[str], Optional[str]] = (None, None)
     yolox_infer_protocol: str = ""
@@ -123,7 +123,7 @@ class NemoRetrieverParseConfigSchema(BaseModel):
         Pydantic config option to forbid extra fields.
     """
 
-    auth_token: Optional[str] = None
+    auth_token: Optional[str] = Field(default=None, repr=False)
 
     yolox_endpoints: Tuple[Optional[str], Optional[str]] = (None, None)
     yolox_infer_protocol: str = ""

{nv_ingest_api-2025.8.20.dev20250820 → nv_ingest_api-2025.8.22.dev20250822}/src/nv_ingest_api/internal/schemas/extract/extract_pptx_schema.py

@@ -7,7 +7,7 @@ import logging
 from typing import Optional
 from typing import Tuple
 
-from pydantic import model_validator, ConfigDict, BaseModel
+from pydantic import model_validator, ConfigDict, BaseModel, Field
 
 logger = logging.getLogger(__name__)
 
@@ -41,7 +41,7 @@ class PPTXConfigSchema(BaseModel):
         Pydantic config option to forbid extra fields.
     """
 
-    auth_token: Optional[str] = None
+    auth_token: Optional[str] = Field(default=None, repr=False)
 
     yolox_endpoints: Tuple[Optional[str], Optional[str]] = (None, None)
     yolox_infer_protocol: str = ""

{nv_ingest_api-2025.8.20.dev20250820 → nv_ingest_api-2025.8.22.dev20250822}/src/nv_ingest_api/internal/schemas/extract/extract_table_schema.py

@@ -7,7 +7,7 @@ import logging
 from typing import Optional
 from typing import Tuple
 
-from pydantic import field_validator, model_validator, ConfigDict, BaseModel
+from pydantic import field_validator, model_validator, ConfigDict, BaseModel, Field
 
 
 logger = logging.getLogger(__name__)
@@ -42,7 +42,7 @@ class TableExtractorConfigSchema(BaseModel):
         Pydantic config option to forbid extra fields.
     """
 
-    auth_token: Optional[str] = None
+    auth_token: Optional[str] = Field(default=None, repr=False)
 
     yolox_endpoints: Tuple[Optional[str], Optional[str]] = (None, None)
     yolox_infer_protocol: str = ""

{nv_ingest_api-2025.8.20.dev20250820 → nv_ingest_api-2025.8.22.dev20250822}/src/nv_ingest_api/internal/schemas/meta/ingest_job_schema.py

@@ -73,7 +73,7 @@ class IngestTaskStoreSchema(BaseModelNoExt):
 
 # Captioning: All fields are optional and override default parameters.
 class IngestTaskCaptionSchema(BaseModelNoExt):
-    api_key: Optional[str] = None
+    api_key: Optional[str] = Field(default=None, repr=False)
     endpoint_url: Optional[str] = None
     prompt: Optional[str] = None
     model_name: Optional[str] = None
@@ -105,7 +105,7 @@ class IngestTaskDedupSchema(BaseModelNoExt):
 class IngestTaskEmbedSchema(BaseModelNoExt):
     endpoint_url: Optional[str] = None
     model_name: Optional[str] = None
-    api_key: Optional[str] = None
+    api_key: Optional[str] = Field(default=None, repr=False)
     filter_errors: bool = False
     text_elements_modality: Optional[str] = None
     image_elements_modality: Optional[str] = None
@@ -121,13 +121,13 @@ class IngestTaskVdbUploadSchema(BaseModelNoExt):
 
 
 class IngestTaskAudioExtraction(BaseModelNoExt):
-    auth_token: Optional[str] = None
+    auth_token: Optional[str] = Field(default=None, repr=False)
     grpc_endpoint: Optional[str] = None
     http_endpoint: Optional[str] = None
     infer_protocol: Optional[str] = None
     function_id: Optional[str] = None
     use_ssl: Optional[bool] = None
-    ssl_cert: Optional[str] = None
+    ssl_cert: Optional[str] = Field(default=None, repr=False)
     segment_audio: Optional[bool] = None
 
 

{nv_ingest_api-2025.8.20.dev20250820 → nv_ingest_api-2025.8.22.dev20250822}/src/nv_ingest_api/internal/schemas/transform/transform_image_caption_schema.py

@@ -3,11 +3,11 @@
 # SPDX-License-Identifier: Apache-2.0
 
 
-from pydantic import ConfigDict, BaseModel, model_validator, field_validator
+from pydantic import ConfigDict, BaseModel, model_validator, field_validator, Field
 
 
 class ImageCaptionExtractionSchema(BaseModel):
-    api_key: str = ""
+    api_key: str = Field(default="", repr=False)
     endpoint_url: str = "https://integrate.api.nvidia.com/v1/chat/completions"
     prompt: str = "Caption the content of this image:"
     model_name: str = "nvidia/llama-3.1-nemotron-nano-vl-8b-v1"

{nv_ingest_api-2025.8.20.dev20250820 → nv_ingest_api-2025.8.22.dev20250822}/src/nv_ingest_api/internal/schemas/transform/transform_text_embedding_schema.py

@@ -13,7 +13,7 @@ logger = logging.getLogger(__name__)
 
 
 class TextEmbeddingSchema(BaseModel):
-    api_key: str = Field(default="")
+    api_key: str = Field(default="", repr=False)
     batch_size: int = Field(default=4)
     embedding_model: str = Field(default="nvidia/llama-3.2-nv-embedqa-1b-v2")
     embedding_nim_endpoint: str = Field(default="http://embedding:8000/v1")

nv_ingest_api-2025.8.22.dev20250822/src/nv_ingest_api/util/logging/sanitize.py

@@ -0,0 +1,84 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from typing import Any, Mapping, MutableMapping, Sequence, Set
+
+try:
+    # Pydantic is optional at runtime for this helper; import if available
+    from pydantic import BaseModel  # type: ignore
+except Exception:  # pragma: no cover - pydantic always present in this repo
+    BaseModel = None  # type: ignore
+
+
+_DEFAULT_SENSITIVE_KEYS: Set[str] = {
+    "access_token",
+    "api_key",
+    "authorization",
+    "auth_token",
+    "client_secret",
+    "hf_access_token",
+    "hugging_face_access_token",
+    "password",
+    "refresh_token",
+    "secret",
+    "ssl_cert",
+    "x-api-key",
+}
+
+_REDACTION = "***REDACTED***"
+
+
+def _is_mapping(obj: Any) -> bool:
+    try:
+        return isinstance(obj, Mapping)
+    except Exception:
+        return False
+
+
+def _is_sequence(obj: Any) -> bool:
+    # Exclude strings/bytes from sequences we want to traverse
+    return isinstance(obj, Sequence) and not isinstance(obj, (str, bytes, bytearray))
+
+
+def sanitize_for_logging(
+    data: Any,
+    sensitive_keys: Set[str] | None = None,
+    redaction: str = _REDACTION,
+) -> Any:
+    """
+    Recursively sanitize common secret fields from dicts, lists, tuples, and Pydantic models.
+
+    - Key comparison is case-insensitive and matches exact keys only.
+    - Does not mutate input; returns a sanitized deep copy.
+    - For Pydantic BaseModel instances, uses model_dump() before redaction.
+    """
+    keys = {k.lower() for k in (sensitive_keys or _DEFAULT_SENSITIVE_KEYS)}
+
+    # Handle Pydantic models without importing pydantic at module import time
+    if BaseModel is not None and isinstance(data, BaseModel):  # type: ignore[arg-type]
+        try:
+            return sanitize_for_logging(data.model_dump(), keys, redaction)
+        except Exception:
+            # Fall through and try generic handling below
+            pass
+
+    # Dict-like
+    if _is_mapping(data):
+        out: MutableMapping[str, Any] = type(data)()  # preserve mapping type where possible
+        for k, v in data.items():  # type: ignore[assignment]
+            key_lower = str(k).lower()
+            if key_lower in keys:
+                out[k] = redaction
+            else:
+                out[k] = sanitize_for_logging(v, keys, redaction)
+        return out
+
+    # List/Tuple/Sequence
+    if _is_sequence(data):
+        return type(data)(sanitize_for_logging(v, keys, redaction) for v in data)
+
+    # Fallback: return as-is
+    return data

{nv_ingest_api-2025.8.20.dev20250820 → nv_ingest_api-2025.8.22.dev20250822/src/nv_ingest_api.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest-api
-Version: 2025.8.20.dev20250820
+Version: 2025.8.22.dev20250822
 Summary: Python module with core document ingestion functions.
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License

{nv_ingest_api-2025.8.20.dev20250820 → nv_ingest_api-2025.8.22.dev20250822}/src/nv_ingest_api.egg-info/SOURCES.txt

@@ -145,6 +145,7 @@ src/nv_ingest_api/util/introspection/class_inspect.py
 src/nv_ingest_api/util/introspection/function_inspect.py
 src/nv_ingest_api/util/logging/__init__.py
 src/nv_ingest_api/util/logging/configuration.py
+src/nv_ingest_api/util/logging/sanitize.py
 src/nv_ingest_api/util/message_brokers/__init__.py
 src/nv_ingest_api/util/message_brokers/simple_message_broker/__init__.py
 src/nv_ingest_api/util/message_brokers/simple_message_broker/broker.py