nv-ingest-api 25.6.2__tar.gz → 25.6.26.dev20250626__tar.gz

{nv_ingest_api-25.6.2/src/nv_ingest_api.egg-info → nv_ingest_api-25.6.26.dev20250626}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest-api
-Version: 25.6.2
+Version: 25.6.26.dev20250626
 Summary: Python module with core document ingestion functions.
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License

{nv_ingest_api-25.6.2 → nv_ingest_api-25.6.26.dev20250626}/src/nv_ingest_api/internal/extract/audio/audio_extraction.py

@@ -5,6 +5,8 @@
 import logging
 
 import pandas as pd
+import functools
+import uuid
 from typing import Any
 from typing import Dict
 from typing import Optional
@@ -21,7 +23,7 @@ logger = logging.getLogger(__name__)
 
 
 @unified_exception_handler
-def _update_audio_metadata(row: pd.Series, audio_client: Any, trace_info: Dict) -> Dict:
+def _extract_from_audio(row: pd.Series, audio_client: Any, trace_info: Dict, segment_audio: bool = False) -> Dict:
     """
     Modifies the metadata of a row if the conditions for table extraction are met.
 
@@ -56,24 +58,42 @@ def _update_audio_metadata(row: pd.Series, audio_client: Any, trace_info: Dict)
     base64_audio = metadata.pop("content")
     content_metadata = metadata.get("content_metadata", {})
 
-    # Only modify if content type is audio
+    # Only extract transcript if content type is audio
     if (content_metadata.get("type") != ContentTypeEnum.AUDIO) or (base64_audio in (None, "")):
-        return metadata
+        return [row.to_list()]
 
-    # Modify audio metadata with the result from the inference model
-    audio_result = audio_client.infer(
+    # Get the result from the inference model
+    segments, transcript = audio_client.infer(
         base64_audio,
         model_name="parakeet",
         trace_info=trace_info,  # traceable_func arg
         stage_name="audio_extraction",
     )
 
-    row["document_type"] = ContentTypeEnum.AUDIO
-    audio_metadata = {"audio_transcript": audio_result}
-    metadata["audio_metadata"] = validate_schema(audio_metadata, AudioMetadataSchema).model_dump()
-    row["metadata"] = validate_schema(metadata, MetadataSchema).model_dump()
+    extracted_data = []
+    if segment_audio:
+        for segment in segments:
+            segment_metadata = metadata.copy()
+            audio_metadata = {"audio_transcript": segment["text"]}
+            segment_metadata["audio_metadata"] = validate_schema(audio_metadata, AudioMetadataSchema).model_dump()
+            segment_metadata["content_metadata"]["start_time"] = segment["start"]
+            segment_metadata["content_metadata"]["end_time"] = segment["end"]
+
+            extracted_data.append(
+                [
+                    ContentTypeEnum.AUDIO,
+                    validate_schema(segment_metadata, MetadataSchema).model_dump(),
+                    str(uuid.uuid4()),
+                ]
+            )
+    else:
+        audio_metadata = {"audio_transcript": transcript}
+        metadata["audio_metadata"] = validate_schema(audio_metadata, AudioMetadataSchema).model_dump()
+        extracted_data.append(
+            [ContentTypeEnum.AUDIO, validate_schema(metadata, MetadataSchema).model_dump(), str(uuid.uuid4())]
+        )
 
-    return metadata
+    return extracted_data
 
 
 def extract_text_from_audio_internal(
@@ -121,6 +141,7 @@ def extract_text_from_audio_internal(
     function_id = extract_params.get("function_id") or audio_extraction_config.function_id
     use_ssl = extract_params.get("use_ssl") or audio_extraction_config.use_ssl
     ssl_cert = extract_params.get("ssl_cert") or audio_extraction_config.ssl_cert
+    segment_audio = extract_params.get("segment_audio") or audio_extraction_config.segment_audio
 
     parakeet_client = create_audio_inference_client(
         (grpc_endpoint, http_endpoint),
@@ -136,12 +157,27 @@ def extract_text_from_audio_internal(
         logger.debug("No trace_info provided. Initialized empty trace_info dictionary.")
 
     try:
-        # Apply the _update_metadata function to each row in the DataFrame
-        df_extraction_ledger["metadata"] = df_extraction_ledger.apply(
-            _update_audio_metadata, axis=1, args=(parakeet_client, execution_trace_log)
+        # Create a partial function to extract using the provided configurations.
+        _extract_from_audio_partial = functools.partial(
+            _extract_from_audio,
+            audio_client=parakeet_client,
+            trace_info=execution_trace_log,
+            segment_audio=segment_audio,
         )
 
-        return df_extraction_ledger, execution_trace_log
+        # Apply the _extract_from_audio_partial function to each row in the DataFrame
+        extraction_series = df_extraction_ledger.apply(_extract_from_audio_partial, axis=1)
+
+        # Explode the results if the extraction returns lists.
+        extraction_series = extraction_series.explode().dropna()
+
+        # Convert the extracted results into a DataFrame.
+        if not extraction_series.empty:
+            extracted_df = pd.DataFrame(extraction_series.to_list(), columns=["document_type", "metadata", "uuid"])
+        else:
+            extracted_df = pd.DataFrame({"document_type": [], "metadata": [], "uuid": []})
+
+        return extracted_df, execution_trace_log
 
     except Exception as e:
         logger.exception(f"Error occurred while extracting audio data: {e}", exc_info=True)

{nv_ingest_api-25.6.2 → nv_ingest_api-25.6.26.dev20250626}/src/nv_ingest_api/internal/primitives/nim/model_interface/parakeet.py

@@ -101,7 +101,7 @@ class ParakeetClient:
         segments, transcript = process_transcription_response(response)
         logger.debug("Processing Parakeet inference results (pass-through).")
 
-        return transcript
+        return segments, transcript
 
     def transcribe(
         self,

{nv_ingest_api-25.6.2 → nv_ingest_api-25.6.26.dev20250626}/src/nv_ingest_api/internal/schemas/extract/extract_audio_schema.py

@@ -48,6 +48,7 @@ class AudioConfigSchema(BaseModel):
     function_id: Optional[str] = None
     use_ssl: Optional[bool] = None
     ssl_cert: Optional[str] = None
+    segment_audio: Optional[bool] = None
 
     @root_validator(pre=True)
     def validate_endpoints(cls, values):

{nv_ingest_api-25.6.2 → nv_ingest_api-25.6.26.dev20250626}/src/nv_ingest_api/internal/schemas/meta/ingest_job_schema.py

@@ -124,6 +124,7 @@ class IngestTaskAudioExtraction(BaseModelNoExt):
     function_id: Optional[str] = None
     use_ssl: Optional[bool] = None
     ssl_cert: Optional[str] = None
+    segment_audio: Optional[bool] = None
 
 
 class IngestTaskTableExtraction(BaseModelNoExt):

{nv_ingest_api-25.6.2 → nv_ingest_api-25.6.26.dev20250626}/src/nv_ingest_api/internal/schemas/meta/metadata_schema.py

@@ -97,6 +97,8 @@ class ContentMetadataSchema(BaseModelNoExt):
     page_number: int = -1
     hierarchy: ContentHierarchySchema = ContentHierarchySchema()
     subtype: Union[ContentTypeEnum, str] = ""
+    start_time: int = -1
+    end_time: int = -1
 
 
 class TextMetadataSchema(BaseModelNoExt):

{nv_ingest_api-25.6.2 → nv_ingest_api-25.6.26.dev20250626}/src/nv_ingest_api/internal/transform/split_text.py

@@ -31,9 +31,16 @@ def _build_split_documents(row, chunks: List[str]) -> List[dict[str, Any]]:
         metadata = row.metadata if hasattr(row, "metadata") and isinstance(row.metadata, dict) else {}
         metadata = copy.deepcopy(metadata)
 
-        metadata["content"] = text
-
-        documents.append({"document_type": ContentTypeEnum.TEXT.value, "metadata": metadata, "uuid": str(uuid.uuid4())})
+        if row.document_type == ContentTypeEnum.AUDIO:
+            metadata["audio_metadata"]["audio_transcript"] = text
+            documents.append(
+                {"document_type": ContentTypeEnum.AUDIO.value, "metadata": metadata, "uuid": str(uuid.uuid4())}
+            )
+        else:
+            metadata["content"] = text
+            documents.append(
+                {"document_type": ContentTypeEnum.TEXT.value, "metadata": metadata, "uuid": str(uuid.uuid4())}
+            )
 
     return documents
 
@@ -118,7 +125,7 @@ def transform_text_split_and_tokenize_internal(
     )
 
     # Filter to documents with text content.
-    text_type_condition = df_transform_ledger["document_type"] == ContentTypeEnum.TEXT
+    text_type_condition = df_transform_ledger["document_type"].isin([ContentTypeEnum.TEXT, ContentTypeEnum.AUDIO])
 
     normalized_meta_df = pd.json_normalize(df_transform_ledger["metadata"], errors="ignore")
     if "source_metadata.source_type" in normalized_meta_df.columns:
@@ -147,7 +154,14 @@ def transform_text_split_and_tokenize_internal(
 
     split_docs: List[Dict[str, Any]] = []
     for _, row in df_filtered.iterrows():
-        content: str = row["metadata"]["content"] if row["metadata"]["content"] is not None else ""
+        if row["document_type"] == ContentTypeEnum.AUDIO:
+            content: str = (
+                row["metadata"]["audio_metadata"]["audio_transcript"]
+                if row["metadata"]["audio_metadata"]["audio_transcript"] is not None
+                else ""
+            )
+        else:
+            content: str = row["metadata"]["content"] if row["metadata"]["content"] is not None else ""
         chunks: List[str] = _split_into_chunks(content, tokenizer_model, chunk_size, chunk_overlap)
         split_docs.extend(_build_split_documents(row, chunks))
 

{nv_ingest_api-25.6.2 → nv_ingest_api-25.6.26.dev20250626/src/nv_ingest_api.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest-api
-Version: 25.6.2
+Version: 25.6.26.dev20250626
 Summary: Python module with core document ingestion functions.
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License

{nv_ingest_api-25.6.2 → nv_ingest_api-25.6.26.dev20250626}/src/version.py

@@ -5,7 +5,6 @@
 
 import datetime
 import os
-import re
 
 
 def get_version():
@@ -16,13 +15,6 @@ def get_version():
     if not version:
         version = f"{datetime.datetime.now().strftime('%Y.%m.%d')}"
 
-    # We only check this for dev, we assume for release the user knows what they are doing
-    if release_type != "release":
-        # Ensure the version is PEP 440 compatible
-        pep440_regex = r"^\d{4}\.\d{1,2}\.\d{1,2}$"
-        if not re.match(pep440_regex, version):
-            raise ValueError(f"Version '{version}' is not PEP 440 compatible")
-
     # Construct the final version string
     if release_type == "dev":
         # If rev is not specified and defaults to 0 lets create a more meaningful development