nv-ingest-api 2025.10.4.dev20251004__py3-none-any.whl → 2025.11.2.dev20251102__py3-none-any.whl

{nv_ingest_api-2025.10.4.dev20251004.dist-info → nv_ingest_api-2025.11.2.dev20251102.dist-info}/RECORD

@@ -20,10 +20,10 @@ nv_ingest_api/internal/extract/docx/engines/docxreader_helpers/docxreader.py,sha
 nv_ingest_api/internal/extract/html/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest_api/internal/extract/html/html_extractor.py,sha256=I9oWfj6_As4898GDDh0zsSuKxO3lBsvyYzhvUotjzJI,3282
 nv_ingest_api/internal/extract/image/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
-nv_ingest_api/internal/extract/image/chart_extractor.py,sha256=Pojiu7R1BT8tUUzD5DsF-dDEwakz1ZfUrL_agalUsNc,13591
+nv_ingest_api/internal/extract/image/chart_extractor.py,sha256=V-unsKvSAJo7QqsDDm_OsZxzGpRwhVGFbc_47zX3hHs,13764
 nv_ingest_api/internal/extract/image/image_extractor.py,sha256=gBKjlx28hA_e-dupatu46YQgOHJ0DLpAWxREiLaZLyo,9039
-nv_ingest_api/internal/extract/image/infographic_extractor.py,sha256=gP-WiBIHruDmNFchq4BbVAci3XStMtyeN99M8dLm1j4,10225
-nv_ingest_api/internal/extract/image/table_extractor.py,sha256=T80-Smkf54Y5OkSaOquXpcoLbAf5uMnV-LOsBgD0L7E,14440
+nv_ingest_api/internal/extract/image/infographic_extractor.py,sha256=UVYkxau53XsTDtYO8Pjt76f4lGLBHltXSCgkp1NBsIs,10398
+nv_ingest_api/internal/extract/image/table_extractor.py,sha256=1z79OM_a9V-91oPXVNw6I388m68HZ4kXwAJw07FLicY,14613
 nv_ingest_api/internal/extract/image/image_helpers/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest_api/internal/extract/image/image_helpers/common.py,sha256=VhqjsBqvUz-2y92t6iryVERTuRfcGUdTHOOScYr8GLo,14916
 nv_ingest_api/internal/extract/pdf/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
@@ -32,7 +32,7 @@ nv_ingest_api/internal/extract/pdf/engines/__init__.py,sha256=u4GnAZmDKRl0RwYGIR
 nv_ingest_api/internal/extract/pdf/engines/adobe.py,sha256=VT0dEqkU-y2uGkaCqxtKYov_Q8R1028UQVBchgMLca4,17466
 nv_ingest_api/internal/extract/pdf/engines/llama.py,sha256=MwzM-n2tu0FHM0wDe_0mONLlzHrPte7EOTuPtzCh7Zs,8384
 nv_ingest_api/internal/extract/pdf/engines/nemoretriever.py,sha256=IVbNcH_phMiRSxnkZ04pGfQrPJ-x1zVR3hXyhxv7juc,22977
-nv_ingest_api/internal/extract/pdf/engines/pdfium.py,sha256=CCfxcHAS3mED8zD6GKTGNUi02CzBMs7FsSopevhsiyk,22720
+nv_ingest_api/internal/extract/pdf/engines/pdfium.py,sha256=yAndWwh_k00nP0spYGxlewP3RBPxE4QR-b3U3VgXnBo,22852
 nv_ingest_api/internal/extract/pdf/engines/tika.py,sha256=6GyR2l6EsgNZl9jnYDXLeKNK9Fj2Mw9y2UWDq-eSkOc,3169
 nv_ingest_api/internal/extract/pdf/engines/unstructured_io.py,sha256=jrv2B4VZAH4PevAQrFz965qz8UyXq3rViiOTbGLejec,14908
 nv_ingest_api/internal/extract/pdf/engines/pdf_helpers/__init__.py,sha256=uTPTUTWQsGM1oeTUo49_hzwC5Yy9iEokrnS3z3WvtIo,5988
@@ -50,7 +50,7 @@ nv_ingest_api/internal/primitives/control_message_task.py,sha256=nWVB3QsP6p8BKwH
 nv_ingest_api/internal/primitives/ingest_control_message.py,sha256=8rA0UbPDSB3avReAKNxiUa_FCy7fIQpqk6tfmcYUibA,9879
 nv_ingest_api/internal/primitives/nim/__init__.py,sha256=-dFBTHQnMKV0yc5tfSqIT-rkJXKtpcmyUfTPs8TJAi8,339
 nv_ingest_api/internal/primitives/nim/default_values.py,sha256=W92XjfyeC6uuVxut6J7p00x1kpNsnXIDb97gSVytZJk,380
-nv_ingest_api/internal/primitives/nim/nim_client.py,sha256=kQAHWwZ6kjTVYZSfa0qRyIOFcqrhMe8LUygGtgzAly0,26321
+nv_ingest_api/internal/primitives/nim/nim_client.py,sha256=kT-JP9jbkXzotS7EeajTgfMbFWhMoD8o2JtOLYu1JuU,32770
 nv_ingest_api/internal/primitives/nim/nim_model_interface.py,sha256=gWhyR33mIgEOYirq53WOk1bRl1SL0C_SVrM4w1-JmKU,4166
 nv_ingest_api/internal/primitives/nim/model_interface/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest_api/internal/primitives/nim/model_interface/cached.py,sha256=b1HX-PY1ExW5V6pXC1ZiHdobeG_BmbPr3rBbVJef13s,11003
@@ -58,8 +58,8 @@ nv_ingest_api/internal/primitives/nim/model_interface/decorators.py,sha256=qwubk
 nv_ingest_api/internal/primitives/nim/model_interface/deplot.py,sha256=TvKdk6PTuI1WNhRmNNrvygaI_DIutkJkDL-XdtLZQac,10787
 nv_ingest_api/internal/primitives/nim/model_interface/helpers.py,sha256=jqbEbavvr9giODpzsGQSRDu5yZ4YfNfKAQfqUm9yUDI,11698
 nv_ingest_api/internal/primitives/nim/model_interface/nemoretriever_parse.py,sha256=WysjDZeegclO3mZgVcGOwzWbr8wSI4pWRiYD4iC2EXo,7098
-nv_ingest_api/internal/primitives/nim/model_interface/ocr.py,sha256=QOjKEJaL7Z_aT-luyV4eJSNQX4o-a9-P0CB0ZwSxFk4,29282
-nv_ingest_api/internal/primitives/nim/model_interface/parakeet.py,sha256=3XXJkeJaVFe_iIfNn_bDYn79JN20besjZHiNZ5dEnZQ,12778
+nv_ingest_api/internal/primitives/nim/model_interface/ocr.py,sha256=WuX-veTC510TbvMWYGYP6WCzjYCbUBAUc5ovJUWCrFU,29607
+nv_ingest_api/internal/primitives/nim/model_interface/parakeet.py,sha256=6M3yxZHEyRX5sCDz-82SIPKw00TabYWxx-ZUEvGKrQQ,12920
 nv_ingest_api/internal/primitives/nim/model_interface/text_embedding.py,sha256=lFhppNqrq5X_fzbCWKphvZQMzaJd3gHrkWsyJORzFrU,5010
 nv_ingest_api/internal/primitives/nim/model_interface/vlm.py,sha256=sUDKKlVqKjiHEGr2D04I7S4pDfnLR8b-NplV1pf5GVQ,6240
 nv_ingest_api/internal/primitives/nim/model_interface/yolox.py,sha256=zpfEZIPctWhNfREnP6e77zffU8vs_RfnMprBj-2jXXk,42847
@@ -68,24 +68,25 @@ nv_ingest_api/internal/primitives/tracing/latency.py,sha256=5kVTeYRbRdTlT_aI4MeS
 nv_ingest_api/internal/primitives/tracing/logging.py,sha256=SSzIgS7afLH-e1C7VagYDmkkA6rTXmQ-bmtLjoEguhg,3851
 nv_ingest_api/internal/primitives/tracing/tagging.py,sha256=xU534rb94uKnsSu0_DzyZcCSkIpa5SWTMxX7NSA3HoE,11671
 nv_ingest_api/internal/schemas/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
+nv_ingest_api/internal/schemas/mixins.py,sha256=QyAhZATodR0qWgcq6HxO3yTd31X1nxZfxvpNrXfaY9w,1236
 nv_ingest_api/internal/schemas/extract/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
-nv_ingest_api/internal/schemas/extract/extract_audio_schema.py,sha256=3cjLcw5zFUb7WNbGPLDJN2KukF_hoOM7PE33UHici6w,3873
-nv_ingest_api/internal/schemas/extract/extract_chart_schema.py,sha256=PZFJPLrLs8k5I5ufnp0XWrBjmbQVkkaxjb-xq-2rn2Q,4317
-nv_ingest_api/internal/schemas/extract/extract_docx_schema.py,sha256=Bafw6lIXLS2PcEpU82D4Vb0OPD_FvGSr546IedsfR8o,3795
+nv_ingest_api/internal/schemas/extract/extract_audio_schema.py,sha256=fl08h6XBjKRtm1LJlTJpH3s_LGoaKmuq8duBmHrXzXc,3974
+nv_ingest_api/internal/schemas/extract/extract_chart_schema.py,sha256=24zXz08DD-OXiOhjZPfv9fxIki8-c50xt3kH5JemE8I,4787
+nv_ingest_api/internal/schemas/extract/extract_docx_schema.py,sha256=qSHCIFWx6w5NTguWrgTXRD3i-H2nRMxjoNTw2IsF5Zg,3896
 nv_ingest_api/internal/schemas/extract/extract_html_schema.py,sha256=lazpONTGZ6Fl420BGBAr6rogFGtlzBiZTc1uA694OIs,841
-nv_ingest_api/internal/schemas/extract/extract_image_schema.py,sha256=hXiHIKIZS2qb4u8g10m_S5CpeAzHvH3tSB_He5LYOfU,3800
-nv_ingest_api/internal/schemas/extract/extract_infographic_schema.py,sha256=Yb580-k1oYXanWMA17u0LGO5-AzUP80j6aTPGpjn920,4007
-nv_ingest_api/internal/schemas/extract/extract_pdf_schema.py,sha256=TTFzlPG5r1QFO2B5VWBPF69IXP1eQKrg16wo49hHDV0,6650
-nv_ingest_api/internal/schemas/extract/extract_pptx_schema.py,sha256=6KomEL6wdhpZVnE2SiOeZJNhyfHIQ2sARdfm0R16uCM,3795
-nv_ingest_api/internal/schemas/extract/extract_table_schema.py,sha256=npBEGXkVaShOiHrc8FQ-25kHhPRD8WB6dRn-T5TScKc,3969
+nv_ingest_api/internal/schemas/extract/extract_image_schema.py,sha256=NJAg1m-CPHAs4BMW8P09qL9sxS3ARBZOMpeNewFc5I8,3901
+nv_ingest_api/internal/schemas/extract/extract_infographic_schema.py,sha256=e65tQs6zSo7P7Wl9KZJznomvBrS1PmLX9f0IjytszO4,4477
+nv_ingest_api/internal/schemas/extract/extract_pdf_schema.py,sha256=J7J-rO6RO31777m9RKSdcOZhIwgIS7P9Y4M6WBaUYWs,6778
+nv_ingest_api/internal/schemas/extract/extract_pptx_schema.py,sha256=vuHK2qf-I41iDgnuXc08dol45LR5qcb0lYi1BQbZa74,3896
+nv_ingest_api/internal/schemas/extract/extract_table_schema.py,sha256=ZMH0aR4fl4BMP5ZJq_J6R9Bq5VuHPxaS76bm9cMZn84,4438
 nv_ingest_api/internal/schemas/message_brokers/__init__.py,sha256=uLsBITo_XfgbwpzqXUm1IYX6XlZrTfx6T1cIhdILwG8,140
 nv_ingest_api/internal/schemas/message_brokers/message_broker_client_schema.py,sha256=4xTSFE_vH7yZE9RRJRflFAG9hNXIaF6K020M_xA7ylw,1351
 nv_ingest_api/internal/schemas/message_brokers/request_schema.py,sha256=LZX_wXDxTamVFqTQs2Yd8uvWyPE5mddHAWSU4PtfEIQ,966
 nv_ingest_api/internal/schemas/message_brokers/response_schema.py,sha256=4b275HlzBSzpmuE2wdoeaGKPCdKki3wuWldtRIfrj8w,727
 nv_ingest_api/internal/schemas/meta/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest_api/internal/schemas/meta/base_model_noext.py,sha256=8hXU1uuiqZ6t8EsoZ8vlC5EFf2zSZrKEX133FcfZMwI,316
-nv_ingest_api/internal/schemas/meta/ingest_job_schema.py,sha256=cIpoesvIs0dR6s8dGjGHL246k5kf7hDmdhA48i8Si7s,10253
-nv_ingest_api/internal/schemas/meta/metadata_schema.py,sha256=FDD6yq-QxW8yDwn0Bq6bmWakX41ABMn3cytrvCbT-Po,11961
+nv_ingest_api/internal/schemas/meta/ingest_job_schema.py,sha256=auvKHFJm9FquYRS6Ro7GawvgNhszT-1uG3ADMy4E_B8,12240
+nv_ingest_api/internal/schemas/meta/metadata_schema.py,sha256=nHS2PwYE7YwuTUotvUd0hP8a-5f9uefy6_G3mMH4UyQ,12321
 nv_ingest_api/internal/schemas/meta/udf.py,sha256=GgzqbZOlipQgMpDhbXLqbF8xrHenj_hMNqhR_P-1ynw,779
 nv_ingest_api/internal/schemas/mutate/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest_api/internal/schemas/mutate/mutate_image_dedup_schema.py,sha256=k1JOdlPPpsipc0XhHf-9YxJ_-W0HvpVE1ZhYmr7fzj0,395
@@ -95,14 +96,14 @@ nv_ingest_api/internal/schemas/store/store_image_schema.py,sha256=p2LGij9i6sG6RY
 nv_ingest_api/internal/schemas/transform/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest_api/internal/schemas/transform/transform_image_caption_schema.py,sha256=fRMRwcWP-L8sfv2enNDt_W_CL0eC2i3b_1VCCtmr1K8,1188
 nv_ingest_api/internal/schemas/transform/transform_image_filter_schema.py,sha256=31ThI5fr0yyENeJeE1xMAA-pxk1QVJLwM842zMate_k,429
-nv_ingest_api/internal/schemas/transform/transform_text_embedding_schema.py,sha256=rzdhRANCqG9mOEoLargznuBwj1-MbEQUu2LDVi5vl50,1616
+nv_ingest_api/internal/schemas/transform/transform_text_embedding_schema.py,sha256=gWED1Q861onhSmDIESX1ZG1BCyelhzNTepyb5ZZuFXc,1738
 nv_ingest_api/internal/schemas/transform/transform_text_splitter_schema.py,sha256=D9K8tvu-tkEBQkZo7uuRzgrHdGyM3ZcNycHbHy5HV2E,791
 nv_ingest_api/internal/store/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest_api/internal/store/embed_text_upload.py,sha256=maxb4FPsBvWgvlrjAPEBlRZEFdJX5NxPG-p8kUbzV7I,9898
 nv_ingest_api/internal/store/image_upload.py,sha256=GNlY4k3pfcHv3lzXxkbmGLeHFsf9PI25bkBn6Xn9h3I,9654
 nv_ingest_api/internal/transform/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest_api/internal/transform/caption_image.py,sha256=0ILCG2F8ESqKtZiPUM-6F1BHUflFZ76Dzi2GNzkE-lU,8517
-nv_ingest_api/internal/transform/embed_text.py,sha256=LB_2Zvw6plc7uOWT2QN13aDu2qFumXzl_RB3ZcZbLGs,20191
+nv_ingest_api/internal/transform/embed_text.py,sha256=F3G1zVFDJMYYZkly7bb6w5bgbUed9sv5sDd02JOF3no,23163
 nv_ingest_api/internal/transform/split_text.py,sha256=LAtInGVuydH43UwjNMQWFVC1A6NdhXP_dZup2xX4qEo,7745
 nv_ingest_api/util/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest_api/util/control_message/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -115,7 +116,7 @@ nv_ingest_api/util/converters/dftools.py,sha256=FjHjazIeiUd1LdFwWuummJmraqZe1a90
 nv_ingest_api/util/converters/formats.py,sha256=L11FtormO2SeHSebbwsGE_uuCv6Jk0D3VvVW2avU0vI,2258
 nv_ingest_api/util/converters/type_mappings.py,sha256=5TVXRyU6BlQvFOdqknEuQw3ss4PXeCvSUynJnjvgQpA,1102
 nv_ingest_api/util/dataloader/__init__.py,sha256=B6ybDORMI9IzXGdhM7w_agcVj1BNYgAlcfTA0lG5jng,308
-nv_ingest_api/util/dataloader/dataloader.py,sha256=r_TU-RfdYerl3k3jRsGIVByxejwz-UQDuallx5-YAGM,14790
+nv_ingest_api/util/dataloader/dataloader.py,sha256=1SG0cHKo7X_eBRTVMJ9EFJOfpqe37QqfEoMxeAWxkEU,15124
 nv_ingest_api/util/detectors/__init__.py,sha256=HIHfzSig66GT0Uk8qsGBm_f13fKYcPtItBicRUWOOVA,183
 nv_ingest_api/util/detectors/language.py,sha256=TvzcESYY0bn0U4aLY6GjB4VaCWA6XrXxAGZbVzHTMuE,965
 nv_ingest_api/util/exception_handlers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -128,7 +129,7 @@ nv_ingest_api/util/image_processing/__init__.py,sha256=Jiy8C1ZuSrNb_eBM1ZTV9IKFI
 nv_ingest_api/util/image_processing/clustering.py,sha256=sUGlZI4cx1q8h4Pns1N9JVpdfSM2BOH8zRmn9QFCtzI,9236
 nv_ingest_api/util/image_processing/processing.py,sha256=LSoDDEmahr7a-qSS12McVcowRe3dOrAZwa1h-PD_JPQ,6554
 nv_ingest_api/util/image_processing/table_and_chart.py,sha256=idCIjiLkY-usI2EARchg3omWLtIYmYA-1tdUUV2lbno,16338
-nv_ingest_api/util/image_processing/transforms.py,sha256=ygIBf-EWm4mqi1qcnLb-l6TZps8gjotZakxj8ktxdYU,27730
+nv_ingest_api/util/image_processing/transforms.py,sha256=Mj0ry3DzCKY83ZNfvNuAIQBSkRvsxHPe0VAHRJb2BfA,30136
 nv_ingest_api/util/imports/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
 nv_ingest_api/util/imports/callable_signatures.py,sha256=ipzXNZJpfu7oeTBrQz2h6zrFVIQaqb2KBpzSuIX3u-Y,4138
 nv_ingest_api/util/imports/dynamic_resolvers.py,sha256=qy7RpmBZrXJarOQl3J7jiCKnbZMNChXTL_Z-H4c9zlc,6170
@@ -139,14 +140,15 @@ nv_ingest_api/util/logging/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 nv_ingest_api/util/logging/configuration.py,sha256=05KR3LOS-PCqU-Io__iiKG_Ds730eKxciklFfNeId3w,3126
 nv_ingest_api/util/logging/sanitize.py,sha256=-dIbmvLTevrTRd18QKUQQMV4hBk6pStWP_7_VtDDctg,2584
 nv_ingest_api/util/message_brokers/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
+nv_ingest_api/util/message_brokers/qos_scheduler.py,sha256=TdpjRyUfqR9y1v9SNxZaIN9ZgxVMlEGvLFhfUD7jjO8,10339
 nv_ingest_api/util/message_brokers/simple_message_broker/__init__.py,sha256=WaQ3CWIpIKWEivT5kL-bkmzcSQKLGFNFHdXHUJjqZFs,325
 nv_ingest_api/util/message_brokers/simple_message_broker/broker.py,sha256=PekxaxVcAa9k1wgUtozlr04SW3sAeqYJE-wdVBZf9eo,17264
 nv_ingest_api/util/message_brokers/simple_message_broker/ordered_message_queue.py,sha256=3p-LRqG8qLnsfEhBNf73_DG22C08JKahTqUvPLS2Apg,2554
-nv_ingest_api/util/message_brokers/simple_message_broker/simple_client.py,sha256=CCRAbq2EBH2quX9UTfuBbz3tTMDnWqhEF33roFwbyuk,16484
+nv_ingest_api/util/message_brokers/simple_message_broker/simple_client.py,sha256=BFuegFsbU_YB_98gzhs8oU2by4_iVzIDanT0nJdjJ7g,16517
 nv_ingest_api/util/metadata/__init__.py,sha256=HIHfzSig66GT0Uk8qsGBm_f13fKYcPtItBicRUWOOVA,183
 nv_ingest_api/util/metadata/aggregators.py,sha256=YYdvJ1E04eGFZKKHUxXoH6mzLg8nor9Smvnv0qzqK5w,15988
 nv_ingest_api/util/multi_processing/__init__.py,sha256=4fojP8Rp_5Hu1YAkqGylqTyEZ-HBVVEunn5Z9I99swA,242
-nv_ingest_api/util/multi_processing/mp_pool_singleton.py,sha256=dTfP82DgGPaXEJH3jywTO8rNlLZUniD4FFzwv84_giE,7372
+nv_ingest_api/util/multi_processing/mp_pool_singleton.py,sha256=34O7I8Lin5GvO_zNZGbsqEGkDvIbqy_0Eh3ejoPNDVE,7501
 nv_ingest_api/util/nim/__init__.py,sha256=No45pMstom1Jo0EENT6VEFkZn3YmTha7lYaBZU7xtHk,2116
 nv_ingest_api/util/pdf/__init__.py,sha256=uLsBITo_XfgbwpzqXUm1IYX6XlZrTfx6T1cIhdILwG8,140
 nv_ingest_api/util/pdf/pdfium.py,sha256=1aPCnPKXHWnncYoMO8HllYjrhODSXIeRBIsSLDevpYs,15667
@@ -156,18 +158,18 @@ nv_ingest_api/util/service_clients/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusX
 nv_ingest_api/util/service_clients/client_base.py,sha256=eCOeq3Rr6Xnnsh-oHszYlQTOffQyzsT8s43V4V8H_h8,2716
 nv_ingest_api/util/service_clients/kafka/__init__.py,sha256=uLsBITo_XfgbwpzqXUm1IYX6XlZrTfx6T1cIhdILwG8,140
 nv_ingest_api/util/service_clients/redis/__init__.py,sha256=wQSlVx3T14ZgQAt-EPzEczQusXVW0W8yynnUaFFGE3s,143
-nv_ingest_api/util/service_clients/redis/redis_client.py,sha256=3NLecvIvVN1v-sA7d7G-_f6qJVZyfJE2H8Iu5KG3Aew,37417
+nv_ingest_api/util/service_clients/redis/redis_client.py,sha256=b7rqJKYW27lmuSjTTho1sO2-q093cfeXARx8JgCHZ-o,44042
 nv_ingest_api/util/service_clients/rest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-nv_ingest_api/util/service_clients/rest/rest_client.py,sha256=dZ-jrk7IK7oNtHoXFSNTf7psoOpLREiLN5ezpHFW0HI,21732
+nv_ingest_api/util/service_clients/rest/rest_client.py,sha256=7ymPxhuN9SP8nPSVepqqbvUxXPaTVunq2aC2bDbg98g,23684
 nv_ingest_api/util/string_processing/__init__.py,sha256=mkwHthyS-IILcLcL1tJYeF6mpqX3pxEw5aUzDGjTSeU,1411
 nv_ingest_api/util/string_processing/configuration.py,sha256=2HS08msccuPCT0fn_jfXRo9_M6hCZ59OxKLxG_47HRY,29888
-nv_ingest_api/util/string_processing/yaml.py,sha256=6SW2O6wbXRhGbhETMbtXjYCZn53HeCNOP6a96AaxlHs,1454
+nv_ingest_api/util/string_processing/yaml.py,sha256=4Zdmc4474lUZn6kznqaNTlQJwsmRnnJQZ-DvAWLu-zo,2678
 nv_ingest_api/util/system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nv_ingest_api/util/system/hardware_info.py,sha256=1UFM8XE6M3pgQcpbVsCsqDQ7Dj-zzptL-XRE-DEu9UA,27213
-nv_ingest_api-2025.10.4.dev20251004.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+nv_ingest_api-2025.11.2.dev20251102.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
 udfs/__init__.py,sha256=pXFqPgXIUqHDfj7SAR1Q19tt8KwGv_iMvhHyziz4AYM,205
-udfs/llm_summarizer_udf.py,sha256=sIMfcH4GRyciTKUtq4dmhd6fZmAp07X32irIC4k7nEI,7316
-nv_ingest_api-2025.10.4.dev20251004.dist-info/METADATA,sha256=we6OaOQesMnj6J87Fg0W5ZdYcsJ6zOE-30dbrVnoLrI,14085
-nv_ingest_api-2025.10.4.dev20251004.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nv_ingest_api-2025.10.4.dev20251004.dist-info/top_level.txt,sha256=I1lseG9FF0CH93SPx4kFblsxFuv190cfzaas_CLNIiw,19
-nv_ingest_api-2025.10.4.dev20251004.dist-info/RECORD,,
+udfs/llm_summarizer_udf.py,sha256=lH5c5NHoT-5ecHC3og_40u1Ujta8SpsKU4X0e4wzbMU,7314
+nv_ingest_api-2025.11.2.dev20251102.dist-info/METADATA,sha256=tE2TOo_c9GVqOG_deBDe46ps5gTi2Mr5_l1FupFhG2I,14105
+nv_ingest_api-2025.11.2.dev20251102.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nv_ingest_api-2025.11.2.dev20251102.dist-info/top_level.txt,sha256=I1lseG9FF0CH93SPx4kFblsxFuv190cfzaas_CLNIiw,19
+nv_ingest_api-2025.11.2.dev20251102.dist-info/RECORD,,

udfs/llm_summarizer_udf.py

@@ -2,22 +2,39 @@
 """
 LLM Content Summarizer UDF for NV-Ingest Pipeline
 
-This UDF uses an LLM API to generate concise summaries
-of text content chunks, adding AI-generated summaries to the metadata for
-enhanced downstream processing and search capabilities.
+Generates document summaries using NVIDIA-hosted LLMs. This production UDF demonstrates how to extract the pipeline
+payload, run custom code (summarization), and inject results into the metadata for downstream usecases (such as
+retrieval).
 
-Environment Variables:
+These variables can be set in the environment before running the pipeline. These can be treated as kwargs.
 - NVIDIA_API_KEY: API key for NVIDIA NIM endpoints (required)
 - LLM_SUMMARIZATION_MODEL: Model to use (default: nvidia/llama-3.1-nemotron-70b-instruct)
-- LLM_SUMMARIZATION_BASE_URL: API base URL (default: https://integrate.api.nvidia.com/v1)
-- LLM_SUMMARIZATION_TIMEOUT: API timeout in seconds (default: 60)
-- LLM_MIN_CONTENT_LENGTH: Minimum content length to summarize (default: 50)
-- LLM_MAX_CONTENT_LENGTH: Maximum content length to send to API (default: 12000)
+- LLM_BASE_URL: base URL (default: https://integrate.api.nvidia.com/v1)
+- TIMEOUT: API timeout in seconds (default: 60)
+- MIN_CONTENT_LENGTH: Minimum content length to summarize (default: 50)
+- MAX_CONTENT_LENGTH: Maximum content length to send to API (default: 12000)
+TODO: Implement this
+- NUM_CHUNKS: (Optional) Number of first and last pages to summarize. default=1
+
+More info can be found in `examples/udfs/README.md`
 """
 
-import os
 import logging
-from typing import Optional
+import os
+import time
+
+
+logger = logging.getLogger(__name__)
+
+PROMPT = """
+Here are the contents from the first and last page of a document. Focus on the main purpose, key topics,
+and important details. Just return the summary as a paragraph. Do not add special characters for formatting.
+This summary will be used for document search and understanding.
+
+[CONTENT]
+{content}
+[END CONTENT]
+"""
 
 
 def content_summarizer(control_message: "IngestControlMessage") -> "IngestControlMessage":  # noqa: F821
@@ -27,13 +44,6 @@ def content_summarizer(control_message: "IngestControlMessage") -> "IngestContro
     This function processes text primitives and generates concise summaries using
     an LLM API, storing the results in the metadata's custom_content field.
 
-    Features:
-    - Flexible content detection across multiple metadata locations
-    - Robust error handling with graceful fallbacks
-    - Comprehensive logging for monitoring and debugging
-    - Configurable content length thresholds
-    - Safe metadata manipulation preserving existing data
-
     Parameters
     ----------
     control_message : IngestControlMessage
@@ -44,167 +54,152 @@ def content_summarizer(control_message: "IngestControlMessage") -> "IngestContro
     IngestControlMessage
         The modified control message with LLM summaries added to metadata
     """
-    from openai import OpenAI
-
-    logger = logging.getLogger(__name__)
     logger.info("UDF: Starting LLM content summarization")
 
-    # Get configuration from environment
-    api_key = os.getenv("NVIDIA_API_KEY", "")
+    api_key = os.getenv("NVIDIA_API_KEY")
     model_name = os.getenv("LLM_SUMMARIZATION_MODEL", "nvidia/llama-3.1-nemotron-70b-instruct")
     base_url = os.getenv("LLM_SUMMARIZATION_BASE_URL", "https://integrate.api.nvidia.com/v1")
-    timeout = int(os.getenv("LLM_SUMMARIZATION_TIMEOUT", "60"))
-    min_content_length = int(os.getenv("LLM_MIN_CONTENT_LENGTH", "50"))
-    max_content_length = int(os.getenv("LLM_MAX_CONTENT_LENGTH", "12000"))
+    min_content_length = int(os.getenv("LLM_MIN_CONTENT_LENGTH", 50))
+    max_content_length = int(os.getenv("LLM_MAX_CONTENT_LENGTH", 12000))
+    timeout = int(os.getenv("LLM_SUMMARIZATION_TIMEOUT", 60))
+
+    stats = {
+        "skipped": False,
+        "failed": False,
+        "tokens": 0,
+        "duration": 0.0,
+    }
 
     if not api_key:
-        logger.warning("NVIDIA_API_KEY not found, skipping summarization")
+        logger.error("NVIDIA_API_KEY not set. Skipping...")
         return control_message
 
-    # Get the DataFrame payload
     df = control_message.payload()
-    if df is None or len(df) == 0:
-        logger.warning("No payload found in control message")
-        return control_message
-
-    logger.info(f"Processing {len(df)} rows for LLM summarization")
 
-    # Initialize OpenAI client with error handling
-    try:
-        client = OpenAI(base_url=base_url, api_key=api_key, timeout=timeout)
-    except Exception as e:
-        logger.error(f"Failed to initialize OpenAI client: {e}")
+    if df is None or df.empty:
+        logger.warning("No payload found. Nothing to summarize.")
         return control_message
 
-    # Stats for reporting
-    stats = {"processed": 0, "summarized": 0, "skipped": 0, "failed": 0}
-
-    # Process each row
-    for idx, row in df.iterrows():
-        stats["processed"] += 1
-
-        try:
-            # Extract content - be more flexible about where it comes from
-            content = _extract_content(row, logger)
-
-            if not content:
-                stats["skipped"] += 1
-                continue
-
-            content = content.strip()
-            if len(content) < min_content_length:
-                stats["skipped"] += 1
-                continue
-
-            # Truncate if needed
-            if len(content) > max_content_length:
-                content = content[:max_content_length]
-
-            # Generate summary
-            summary = _generate_summary(client, content, model_name, logger)
+    # Select first and last chunk for summarization
+    # According to docs/docs/extraction/user_defined_functions.md#understanding-the-dataframe-payload
+    # the rows are not necessarily pages. they are chunks of data extracted from the document. in order to select
+    # pages, it must require parsing the payload to see which chunks correspond to which pages
+    original_df = df.copy()
+    if len(df) > 1:
+        # TODO: add feature to select N first and last chunks
+        df = df.iloc[[0, -1]]
+    else:
+        logger.info("Document has only one chunk")
+
+    # Combine all content into a single string
+    content_list = df.apply(
+        _extract_content,
+        axis=1,
+        min_content_length=min_content_length,
+        max_content_length=max_content_length,
+        stats=stats,
+    )
+    content = " ".join(content_list)
 
-            if summary:
-                # Add to metadata
-                _add_summary(df, idx, row, summary, model_name, logger)
-                stats["summarized"] += 1
-            else:
-                stats["failed"] += 1
+    # Nicely ask LLM to summarize content
+    summary, stats["duration"] = _generate_llm_summary(content, model_name, base_url, api_key, timeout)
 
-        except Exception as e:
-            stats["failed"] += 1
-            logger.error(f"Row {idx}: Error processing content: {e}")
+    stats["failed"] = summary is None
+    if not stats["failed"]:
+        stats["tokens"] = _estimate_tokens(content)
+        logger.info("Summarized %d tokens in %f seconds using %s", stats["tokens"], stats["duration"], model_name)
+        _store_summary(original_df, summary, model_name)
 
-    # Update the control message with modified DataFrame
-    control_message.payload(df)
+        # Update the control message with modified DataFrame
+        control_message.payload(original_df)
 
-    logger.info(
-        f"LLM summarization complete: {stats['summarized']}/{stats['processed']} documents summarized, "
-        f"{stats['skipped']} skipped, {stats['failed']} failed"
-    )
+    else:
+        logger.warning("%s failed to summarize content", model_name)
 
     return control_message
 
 
-def _extract_content(row, logger) -> Optional[str]:
-    """Extract text content from row, trying multiple locations."""
-    content = ""
-
-    # Try different locations for content
-    if isinstance(row.get("metadata"), dict):
-        metadata = row["metadata"]
-
-        # Primary location: metadata.content
-        content = metadata.get("content", "")
+def _extract_content(row, stats: dict, min_content_length: int = 50, max_content_length: int = 12000) -> str | None:
+    """Extract text content from row"""
+    metadata = row.get("metadata")
 
-        # If no content, try other locations
-        if not content:
-            # Try in text_metadata
-            text_metadata = metadata.get("text_metadata", {})
-            content = text_metadata.get("text", "") or text_metadata.get("content", "")
-
-    # Try top-level content field
-    if not content:
-        content = row.get("content", "")
+    if isinstance(metadata, dict):
+        content = metadata.get("content")
+        if content is not None:
+            content = content.strip()
+            if len(content) < min_content_length:
+                stats["skipped"] = True
+                logger.warning(f"Content less than min={min_content_length}. Skipping...")
+                content = ""
+            elif len(content) > max_content_length:
+                logger.warning(f"Truncating content to {max_content_length} characters")
+                content = content[:max_content_length]
+        else:
+            stats["skipped"] = True
+            content = ""
 
-    if not content:
-        return None
+    else:
+        stats["skipped"] = True
+        logger.warning("No metadata found. Skipping...")
+        content = ""
 
     return content
 
 
-def _generate_summary(client, content: str, model_name: str, logger) -> Optional[str]:
-    """Generate summary with robust error handling."""
-    prompt = f"""Please provide a comprehensive 3-4 sentence summary of the following document:
-
-{content}
-
-Focus on the main purpose, key topics, and important details.
-This summary will be used for document search and understanding.
-
-Summary:"""
+def _generate_llm_summary(
+    content: str,
+    model_name: str,
+    base_url: str,
+    api_key: str,
+    timeout: int,
+) -> tuple[str | None, float]:
+    """Ask an LLM to summarize content extracted from doc."""
 
+    start_time = time.time()
     try:
+        from openai import OpenAI
+
+        client = OpenAI(base_url=base_url, api_key=api_key, timeout=timeout)
+        start_time = time.time()
         completion = client.chat.completions.create(
             model=model_name,
-            messages=[{"role": "user", "content": prompt}],
+            messages=[{"role": "user", "content": PROMPT.format(content=content)}],
             max_tokens=400,  # Increased for more comprehensive summaries
             temperature=0.7,
         )
+        duration = time.time() - start_time
 
-        if completion.choices and len(completion.choices) > 0:
+        if completion.choices:
             summary = completion.choices[0].message.content.strip()
-            return summary
-        else:
-            return None
+            return summary, duration
+        return None, duration
 
     except Exception as e:
         logger.error(f"API call failed: {e}")
-        return None
+        # TODO: GitHub Thread
+        # Reviewers, tell me if this is a bad idea.
+        # I think the convention is to return timestamp for time even if it fails
+        return None, time.time() - start_time
 
 
-def _add_summary(df, idx: int, row, summary: str, model_name: str, logger):
-    """Add summary to metadata with safe handling."""
-    try:
-        # Get current metadata or create new dict - handle None case properly
-        existing_metadata = row.get("metadata")
-        if existing_metadata is not None and isinstance(existing_metadata, dict):
-            metadata = dict(existing_metadata)  # Create a copy
-        else:
-            metadata = {}
+def _store_summary(df, summary: str, model_name: str):
+    """Add summary to metadata and store in df"""
+    # hardcoded heuristic to store everything on chunk 0's metadata
+    row_0 = df.iloc[0]
 
-        # Ensure custom_content exists
-        if "custom_content" not in metadata or metadata["custom_content"] is None:
-            metadata["custom_content"] = {}
+    # this is a reference to a dictionary that is stored in the dataframe
+    # and is modified in place
+    metadata = row_0.get("metadata")
 
-        # Add LLM summary
-        metadata["custom_content"]["llm_summary"] = {"summary": summary, "model": model_name}
+    if metadata.get("custom_content") is None:
+        metadata["custom_content"] = {}
+    metadata["custom_content"]["llm_summarizer_udf"] = {"summary": summary, "model": model_name}
 
-        # Update the DataFrame at the specific index
-        try:
-            df.at[idx, "metadata"] = metadata
-        except Exception:
-            # Alternative approach: update the original row reference
-            df.iloc[idx]["metadata"] = metadata
 
-    except Exception as e:
-        logger.error(f"Failed to add summary to row {idx}: {e}")
+def _estimate_tokens(text: str) -> int:
+    """Rough estimate (~4 characters per token)"""
+    return len(text) // 4
+
+
+def _safe_model_name(name: str) -> str:
+    return name.replace("/", "__").replace("-", "_")