nvidia-nat 1.4.0a20251112__py3-none-any.whl → 1.4.0a20260113__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aiq/__init__.py +1 -1
- nat/{front_ends/mcp → agent/auto_memory_wrapper}/__init__.py +1 -1
- nat/agent/auto_memory_wrapper/agent.py +278 -0
- nat/agent/auto_memory_wrapper/register.py +227 -0
- nat/agent/auto_memory_wrapper/state.py +30 -0
- nat/agent/base.py +1 -1
- nat/agent/dual_node.py +1 -1
- nat/agent/prompt_optimizer/prompt.py +1 -1
- nat/agent/prompt_optimizer/register.py +1 -1
- nat/agent/react_agent/agent.py +16 -9
- nat/agent/react_agent/output_parser.py +2 -2
- nat/agent/react_agent/prompt.py +3 -2
- nat/agent/react_agent/register.py +2 -2
- nat/agent/react_agent/register_per_user_agent.py +104 -0
- nat/agent/reasoning_agent/reasoning_agent.py +1 -1
- nat/agent/register.py +3 -1
- nat/agent/responses_api_agent/__init__.py +1 -1
- nat/agent/responses_api_agent/register.py +1 -1
- nat/agent/rewoo_agent/agent.py +9 -4
- nat/agent/rewoo_agent/prompt.py +1 -1
- nat/agent/rewoo_agent/register.py +1 -1
- nat/agent/tool_calling_agent/agent.py +5 -4
- nat/agent/tool_calling_agent/register.py +1 -1
- nat/authentication/__init__.py +1 -1
- nat/authentication/api_key/__init__.py +1 -1
- nat/authentication/api_key/api_key_auth_provider.py +1 -1
- nat/authentication/api_key/api_key_auth_provider_config.py +22 -7
- nat/authentication/api_key/register.py +1 -1
- nat/authentication/credential_validator/__init__.py +1 -1
- nat/authentication/credential_validator/bearer_token_validator.py +1 -1
- nat/authentication/exceptions/__init__.py +1 -1
- nat/authentication/exceptions/api_key_exceptions.py +1 -1
- nat/authentication/http_basic_auth/http_basic_auth_provider.py +1 -1
- nat/authentication/http_basic_auth/register.py +1 -1
- nat/authentication/interfaces.py +1 -1
- nat/authentication/oauth2/__init__.py +1 -1
- nat/authentication/oauth2/oauth2_auth_code_flow_provider.py +1 -1
- nat/authentication/oauth2/oauth2_auth_code_flow_provider_config.py +1 -1
- nat/authentication/oauth2/oauth2_resource_server_config.py +1 -1
- nat/authentication/oauth2/register.py +1 -1
- nat/authentication/register.py +1 -1
- nat/builder/builder.py +563 -1
- nat/builder/child_builder.py +385 -0
- nat/builder/component_utils.py +34 -4
- nat/builder/context.py +34 -1
- nat/builder/embedder.py +1 -1
- nat/builder/eval_builder.py +19 -7
- nat/builder/evaluator.py +1 -1
- nat/builder/framework_enum.py +3 -1
- nat/builder/front_end.py +1 -1
- nat/builder/function.py +113 -5
- nat/builder/function_base.py +1 -1
- nat/builder/function_info.py +1 -1
- nat/builder/intermediate_step_manager.py +1 -1
- nat/builder/llm.py +1 -1
- nat/builder/per_user_workflow_builder.py +843 -0
- nat/builder/retriever.py +1 -1
- nat/builder/sync_builder.py +571 -0
- nat/builder/user_interaction_manager.py +1 -1
- nat/builder/workflow.py +5 -3
- nat/builder/workflow_builder.py +619 -378
- nat/cli/__init__.py +1 -1
- nat/cli/cli_utils/config_override.py +1 -1
- nat/cli/cli_utils/validation.py +32 -1
- nat/cli/commands/configure/channel/add.py +1 -1
- nat/cli/commands/configure/channel/channel.py +1 -1
- nat/cli/commands/configure/channel/remove.py +1 -1
- nat/cli/commands/configure/channel/update.py +1 -1
- nat/cli/commands/configure/configure.py +1 -1
- nat/cli/commands/evaluate.py +87 -13
- nat/cli/commands/finetune.py +132 -0
- nat/cli/commands/info/__init__.py +1 -1
- nat/cli/commands/info/info.py +1 -1
- nat/cli/commands/info/list_channels.py +1 -1
- nat/cli/commands/info/list_components.py +1 -1
- nat/cli/commands/object_store/__init__.py +1 -1
- nat/cli/commands/object_store/object_store.py +1 -1
- nat/cli/commands/optimize.py +1 -1
- nat/cli/commands/{mcp → red_teaming}/__init__.py +1 -1
- nat/cli/commands/red_teaming/red_teaming.py +138 -0
- nat/cli/commands/red_teaming/red_teaming_utils.py +73 -0
- nat/cli/commands/registry/__init__.py +1 -1
- nat/cli/commands/registry/publish.py +1 -1
- nat/cli/commands/registry/pull.py +1 -1
- nat/cli/commands/registry/registry.py +1 -1
- nat/cli/commands/registry/remove.py +1 -1
- nat/cli/commands/registry/search.py +1 -1
- nat/cli/commands/sizing/__init__.py +1 -1
- nat/cli/commands/sizing/calc.py +1 -1
- nat/cli/commands/sizing/sizing.py +1 -1
- nat/cli/commands/start.py +1 -1
- nat/cli/commands/uninstall.py +1 -1
- nat/cli/commands/validate.py +1 -1
- nat/cli/commands/workflow/__init__.py +1 -1
- nat/cli/commands/workflow/workflow.py +1 -1
- nat/cli/commands/workflow/workflow_commands.py +3 -2
- nat/cli/entrypoint.py +15 -37
- nat/cli/main.py +2 -2
- nat/cli/plugin_loader.py +69 -0
- nat/cli/register_workflow.py +233 -5
- nat/cli/type_registry.py +237 -3
- nat/control_flow/register.py +1 -1
- nat/control_flow/router_agent/agent.py +1 -1
- nat/control_flow/router_agent/prompt.py +1 -1
- nat/control_flow/router_agent/register.py +1 -1
- nat/control_flow/sequential_executor.py +28 -7
- nat/data_models/__init__.py +1 -1
- nat/data_models/agent.py +1 -1
- nat/data_models/api_server.py +38 -3
- nat/data_models/authentication.py +1 -1
- nat/data_models/common.py +1 -1
- nat/data_models/component.py +9 -1
- nat/data_models/component_ref.py +45 -1
- nat/data_models/config.py +78 -1
- nat/data_models/dataset_handler.py +15 -2
- nat/data_models/discovery_metadata.py +1 -1
- nat/data_models/embedder.py +1 -1
- nat/data_models/evaluate.py +6 -1
- nat/data_models/evaluator.py +1 -1
- nat/data_models/finetuning.py +260 -0
- nat/data_models/front_end.py +1 -1
- nat/data_models/function.py +15 -2
- nat/data_models/function_dependencies.py +1 -1
- nat/data_models/gated_field_mixin.py +1 -1
- nat/data_models/interactive.py +1 -1
- nat/data_models/intermediate_step.py +29 -2
- nat/data_models/invocation_node.py +1 -1
- nat/data_models/llm.py +1 -1
- nat/data_models/logging.py +1 -1
- nat/data_models/memory.py +1 -1
- nat/data_models/middleware.py +37 -0
- nat/data_models/object_store.py +1 -1
- nat/data_models/openai_mcp.py +1 -1
- nat/data_models/optimizable.py +1 -1
- nat/data_models/optimizer.py +1 -1
- nat/data_models/profiler.py +1 -1
- nat/data_models/registry_handler.py +1 -1
- nat/data_models/retriever.py +1 -1
- nat/data_models/retry_mixin.py +1 -1
- nat/data_models/runtime_enum.py +26 -0
- nat/data_models/span.py +1 -1
- nat/data_models/step_adaptor.py +1 -1
- nat/data_models/streaming.py +1 -1
- nat/data_models/swe_bench_model.py +1 -1
- nat/data_models/telemetry_exporter.py +1 -1
- nat/data_models/thinking_mixin.py +1 -1
- nat/data_models/ttc_strategy.py +1 -1
- nat/embedder/azure_openai_embedder.py +1 -1
- nat/embedder/nim_embedder.py +1 -1
- nat/embedder/openai_embedder.py +1 -1
- nat/embedder/register.py +1 -1
- nat/eval/__init__.py +1 -1
- nat/eval/config.py +8 -1
- nat/eval/dataset_handler/dataset_downloader.py +1 -1
- nat/eval/dataset_handler/dataset_filter.py +1 -1
- nat/eval/dataset_handler/dataset_handler.py +4 -2
- nat/eval/evaluate.py +226 -81
- nat/eval/evaluator/__init__.py +1 -1
- nat/eval/evaluator/base_evaluator.py +2 -2
- nat/eval/evaluator/evaluator_model.py +3 -2
- nat/eval/intermediate_step_adapter.py +1 -1
- nat/eval/llm_validator.py +336 -0
- nat/eval/rag_evaluator/evaluate.py +17 -10
- nat/eval/rag_evaluator/register.py +1 -1
- nat/eval/red_teaming_evaluator/__init__.py +14 -0
- nat/eval/red_teaming_evaluator/data_models.py +66 -0
- nat/eval/red_teaming_evaluator/evaluate.py +327 -0
- nat/eval/red_teaming_evaluator/filter_conditions.py +75 -0
- nat/eval/red_teaming_evaluator/register.py +55 -0
- nat/eval/register.py +2 -1
- nat/eval/remote_workflow.py +1 -1
- nat/eval/runners/__init__.py +1 -1
- nat/eval/runners/config.py +1 -1
- nat/eval/runners/multi_eval_runner.py +1 -1
- nat/eval/runners/red_teaming_runner/__init__.py +24 -0
- nat/eval/runners/red_teaming_runner/config.py +282 -0
- nat/eval/runners/red_teaming_runner/report_utils.py +707 -0
- nat/eval/runners/red_teaming_runner/runner.py +867 -0
- nat/eval/runtime_evaluator/__init__.py +1 -1
- nat/eval/runtime_evaluator/evaluate.py +1 -1
- nat/eval/runtime_evaluator/register.py +1 -1
- nat/eval/runtime_event_subscriber.py +1 -1
- nat/eval/swe_bench_evaluator/evaluate.py +1 -1
- nat/eval/swe_bench_evaluator/register.py +1 -1
- nat/eval/trajectory_evaluator/evaluate.py +2 -2
- nat/eval/trajectory_evaluator/register.py +1 -1
- nat/eval/tunable_rag_evaluator/evaluate.py +5 -5
- nat/eval/tunable_rag_evaluator/register.py +1 -1
- nat/eval/usage_stats.py +1 -1
- nat/eval/utils/eval_trace_ctx.py +1 -1
- nat/eval/utils/output_uploader.py +1 -1
- nat/eval/utils/tqdm_position_registry.py +1 -1
- nat/eval/utils/weave_eval.py +1 -1
- nat/experimental/decorators/experimental_warning_decorator.py +1 -1
- nat/experimental/test_time_compute/editing/iterative_plan_refinement_editor.py +1 -1
- nat/experimental/test_time_compute/editing/llm_as_a_judge_editor.py +1 -1
- nat/experimental/test_time_compute/editing/motivation_aware_summarization.py +1 -1
- nat/experimental/test_time_compute/functions/execute_score_select_function.py +1 -1
- nat/experimental/test_time_compute/functions/multi_llm_judge_function.py +88 -0
- nat/experimental/test_time_compute/functions/plan_select_execute_function.py +1 -1
- nat/experimental/test_time_compute/functions/ttc_tool_orchestration_function.py +1 -1
- nat/experimental/test_time_compute/functions/ttc_tool_wrapper_function.py +1 -1
- nat/experimental/test_time_compute/models/editor_config.py +1 -1
- nat/experimental/test_time_compute/models/scoring_config.py +1 -1
- nat/experimental/test_time_compute/models/search_config.py +20 -2
- nat/experimental/test_time_compute/models/selection_config.py +33 -2
- nat/experimental/test_time_compute/models/stage_enums.py +1 -1
- nat/experimental/test_time_compute/models/strategy_base.py +1 -1
- nat/experimental/test_time_compute/models/tool_use_config.py +1 -1
- nat/experimental/test_time_compute/models/ttc_item.py +1 -1
- nat/experimental/test_time_compute/register.py +4 -1
- nat/experimental/test_time_compute/scoring/llm_based_agent_scorer.py +1 -1
- nat/experimental/test_time_compute/scoring/llm_based_plan_scorer.py +1 -1
- nat/experimental/test_time_compute/scoring/motivation_aware_scorer.py +1 -1
- nat/experimental/test_time_compute/search/multi_llm_generation.py +115 -0
- nat/experimental/test_time_compute/search/multi_llm_planner.py +1 -1
- nat/experimental/test_time_compute/search/multi_query_retrieval_search.py +1 -1
- nat/experimental/test_time_compute/search/single_shot_multi_plan_planner.py +1 -1
- nat/experimental/test_time_compute/selection/best_of_n_selector.py +1 -1
- nat/experimental/test_time_compute/selection/llm_based_agent_output_selector.py +1 -1
- nat/experimental/test_time_compute/selection/llm_based_output_merging_selector.py +1 -1
- nat/experimental/test_time_compute/selection/llm_based_plan_selector.py +1 -1
- nat/experimental/test_time_compute/selection/llm_judge_selection.py +127 -0
- nat/experimental/test_time_compute/selection/threshold_selector.py +1 -1
- nat/finetuning/__init__.py +24 -0
- nat/finetuning/finetuning_runtime.py +143 -0
- nat/finetuning/interfaces/__init__.py +24 -0
- nat/finetuning/interfaces/finetuning_runner.py +261 -0
- nat/finetuning/interfaces/trainer_adapter.py +103 -0
- nat/finetuning/interfaces/trajectory_builder.py +115 -0
- nat/finetuning/utils/__init__.py +15 -0
- nat/finetuning/utils/parsers/__init__.py +15 -0
- nat/finetuning/utils/parsers/adk_parser.py +141 -0
- nat/finetuning/utils/parsers/base_parser.py +238 -0
- nat/finetuning/utils/parsers/common.py +91 -0
- nat/finetuning/utils/parsers/langchain_parser.py +267 -0
- nat/finetuning/utils/parsers/llama_index_parser.py +218 -0
- nat/front_ends/__init__.py +1 -1
- nat/front_ends/console/__init__.py +1 -1
- nat/front_ends/console/authentication_flow_handler.py +1 -1
- nat/front_ends/console/console_front_end_config.py +4 -1
- nat/front_ends/console/console_front_end_plugin.py +5 -4
- nat/front_ends/console/register.py +1 -1
- nat/front_ends/cron/__init__.py +1 -1
- nat/front_ends/fastapi/__init__.py +1 -1
- nat/front_ends/fastapi/async_job.py +128 -0
- nat/front_ends/fastapi/auth_flow_handlers/http_flow_handler.py +1 -1
- nat/front_ends/fastapi/auth_flow_handlers/websocket_flow_handler.py +13 -9
- nat/front_ends/fastapi/dask_client_mixin.py +1 -1
- nat/front_ends/fastapi/fastapi_front_end_config.py +23 -1
- nat/front_ends/fastapi/fastapi_front_end_controller.py +1 -1
- nat/front_ends/fastapi/fastapi_front_end_plugin.py +25 -30
- nat/front_ends/fastapi/fastapi_front_end_plugin_worker.py +318 -59
- nat/front_ends/fastapi/html_snippets/__init__.py +1 -1
- nat/front_ends/fastapi/html_snippets/auth_code_grant_success.py +1 -1
- nat/front_ends/fastapi/intermediate_steps_subscriber.py +12 -1
- nat/front_ends/fastapi/job_store.py +23 -11
- nat/front_ends/fastapi/main.py +1 -1
- nat/front_ends/fastapi/message_handler.py +27 -4
- nat/front_ends/fastapi/message_validator.py +54 -2
- nat/front_ends/fastapi/register.py +1 -1
- nat/front_ends/fastapi/response_helpers.py +16 -15
- nat/front_ends/fastapi/step_adaptor.py +1 -1
- nat/front_ends/fastapi/utils.py +1 -1
- nat/front_ends/register.py +1 -2
- nat/front_ends/simple_base/__init__.py +1 -1
- nat/front_ends/simple_base/simple_front_end_plugin_base.py +6 -4
- nat/llm/aws_bedrock_llm.py +1 -1
- nat/llm/azure_openai_llm.py +10 -1
- nat/llm/dynamo_llm.py +363 -0
- nat/llm/huggingface_llm.py +177 -0
- nat/llm/litellm_llm.py +1 -1
- nat/llm/nim_llm.py +1 -1
- nat/llm/openai_llm.py +1 -1
- nat/llm/register.py +3 -1
- nat/llm/utils/__init__.py +1 -1
- nat/llm/utils/env_config_value.py +1 -1
- nat/llm/utils/error.py +1 -1
- nat/llm/utils/thinking.py +1 -1
- nat/memory/__init__.py +1 -1
- nat/memory/interfaces.py +1 -1
- nat/memory/models.py +1 -1
- nat/meta/pypi.md +1 -1
- nat/middleware/__init__.py +35 -0
- nat/middleware/cache/__init__.py +14 -0
- nat/middleware/cache/cache_middleware.py +253 -0
- nat/middleware/cache/cache_middleware_config.py +44 -0
- nat/middleware/cache/register.py +33 -0
- nat/middleware/defense/__init__.py +14 -0
- nat/middleware/defense/defense_middleware.py +362 -0
- nat/middleware/defense/defense_middleware_content_guard.py +455 -0
- nat/middleware/defense/defense_middleware_data_models.py +91 -0
- nat/middleware/defense/defense_middleware_output_verifier.py +440 -0
- nat/middleware/defense/defense_middleware_pii.py +356 -0
- nat/middleware/defense/register.py +82 -0
- nat/middleware/dynamic/__init__.py +14 -0
- nat/middleware/dynamic/dynamic_function_middleware.py +962 -0
- nat/middleware/dynamic/dynamic_middleware_config.py +132 -0
- nat/middleware/dynamic/register.py +34 -0
- nat/middleware/function_middleware.py +370 -0
- nat/middleware/logging/__init__.py +14 -0
- nat/middleware/logging/logging_middleware.py +67 -0
- nat/middleware/logging/logging_middleware_config.py +28 -0
- nat/middleware/logging/register.py +33 -0
- nat/middleware/middleware.py +298 -0
- nat/middleware/red_teaming/__init__.py +14 -0
- nat/middleware/red_teaming/red_teaming_middleware.py +344 -0
- nat/middleware/red_teaming/red_teaming_middleware_config.py +112 -0
- nat/middleware/red_teaming/register.py +47 -0
- nat/middleware/register.py +22 -0
- nat/middleware/utils/__init__.py +14 -0
- nat/middleware/utils/workflow_inventory.py +155 -0
- nat/object_store/__init__.py +1 -1
- nat/object_store/in_memory_object_store.py +1 -1
- nat/object_store/interfaces.py +1 -1
- nat/object_store/models.py +1 -1
- nat/object_store/register.py +1 -1
- nat/observability/__init__.py +1 -1
- nat/observability/exporter/__init__.py +1 -1
- nat/observability/exporter/base_exporter.py +1 -1
- nat/observability/exporter/exporter.py +1 -1
- nat/observability/exporter/file_exporter.py +1 -1
- nat/observability/exporter/processing_exporter.py +1 -1
- nat/observability/exporter/raw_exporter.py +1 -1
- nat/observability/exporter/span_exporter.py +7 -1
- nat/observability/exporter_manager.py +1 -1
- nat/observability/mixin/__init__.py +1 -1
- nat/observability/mixin/batch_config_mixin.py +1 -1
- nat/observability/mixin/collector_config_mixin.py +1 -1
- nat/observability/mixin/file_mixin.py +1 -1
- nat/observability/mixin/file_mode.py +1 -1
- nat/observability/mixin/redaction_config_mixin.py +1 -1
- nat/observability/mixin/resource_conflict_mixin.py +1 -1
- nat/observability/mixin/serialize_mixin.py +1 -1
- nat/observability/mixin/tagging_config_mixin.py +1 -1
- nat/observability/mixin/type_introspection_mixin.py +1 -1
- nat/observability/processor/__init__.py +1 -1
- nat/observability/processor/batching_processor.py +1 -1
- nat/observability/processor/callback_processor.py +1 -1
- nat/observability/processor/falsy_batch_filter_processor.py +1 -1
- nat/observability/processor/intermediate_step_serializer.py +1 -1
- nat/observability/processor/processor.py +1 -1
- nat/observability/processor/processor_factory.py +1 -1
- nat/observability/processor/redaction/__init__.py +1 -1
- nat/observability/processor/redaction/contextual_redaction_processor.py +1 -1
- nat/observability/processor/redaction/contextual_span_redaction_processor.py +1 -1
- nat/observability/processor/redaction/redaction_processor.py +1 -1
- nat/observability/processor/redaction/span_header_redaction_processor.py +1 -1
- nat/observability/processor/span_tagging_processor.py +1 -1
- nat/observability/register.py +1 -1
- nat/observability/utils/__init__.py +1 -1
- nat/observability/utils/dict_utils.py +1 -1
- nat/observability/utils/time_utils.py +1 -1
- nat/profiler/calc/__init__.py +1 -1
- nat/profiler/calc/calc_runner.py +3 -3
- nat/profiler/calc/calculations.py +1 -1
- nat/profiler/calc/data_models.py +1 -1
- nat/profiler/calc/plot.py +30 -3
- nat/profiler/callbacks/agno_callback_handler.py +1 -1
- nat/profiler/callbacks/base_callback_class.py +1 -1
- nat/profiler/callbacks/langchain_callback_handler.py +33 -3
- nat/profiler/callbacks/llama_index_callback_handler.py +13 -10
- nat/profiler/callbacks/semantic_kernel_callback_handler.py +1 -1
- nat/profiler/callbacks/token_usage_base_model.py +1 -1
- nat/profiler/data_frame_row.py +1 -1
- nat/profiler/data_models.py +1 -1
- nat/profiler/decorators/framework_wrapper.py +32 -1
- nat/profiler/decorators/function_tracking.py +1 -1
- nat/profiler/forecasting/config.py +1 -1
- nat/profiler/forecasting/model_trainer.py +1 -1
- nat/profiler/forecasting/models/__init__.py +1 -1
- nat/profiler/forecasting/models/forecasting_base_model.py +1 -1
- nat/profiler/forecasting/models/linear_model.py +1 -1
- nat/profiler/forecasting/models/random_forest_regressor.py +1 -1
- nat/profiler/inference_metrics_model.py +1 -1
- nat/profiler/inference_optimization/bottleneck_analysis/nested_stack_analysis.py +1 -1
- nat/profiler/inference_optimization/bottleneck_analysis/simple_stack_analysis.py +1 -1
- nat/profiler/inference_optimization/data_models.py +1 -1
- nat/profiler/inference_optimization/experimental/concurrency_spike_analysis.py +1 -1
- nat/profiler/inference_optimization/experimental/prefix_span_analysis.py +1 -1
- nat/profiler/inference_optimization/llm_metrics.py +1 -1
- nat/profiler/inference_optimization/prompt_caching.py +1 -1
- nat/profiler/inference_optimization/token_uniqueness.py +1 -1
- nat/profiler/inference_optimization/workflow_runtimes.py +1 -1
- nat/profiler/intermediate_property_adapter.py +1 -1
- nat/profiler/parameter_optimization/optimizable_utils.py +1 -1
- nat/profiler/parameter_optimization/optimizer_runtime.py +1 -1
- nat/profiler/parameter_optimization/parameter_optimizer.py +1 -1
- nat/profiler/parameter_optimization/parameter_selection.py +1 -1
- nat/profiler/parameter_optimization/pareto_visualizer.py +1 -1
- nat/profiler/parameter_optimization/prompt_optimizer.py +1 -1
- nat/profiler/parameter_optimization/update_helpers.py +1 -1
- nat/profiler/profile_runner.py +1 -1
- nat/profiler/utils.py +1 -1
- nat/registry_handlers/local/local_handler.py +1 -1
- nat/registry_handlers/local/register_local.py +1 -1
- nat/registry_handlers/metadata_factory.py +1 -1
- nat/registry_handlers/package_utils.py +1 -1
- nat/registry_handlers/pypi/pypi_handler.py +1 -1
- nat/registry_handlers/pypi/register_pypi.py +1 -1
- nat/registry_handlers/register.py +1 -1
- nat/registry_handlers/registry_handler_base.py +1 -1
- nat/registry_handlers/rest/register_rest.py +1 -1
- nat/registry_handlers/rest/rest_handler.py +1 -1
- nat/registry_handlers/schemas/headers.py +1 -1
- nat/registry_handlers/schemas/package.py +1 -1
- nat/registry_handlers/schemas/publish.py +1 -1
- nat/registry_handlers/schemas/pull.py +1 -1
- nat/registry_handlers/schemas/remove.py +1 -1
- nat/registry_handlers/schemas/search.py +1 -1
- nat/registry_handlers/schemas/status.py +1 -1
- nat/retriever/interface.py +1 -1
- nat/retriever/milvus/__init__.py +1 -1
- nat/retriever/milvus/register.py +12 -4
- nat/retriever/milvus/retriever.py +103 -41
- nat/retriever/models.py +1 -1
- nat/retriever/nemo_retriever/__init__.py +1 -1
- nat/retriever/nemo_retriever/register.py +1 -1
- nat/retriever/nemo_retriever/retriever.py +5 -5
- nat/retriever/register.py +1 -1
- nat/runtime/__init__.py +1 -1
- nat/runtime/loader.py +10 -3
- nat/runtime/metrics.py +180 -0
- nat/runtime/runner.py +13 -6
- nat/runtime/session.py +458 -32
- nat/runtime/user_metadata.py +1 -1
- nat/settings/global_settings.py +1 -1
- nat/tool/chat_completion.py +1 -1
- nat/tool/code_execution/README.md +1 -1
- nat/tool/code_execution/code_sandbox.py +2 -2
- nat/tool/code_execution/local_sandbox/Dockerfile.sandbox +1 -1
- nat/tool/code_execution/local_sandbox/__init__.py +1 -1
- nat/tool/code_execution/local_sandbox/local_sandbox_server.py +1 -1
- nat/tool/code_execution/local_sandbox/start_local_sandbox.sh +1 -1
- nat/tool/code_execution/register.py +1 -1
- nat/tool/code_execution/utils.py +1 -1
- nat/tool/datetime_tools.py +1 -1
- nat/tool/document_search.py +1 -1
- nat/tool/github_tools.py +1 -1
- nat/tool/memory_tools/add_memory_tool.py +1 -1
- nat/tool/memory_tools/delete_memory_tool.py +1 -1
- nat/tool/memory_tools/get_memory_tool.py +1 -1
- nat/tool/nvidia_rag.py +2 -2
- nat/tool/register.py +1 -1
- nat/tool/retriever.py +1 -1
- nat/tool/server_tools.py +1 -1
- nat/utils/__init__.py +8 -5
- nat/utils/callable_utils.py +1 -1
- nat/utils/data_models/schema_validator.py +1 -1
- nat/utils/debugging_utils.py +1 -1
- nat/utils/decorators.py +1 -1
- nat/utils/dump_distro_mapping.py +1 -1
- nat/utils/exception_handlers/automatic_retries.py +3 -3
- nat/utils/exception_handlers/schemas.py +1 -1
- nat/utils/io/model_processing.py +1 -1
- nat/utils/io/supress_logs.py +33 -0
- nat/utils/io/yaml_tools.py +1 -1
- nat/utils/log_levels.py +1 -1
- nat/utils/log_utils.py +13 -1
- nat/utils/metadata_utils.py +1 -1
- nat/utils/optional_imports.py +1 -1
- nat/utils/producer_consumer_queue.py +1 -1
- nat/utils/reactive/base/observable_base.py +1 -1
- nat/utils/reactive/base/observer_base.py +1 -1
- nat/utils/reactive/base/subject_base.py +1 -1
- nat/utils/reactive/observable.py +1 -1
- nat/utils/reactive/observer.py +1 -1
- nat/utils/reactive/subject.py +1 -1
- nat/utils/reactive/subscription.py +1 -1
- nat/utils/responses_api.py +1 -1
- nat/utils/settings/global_settings.py +1 -1
- nat/utils/string_utils.py +1 -1
- nat/utils/type_converter.py +18 -5
- nat/utils/type_utils.py +1 -1
- nat/utils/url_utils.py +1 -1
- {nvidia_nat-1.4.0a20251112.dist-info → nvidia_nat-1.4.0a20260113.dist-info}/METADATA +46 -15
- nvidia_nat-1.4.0a20260113.dist-info/RECORD +547 -0
- nvidia_nat-1.4.0a20260113.dist-info/entry_points.txt +38 -0
- nat/cli/commands/mcp/mcp.py +0 -986
- nat/front_ends/mcp/introspection_token_verifier.py +0 -73
- nat/front_ends/mcp/mcp_front_end_config.py +0 -109
- nat/front_ends/mcp/mcp_front_end_plugin.py +0 -151
- nat/front_ends/mcp/mcp_front_end_plugin_worker.py +0 -362
- nat/front_ends/mcp/memory_profiler.py +0 -320
- nat/front_ends/mcp/register.py +0 -27
- nat/front_ends/mcp/tool_converter.py +0 -321
- nvidia_nat-1.4.0a20251112.dist-info/RECORD +0 -481
- nvidia_nat-1.4.0a20251112.dist-info/entry_points.txt +0 -22
- {nvidia_nat-1.4.0a20251112.dist-info → nvidia_nat-1.4.0a20260113.dist-info}/WHEEL +0 -0
- {nvidia_nat-1.4.0a20251112.dist-info → nvidia_nat-1.4.0a20260113.dist-info}/licenses/LICENSE-3rd-party.txt +0 -0
- {nvidia_nat-1.4.0a20251112.dist-info → nvidia_nat-1.4.0a20260113.dist-info}/licenses/LICENSE.md +0 -0
- {nvidia_nat-1.4.0a20251112.dist-info → nvidia_nat-1.4.0a20260113.dist-info}/top_level.txt +0 -0
nat/runtime/session.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
2
|
# SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -14,15 +14,20 @@
|
|
|
14
14
|
# limitations under the License.
|
|
15
15
|
|
|
16
16
|
import asyncio
|
|
17
|
-
import
|
|
17
|
+
import logging
|
|
18
|
+
import time
|
|
18
19
|
import typing
|
|
19
20
|
import uuid
|
|
20
21
|
from collections.abc import Awaitable
|
|
21
22
|
from collections.abc import Callable
|
|
22
23
|
from contextlib import asynccontextmanager
|
|
23
24
|
from contextlib import nullcontext
|
|
25
|
+
from datetime import datetime
|
|
24
26
|
|
|
25
27
|
from fastapi import WebSocket
|
|
28
|
+
from pydantic import BaseModel
|
|
29
|
+
from pydantic import ConfigDict
|
|
30
|
+
from pydantic import Field
|
|
26
31
|
from starlette.requests import HTTPConnection
|
|
27
32
|
from starlette.requests import Request
|
|
28
33
|
|
|
@@ -35,62 +40,394 @@ from nat.data_models.authentication import AuthProviderBaseConfig
|
|
|
35
40
|
from nat.data_models.config import Config
|
|
36
41
|
from nat.data_models.interactive import HumanResponse
|
|
37
42
|
from nat.data_models.interactive import InteractionPrompt
|
|
43
|
+
from nat.data_models.runtime_enum import RuntimeTypeEnum
|
|
38
44
|
|
|
39
|
-
|
|
45
|
+
if typing.TYPE_CHECKING:
|
|
46
|
+
from nat.builder.per_user_workflow_builder import PerUserWorkflowBuilder
|
|
47
|
+
from nat.builder.workflow_builder import WorkflowBuilder
|
|
40
48
|
|
|
49
|
+
logger = logging.getLogger(__name__)
|
|
41
50
|
|
|
42
|
-
|
|
43
|
-
|
|
51
|
+
|
|
52
|
+
class PerUserBuilderInfo(BaseModel):
|
|
53
|
+
"""
|
|
54
|
+
Container for per-user builder data with activity tracking.
|
|
55
|
+
|
|
56
|
+
Tracks lifecycle and usage of per-user builders for automatic cleanup.
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
model_config = ConfigDict(arbitrary_types_allowed=True, validate_assignment=False)
|
|
60
|
+
|
|
61
|
+
builder: typing.Any = Field(description="The per-user workflow builder instance")
|
|
62
|
+
workflow: typing.Any = Field(description="The cached per-user workflow instance")
|
|
63
|
+
semaphore: typing.Any = Field(description="Per-user semaphore for concurrency control")
|
|
64
|
+
last_activity: datetime = Field(default_factory=datetime.now,
|
|
65
|
+
description="The timestamp of the last access to this builder")
|
|
66
|
+
ref_count: int = Field(default=0, ge=0, description="The reference count of this builder")
|
|
67
|
+
lock: asyncio.Lock = Field(default_factory=asyncio.Lock, description="Lock for thread-safe ref_count updates")
|
|
68
|
+
|
|
69
|
+
# Monitoring metrics
|
|
70
|
+
created_at: datetime = Field(default_factory=datetime.now, description="When the per-user workflow was created")
|
|
71
|
+
total_requests: int = Field(default=0, ge=0, description="Total number of requests processed")
|
|
72
|
+
error_count: int = Field(default=0, ge=0, description="Total number of failed requests")
|
|
73
|
+
total_latency_ms: float = Field(default=0.0, ge=0, description="Total latency of all requests in milliseconds")
|
|
74
|
+
|
|
75
|
+
def record_request(self, latency_ms: float, success: bool) -> None:
|
|
76
|
+
"""Record metrics for a completed request.
|
|
77
|
+
|
|
78
|
+
Args:
|
|
79
|
+
latency_ms: Request latency in milliseconds
|
|
80
|
+
success: Whether the request was successful
|
|
81
|
+
"""
|
|
82
|
+
self.total_requests += 1
|
|
83
|
+
self.total_latency_ms += latency_ms
|
|
84
|
+
if not success:
|
|
85
|
+
self.error_count += 1
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class Session:
|
|
89
|
+
"""
|
|
90
|
+
Represents an active session with access to workflow and builders.
|
|
91
|
+
|
|
92
|
+
Each session is tied to a specific request, and provides access to the appropriate workflow
|
|
93
|
+
instance (shared or per-user).
|
|
94
|
+
|
|
95
|
+
Lifecycle:
|
|
96
|
+
- Created for each request via SessionManager.session()
|
|
97
|
+
- Automatically manages ref_count for per-user builder tracking
|
|
98
|
+
- Cleans up context variables on exit
|
|
99
|
+
|
|
100
|
+
Concurrency:
|
|
101
|
+
- Each session has its own semaphore for concurrency control
|
|
102
|
+
- For per-user workflows: each user has an independent concurrency limit
|
|
103
|
+
- For shared workflows: all sessions share the SessionManager's semaphore
|
|
104
|
+
"""
|
|
105
|
+
|
|
106
|
+
def __init__(self,
|
|
107
|
+
session_manager: "SessionManager",
|
|
108
|
+
workflow: Workflow,
|
|
109
|
+
semaphore: asyncio.Semaphore | nullcontext,
|
|
110
|
+
user_id: str | None = None):
|
|
111
|
+
self._session_manager = session_manager
|
|
112
|
+
self._workflow = workflow
|
|
113
|
+
self._semaphore = semaphore
|
|
114
|
+
self._user_id = user_id
|
|
115
|
+
|
|
116
|
+
@property
|
|
117
|
+
def user_id(self) -> str | None:
|
|
118
|
+
return self._user_id
|
|
119
|
+
|
|
120
|
+
@property
|
|
121
|
+
def workflow(self) -> Workflow:
|
|
122
|
+
return self._workflow
|
|
123
|
+
|
|
124
|
+
@property
|
|
125
|
+
def session_manager(self) -> "SessionManager":
|
|
126
|
+
return self._session_manager
|
|
127
|
+
|
|
128
|
+
@asynccontextmanager
|
|
129
|
+
async def run(self, message, runtime_type: RuntimeTypeEnum = RuntimeTypeEnum.RUN_OR_SERVE):
|
|
130
|
+
"""
|
|
131
|
+
Start a workflow run using this session's workflow.
|
|
132
|
+
|
|
133
|
+
Args:
|
|
134
|
+
message: Input message for the workflow
|
|
135
|
+
runtime_type: Runtime type (defaults to SessionManager's runtime_type)
|
|
136
|
+
|
|
137
|
+
Yields:
|
|
138
|
+
Runner instance for the workflow execution
|
|
139
|
+
"""
|
|
140
|
+
async with self._semaphore:
|
|
141
|
+
async with self._workflow.run(message, runtime_type=runtime_type) as runner:
|
|
142
|
+
yield runner
|
|
44
143
|
|
|
45
144
|
|
|
46
145
|
class SessionManager:
|
|
47
146
|
|
|
48
|
-
def __init__(self,
|
|
147
|
+
def __init__(self,
|
|
148
|
+
config: Config,
|
|
149
|
+
shared_builder: "WorkflowBuilder",
|
|
150
|
+
entry_function: str | None = None,
|
|
151
|
+
shared_workflow: Workflow | None = None,
|
|
152
|
+
max_concurrency: int = 8,
|
|
153
|
+
runtime_type: RuntimeTypeEnum = RuntimeTypeEnum.RUN_OR_SERVE):
|
|
49
154
|
"""
|
|
50
|
-
The SessionManager class is used to
|
|
51
|
-
|
|
155
|
+
The SessionManager class is used to manage workflow builders and sessions.
|
|
156
|
+
It manages workflow sessions and per-user builders with lifecycle management.
|
|
157
|
+
|
|
158
|
+
Architecture:
|
|
159
|
+
- One SessionManager per FastAPI server
|
|
160
|
+
- Creates/caches PerUserWorkflowBuilder instances per user
|
|
161
|
+
- Cleans up inactive builders based on timeout
|
|
52
162
|
|
|
53
163
|
Parameters
|
|
54
164
|
----------
|
|
55
|
-
|
|
56
|
-
The
|
|
165
|
+
config : Config
|
|
166
|
+
The configuration for the workflow
|
|
167
|
+
shared_builder : WorkflowBuilder
|
|
168
|
+
The shared workflow builder
|
|
169
|
+
entry_function : str | None, optional
|
|
170
|
+
The entry function for this SessionManager's workflows, by default None
|
|
171
|
+
shared_workflow : Workflow, optional
|
|
172
|
+
The shared workflow, by default None
|
|
57
173
|
max_concurrency : int, optional
|
|
58
174
|
The maximum number of simultaneous workflow invocations, by default 8
|
|
175
|
+
runtime_type : RuntimeTypeEnum, optional
|
|
176
|
+
The type of runtime the session manager is operating in, by default RuntimeTypeEnum.RUN_OR_SERVE
|
|
59
177
|
"""
|
|
60
178
|
|
|
61
|
-
|
|
62
|
-
raise ValueError("Workflow cannot be None")
|
|
63
|
-
|
|
64
|
-
self._workflow: Workflow = workflow
|
|
179
|
+
from nat.cli.type_registry import GlobalTypeRegistry
|
|
65
180
|
|
|
181
|
+
self._config = config
|
|
66
182
|
self._max_concurrency = max_concurrency
|
|
67
|
-
self.
|
|
68
|
-
self._context = Context(self._context_state)
|
|
69
|
-
|
|
70
|
-
# We save the context because Uvicorn spawns a new process
|
|
71
|
-
# for each request, and we need to restore the context vars
|
|
72
|
-
self._saved_context = contextvars.copy_context()
|
|
183
|
+
self._entry_function = entry_function
|
|
73
184
|
|
|
74
|
-
|
|
185
|
+
# Semaphore for limiting concurrency
|
|
186
|
+
if max_concurrency > 0:
|
|
75
187
|
self._semaphore = asyncio.Semaphore(max_concurrency)
|
|
76
188
|
else:
|
|
77
189
|
# If max_concurrency is 0, then we don't need to limit the concurrency but we still need a context
|
|
78
190
|
self._semaphore = nullcontext()
|
|
79
191
|
|
|
192
|
+
self._runtime_type = runtime_type
|
|
193
|
+
|
|
194
|
+
# Context state for per-request context variables
|
|
195
|
+
self._context_state = ContextState.get()
|
|
196
|
+
self._context = Context(self._context_state)
|
|
197
|
+
|
|
198
|
+
# Track if workflow is shared or per-user
|
|
199
|
+
workflow_registration = GlobalTypeRegistry.get().get_function(type(config.workflow))
|
|
200
|
+
self._is_workflow_per_user = workflow_registration.is_per_user
|
|
201
|
+
|
|
202
|
+
# Shared components
|
|
203
|
+
self._shared_builder = shared_builder
|
|
204
|
+
self._shared_workflow = shared_workflow
|
|
205
|
+
|
|
206
|
+
# Per-user management
|
|
207
|
+
self._per_user_builders: dict[str, PerUserBuilderInfo] = {}
|
|
208
|
+
self._per_user_builders_lock = asyncio.Lock()
|
|
209
|
+
self._per_user_builders_cleanup_task: asyncio.Task | None = None
|
|
210
|
+
self._per_user_session_timeout = config.general.per_user_workflow_timeout
|
|
211
|
+
self._per_user_session_cleanup_interval = config.general.per_user_workflow_cleanup_interval
|
|
212
|
+
self._shutdown_event = asyncio.Event()
|
|
213
|
+
|
|
214
|
+
# Cache schemas for per-user workflows
|
|
215
|
+
if self._is_workflow_per_user:
|
|
216
|
+
self._per_user_workflow_input_schema = workflow_registration.per_user_function_input_schema
|
|
217
|
+
self._per_user_workflow_single_output_schema = workflow_registration.per_user_function_single_output_schema
|
|
218
|
+
self._per_user_workflow_streaming_output_schema = \
|
|
219
|
+
workflow_registration.per_user_function_streaming_output_schema
|
|
220
|
+
else:
|
|
221
|
+
self._per_user_workflow_input_schema = None
|
|
222
|
+
self._per_user_workflow_single_output_schema = None
|
|
223
|
+
self._per_user_workflow_streaming_output_schema = None
|
|
224
|
+
|
|
80
225
|
@property
|
|
81
226
|
def config(self) -> Config:
|
|
82
|
-
return self.
|
|
227
|
+
return self._config
|
|
83
228
|
|
|
84
229
|
@property
|
|
85
230
|
def workflow(self) -> Workflow:
|
|
86
|
-
|
|
231
|
+
"""
|
|
232
|
+
Get workflow for backward compatibility.
|
|
233
|
+
|
|
234
|
+
Only works for shared workflows. For per-user workflows, use session.workflow.
|
|
235
|
+
|
|
236
|
+
Raises:
|
|
237
|
+
ValueError: If workflow is per-user
|
|
238
|
+
"""
|
|
239
|
+
if self._is_workflow_per_user:
|
|
240
|
+
raise ValueError("Workflow is per-user. Access workflow through session.workflow instead.")
|
|
241
|
+
if self._shared_workflow is None:
|
|
242
|
+
raise ValueError("No shared workflow available")
|
|
243
|
+
return self._shared_workflow
|
|
244
|
+
|
|
245
|
+
@property
|
|
246
|
+
def shared_builder(self) -> "WorkflowBuilder":
|
|
247
|
+
return self._shared_builder
|
|
87
248
|
|
|
88
249
|
@property
|
|
89
|
-
def
|
|
90
|
-
return self.
|
|
250
|
+
def is_workflow_per_user(self) -> bool:
|
|
251
|
+
return self._is_workflow_per_user
|
|
252
|
+
|
|
253
|
+
def get_workflow_input_schema(self) -> type[BaseModel]:
|
|
254
|
+
"""Get workflow input schema for OpenAPI documentation."""
|
|
255
|
+
|
|
256
|
+
if self._is_workflow_per_user:
|
|
257
|
+
return self._per_user_workflow_input_schema
|
|
258
|
+
|
|
259
|
+
return self._shared_workflow.input_schema
|
|
260
|
+
|
|
261
|
+
def get_workflow_single_output_schema(self) -> type[BaseModel]:
|
|
262
|
+
"""Get workflow single output schema for OpenAPI documentation."""
|
|
263
|
+
|
|
264
|
+
if self._is_workflow_per_user:
|
|
265
|
+
return self._per_user_workflow_single_output_schema
|
|
266
|
+
|
|
267
|
+
return self._shared_workflow.single_output_schema
|
|
268
|
+
|
|
269
|
+
def get_workflow_streaming_output_schema(self) -> type[BaseModel]:
|
|
270
|
+
"""Get workflow streaming output schema for OpenAPI documentation."""
|
|
271
|
+
|
|
272
|
+
if self._is_workflow_per_user:
|
|
273
|
+
return self._per_user_workflow_streaming_output_schema
|
|
274
|
+
|
|
275
|
+
return self._shared_workflow.streaming_output_schema
|
|
276
|
+
|
|
277
|
+
@classmethod
|
|
278
|
+
async def create(cls,
|
|
279
|
+
config: Config,
|
|
280
|
+
shared_builder: "WorkflowBuilder",
|
|
281
|
+
entry_function: str | None = None,
|
|
282
|
+
max_concurrency: int = 8,
|
|
283
|
+
runtime_type: RuntimeTypeEnum = RuntimeTypeEnum.RUN_OR_SERVE) -> "SessionManager":
|
|
284
|
+
"""
|
|
285
|
+
Create a SessionManager. This is the preferred way to instantiate.
|
|
286
|
+
|
|
287
|
+
Handles async workflow building and starts cleanup task if per-user.
|
|
288
|
+
"""
|
|
289
|
+
from nat.cli.type_registry import GlobalTypeRegistry
|
|
290
|
+
|
|
291
|
+
workflow_registration = GlobalTypeRegistry.get().get_function(type(config.workflow))
|
|
292
|
+
|
|
293
|
+
if workflow_registration.is_per_user:
|
|
294
|
+
shared_workflow = None
|
|
295
|
+
logger.info(f"Workflow is per-user (entry_function={entry_function})")
|
|
296
|
+
else:
|
|
297
|
+
shared_workflow = await shared_builder.build(entry_function=entry_function)
|
|
298
|
+
logger.info(f"Shared workflow built (entry_function={entry_function})")
|
|
299
|
+
|
|
300
|
+
session_manager = cls(config=config,
|
|
301
|
+
shared_builder=shared_builder,
|
|
302
|
+
entry_function=entry_function,
|
|
303
|
+
shared_workflow=shared_workflow,
|
|
304
|
+
max_concurrency=max_concurrency,
|
|
305
|
+
runtime_type=runtime_type)
|
|
306
|
+
|
|
307
|
+
# Start cleanup task for per-user workflows
|
|
308
|
+
if session_manager._is_workflow_per_user:
|
|
309
|
+
session_manager._per_user_builders_cleanup_task = asyncio.create_task(
|
|
310
|
+
session_manager._run_periodic_cleanup())
|
|
311
|
+
|
|
312
|
+
return session_manager
|
|
313
|
+
|
|
314
|
+
async def _run_periodic_cleanup(self):
|
|
315
|
+
|
|
316
|
+
logger.debug("Running periodic cleanup of per-user builders")
|
|
317
|
+
while not self._shutdown_event.is_set():
|
|
318
|
+
try:
|
|
319
|
+
# Wait for either cleanup interval or shutdown
|
|
320
|
+
await asyncio.wait_for(self._shutdown_event.wait(),
|
|
321
|
+
timeout=self._per_user_session_cleanup_interval.total_seconds())
|
|
322
|
+
# If we get here, shutdown was signaled
|
|
323
|
+
break
|
|
324
|
+
except TimeoutError:
|
|
325
|
+
# Timeout means it's time to run cleanup
|
|
326
|
+
try:
|
|
327
|
+
await self._cleanup_inactive_per_user_builders()
|
|
328
|
+
except Exception:
|
|
329
|
+
logger.exception("Error during periodic cleanup")
|
|
330
|
+
|
|
331
|
+
logger.debug("Periodic cleanup task shutting down")
|
|
332
|
+
|
|
333
|
+
async def _cleanup_inactive_per_user_builders(self) -> int:
|
|
334
|
+
|
|
335
|
+
now = datetime.now()
|
|
336
|
+
threshold = now - self._per_user_session_timeout
|
|
337
|
+
builders_to_cleanup: list[tuple[str, PerUserBuilderInfo]] = []
|
|
338
|
+
|
|
339
|
+
# Identify builders to cleanup (under lock)
|
|
340
|
+
async with self._per_user_builders_lock:
|
|
341
|
+
for user_id, builder_info in list(self._per_user_builders.items()):
|
|
342
|
+
if builder_info.ref_count == 0 and builder_info.last_activity < threshold:
|
|
343
|
+
# Remove from dict and add to cleanup list
|
|
344
|
+
builders_to_cleanup.append((user_id, builder_info))
|
|
345
|
+
del self._per_user_builders[user_id]
|
|
346
|
+
logger.debug(f"Marked per-user builder for user {user_id} for cleanup "
|
|
347
|
+
f"(inactive since {builder_info.last_activity.isoformat()})")
|
|
348
|
+
# Cleanup builders (outside lock to avoid blocking)
|
|
349
|
+
for user_id, builder_info in builders_to_cleanup:
|
|
350
|
+
try:
|
|
351
|
+
await builder_info.builder.__aexit__(None, None, None)
|
|
352
|
+
logger.info(f"Cleaned up inactive per-user builder for user={user_id} "
|
|
353
|
+
f"(remaining users: {len(self._per_user_builders)})")
|
|
354
|
+
except Exception:
|
|
355
|
+
logger.exception(f"Error cleaning up per-user builder for user {user_id}")
|
|
356
|
+
|
|
357
|
+
return len(builders_to_cleanup)
|
|
358
|
+
|
|
359
|
+
def _get_user_id_from_context(self) -> str | None:
|
|
360
|
+
"""
|
|
361
|
+
Get user ID from current context.
|
|
362
|
+
|
|
363
|
+
Extraction order:
|
|
364
|
+
1. From context user_id (set from nat-session cookie)
|
|
365
|
+
2. From context user_manager if set
|
|
366
|
+
3. None (for shared workflow or unauthenticated access)
|
|
367
|
+
|
|
368
|
+
"""
|
|
369
|
+
try:
|
|
370
|
+
# Primary: Get from context user_id (already extracted from nat-session cookie)
|
|
371
|
+
user_id = self._context.user_id
|
|
372
|
+
if user_id:
|
|
373
|
+
return user_id
|
|
374
|
+
|
|
375
|
+
# Fallback: Get from user_manager if set
|
|
376
|
+
user_manager = self._context.user_manager
|
|
377
|
+
if user_manager:
|
|
378
|
+
return user_manager.get_id()
|
|
379
|
+
return None
|
|
380
|
+
except Exception as e:
|
|
381
|
+
logger.debug(f"Could not extract user_id from context: {e}")
|
|
382
|
+
return None
|
|
383
|
+
|
|
384
|
+
async def _get_or_create_per_user_builder(self, user_id: str) -> tuple["PerUserWorkflowBuilder", Workflow]:
|
|
385
|
+
from nat.builder.per_user_workflow_builder import PerUserWorkflowBuilder
|
|
386
|
+
|
|
387
|
+
async with self._per_user_builders_lock:
|
|
388
|
+
if user_id in self._per_user_builders:
|
|
389
|
+
builder_info = self._per_user_builders[user_id]
|
|
390
|
+
builder_info.last_activity = datetime.now()
|
|
391
|
+
|
|
392
|
+
return builder_info.builder, builder_info.workflow
|
|
393
|
+
|
|
394
|
+
logger.info(f"Creating per-user builder for user={user_id}, entry_function={self._entry_function}")
|
|
395
|
+
builder = PerUserWorkflowBuilder(user_id=user_id, shared_builder=self._shared_builder)
|
|
396
|
+
# Enter the builder's context manually to avoid exiting the context manager
|
|
397
|
+
# Exit the context when cleaning up the builder
|
|
398
|
+
await builder.__aenter__()
|
|
399
|
+
|
|
400
|
+
try:
|
|
401
|
+
await builder.populate_builder(self._config)
|
|
402
|
+
workflow = await builder.build(entry_function=self._entry_function)
|
|
403
|
+
|
|
404
|
+
# Create per-user semaphore for concurrency control
|
|
405
|
+
if self._max_concurrency > 0:
|
|
406
|
+
per_user_semaphore = asyncio.Semaphore(self._max_concurrency)
|
|
407
|
+
else:
|
|
408
|
+
per_user_semaphore = nullcontext()
|
|
409
|
+
|
|
410
|
+
builder_info = PerUserBuilderInfo(builder=builder,
|
|
411
|
+
workflow=workflow,
|
|
412
|
+
semaphore=per_user_semaphore,
|
|
413
|
+
last_activity=datetime.now(),
|
|
414
|
+
ref_count=0,
|
|
415
|
+
lock=asyncio.Lock())
|
|
416
|
+
self._per_user_builders[user_id] = builder_info
|
|
417
|
+
logger.info(
|
|
418
|
+
f"Created per-user builder for user={user_id} (total users: {len(self._per_user_builders)})")
|
|
419
|
+
return builder_info.builder, builder_info.workflow
|
|
420
|
+
except Exception:
|
|
421
|
+
logger.exception(f"Error creating per-user builder for user {user_id}")
|
|
422
|
+
try:
|
|
423
|
+
await builder.__aexit__(None, None, None)
|
|
424
|
+
except Exception:
|
|
425
|
+
logger.exception("Error during builder cleanup after failed creation")
|
|
426
|
+
raise
|
|
91
427
|
|
|
92
428
|
@asynccontextmanager
|
|
93
429
|
async def session(self,
|
|
430
|
+
user_id: str | None = None,
|
|
94
431
|
user_manager=None,
|
|
95
432
|
http_connection: HTTPConnection | None = None,
|
|
96
433
|
user_message_id: str | None = None,
|
|
@@ -117,9 +454,65 @@ class SessionManager:
|
|
|
117
454
|
if isinstance(http_connection, Request):
|
|
118
455
|
self.set_metadata_from_http_request(http_connection)
|
|
119
456
|
|
|
457
|
+
builder_info: PerUserBuilderInfo | None = None
|
|
458
|
+
request_start_time: float | None = None
|
|
459
|
+
request_success = True
|
|
460
|
+
|
|
461
|
+
if self._is_workflow_per_user:
|
|
462
|
+
# Resolve user_id: explicit param > context
|
|
463
|
+
if user_id is None:
|
|
464
|
+
user_id = self._get_user_id_from_context()
|
|
465
|
+
if user_id is None:
|
|
466
|
+
raise ValueError("user_id is required for per-user workflow but could not be determined. "
|
|
467
|
+
"Ensure 'nat-session' cookie is set or pass user_id explicitly.")
|
|
468
|
+
|
|
469
|
+
# To ensure the user_id is set in the context before the per-user builder is created
|
|
470
|
+
self._context_state.user_id.set(user_id)
|
|
471
|
+
|
|
472
|
+
# Get or create per-user builder
|
|
473
|
+
logger.debug(f"Getting or creating per-user builder for user {user_id}")
|
|
474
|
+
_, workflow = await self._get_or_create_per_user_builder(user_id)
|
|
475
|
+
builder_info = self._per_user_builders[user_id]
|
|
476
|
+
async with builder_info.lock:
|
|
477
|
+
builder_info.ref_count += 1
|
|
478
|
+
logger.debug(f"Incremented ref_count for user {user_id} to {builder_info.ref_count}")
|
|
479
|
+
# Use per-user semaphore for concurrency control
|
|
480
|
+
semaphore = builder_info.semaphore
|
|
481
|
+
# Start request timing for metrics
|
|
482
|
+
request_start_time = time.perf_counter()
|
|
483
|
+
else:
|
|
484
|
+
workflow = self._shared_workflow
|
|
485
|
+
# Use shared semaphore for concurrency control
|
|
486
|
+
semaphore = self._semaphore
|
|
487
|
+
|
|
488
|
+
# TODO: this logic needs to be cleaned up since it is a duplicated setting of the user_id
|
|
489
|
+
# But we need to keep it for now to maintain the token_user_id
|
|
490
|
+
token_user_id = None
|
|
491
|
+
if user_id is not None:
|
|
492
|
+
token_user_id = self._context_state.user_id.set(user_id)
|
|
493
|
+
|
|
120
494
|
try:
|
|
121
|
-
|
|
495
|
+
session = Session(session_manager=self, user_id=user_id, workflow=workflow, semaphore=semaphore)
|
|
496
|
+
|
|
497
|
+
yield session
|
|
498
|
+
|
|
499
|
+
except Exception:
|
|
500
|
+
request_success = False
|
|
501
|
+
raise
|
|
502
|
+
|
|
122
503
|
finally:
|
|
504
|
+
if builder_info is not None:
|
|
505
|
+
async with builder_info.lock:
|
|
506
|
+
builder_info.ref_count -= 1
|
|
507
|
+
builder_info.last_activity = datetime.now()
|
|
508
|
+
|
|
509
|
+
# Record request metrics
|
|
510
|
+
if request_start_time is not None:
|
|
511
|
+
latency_ms = (time.perf_counter() - request_start_time) * 1000
|
|
512
|
+
builder_info.record_request(latency_ms, request_success)
|
|
513
|
+
|
|
514
|
+
if token_user_id is not None:
|
|
515
|
+
self._context_state.user_id.reset(token_user_id)
|
|
123
516
|
if token_user_manager is not None:
|
|
124
517
|
self._context_state.user_manager.reset(token_user_manager)
|
|
125
518
|
if token_user_input is not None:
|
|
@@ -128,18 +521,43 @@ class SessionManager:
|
|
|
128
521
|
self._context_state.user_auth_callback.reset(token_user_authentication)
|
|
129
522
|
|
|
130
523
|
@asynccontextmanager
|
|
131
|
-
async def run(self, message):
|
|
524
|
+
async def run(self, message, runtime_type: RuntimeTypeEnum = RuntimeTypeEnum.RUN_OR_SERVE):
|
|
132
525
|
"""
|
|
133
526
|
Start a workflow run
|
|
134
527
|
"""
|
|
528
|
+
if self._is_workflow_per_user:
|
|
529
|
+
raise ValueError("Cannot use SessionManager.run() with per-user workflows. "
|
|
530
|
+
"Use 'async with session_manager.session() as session' then 'session.run()' instead.")
|
|
135
531
|
async with self._semaphore:
|
|
136
|
-
|
|
137
|
-
for k, v in self._saved_context.items():
|
|
138
|
-
k.set(v)
|
|
139
|
-
|
|
140
|
-
async with self._workflow.run(message) as runner:
|
|
532
|
+
async with self._shared_workflow.run(message, runtime_type=runtime_type) as runner:
|
|
141
533
|
yield runner
|
|
142
534
|
|
|
535
|
+
async def shutdown(self) -> None:
|
|
536
|
+
"""
|
|
537
|
+
Shutdown the SessionManager and cleanup resources.
|
|
538
|
+
|
|
539
|
+
Call this when the SessionManager is no longer needed.
|
|
540
|
+
"""
|
|
541
|
+
if self._is_workflow_per_user:
|
|
542
|
+
# Shutdown cleanup task
|
|
543
|
+
self._shutdown_event.set()
|
|
544
|
+
if self._per_user_builders_cleanup_task:
|
|
545
|
+
try:
|
|
546
|
+
await asyncio.wait_for(self._per_user_builders_cleanup_task, timeout=5.0)
|
|
547
|
+
except TimeoutError:
|
|
548
|
+
logger.warning("Cleanup task did not finish in time, cancelling")
|
|
549
|
+
self._per_user_builders_cleanup_task.cancel()
|
|
550
|
+
|
|
551
|
+
# Cleanup all per-user builders
|
|
552
|
+
async with self._per_user_builders_lock:
|
|
553
|
+
for user_id, builder_info in list(self._per_user_builders.items()):
|
|
554
|
+
logger.debug(f"Cleaning up per-user builder for user {user_id}")
|
|
555
|
+
try:
|
|
556
|
+
await builder_info.builder.__aexit__(None, None, None)
|
|
557
|
+
except Exception:
|
|
558
|
+
logger.exception(f"Error cleaning up builder for user {user_id}")
|
|
559
|
+
self._per_user_builders.clear()
|
|
560
|
+
|
|
143
561
|
def set_metadata_from_http_request(self, request: Request) -> None:
|
|
144
562
|
"""
|
|
145
563
|
Extracts and sets user metadata request attributes from a HTTP request.
|
|
@@ -162,6 +580,10 @@ class SessionManager:
|
|
|
162
580
|
if request.headers.get("user-message-id"):
|
|
163
581
|
self._context_state.user_message_id.set(request.headers["user-message-id"])
|
|
164
582
|
|
|
583
|
+
# Set user_id from nat-session cookie
|
|
584
|
+
if request.cookies.get("nat-session"):
|
|
585
|
+
self._context_state.user_id.set(request.cookies["nat-session"])
|
|
586
|
+
|
|
165
587
|
# W3C Trace Context header: traceparent: 00-<trace-id>-<span-id>-<flags>
|
|
166
588
|
traceparent = request.headers.get("traceparent")
|
|
167
589
|
if traceparent:
|
|
@@ -212,6 +634,10 @@ class SessionManager:
|
|
|
212
634
|
self._context.metadata._request.cookies = cookies
|
|
213
635
|
self._context_state.metadata.set(self._context.metadata)
|
|
214
636
|
|
|
637
|
+
# Set user_id from nat-session cookie
|
|
638
|
+
if cookies.get("nat-session"):
|
|
639
|
+
self._context_state.user_id.set(cookies["nat-session"])
|
|
640
|
+
|
|
215
641
|
if conversation_id is not None:
|
|
216
642
|
self._context_state.conversation_id.set(conversation_id)
|
|
217
643
|
|
nat/runtime/user_metadata.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# SPDX-FileCopyrightText: Copyright (c) 2024-
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
2
|
# SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
nat/settings/global_settings.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# SPDX-FileCopyrightText: Copyright (c) 2024-
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
2
|
# SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
nat/tool/chat_completion.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# SPDX-FileCopyrightText: Copyright (c) 2024-
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
2
|
# SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
<!--
|
|
2
|
-
SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
3
3
|
SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
5
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright (c) 2024-
|
|
1
|
+
# Copyright (c) 2024-2026, NVIDIA CORPORATION. All rights reserved.
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -152,7 +152,7 @@ class LocalSandbox(Sandbox):
|
|
|
152
152
|
output_json = output.json()
|
|
153
153
|
assert isinstance(output_json, dict)
|
|
154
154
|
return output_json
|
|
155
|
-
except
|
|
155
|
+
except (requests.exceptions.JSONDecodeError, AssertionError) as e:
|
|
156
156
|
logger.exception("Error parsing output: %s. %s", output.text, e)
|
|
157
157
|
return {'process_status': 'error', 'stdout': '', 'stderr': f'Unknown error: {e} \"{output.text}\"'}
|
|
158
158
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright (c) 2024-
|
|
1
|
+
# Copyright (c) 2024-2026, NVIDIA CORPORATION. All rights reserved.
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright (c) 2024-
|
|
1
|
+
# Copyright (c) 2024-2026, NVIDIA CORPORATION. All rights reserved.
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright (c) 2024-
|
|
1
|
+
# Copyright (c) 2024-2026, NVIDIA CORPORATION. All rights reserved.
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/bin/bash
|
|
2
2
|
|
|
3
|
-
# Copyright (c) 2024-
|
|
3
|
+
# Copyright (c) 2024-2026, NVIDIA CORPORATION. All rights reserved.
|
|
4
4
|
#
|
|
5
5
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
6
|
# you may not use this file except in compliance with the License.
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
2
|
# SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
#
|
|
4
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
nat/tool/code_execution/utils.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright (c) 2024-
|
|
1
|
+
# Copyright (c) 2024-2026, NVIDIA CORPORATION. All rights reserved.
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|