nvidia-nat 1.3.0a20250910__py3-none-any.whl → 1.4.0a20251112__py3-none-any.whl

nat/eval/runtime_evaluator/register.py

@@ -0,0 +1,100 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pydantic import Field
+
+from nat.builder.builder import EvalBuilder
+from nat.builder.evaluator import EvaluatorInfo
+from nat.cli.register_workflow import register_evaluator
+from nat.data_models.evaluator import EvaluatorBaseConfig
+from nat.eval.evaluator.evaluator_model import EvalInput
+from nat.eval.evaluator.evaluator_model import EvalOutput
+
+
+class AverageLLMLatencyConfig(EvaluatorBaseConfig, name="avg_llm_latency"):
+    """Mean difference between connected LLM_START and LLM_END events (same UUID)."""
+
+    max_concurrency: int = Field(default=8, description="Max concurrency for evaluation.")
+
+
+class AverageWorkflowRuntimeConfig(EvaluatorBaseConfig, name="avg_workflow_runtime"):
+    """Average workflow runtime per item (max timestamp - min timestamp)."""
+
+    max_concurrency: int = Field(default=8, description="Max concurrency for evaluation.")
+
+
+class AverageNumberOfLLMCallsConfig(EvaluatorBaseConfig, name="avg_num_llm_calls"):
+    """Average number of LLM calls per item (count of LLM_END)."""
+
+    max_concurrency: int = Field(default=8, description="Max concurrency for evaluation.")
+
+
+class AverageTokensPerLLMEndConfig(EvaluatorBaseConfig, name="avg_tokens_per_llm_end"):
+    """Average total tokens per LLM_END event (prompt + completion if available)."""
+
+    max_concurrency: int = Field(default=8, description="Max concurrency for evaluation.")
+
+
+@register_evaluator(config_type=AverageLLMLatencyConfig)
+async def register_avg_llm_latency_evaluator(config: AverageLLMLatencyConfig, builder: EvalBuilder):
+    from .evaluate import AverageLLMLatencyEvaluator
+
+    evaluator = AverageLLMLatencyEvaluator(max_concurrency=config.max_concurrency or builder.get_max_concurrency())
+
+    async def evaluate_fn(eval_input: EvalInput) -> EvalOutput:
+        return await evaluator.evaluate(eval_input)
+
+    yield EvaluatorInfo(config=config,
+                        evaluate_fn=evaluate_fn,
+                        description="Average LLM latency (s) from LLM_START to LLM_END")
+
+
+@register_evaluator(config_type=AverageWorkflowRuntimeConfig)
+async def register_avg_workflow_runtime_evaluator(config: AverageWorkflowRuntimeConfig, builder: EvalBuilder):
+    from .evaluate import AverageWorkflowRuntimeEvaluator
+
+    evaluator = AverageWorkflowRuntimeEvaluator(max_concurrency=config.max_concurrency or builder.get_max_concurrency())
+
+    async def evaluate_fn(eval_input: EvalInput) -> EvalOutput:
+        return await evaluator.evaluate(eval_input)
+
+    yield EvaluatorInfo(config=config, evaluate_fn=evaluate_fn, description="Average workflow runtime (s)")
+
+
+@register_evaluator(config_type=AverageNumberOfLLMCallsConfig)
+async def register_avg_num_llm_calls_evaluator(config: AverageNumberOfLLMCallsConfig, builder: EvalBuilder):
+    from .evaluate import AverageNumberOfLLMCallsEvaluator
+
+    evaluator = AverageNumberOfLLMCallsEvaluator(
+        max_concurrency=config.max_concurrency or builder.get_max_concurrency())
+
+    async def evaluate_fn(eval_input: EvalInput) -> EvalOutput:
+        return await evaluator.evaluate(eval_input)
+
+    yield EvaluatorInfo(config=config, evaluate_fn=evaluate_fn, description="Average number of LLM calls")
+
+
+@register_evaluator(config_type=AverageTokensPerLLMEndConfig)
+async def register_avg_tokens_per_llm_end_evaluator(config: AverageTokensPerLLMEndConfig, builder: EvalBuilder):
+    from .evaluate import AverageTokensPerLLMEndEvaluator
+
+    evaluator = AverageTokensPerLLMEndEvaluator(max_concurrency=config.max_concurrency or builder.get_max_concurrency())
+
+    async def evaluate_fn(eval_input: EvalInput) -> EvalOutput:
+        return await evaluator.evaluate(eval_input)
+
+    yield EvaluatorInfo(config=config,
+                        evaluate_fn=evaluate_fn,
+                        description="Average total tokens per LLM_END (prompt + completion)")

nat/eval/swe_bench_evaluator/evaluate.py

@@ -204,7 +204,7 @@ class SweBenchEvaluator:
         # if report file is not present, return empty EvalOutput
         avg_score = 0.0
         if report_file.exists():
-            with open(report_file, "r", encoding="utf-8") as f:
+            with open(report_file, encoding="utf-8") as f:
                 report = json.load(f)
                 resolved_instances = report.get("resolved_instances", 0)
                 total_instances = report.get("total_instances", 0)

nat/eval/trajectory_evaluator/register.py

@@ -33,7 +33,7 @@ async def register_trajectory_evaluator(config: TrajectoryEvaluatorConfig, build
 
     from .evaluate import TrajectoryEvaluator
     llm = await builder.get_llm(config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN)
-    tools = builder.get_all_tools(wrapper_type=LLMFrameworkEnum.LANGCHAIN)
+    tools = await builder.get_all_tools(wrapper_type=LLMFrameworkEnum.LANGCHAIN)
 
     _evaluator = TrajectoryEvaluator(llm, tools, builder.get_max_concurrency())
 

nat/eval/tunable_rag_evaluator/evaluate.py

@@ -14,7 +14,7 @@
 # limitations under the License.
 
 import logging
-from typing import Callable
+from collections.abc import Callable
 
 from langchain.output_parsers import ResponseSchema
 from langchain.output_parsers import StructuredOutputParser

nat/eval/usage_stats.py

@@ -21,6 +21,8 @@ from pydantic import BaseModel
 class UsageStatsLLM(BaseModel):
     prompt_tokens: int = 0
     completion_tokens: int = 0
+    cached_tokens: int = 0
+    reasoning_tokens: int = 0
     total_tokens: int = 0
 
 

nat/eval/utils/output_uploader.py

@@ -24,6 +24,7 @@ import aioboto3
 from botocore.exceptions import NoCredentialsError
 from tqdm import tqdm
 
+from nat.data_models.common import get_secret_value
 from nat.data_models.evaluate import EvalOutputConfig
 
 logger = logging.getLogger(__name__)
@@ -90,8 +91,8 @@ class OutputUploader:
                     "s3",
                     endpoint_url=endpoint_url,
                     region_name=region_name,
-                    aws_access_key_id=self.s3_config.access_key,
-                    aws_secret_access_key=self.s3_config.secret_key,
+                    aws_access_key_id=get_secret_value(self.s3_config.access_key),
+                    aws_secret_access_key=get_secret_value(self.s3_config.secret_key),
             ) as s3_client:
                 with tqdm(total=len(file_entries), desc="Uploading files to S3") as pbar:
                     upload_tasks = [

nat/eval/utils/weave_eval.py

@@ -82,7 +82,7 @@ class WeaveEvaluationIntegration:
         """Get the full dataset for Weave."""
         return [item.full_dataset_entry for item in eval_input.eval_input_items]
 
-    def initialize_logger(self, workflow_alias: str, eval_input: EvalInput, config: Any):
+    def initialize_logger(self, workflow_alias: str, eval_input: EvalInput, config: Any, job_id: str | None = None):
         """Initialize the Weave evaluation logger."""
         if not self.client and not self.initialize_client():
             # lazy init the client
@@ -92,10 +92,16 @@ class WeaveEvaluationIntegration:
             weave_dataset = self._get_weave_dataset(eval_input)
             config_dict = config.model_dump(mode="json")
             config_dict["name"] = workflow_alias
+
+            # Include job_id in eval_attributes if provided
+            eval_attributes = {}
+            if job_id:
+                eval_attributes["job_id"] = job_id
+
             self.eval_logger = self.evaluation_logger_cls(model=config_dict,
                                                           dataset=weave_dataset,
                                                           name=workflow_alias,
-                                                          eval_attributes={})
+                                                          eval_attributes=eval_attributes)
             self.pred_loggers = {}
 
             # Capture the current evaluation call for context propagation
@@ -136,9 +142,17 @@ class WeaveEvaluationIntegration:
         coros = []
         for eval_output_item in eval_output.eval_output_items:
             if eval_output_item.id in self.pred_loggers:
+                # Structure the score as a dict and include reasoning if available
+                score_value = {
+                    "score": eval_output_item.score,
+                }
+
+                if eval_output_item.reasoning is not None:
+                    score_value["reasoning"] = eval_output_item.reasoning
+
                 coros.append(self.pred_loggers[eval_output_item.id].alog_score(
                     scorer=evaluator_name,
-                    score=eval_output_item.score,
+                    score=score_value,
                 ))
 
         # Execute all coroutines concurrently

nat/experimental/decorators/experimental_warning_decorator.py

@@ -16,7 +16,12 @@
 import functools
 import inspect
 import logging
+from collections.abc import AsyncGenerator
+from collections.abc import Callable
+from collections.abc import Generator
 from typing import Any
+from typing import TypeVar
+from typing import overload
 
 logger = logging.getLogger(__name__)
 
@@ -25,6 +30,9 @@ BASE_WARNING_MESSAGE = ("is experimental and the API may change in future releas
 
 _warning_issued = set()
 
+# Type variables for overloads
+F = TypeVar('F', bound=Callable[..., Any])
+
 
 def issue_experimental_warning(function_name: str,
                                feature_name: str | None = None,
@@ -53,7 +61,20 @@ def issue_experimental_warning(function_name: str,
         _warning_issued.add(function_name)
 
 
-def experimental(func: Any = None, *, feature_name: str | None = None, metadata: dict[str, Any] | None = None):
+# Overloads for different function types
+@overload
+def experimental(func: F, *, feature_name: str | None = None, metadata: dict[str, Any] | None = None) -> F:
+    """Overload for when a function is passed directly."""
+    ...
+
+
+@overload
+def experimental(*, feature_name: str | None = None, metadata: dict[str, Any] | None = None) -> Callable[[F], F]:
+    """Overload for decorator factory usage (when called with parentheses)."""
+    ...
+
+
+def experimental(func: Any = None, *, feature_name: str | None = None, metadata: dict[str, Any] | None = None) -> Any:
     """
     Decorator that can wrap any type of function (sync, async, generator,
     async generator) and logs a warning that the function is experimental.
@@ -90,7 +111,7 @@ def experimental(func: Any = None, *, feature_name: str | None = None, metadata:
         # ---------------------
 
         @functools.wraps(func)
-        async def async_gen_wrapper(*args, **kwargs):
+        async def async_gen_wrapper(*args, **kwargs) -> AsyncGenerator[Any, Any]:
             issue_experimental_warning(function_name, feature_name, metadata)
             async for item in func(*args, **kwargs):
                 yield item  # yield the original item
@@ -102,7 +123,7 @@ def experimental(func: Any = None, *, feature_name: str | None = None, metadata:
         # ASYNC FUNCTION
         # ---------------------
         @functools.wraps(func)
-        async def async_wrapper(*args, **kwargs):
+        async def async_wrapper(*args, **kwargs) -> Any:
             issue_experimental_warning(function_name, feature_name, metadata)
             result = await func(*args, **kwargs)
             return result
@@ -114,15 +135,14 @@ def experimental(func: Any = None, *, feature_name: str | None = None, metadata:
         # SYNC GENERATOR
         # ---------------------
         @functools.wraps(func)
-        def sync_gen_wrapper(*args, **kwargs):
+        def sync_gen_wrapper(*args, **kwargs) -> Generator[Any, Any, Any]:
             issue_experimental_warning(function_name, feature_name, metadata)
-            for item in func(*args, **kwargs):
-                yield item  # yield the original item
+            yield from func(*args, **kwargs)  # yield the original item
 
         return sync_gen_wrapper
 
     @functools.wraps(func)
-    def sync_wrapper(*args, **kwargs):
+    def sync_wrapper(*args, **kwargs) -> Any:
         issue_experimental_warning(function_name, feature_name, metadata)
         result = func(*args, **kwargs)
         return result

nat/experimental/test_time_compute/functions/execute_score_select_function.py

@@ -46,7 +46,7 @@ async def execute_score_select_function(config: ExecuteScoreSelectFunctionConfig
 
     from pydantic import BaseModel
 
-    executable_fn: Function = builder.get_function(name=config.augmented_fn)
+    executable_fn: Function = await builder.get_function(name=config.augmented_fn)
 
     if config.scorer:
         scorer = await builder.get_ttc_strategy(strategy_name=config.scorer,

nat/experimental/test_time_compute/functions/plan_select_execute_function.py

@@ -86,7 +86,7 @@ async def plan_select_execute_function(config: PlanSelectExecuteFunctionConfig,
                           "This error can be resolved by installing nvidia-nat-langchain.")
 
     # Get the augmented function's description
-    augmented_function = builder.get_function(config.augmented_fn)
+    augmented_function = await builder.get_function(config.augmented_fn)
 
     # For now, we rely on runtime checking for type conversion
 
@@ -97,11 +97,15 @@ async def plan_select_execute_function(config: PlanSelectExecuteFunctionConfig,
                          f"function without a description.")
 
     # Get the function dependencies of the augmented function
-    function_used_tools = builder.get_function_dependencies(config.augmented_fn).functions
+    function_dependencies = builder.get_function_dependencies(config.augmented_fn)
+    function_used_tools = set(function_dependencies.functions)
+    for function_group in function_dependencies.function_groups:
+        function_used_tools.update(builder.get_function_group_dependencies(function_group).functions)
+
     tool_list = "Tool: Description\n"
 
     for tool in function_used_tools:
-        tool_impl = builder.get_function(tool)
+        tool_impl = await builder.get_function(tool)
         tool_list += f"- {tool}: {tool_impl.description if hasattr(tool_impl, 'description') else ''}\n"
 
     # Draft the reasoning prompt for the augmented function

nat/experimental/test_time_compute/functions/ttc_tool_orchestration_function.py

@@ -82,7 +82,7 @@ async def register_ttc_tool_orchestration_function(
     function_map = {}
     for fn_ref in config.augmented_fns:
         # Retrieve the actual function from the builder
-        fn_obj = builder.get_function(fn_ref)
+        fn_obj = await builder.get_function(fn_ref)
         function_map[fn_ref] = fn_obj
 
     # 2) Instantiate search, editing, scoring, selection strategies (if any)

nat/experimental/test_time_compute/functions/ttc_tool_wrapper_function.py

@@ -80,7 +80,7 @@ async def register_ttc_tool_wrapper_function(
         raise ImportError("langchain-core is not installed. Please install it to use SingleShotMultiPlanPlanner.\n"
                           "This error can be resolved by installing nvidia-nat-langchain.")
 
-    augmented_function: Function = builder.get_function(config.augmented_fn)
+    augmented_function: Function = await builder.get_function(config.augmented_fn)
     input_llm: BaseChatModel = await builder.get_llm(config.input_llm, wrapper_type=LLMFrameworkEnum.LANGCHAIN)
 
     if not augmented_function.has_single_output:
@@ -98,8 +98,8 @@ async def register_ttc_tool_wrapper_function(
 
         augmented_function_desc = config.tool_description
 
-    fn_input_schema: BaseModel = augmented_function.input_schema
-    fn_output_schema: BaseModel = augmented_function.single_output_schema
+    fn_input_schema: type[BaseModel] = augmented_function.input_schema
+    fn_output_schema: type[BaseModel] | type[None] = augmented_function.single_output_schema
 
     runnable_llm = input_llm.with_structured_output(schema=fn_input_schema)
 

nat/experimental/test_time_compute/models/strategy_base.py

@@ -46,11 +46,11 @@ class StrategyBase(ABC):
                       items: list[TTCItem],
                       original_prompt: str | None = None,
                       agent_context: str | None = None,
-                      **kwargs) -> [TTCItem]:
+                      **kwargs) -> list[TTCItem]:
         pass
 
     @abstractmethod
-    def supported_pipeline_types(self) -> [PipelineTypeEnum]:
+    def supported_pipeline_types(self) -> list[PipelineTypeEnum]:
         """Return the stage types supported by this selector."""
         pass
 

nat/experimental/test_time_compute/selection/llm_based_output_merging_selector.py

@@ -71,7 +71,7 @@ class LLMBasedOutputMergingSelector(StrategyBase):
             raise ImportError("langchain-core is not installed. Please install it to use SingleShotMultiPlanPlanner.\n"
                               "This error can be resolved by installing nvidia-nat-langchain.")
 
-        from typing import Callable
+        from collections.abc import Callable
 
         from pydantic import BaseModel
 

nat/front_ends/console/authentication_flow_handler.py

@@ -14,13 +14,16 @@
 # limitations under the License.
 
 import asyncio
+import logging
 import secrets
 import webbrowser
 from dataclasses import dataclass
 from dataclasses import field
 
 import click
+import httpx
 import pkce
+from authlib.common.errors import AuthlibBaseError as OAuthError
 from authlib.integrations.httpx_client import AsyncOAuth2Client
 from fastapi import FastAPI
 from fastapi import Request
@@ -32,6 +35,8 @@ from nat.data_models.authentication import AuthFlowType
 from nat.data_models.authentication import AuthProviderBaseConfig
 from nat.front_ends.fastapi.fastapi_front_end_controller import _FastApiFrontEndController
 
+logger = logging.getLogger(__name__)
+
 
 # --------------------------------------------------------------------------- #
 # Helpers                                                                     #
@@ -87,17 +92,53 @@ class ConsoleAuthenticationFlowHandler(FlowHandlerBase):
         """
         Separated for easy overriding in tests (to inject ASGITransport).
         """
-        client = AsyncOAuth2Client(
-            client_id=cfg.client_id,
-            client_secret=cfg.client_secret,
-            redirect_uri=cfg.redirect_uri,
-            scope=" ".join(cfg.scopes) if cfg.scopes else None,
-            token_endpoint=cfg.token_url,
-            token_endpoint_auth_method=cfg.token_endpoint_auth_method,
-            code_challenge_method="S256" if cfg.use_pkce else None,
-        )
-        self._oauth_client = client
-        return client
+        try:
+            client = AsyncOAuth2Client(
+                client_id=cfg.client_id,
+                client_secret=cfg.client_secret.get_secret_value(),
+                redirect_uri=cfg.redirect_uri,
+                scope=" ".join(cfg.scopes) if cfg.scopes else None,
+                token_endpoint=cfg.token_url,
+                token_endpoint_auth_method=cfg.token_endpoint_auth_method,
+                code_challenge_method="S256" if cfg.use_pkce else None,
+            )
+            self._oauth_client = client
+            return client
+        except (OAuthError, ValueError, TypeError) as e:
+            raise RuntimeError(f"Invalid OAuth2 configuration: {e}") from e
+        except Exception as e:
+            raise RuntimeError(f"Failed to create OAuth2 client: {e}") from e
+
+    def _create_authorization_url(self,
+                                  client: AsyncOAuth2Client,
+                                  config: OAuth2AuthCodeFlowProviderConfig,
+                                  state: str,
+                                  verifier: str | None = None,
+                                  challenge: str | None = None) -> str:
+        """
+        Create OAuth authorization URL with proper error handling.
+
+        Args:
+            client: The OAuth2 client instance
+            config: OAuth2 configuration
+            state: OAuth state parameter
+            verifier: PKCE verifier (if using PKCE)
+            challenge: PKCE challenge (if using PKCE)
+
+        Returns:
+            The authorization URL
+        """
+        try:
+            auth_url, _ = client.create_authorization_url(
+                config.authorization_url,
+                state=state,
+                code_verifier=verifier if config.use_pkce else None,
+                code_challenge=challenge if config.use_pkce else None,
+                **(config.authorization_kwargs or {})
+            )
+            return auth_url
+        except (OAuthError, ValueError, TypeError) as e:
+            raise RuntimeError(f"Error creating OAuth authorization URL: {e}") from e
 
     # --------------------------- HTTP Basic ------------------------------ #
     @staticmethod
@@ -131,13 +172,12 @@ class ConsoleAuthenticationFlowHandler(FlowHandlerBase):
             flow_state.verifier = verifier
             flow_state.challenge = challenge
 
-        auth_url, _ = client.create_authorization_url(
-            cfg.authorization_url,
-            state=state,
-            code_verifier=flow_state.verifier if cfg.use_pkce else None,
-            code_challenge=flow_state.challenge if cfg.use_pkce else None,
-            **(cfg.authorization_kwargs or {})
-        )
+        # Create authorization URL using helper function
+        auth_url = self._create_authorization_url(client=client,
+                                                  config=cfg,
+                                                  state=state,
+                                                  verifier=flow_state.verifier,
+                                                  challenge=flow_state.challenge)
 
         # Register flow + maybe spin up redirect handler
         async with self._server_lock:
@@ -149,14 +189,18 @@ class ConsoleAuthenticationFlowHandler(FlowHandlerBase):
             self._flows[state] = flow_state
             self._active_flows += 1
 
-        click.echo("Your browser has been opened for authentication.")
-        webbrowser.open(auth_url)
+        try:
+            webbrowser.open(auth_url)
+            click.echo("Your browser has been opened for authentication.")
+        except Exception as e:
+            logger.error("Browser open failed: %s", e)
+            raise RuntimeError(f"Browser open failed: {e}") from e
 
         # Wait for the redirect to land
         try:
             token = await asyncio.wait_for(flow_state.future, timeout=300)
-        except asyncio.TimeoutError:
-            raise RuntimeError("Authentication timed out (5 min).")
+        except TimeoutError as exc:
+            raise RuntimeError("Authentication timed out (5 min).") from exc
         finally:
             async with self._server_lock:
                 self._flows.pop(state, None)
@@ -175,9 +219,9 @@ class ConsoleAuthenticationFlowHandler(FlowHandlerBase):
     # --------------- redirect server / in‑process app -------------------- #
     async def _build_redirect_app(self) -> FastAPI:
         """
-        * If cfg.run_redirect_local_server == True → start a uvicorn server (old behaviour).
-        * Else → only build the FastAPI app and save it to `self._redirect_app`
-                 for in‑process testing with ASGITransport.
+        * If cfg.run_redirect_local_server == True → start a local server.
+        * Else → only build the redirect app and save it to `self._redirect_app`
+                 for in‑process testing.
         """
         app = FastAPI()
 
@@ -195,8 +239,16 @@ class ConsoleAuthenticationFlowHandler(FlowHandlerBase):
                     state=state,
                 )
                 flow_state.future.set_result(token)
-            except Exception as exc:  # noqa: BLE001
-                flow_state.future.set_exception(exc)
+            except OAuthError as e:
+                flow_state.future.set_exception(
+                    RuntimeError(f"Authorization server rejected request: {e.error} ({e.description})"))
+                return "Authentication failed: Authorization server rejected the request. You may close this tab."
+            except httpx.HTTPError as e:
+                flow_state.future.set_exception(RuntimeError(f"Network error during token fetch: {e}"))
+                return "Authentication failed: Network error occurred. You may close this tab."
+            except Exception as e:
+                flow_state.future.set_exception(RuntimeError(f"Authentication failed: {e}"))
+                return "Authentication failed: An unexpected error occurred. You may close this tab."
             return "Authentication successful – you may close this tab."
 
         return app
@@ -213,7 +265,7 @@ class ConsoleAuthenticationFlowHandler(FlowHandlerBase):
 
             asyncio.create_task(self._server_controller.start_server(host="localhost", port=8000))
 
-            # Give uvicorn a moment to bind sockets before we return
+            # Give the server a moment to bind sockets before we return
             await asyncio.sleep(0.3)
         except Exception as exc:  # noqa: BLE001
             raise RuntimeError(f"Failed to start redirect server: {exc}") from exc
@@ -227,7 +279,7 @@ class ConsoleAuthenticationFlowHandler(FlowHandlerBase):
     @property
     def redirect_app(self) -> FastAPI | None:
         """
-        In “test‑mode” (run_redirect_local_server=False) the in‑memory FastAPI
-        app is exposed so you can mount it on `httpx.ASGITransport`.
+        In test mode (run_redirect_local_server=False) the in‑memory redirect
+        app is exposed for testing purposes.
         """
         return self._redirect_app

nat/front_ends/console/console_front_end_plugin.py

@@ -55,9 +55,10 @@ class ConsoleFrontEndPlugin(SimpleFrontEndPluginBase[ConsoleFrontEndConfig]):
         self.auth_flow_handler = ConsoleAuthenticationFlowHandler()
 
     async def pre_run(self):
-
-        if (not self.front_end_config.input_query and not self.front_end_config.input_file):
-            raise click.UsageError("Must specify either --input_query or --input_file")
+        if (self.front_end_config.input_query is not None and self.front_end_config.input_file is not None):
+            raise click.UsageError("Must specify either --input or --input_file, not both")
+        if (self.front_end_config.input_query is None and self.front_end_config.input_file is None):
+            raise click.UsageError("Must specify either --input or --input_file")
 
     async def run_workflow(self, session_manager: SessionManager):
 
@@ -80,17 +81,28 @@ class ConsoleFrontEndPlugin(SimpleFrontEndPluginBase[ConsoleFrontEndConfig]):
             input_list = list(self.front_end_config.input_query)
             logger.debug("Processing input: %s", self.front_end_config.input_query)
 
-            runner_outputs = await asyncio.gather(*[run_single_query(query) for query in input_list])
+            # Make `return_exceptions=False` explicit; all exceptions are raised instead of being silenced
+            runner_outputs = await asyncio.gather(*[run_single_query(query) for query in input_list],
+                                                  return_exceptions=False)
 
         elif (self.front_end_config.input_file):
 
             # Run the workflow
-            with open(self.front_end_config.input_file, "r", encoding="utf-8") as f:
+            with open(self.front_end_config.input_file, encoding="utf-8") as f:
 
                 async with session_manager.workflow.run(f) as runner:
                     runner_outputs = await runner.result(to_type=str)
         else:
             assert False, "Should not reach here. Should have been caught by pre_run"
 
-        # Print result
-        logger.info(f"\n{'-' * 50}\n{Fore.GREEN}Workflow Result:\n%s{Fore.RESET}\n{'-' * 50}", runner_outputs)
+        line = f"{'-' * 50}"
+        prefix = f"{line}\n{Fore.GREEN}Workflow Result:\n"
+        suffix = f"{Fore.RESET}\n{line}"
+
+        logger.info(f"{prefix}%s{suffix}", runner_outputs)
+
+        # (handler is a stream handler) => (level > INFO)
+        effective_level_too_high = all(
+            type(h) is not logging.StreamHandler or h.level > logging.INFO for h in logging.getLogger().handlers)
+        if effective_level_too_high:
+            print(f"{prefix}{runner_outputs}{suffix}")

nat/front_ends/fastapi/auth_flow_handlers/http_flow_handler.py

@@ -24,4 +24,4 @@ class HTTPAuthenticationFlowHandler(FlowHandlerBase):
     async def authenticate(self, config: AuthProviderBaseConfig, method: AuthFlowType) -> AuthenticatedContext:
 
         raise NotImplementedError(f"Authentication method '{method}' is not supported by the HTTP frontend."
-                                  f" Do you have Websockets enabled?")
+                                  f" Do you have WebSockets enabled?")