nvidia-nat-security 1.6.0__py3-none-any.whl

nat/meta/pypi.md

@@ -0,0 +1,24 @@
+<!--
+SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+![NVIDIA NeMo Agent Toolkit](https://media.githubusercontent.com/media/NVIDIA/NeMo-Agent-Toolkit/refs/heads/main/docs/source/_static/banner.png "NeMo Agent Toolkit banner image")
+
+# NVIDIA NeMo Agent Toolkit Security Subpackage
+
+Subpackage for security and red-team interfaces in NeMo Agent Toolkit.
+
+For more information about the NVIDIA NeMo Agent Toolkit, please visit the [NeMo Agent Toolkit GitHub Repo](https://github.com/NVIDIA/NeMo-Agent-Toolkit).

nat/plugins/security/__init__.py

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nat/plugins/security/cli/__init__.py

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nat/plugins/security/cli/commands.py

@@ -0,0 +1,19 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Security CLI command exports."""
+
+from nat.plugins.security.cli.red_teaming.red_teaming import red_team_command
+
+__all__ = ["red_team_command"]

nat/plugins/security/cli/red_teaming/__init__.py

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nat/plugins/security/cli/red_teaming/red_teaming.py

@@ -0,0 +1,138 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Red teaming CLI command."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from pathlib import Path
+
+import click
+
+logger = logging.getLogger(__name__)
+
+
+@click.group(name=__name__, invoke_without_command=True, help="Run red teaming evaluation with multiple scenarios.")
+@click.option(
+    "--red_team_config",
+    type=click.Path(exists=True, file_okay=True, dir_okay=False, path_type=Path),
+    required=False,
+    help="A YAML/JSON file containing red teaming configuration (evaluator, scenarios, etc.).",
+)
+@click.option(
+    "--config_file",
+    type=click.Path(exists=True, file_okay=True, dir_okay=False, path_type=Path),
+    required=False,
+    help="A JSON/YAML file that sets the parameters for the base workflow. "
+    "Overrides base_workflow in red_team_config if both are provided.",
+)
+@click.option(
+    "--dataset",
+    type=click.Path(exists=True, file_okay=True, dir_okay=False, path_type=Path),
+    required=False,
+    help="A JSON file with questions and ground truth answers. "
+    "Overrides the dataset path in the config file.",
+)
+@click.option(
+    "--result_json_path",
+    type=str,
+    default="$",
+    help="A JSON path to extract the result from the workflow. "
+    "For example, '$.output' extracts the 'output' field.",
+)
+@click.option(
+    "--endpoint",
+    type=str,
+    default=None,
+    help="Use endpoint for running the workflow. Example: http://localhost:8000/generate",
+)
+@click.option(
+    "--endpoint_timeout",
+    type=int,
+    default=300,
+    help="HTTP response timeout in seconds. Only relevant if endpoint is specified.",
+)
+@click.option(
+    "--reps",
+    type=int,
+    default=1,
+    help="Number of repetitions for the evaluation.",
+)
+@click.option(
+    "--override",
+    type=(str, str),
+    multiple=True,
+    help="Override config values for the base workflow config using dot notation "
+    "(e.g., --override llms.nim_llm.temperature 0.7)",
+)
+@click.pass_context
+def red_team_command(ctx, **kwargs) -> None:
+    """Run red teaming evaluation with multiple scenarios."""
+    pass
+
+
+@red_team_command.result_callback(replace=True)
+def process_red_team_eval(
+    processors,
+    *,
+    red_team_config: Path | None,
+    config_file: Path | None,
+    dataset: Path | None,
+    result_json_path: str,
+    endpoint: str | None,
+    endpoint_timeout: int,
+    reps: int,
+    override: tuple[tuple[str, str], ...],
+):
+    """Process the red team eval command and execute the evaluation."""
+    from nat.plugins.security.eval.runners.red_teaming_runner import RedTeamingRunner
+    from nat.runtime.loader import load_config
+
+    from .red_teaming_utils import load_red_teaming_config
+
+    # Must have at least one of these
+    if red_team_config is None and config_file is None:
+        raise click.ClickException("Either --red_team_config or --config_file must be provided.")
+
+    # Load configs
+    rt_config = None
+    if red_team_config is not None:
+        rt_config = load_red_teaming_config(red_team_config)
+        base_workflow_path = config_file or rt_config.base_workflow
+        if base_workflow_path is None:
+            raise click.ClickException(
+                "No base workflow specified. Set 'base_workflow' in red_team_config or provide --config_file.")
+        base_workflow_config = load_config(base_workflow_path)
+    else:
+        assert config_file is not None
+        base_workflow_config = load_config(config_file)
+
+    # Create and run the runner
+    runner = RedTeamingRunner(
+        config=rt_config,
+        base_workflow_config=base_workflow_config,
+        dataset_path=str(dataset) if dataset else None,
+        result_json_path=result_json_path,
+        endpoint=endpoint,
+        endpoint_timeout=endpoint_timeout,
+        reps=reps,
+        overrides=override,
+    )
+
+    try:
+        _ = asyncio.run(runner.run())
+    except ValueError as e:
+        raise click.ClickException(str(e)) from e

nat/plugins/security/cli/red_teaming/red_teaming_utils.py

@@ -0,0 +1,73 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utility functions for red team evaluation CLI."""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+
+import yaml
+
+from nat.plugins.security.eval.runners.red_teaming_runner import RedTeamingRunnerConfig
+
+logger = logging.getLogger(__name__)
+
+
+def load_red_teaming_config(config_file: Path) -> RedTeamingRunnerConfig:
+    """Load a RedTeamingRunnerConfig from a YAML or JSON file.
+
+    Args:
+        config_file: Path to the configuration file (YAML or JSON)
+
+    Returns:
+        Parsed RedTeamingRunnerConfig object
+
+    Raises:
+        ValueError: If the file format is invalid or parsing fails
+        FileNotFoundError: If the file doesn't exist
+    """
+    # Ensure plugins are discovered and registered before parsing the config.
+    # This triggers rebuild_annotations() which allows Pydantic to resolve
+    # discriminated unions (e.g., _type: nim -> NIMConfig).
+    from nat.runtime.loader import PluginTypes
+    from nat.runtime.loader import discover_and_register_plugins
+    discover_and_register_plugins(PluginTypes.CONFIG_OBJECT)
+
+    logger.info("Loading red teaming config from: %s", config_file)
+
+    if not config_file.exists():
+        raise FileNotFoundError(f"Red teaming config file not found: {config_file}")
+
+    with open(config_file, encoding='utf-8') as f:
+        if config_file.suffix in ('.yml', '.yaml'):
+            config_data = yaml.safe_load(f)
+        elif config_file.suffix == '.json':
+            config_data = json.load(f)
+        else:
+            raise ValueError(f"Unsupported file format: {config_file.suffix}. "
+                             "Use .yml, .yaml, or .json")
+
+    if not isinstance(config_data, dict):
+        raise ValueError(f"Red teaming config file must contain a dictionary, got {type(config_data)}")
+
+    try:
+        config = RedTeamingRunnerConfig(**config_data)
+    except Exception as e:
+        raise ValueError(f"Failed to parse red teaming config: {e}") from e
+
+    logger.info("Loaded red teaming config with %d scenarios", len(config.scenarios))
+    return config

nat/plugins/security/eval/__init__.py

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nat/plugins/security/eval/red_teaming_evaluator/__init__.py

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nat/plugins/security/eval/red_teaming_evaluator/data_models.py

@@ -0,0 +1,66 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Data models for red teaming evaluation output."""
+
+from __future__ import annotations
+
+from pydantic import Field
+
+from nat.data_models.intermediate_step import IntermediateStep
+from nat.plugins.eval.data_models.evaluator_io import EvalOutputItem
+
+
+class ConditionEvalOutputItem(EvalOutputItem):
+    """Evaluation results for a single IntermediateStep that meets the filtering condition.
+
+    Attributes:
+        id: Identifier from the input item.
+        score: Average score across all filter conditions.
+        reasoning: Reasoning for given score.
+        intermediate_step: IntermediateStep selected and evaluated via reduction strategy.
+        error_message: Error message if any step of the evaluation has failed.
+    """
+
+    intermediate_step: IntermediateStep | None = Field(
+        default=None,
+        description="The single IntermediateStep that was selected and evaluated (based on reduction strategy)")
+    error_message: str | None = Field(default=None,
+                                      description="Error message if any step of the evaluation has failed")
+
+    @classmethod
+    def empty(cls, id: str, error: str | None = None) -> ConditionEvalOutputItem:
+        """Create an empty ConditionEvalOutputItem.
+
+        Returns:
+            Empty ConditionEvalOutputItem instance
+        """
+        return cls(id=id, score=0.0, reasoning={}, error_message=error, intermediate_step=None)
+
+
+class RedTeamingEvalOutputItem(EvalOutputItem):
+    """Extended evaluation output item for red teaming evaluations.
+
+    Organizes results by filter condition name, with each condition containing
+    its score, the evaluated output, and the single intermediate step that was selected.
+
+    Attributes:
+        id: Identifier from the input item
+        score: Average score across all filter conditions
+        reasoning: Summary information for compatibility
+        results_by_condition: Map from condition name to evaluation results
+    """
+
+    results_by_condition: dict[str, ConditionEvalOutputItem] = Field(
+        description="Results organized by filter condition name")