nvidia-nat-test 1.4.0a20251023__py3-none-any.whl → 1.4.0a20251129__py3-none-any.whl

nat/test/llm.py

@@ -96,6 +96,14 @@ async def test_llm_langchain(config: TestLLMConfig, builder: Builder):
             await chooser.async_sleep()
             yield chooser.next_response()
 
+        def bind_tools(self, tools: Any, **_kwargs: Any) -> "LangChainTestLLM":
+            """Bind tools to the LLM. Returns self to maintain fluent interface."""
+            return self
+
+        def bind(self, **_kwargs: Any) -> "LangChainTestLLM":
+            """Bind additional parameters to the LLM. Returns self to maintain fluent interface."""
+            return self
+
     yield LangChainTestLLM()
 
 

nat/test/plugin.py

@@ -14,7 +14,9 @@
 # limitations under the License.
 
 import os
+import random
 import subprocess
+import time
 import types
 import typing
 from collections.abc import AsyncGenerator
@@ -25,6 +27,10 @@ import pytest
 import pytest_asyncio
 
 if typing.TYPE_CHECKING:
+    import galileo.log_streams
+    import galileo.projects
+    import langsmith.client
+
     from docker.client import DockerClient
 
 
@@ -262,6 +268,116 @@ def require_weave_fixture(fail_missing: bool) -> types.ModuleType:
         pytest.skip(reason=reason)
 
 
+@pytest.fixture(name="langsmith_api_key", scope='session')
+def langsmith_api_key_fixture(fail_missing: bool):
+    """
+    Use for integration tests that require a LangSmith API key.
+    """
+    yield require_env_variables(
+        varnames=["LANGSMITH_API_KEY"],
+        reason="LangSmith integration tests require the `LANGSMITH_API_KEY` environment variable to be defined.",
+        fail_missing=fail_missing)
+
+
+@pytest.fixture(name="langsmith_client")
+def langsmith_client_fixture(langsmith_api_key: str, fail_missing: bool) -> "langsmith.client.Client":
+    try:
+        import langsmith.client
+        client = langsmith.client.Client()
+        return client
+    except ImportError:
+        reason = "LangSmith integration tests require the `langsmith` package to be installed."
+        if fail_missing:
+            raise RuntimeError(reason)
+        pytest.skip(reason=reason)
+
+
+@pytest.fixture(name="project_name")
+def project_name_fixture() -> str:
+    # Create a unique project name for each test run
+    return f"nat-e2e-test-{time.time()}-{random.random()}"
+
+
+@pytest.fixture(name="langsmith_project_name")
+def langsmith_project_name_fixture(langsmith_client: "langsmith.client.Client", project_name: str) -> Generator[str]:
+    langsmith_client.create_project(project_name)
+    yield project_name
+
+    langsmith_client.delete_project(project_name=project_name)
+
+
+@pytest.fixture(name="galileo_api_key", scope='session')
+def galileo_api_key_fixture(fail_missing: bool):
+    """
+    Use for integration tests that require a Galileo API key.
+    """
+    yield require_env_variables(
+        varnames=["GALILEO_API_KEY"],
+        reason="Galileo integration tests require the `GALILEO_API_KEY` environment variable to be defined.",
+        fail_missing=fail_missing)
+
+
+@pytest.fixture(name="galileo_project")
+def galileo_project_fixture(galileo_api_key: str, fail_missing: bool,
+                            project_name: str) -> Generator["galileo.projects.Project"]:
+    """
+    Creates a unique Galileo project and deletes it after the test run.
+    """
+    try:
+        import galileo.projects
+        project = galileo.projects.create_project(name=project_name)
+        yield project
+
+        galileo.projects.delete_project(id=project.id)
+    except ImportError as e:
+        reason = "Galileo integration tests require the `galileo` package to be installed."
+        if fail_missing:
+            raise RuntimeError(reason) from e
+        pytest.skip(reason=reason)
+
+
+@pytest.fixture(name="galileo_log_stream")
+def galileo_log_stream_fixture(galileo_project: "galileo.projects.Project") -> "galileo.log_streams.LogStream":
+    """
+    Creates a Galileo log stream for integration tests.
+
+    The log stream is automatically deleted when the associated project is deleted.
+    """
+    import galileo.log_streams
+    return galileo.log_streams.create_log_stream(project_id=galileo_project.id, name="test")
+
+
+@pytest.fixture(name="catalyst_keys", scope='session')
+def catalyst_keys_fixture(fail_missing: bool):
+    """
+    Use for integration tests that require RagaAI Catalyst credentials.
+    """
+    yield require_env_variables(
+        varnames=["CATALYST_ACCESS_KEY", "CATALYST_SECRET_KEY"],
+        reason="Catalyst integration tests require the `CATALYST_ACCESS_KEY` and `CATALYST_SECRET_KEY` environment "
+        "variables to be defined.",
+        fail_missing=fail_missing)
+
+
+@pytest.fixture(name="catalyst_project_name")
+def catalyst_project_name_fixture(catalyst_keys) -> str:
+    return os.environ.get("NAT_CI_CATALYST_PROJECT_NAME", "nat-e2e")
+
+
+@pytest.fixture(name="catalyst_dataset_name")
+def catalyst_dataset_name_fixture(catalyst_project_name: str, project_name: str) -> str:
+    """
+    We can't create and delete projects, but we can create and delete datasets, so use a unique dataset name
+    """
+    dataset_name = project_name.replace('.', '-')
+    yield dataset_name
+
+    from ragaai_catalyst import Dataset
+    ds = Dataset(catalyst_project_name)
+    if dataset_name in ds.list_datasets():
+        ds.delete_dataset(dataset_name)
+
+
 @pytest.fixture(name="require_docker", scope='session')
 def require_docker_fixture(fail_missing: bool) -> "DockerClient":
     """
@@ -400,14 +516,22 @@ def populate_milvus_fixture(milvus_uri: str, root_repo_dir: Path):
                    check=True)
 
 
-@pytest.fixture(name="require_nest_asyncio", scope="session")
+@pytest.fixture(name="require_nest_asyncio", scope="session", autouse=True)
 def require_nest_asyncio_fixture():
     """
-    Some tests require nest_asyncio to be installed to allow nested event loops, calling nest_asyncio.apply() more than
-    once is a no-op so it's safe to call this fixture even if one of our dependencies already called it.
+    Some tests require the nest_asyncio2 patch to be applied to allow nested event loops, calling
+    `nest_asyncio2.apply()` more than once is a no-op. However we need to ensure that the nest_asyncio2 patch is
+    applied prior to the older nest_asyncio patch is applied. Requiring us to ensure that any library which will apply
+    the patch on import is lazily imported.
     """
-    import nest_asyncio
-    nest_asyncio.apply()
+    import nest_asyncio2
+    try:
+        nest_asyncio2.apply(error_on_mispatched=True)
+    except RuntimeError as e:
+        raise RuntimeError(
+            "nest_asyncio2 fixture called but asyncio is already patched, most likely this is due to the nest_asyncio "
+            "being applied first, which is not compatible with Python 3.12+. Please ensure that any libraries which "
+            "apply nest_asyncio on import are lazily imported.") from e
 
 
 @pytest.fixture(name="phoenix_url", scope="session")
@@ -447,14 +571,15 @@ def fixture_redis_server(fail_missing: bool) -> Generator[dict[str, str | int]]:
     host = os.environ.get("NAT_CI_REDIS_HOST", "localhost")
     port = int(os.environ.get("NAT_CI_REDIS_PORT", "6379"))
     db = int(os.environ.get("NAT_CI_REDIS_DB", "0"))
+    password = os.environ.get("REDIS_PASSWORD", "redis")
     bucket_name = os.environ.get("NAT_CI_REDIS_BUCKET_NAME", "test")
 
     try:
         import redis
-        client = redis.Redis(host=host, port=port, db=db)
+        client = redis.Redis(host=host, port=port, db=db, password=password)
         if not client.ping():
             raise RuntimeError("Failed to connect to Redis")
-        yield {"host": host, "port": port, "db": db, "bucket_name": bucket_name}
+        yield {"host": host, "port": port, "db": db, "bucket_name": bucket_name, "password": password}
     except ImportError:
         if fail_missing:
             raise
@@ -588,3 +713,162 @@ def langfuse_trace_url_fixture(langfuse_url: str) -> str:
     the trace url which is what this fixture provides.
     """
     return f"{langfuse_url}/api/public/otel/v1/traces"
+
+
+@pytest.fixture(name="oauth2_server_url", scope="session")
+def oauth2_server_url_fixture(fail_missing: bool) -> str:
+    """
+    To run these tests, an oauth2 server must be running.
+    """
+    import requests
+
+    host = os.getenv("NAT_CI_OAUTH2_HOST", "localhost")
+    port = int(os.getenv("NAT_CI_OAUTH2_PORT", "5001"))
+    url = f"http://{host}:{port}"
+    try:
+        response = requests.get(url, timeout=5)
+        response.raise_for_status()
+
+        return url
+    except Exception as e:
+        reason = f"Unable to connect to OAuth2 server at {url}: {e}"
+        if fail_missing:
+            raise RuntimeError(reason)
+        pytest.skip(reason=reason)
+
+
+@pytest.fixture(name="oauth2_client_credentials", scope="session")
+def oauth2_client_credentials_fixture(oauth2_server_url: str, fail_missing: bool) -> dict[str, typing.Any]:
+    """
+    Fixture to provide OAuth2 client credentials for testing
+
+    Simulates the steps a user would take in a web browser to create a new OAuth2 client as documented in:
+    examples/front_ends/simple_auth/README.md
+    """
+
+    try:
+        import requests
+        from bs4 import BeautifulSoup
+        username = os.getenv("NAT_CI_OAUTH2_CLIENT_USERNAME", "Testy Testerson")
+
+        # This post request responds with a cookie that we need for future requests and a 302 redirect, the response
+        # for the redirected url doesn't contain the cookie, so we disable the redirect here to capture the cookie
+        user_create_response = requests.post(oauth2_server_url,
+                                             data=[("username", username)],
+                                             headers={"Content-Type": "application/x-www-form-urlencoded"},
+                                             allow_redirects=False,
+                                             timeout=5)
+        user_create_response.raise_for_status()
+        cookies = user_create_response.cookies
+
+        client_create_response = requests.post(f"{oauth2_server_url}/create_client",
+                                               cookies=cookies,
+                                               headers={"Content-Type": "application/x-www-form-urlencoded"},
+                                               data=[
+                                                   ("client_name", "test"),
+                                                   ("client_uri", "https://test.com"),
+                                                   ("scope", "openid profile email"),
+                                                   ("redirect_uri", "http://localhost:8000/auth/redirect"),
+                                                   ("grant_type", "authorization_code\nrefresh_token"),
+                                                   ("response_type", "code"),
+                                                   ("token_endpoint_auth_method", "client_secret_post"),
+                                               ],
+                                               timeout=5)
+        client_create_response.raise_for_status()
+
+        # Unfortunately the response is HTML so we need to parse it to get the client ID and secret, which are not
+        # locatable via ID tags
+        soup = BeautifulSoup(client_create_response.text, 'html.parser')
+        strong_tags = soup.find_all('strong')
+        i = 0
+        client_id = None
+        client_secret = None
+        while i < len(strong_tags) and None in (client_id, client_secret):
+            tag = strong_tags[i]
+            contents = "".join(tag.contents)
+            if client_id is None and "client_id:" in contents:
+                client_id = tag.next_sibling.strip()
+            elif client_secret is None and "client_secret:" in contents:
+                client_secret = tag.next_sibling.strip()
+
+            i += 1
+
+        assert client_id is not None and client_secret is not None, "Failed to parse client credentials from response"
+
+        return {
+            "id": client_id,
+            "secret": client_secret,
+            "username": username,
+            "url": oauth2_server_url,
+            "cookies": cookies
+        }
+
+    except Exception as e:
+        reason = f"Unable to create OAuth2 client: {e}"
+        if fail_missing:
+            raise RuntimeError(reason)
+        pytest.skip(reason=reason)
+
+
+@pytest.fixture(name="local_sandbox_url", scope="session")
+def local_sandbox_url_fixture(fail_missing: bool) -> str:
+    """Check if sandbox server is running before running tests."""
+    import requests
+    url = os.environ.get("NAT_CI_SANDBOX_URL", "http://127.0.0.1:6000")
+    try:
+        response = requests.get(url, timeout=5)
+        response.raise_for_status()
+        return url
+    except Exception:
+        reason = (f"Sandbox server is not running at {url}. "
+                  "Please start it with: cd src/nat/tool/code_execution/local_sandbox && ./start_local_sandbox.sh")
+        if fail_missing:
+            raise RuntimeError(reason)
+        pytest.skip(reason)
+
+
+@pytest.fixture(name="sandbox_config", scope="session")
+def sandbox_config_fixture(local_sandbox_url: str) -> dict[str, typing.Any]:
+    """Configuration for sandbox testing."""
+    return {
+        "base_url": local_sandbox_url,
+        "execute_url": f"{local_sandbox_url.rstrip('/')}/execute",
+        "timeout": int(os.environ.get("SANDBOX_TIMEOUT", "30")),
+        "connection_timeout": 5
+    }
+
+
+@pytest.fixture(name="piston_url", scope="session")
+def piston_url_fixture(fail_missing: bool) -> str:
+    """
+    Configuration for piston testing.
+
+    The public piston server limits usage to five requests per minute.
+    """
+    import requests
+    url = os.environ.get("NAT_CI_PISTON_URL", "https://emkc.org/api/v2/piston")
+    try:
+        response = requests.get(f"{url.rstrip('/')}/runtimes", timeout=30)
+        response.raise_for_status()
+        return url
+    except Exception:
+        reason = (f"Piston server is not running at {url}. "
+                  "Please start it with: cd src/nat/tool/code_execution/local_sandbox && ./start_local_sandbox.sh")
+        if fail_missing:
+            raise RuntimeError(reason)
+        pytest.skip(reason)
+
+
+@pytest.fixture(autouse=True, scope="session")
+def import_adk_early():
+    """
+    Import ADK early to work-around slow import issue (https://github.com/google/adk-python/issues/2433),
+    when ADK is imported early it takes about 8 seconds, however if we wait until the `packages/nvidia_nat_adk/tests`
+    run the same import will take about 70 seconds.
+
+    Since ADK is an optional dependency, we will ignore any import errors.
+    """
+    try:
+        import google.adk  # noqa: F401
+    except ImportError:
+        pass

nat/test/tool_test_runner.py

@@ -29,18 +29,21 @@ from nat.builder.function import FunctionGroup
 from nat.builder.function_info import FunctionInfo
 from nat.cli.type_registry import GlobalTypeRegistry
 from nat.data_models.authentication import AuthProviderBaseConfig
+from nat.data_models.component_ref import MiddlewareRef
 from nat.data_models.embedder import EmbedderBaseConfig
 from nat.data_models.function import FunctionBaseConfig
 from nat.data_models.function import FunctionGroupBaseConfig
 from nat.data_models.function_dependencies import FunctionDependencies
 from nat.data_models.llm import LLMBaseConfig
 from nat.data_models.memory import MemoryBaseConfig
+from nat.data_models.middleware import FunctionMiddlewareBaseConfig
 from nat.data_models.object_store import ObjectStoreBaseConfig
 from nat.data_models.retriever import RetrieverBaseConfig
 from nat.data_models.ttc_strategy import TTCStrategyBaseConfig
 from nat.experimental.test_time_compute.models.stage_enums import PipelineTypeEnum
 from nat.experimental.test_time_compute.models.stage_enums import StageTypeEnum
 from nat.memory.interfaces import MemoryEditor
+from nat.middleware import FunctionMiddleware
 from nat.object_store.interfaces import ObjectStore
 from nat.runtime.loader import PluginTypes
 from nat.runtime.loader import discover_and_register_plugins
@@ -289,6 +292,19 @@ class MockBuilder(Builder):
         """Mock implementation."""
         return FunctionDependencies()
 
+    async def get_middleware(self, middleware_name: str | MiddlewareRef) -> FunctionMiddleware:
+        """Mock implementation."""
+        return FunctionMiddleware()
+
+    def get_middleware_config(self, middleware_name: str | MiddlewareRef) -> FunctionMiddlewareBaseConfig:
+        """Mock implementation."""
+        return FunctionMiddlewareBaseConfig()
+
+    async def add_middleware(self, name: str | MiddlewareRef,
+                             config: FunctionMiddlewareBaseConfig) -> FunctionMiddleware:
+        """Mock implementation."""
+        return FunctionMiddleware()
+
 
 class ToolTestRunner:
     """

nat/test/utils.py

@@ -68,27 +68,20 @@ def locate_example_config(example_config_class: type,
     return config_path
 
 
-async def run_workflow(
-    *,
-    config: "Config | None" = None,
-    config_file: "StrPath | None" = None,
-    question: str,
-    expected_answer: str,
-    assert_expected_answer: bool = True,
-) -> str:
-    from nat.builder.workflow_builder import WorkflowBuilder
-    from nat.runtime.loader import load_config
-    from nat.runtime.session import SessionManager
-
-    if config is None:
-        assert config_file is not None, "Either config_file or config must be provided"
-        assert Path(config_file).exists(), f"Config file {config_file} does not exist"
-        config = load_config(config_file)
-
-    async with WorkflowBuilder.from_config(config=config) as workflow_builder:
-        workflow = SessionManager(await workflow_builder.build())
-        async with workflow.run(question) as runner:
-            result = await runner.result(to_type=str)
+async def run_workflow(*,
+                       config: "Config | None" = None,
+                       config_file: "StrPath | None" = None,
+                       question: str,
+                       expected_answer: str,
+                       assert_expected_answer: bool = True,
+                       **kwargs) -> str:
+    """
+    Test specific wrapper for `nat.utils.run_workflow` to run a workflow with a question and validate the expected
+    answer. This variant always sets the result type to `str`.
+    """
+    from nat.utils import run_workflow as nat_run_workflow
+
+    result = await nat_run_workflow(config=config, config_file=config_file, prompt=question, to_type=str, **kwargs)
 
     if assert_expected_answer:
         assert expected_answer.lower() in result.lower(), f"Expected '{expected_answer}' in '{result}'"

{nvidia_nat_test-1.4.0a20251023.dist-info → nvidia_nat_test-1.4.0a20251129.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nvidia-nat-test
-Version: 1.4.0a20251023
+Version: 1.4.0a20251129
 Summary: Testing utilities for NeMo Agent toolkit
 Author: NVIDIA Corporation
 Maintainer: NVIDIA Corporation
@@ -16,7 +16,7 @@ Requires-Python: <3.14,>=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE-3rd-party.txt
 License-File: LICENSE.md
-Requires-Dist: nvidia-nat==v1.4.0a20251023
+Requires-Dist: nvidia-nat==v1.4.0a20251129
 Requires-Dist: langchain-community~=0.3
 Requires-Dist: pytest~=8.3
 Dynamic: license-file

{nvidia_nat_test-1.4.0a20251023.dist-info → nvidia_nat_test-1.4.0a20251129.dist-info}/RECORD

@@ -2,17 +2,17 @@ nat/meta/pypi.md,sha256=LLKJHg5oN1-M9Pqfk3Bmphkk4O2TFsyiixuK5T0Y-gw,1100
 nat/test/__init__.py,sha256=_RnTJnsUucHvla_nYKqD4O4g8Bz0tcuDRzWk1bEhcy0,875
 nat/test/embedder.py,sha256=ClDyK1kna4hCBSlz71gK1B-ZjlwcBHTDQRekoNM81Bs,1809
 nat/test/functions.py,sha256=ZxXVzfaLBGOpR5qtmMrKU7q-M9-vVGGj3Xi5mrw4vHY,3557
-nat/test/llm.py,sha256=f6bz6arAQjhjuOKFrLfu_U1LbiyFzQmpM-q8b-WKSrU,9550
+nat/test/llm.py,sha256=dbFoWFrSAlUoKm6QGfS4VJdrhgxwkXzm1oaFd6K7jnM,9926
 nat/test/memory.py,sha256=xki_A2yiMhEZuQk60K7t04QRqf32nQqnfzD5Iv7fkvw,1456
 nat/test/object_store_tests.py,sha256=PyJioOtoSzILPq6LuD-sOZ_89PIcgXWZweoHBQpK2zQ,4281
-nat/test/plugin.py,sha256=NV4JcUnqhBhQzO4sc3P8JDBlwWohmu5TYVCWRbNPwVo,20911
+nat/test/plugin.py,sha256=HF25W2YPTiXaoIJggnZTstiTMaspQckvL_thQSseDEc,32434
 nat/test/register.py,sha256=o1BEA5fyxyFyCxXhQ6ArmtuNpgRyTEfvw6HdBgECPLI,897
-nat/test/tool_test_runner.py,sha256=SxavwXHkvCQDl_PUiiiqgvGfexKJJTeBdI5i1qk6AzI,21712
-nat/test/utils.py,sha256=Lml187P9SUP3IB_HhBaU1XNhiljcpOFFZOAxgQR1vQo,5936
-nvidia_nat_test-1.4.0a20251023.dist-info/licenses/LICENSE-3rd-party.txt,sha256=fOk5jMmCX9YoKWyYzTtfgl-SUy477audFC5hNY4oP7Q,284609
-nvidia_nat_test-1.4.0a20251023.dist-info/licenses/LICENSE.md,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
-nvidia_nat_test-1.4.0a20251023.dist-info/METADATA,sha256=QQJbkpZC8mW0xYQ7mdoxuw2--olf1TaOVpzOtXP3AZ0,1925
-nvidia_nat_test-1.4.0a20251023.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nvidia_nat_test-1.4.0a20251023.dist-info/entry_points.txt,sha256=7dOP9XB6iMDqvav3gYx9VWUwA8RrFzhbAa8nGeC8e4Y,99
-nvidia_nat_test-1.4.0a20251023.dist-info/top_level.txt,sha256=8-CJ2cP6-f0ZReXe5Hzqp-5pvzzHz-5Ds5H2bGqh1-U,4
-nvidia_nat_test-1.4.0a20251023.dist-info/RECORD,,
+nat/test/tool_test_runner.py,sha256=WDwIRo3160raBoEkj1-MgnLSCaaF2Ud_cARRIM3Qdag,22463
+nat/test/utils.py,sha256=GyhxIZ1CcUPcc8RMRyCzpHBEwVifeqiGxT3c9Pp0KAU,5774
+nvidia_nat_test-1.4.0a20251129.dist-info/licenses/LICENSE-3rd-party.txt,sha256=fOk5jMmCX9YoKWyYzTtfgl-SUy477audFC5hNY4oP7Q,284609
+nvidia_nat_test-1.4.0a20251129.dist-info/licenses/LICENSE.md,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
+nvidia_nat_test-1.4.0a20251129.dist-info/METADATA,sha256=9v96TNp_WS6cN3JQN9Aj2ecg2vupk_YPHatlqRYdcj4,1925
+nvidia_nat_test-1.4.0a20251129.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nvidia_nat_test-1.4.0a20251129.dist-info/entry_points.txt,sha256=7dOP9XB6iMDqvav3gYx9VWUwA8RrFzhbAa8nGeC8e4Y,99
+nvidia_nat_test-1.4.0a20251129.dist-info/top_level.txt,sha256=8-CJ2cP6-f0ZReXe5Hzqp-5pvzzHz-5Ds5H2bGqh1-U,4
+nvidia_nat_test-1.4.0a20251129.dist-info/RECORD,,