nvidia-nat 1.4.0a20251105__py3-none-any.whl → 1.4.0a20251107__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nat/eval/dataset_handler/dataset_filter.py +34 -2
- nat/llm/aws_bedrock_llm.py +11 -9
- nat/llm/azure_openai_llm.py +12 -4
- nat/llm/litellm_llm.py +11 -4
- nat/llm/nim_llm.py +11 -9
- nat/llm/openai_llm.py +12 -9
- nat/tool/code_execution/code_sandbox.py +3 -6
- nat/tool/code_execution/local_sandbox/Dockerfile.sandbox +19 -32
- nat/tool/code_execution/local_sandbox/local_sandbox_server.py +5 -0
- nat/tool/code_execution/local_sandbox/sandbox.requirements.txt +2 -0
- nat/tool/code_execution/local_sandbox/start_local_sandbox.sh +10 -4
- nat/utils/__init__.py +8 -4
- {nvidia_nat-1.4.0a20251105.dist-info → nvidia_nat-1.4.0a20251107.dist-info}/METADATA +1 -1
- {nvidia_nat-1.4.0a20251105.dist-info → nvidia_nat-1.4.0a20251107.dist-info}/RECORD +19 -22
- nat/data_models/temperature_mixin.py +0 -44
- nat/data_models/top_p_mixin.py +0 -44
- nat/tool/code_execution/test_code_execution_sandbox.py +0 -414
- {nvidia_nat-1.4.0a20251105.dist-info → nvidia_nat-1.4.0a20251107.dist-info}/WHEEL +0 -0
- {nvidia_nat-1.4.0a20251105.dist-info → nvidia_nat-1.4.0a20251107.dist-info}/entry_points.txt +0 -0
- {nvidia_nat-1.4.0a20251105.dist-info → nvidia_nat-1.4.0a20251107.dist-info}/licenses/LICENSE-3rd-party.txt +0 -0
- {nvidia_nat-1.4.0a20251105.dist-info → nvidia_nat-1.4.0a20251107.dist-info}/licenses/LICENSE.md +0 -0
- {nvidia_nat-1.4.0a20251105.dist-info → nvidia_nat-1.4.0a20251107.dist-info}/top_level.txt +0 -0
|
@@ -13,6 +13,8 @@
|
|
|
13
13
|
# See the License for the specific language governing permissions and
|
|
14
14
|
# limitations under the License.
|
|
15
15
|
|
|
16
|
+
import fnmatch
|
|
17
|
+
|
|
16
18
|
import pandas as pd
|
|
17
19
|
|
|
18
20
|
from nat.data_models.dataset_handler import EvalFilterConfig
|
|
@@ -24,6 +26,7 @@ class DatasetFilter:
|
|
|
24
26
|
- If a allowlist is provided, only keep rows matching the filter values.
|
|
25
27
|
- If a denylist is provided, remove rows matching the filter values.
|
|
26
28
|
- If the filter column does not exist in the DataFrame, the filtering is skipped for that column.
|
|
29
|
+
- Supports Unix shell-style wildcards (``*``, ``?``, ``[seq]``, ``[!seq]``) for string matching.
|
|
27
30
|
|
|
28
31
|
This is a utility class that is dataset agnostic and can be used to filter any DataFrame based on the provided
|
|
29
32
|
filter configuration.
|
|
@@ -33,6 +36,33 @@ class DatasetFilter:
|
|
|
33
36
|
|
|
34
37
|
self.filter_config = filter_config
|
|
35
38
|
|
|
39
|
+
@staticmethod
|
|
40
|
+
def _match_wildcard_patterns(series: pd.Series, patterns: list[str | int | float]) -> pd.Series:
|
|
41
|
+
"""
|
|
42
|
+
Match series values against wildcard patterns and exact values.
|
|
43
|
+
|
|
44
|
+
Args:
|
|
45
|
+
series (pd.Series): pandas Series to match against
|
|
46
|
+
patterns (list[str | int | float]): List of patterns/values
|
|
47
|
+
|
|
48
|
+
Returns:
|
|
49
|
+
pd.Series: Boolean Series indicating matches
|
|
50
|
+
"""
|
|
51
|
+
# Convert series to string for pattern matching
|
|
52
|
+
str_series = series.astype(str)
|
|
53
|
+
|
|
54
|
+
# Initialize boolean mask
|
|
55
|
+
matches = pd.Series([False] * len(series), index=series.index)
|
|
56
|
+
|
|
57
|
+
# Check each pattern using fnmatch with list comprehension to avoid lambda capture
|
|
58
|
+
for pattern in patterns:
|
|
59
|
+
pattern_str = str(pattern)
|
|
60
|
+
pattern_matches = pd.Series([fnmatch.fnmatch(val, pattern_str) for val in str_series],
|
|
61
|
+
index=str_series.index)
|
|
62
|
+
matches |= pattern_matches
|
|
63
|
+
|
|
64
|
+
return matches
|
|
65
|
+
|
|
36
66
|
def apply_filters(self, df) -> pd.DataFrame:
|
|
37
67
|
|
|
38
68
|
filtered_df = df.copy()
|
|
@@ -41,12 +71,14 @@ class DatasetFilter:
|
|
|
41
71
|
if self.filter_config.allowlist:
|
|
42
72
|
for column, values in self.filter_config.allowlist.field.items():
|
|
43
73
|
if column in filtered_df.columns:
|
|
44
|
-
|
|
74
|
+
matches = self._match_wildcard_patterns(filtered_df[column], values)
|
|
75
|
+
filtered_df = filtered_df[matches]
|
|
45
76
|
|
|
46
77
|
# Apply denylist (remove specified rows)
|
|
47
78
|
if self.filter_config.denylist:
|
|
48
79
|
for column, values in self.filter_config.denylist.field.items():
|
|
49
80
|
if column in filtered_df.columns:
|
|
50
|
-
|
|
81
|
+
matches = self._match_wildcard_patterns(filtered_df[column], values)
|
|
82
|
+
filtered_df = filtered_df[~matches]
|
|
51
83
|
|
|
52
84
|
return filtered_df
|
nat/llm/aws_bedrock_llm.py
CHANGED
|
@@ -25,18 +25,10 @@ from nat.data_models.optimizable import OptimizableField
|
|
|
25
25
|
from nat.data_models.optimizable import OptimizableMixin
|
|
26
26
|
from nat.data_models.optimizable import SearchSpace
|
|
27
27
|
from nat.data_models.retry_mixin import RetryMixin
|
|
28
|
-
from nat.data_models.temperature_mixin import TemperatureMixin
|
|
29
28
|
from nat.data_models.thinking_mixin import ThinkingMixin
|
|
30
|
-
from nat.data_models.top_p_mixin import TopPMixin
|
|
31
29
|
|
|
32
30
|
|
|
33
|
-
class AWSBedrockModelConfig(LLMBaseConfig,
|
|
34
|
-
RetryMixin,
|
|
35
|
-
OptimizableMixin,
|
|
36
|
-
TemperatureMixin,
|
|
37
|
-
TopPMixin,
|
|
38
|
-
ThinkingMixin,
|
|
39
|
-
name="aws_bedrock"):
|
|
31
|
+
class AWSBedrockModelConfig(LLMBaseConfig, RetryMixin, OptimizableMixin, ThinkingMixin, name="aws_bedrock"):
|
|
40
32
|
"""An AWS Bedrock llm provider to be used with an LLM client."""
|
|
41
33
|
|
|
42
34
|
model_config = ConfigDict(protected_namespaces=(), extra="allow")
|
|
@@ -61,6 +53,16 @@ class AWSBedrockModelConfig(LLMBaseConfig,
|
|
|
61
53
|
default=None, description="Bedrock endpoint to use. Needed if you don't want to default to us-east-1 endpoint.")
|
|
62
54
|
credentials_profile_name: str | None = Field(
|
|
63
55
|
default=None, description="The name of the profile in the ~/.aws/credentials or ~/.aws/config files.")
|
|
56
|
+
temperature: float | None = OptimizableField(default=None,
|
|
57
|
+
ge=0.0,
|
|
58
|
+
le=1.0,
|
|
59
|
+
description="Sampling temperature in [0, 1].",
|
|
60
|
+
space=SearchSpace(high=0.9, low=0.1, step=0.2))
|
|
61
|
+
top_p: float | None = OptimizableField(default=None,
|
|
62
|
+
ge=0.0,
|
|
63
|
+
le=1.0,
|
|
64
|
+
description="Top-p for distribution sampling.",
|
|
65
|
+
space=SearchSpace(high=1.0, low=0.5, step=0.1))
|
|
64
66
|
|
|
65
67
|
|
|
66
68
|
@register_llm_provider(config_type=AWSBedrockModelConfig)
|
nat/llm/azure_openai_llm.py
CHANGED
|
@@ -22,17 +22,15 @@ from nat.builder.llm import LLMProviderInfo
|
|
|
22
22
|
from nat.cli.register_workflow import register_llm_provider
|
|
23
23
|
from nat.data_models.common import OptionalSecretStr
|
|
24
24
|
from nat.data_models.llm import LLMBaseConfig
|
|
25
|
+
from nat.data_models.optimizable import OptimizableField
|
|
26
|
+
from nat.data_models.optimizable import SearchSpace
|
|
25
27
|
from nat.data_models.retry_mixin import RetryMixin
|
|
26
|
-
from nat.data_models.temperature_mixin import TemperatureMixin
|
|
27
28
|
from nat.data_models.thinking_mixin import ThinkingMixin
|
|
28
|
-
from nat.data_models.top_p_mixin import TopPMixin
|
|
29
29
|
|
|
30
30
|
|
|
31
31
|
class AzureOpenAIModelConfig(
|
|
32
32
|
LLMBaseConfig,
|
|
33
33
|
RetryMixin,
|
|
34
|
-
TemperatureMixin,
|
|
35
|
-
TopPMixin,
|
|
36
34
|
ThinkingMixin,
|
|
37
35
|
name="azure_openai",
|
|
38
36
|
):
|
|
@@ -50,6 +48,16 @@ class AzureOpenAIModelConfig(
|
|
|
50
48
|
serialization_alias="azure_deployment",
|
|
51
49
|
description="The Azure OpenAI hosted model/deployment name.")
|
|
52
50
|
seed: int | None = Field(default=None, description="Random seed to set for generation.")
|
|
51
|
+
temperature: float | None = OptimizableField(default=None,
|
|
52
|
+
ge=0.0,
|
|
53
|
+
le=1.0,
|
|
54
|
+
description="Sampling temperature in [0, 1].",
|
|
55
|
+
space=SearchSpace(high=0.9, low=0.1, step=0.2))
|
|
56
|
+
top_p: float | None = OptimizableField(default=None,
|
|
57
|
+
ge=0.0,
|
|
58
|
+
le=1.0,
|
|
59
|
+
description="Top-p for distribution sampling.",
|
|
60
|
+
space=SearchSpace(high=1.0, low=0.5, step=0.1))
|
|
53
61
|
|
|
54
62
|
|
|
55
63
|
@register_llm_provider(config_type=AzureOpenAIModelConfig)
|
nat/llm/litellm_llm.py
CHANGED
|
@@ -26,18 +26,15 @@ from nat.data_models.common import OptionalSecretStr
|
|
|
26
26
|
from nat.data_models.llm import LLMBaseConfig
|
|
27
27
|
from nat.data_models.optimizable import OptimizableField
|
|
28
28
|
from nat.data_models.optimizable import OptimizableMixin
|
|
29
|
+
from nat.data_models.optimizable import SearchSpace
|
|
29
30
|
from nat.data_models.retry_mixin import RetryMixin
|
|
30
|
-
from nat.data_models.temperature_mixin import TemperatureMixin
|
|
31
31
|
from nat.data_models.thinking_mixin import ThinkingMixin
|
|
32
|
-
from nat.data_models.top_p_mixin import TopPMixin
|
|
33
32
|
|
|
34
33
|
|
|
35
34
|
class LiteLlmModelConfig(
|
|
36
35
|
LLMBaseConfig,
|
|
37
36
|
OptimizableMixin,
|
|
38
37
|
RetryMixin,
|
|
39
|
-
TemperatureMixin,
|
|
40
|
-
TopPMixin,
|
|
41
38
|
ThinkingMixin,
|
|
42
39
|
name="litellm",
|
|
43
40
|
):
|
|
@@ -54,6 +51,16 @@ class LiteLlmModelConfig(
|
|
|
54
51
|
serialization_alias="model",
|
|
55
52
|
description="The LiteLlm hosted model name.")
|
|
56
53
|
seed: int | None = Field(default=None, description="Random seed to set for generation.")
|
|
54
|
+
temperature: float | None = OptimizableField(default=None,
|
|
55
|
+
ge=0.0,
|
|
56
|
+
le=1.0,
|
|
57
|
+
description="Sampling temperature in [0, 1].",
|
|
58
|
+
space=SearchSpace(high=0.9, low=0.1, step=0.2))
|
|
59
|
+
top_p: float | None = OptimizableField(default=None,
|
|
60
|
+
ge=0.0,
|
|
61
|
+
le=1.0,
|
|
62
|
+
description="Top-p for distribution sampling.",
|
|
63
|
+
space=SearchSpace(high=1.0, low=0.5, step=0.1))
|
|
57
64
|
|
|
58
65
|
|
|
59
66
|
@register_llm_provider(config_type=LiteLlmModelConfig)
|
nat/llm/nim_llm.py
CHANGED
|
@@ -27,18 +27,10 @@ from nat.data_models.optimizable import OptimizableField
|
|
|
27
27
|
from nat.data_models.optimizable import OptimizableMixin
|
|
28
28
|
from nat.data_models.optimizable import SearchSpace
|
|
29
29
|
from nat.data_models.retry_mixin import RetryMixin
|
|
30
|
-
from nat.data_models.temperature_mixin import TemperatureMixin
|
|
31
30
|
from nat.data_models.thinking_mixin import ThinkingMixin
|
|
32
|
-
from nat.data_models.top_p_mixin import TopPMixin
|
|
33
31
|
|
|
34
32
|
|
|
35
|
-
class NIMModelConfig(LLMBaseConfig,
|
|
36
|
-
RetryMixin,
|
|
37
|
-
OptimizableMixin,
|
|
38
|
-
TemperatureMixin,
|
|
39
|
-
TopPMixin,
|
|
40
|
-
ThinkingMixin,
|
|
41
|
-
name="nim"):
|
|
33
|
+
class NIMModelConfig(LLMBaseConfig, RetryMixin, OptimizableMixin, ThinkingMixin, name="nim"):
|
|
42
34
|
"""An NVIDIA Inference Microservice (NIM) llm provider to be used with an LLM client."""
|
|
43
35
|
|
|
44
36
|
model_config = ConfigDict(protected_namespaces=(), extra="allow")
|
|
@@ -51,6 +43,16 @@ class NIMModelConfig(LLMBaseConfig,
|
|
|
51
43
|
max_tokens: PositiveInt = OptimizableField(default=300,
|
|
52
44
|
description="Maximum number of tokens to generate.",
|
|
53
45
|
space=SearchSpace(high=2176, low=128, step=512))
|
|
46
|
+
temperature: float | None = OptimizableField(default=None,
|
|
47
|
+
ge=0.0,
|
|
48
|
+
le=1.0,
|
|
49
|
+
description="Sampling temperature in [0, 1].",
|
|
50
|
+
space=SearchSpace(high=0.9, low=0.1, step=0.2))
|
|
51
|
+
top_p: float | None = OptimizableField(default=None,
|
|
52
|
+
ge=0.0,
|
|
53
|
+
le=1.0,
|
|
54
|
+
description="Top-p for distribution sampling.",
|
|
55
|
+
space=SearchSpace(high=1.0, low=0.5, step=0.1))
|
|
54
56
|
|
|
55
57
|
|
|
56
58
|
@register_llm_provider(config_type=NIMModelConfig)
|
nat/llm/openai_llm.py
CHANGED
|
@@ -24,19 +24,12 @@ from nat.data_models.common import OptionalSecretStr
|
|
|
24
24
|
from nat.data_models.llm import LLMBaseConfig
|
|
25
25
|
from nat.data_models.optimizable import OptimizableField
|
|
26
26
|
from nat.data_models.optimizable import OptimizableMixin
|
|
27
|
+
from nat.data_models.optimizable import SearchSpace
|
|
27
28
|
from nat.data_models.retry_mixin import RetryMixin
|
|
28
|
-
from nat.data_models.temperature_mixin import TemperatureMixin
|
|
29
29
|
from nat.data_models.thinking_mixin import ThinkingMixin
|
|
30
|
-
from nat.data_models.top_p_mixin import TopPMixin
|
|
31
30
|
|
|
32
31
|
|
|
33
|
-
class OpenAIModelConfig(LLMBaseConfig,
|
|
34
|
-
RetryMixin,
|
|
35
|
-
OptimizableMixin,
|
|
36
|
-
TemperatureMixin,
|
|
37
|
-
TopPMixin,
|
|
38
|
-
ThinkingMixin,
|
|
39
|
-
name="openai"):
|
|
32
|
+
class OpenAIModelConfig(LLMBaseConfig, RetryMixin, OptimizableMixin, ThinkingMixin, name="openai"):
|
|
40
33
|
"""An OpenAI LLM provider to be used with an LLM client."""
|
|
41
34
|
|
|
42
35
|
model_config = ConfigDict(protected_namespaces=(), extra="allow")
|
|
@@ -48,6 +41,16 @@ class OpenAIModelConfig(LLMBaseConfig,
|
|
|
48
41
|
description="The OpenAI hosted model name.")
|
|
49
42
|
seed: int | None = Field(default=None, description="Random seed to set for generation.")
|
|
50
43
|
max_retries: int = Field(default=10, description="The max number of retries for the request.")
|
|
44
|
+
temperature: float | None = OptimizableField(default=None,
|
|
45
|
+
ge=0.0,
|
|
46
|
+
le=1.0,
|
|
47
|
+
description="Sampling temperature in [0, 1].",
|
|
48
|
+
space=SearchSpace(high=0.9, low=0.1, step=0.2))
|
|
49
|
+
top_p: float | None = OptimizableField(default=None,
|
|
50
|
+
ge=0.0,
|
|
51
|
+
le=1.0,
|
|
52
|
+
description="Top-p for distribution sampling.",
|
|
53
|
+
space=SearchSpace(high=1.0, low=0.5, step=0.1))
|
|
51
54
|
|
|
52
55
|
|
|
53
56
|
@register_llm_provider(config_type=OpenAIModelConfig)
|
|
@@ -92,7 +92,9 @@ class Sandbox(abc.ABC):
|
|
|
92
92
|
raise ValueError(f"Language {language} not supported")
|
|
93
93
|
|
|
94
94
|
generated_code = generated_code.strip().strip("`")
|
|
95
|
-
|
|
95
|
+
# Use json.dumps to properly escape the generated_code instead of repr()
|
|
96
|
+
escaped_code = json.dumps(generated_code)
|
|
97
|
+
code_to_execute = textwrap.dedent(f"""
|
|
96
98
|
import traceback
|
|
97
99
|
import json
|
|
98
100
|
import os
|
|
@@ -101,11 +103,6 @@ class Sandbox(abc.ABC):
|
|
|
101
103
|
import io
|
|
102
104
|
warnings.filterwarnings('ignore')
|
|
103
105
|
os.environ['OPENBLAS_NUM_THREADS'] = '16'
|
|
104
|
-
""").strip()
|
|
105
|
-
|
|
106
|
-
# Use json.dumps to properly escape the generated_code instead of repr()
|
|
107
|
-
escaped_code = json.dumps(generated_code)
|
|
108
|
-
code_to_execute += textwrap.dedent(f"""
|
|
109
106
|
|
|
110
107
|
generated_code = {escaped_code}
|
|
111
108
|
|
|
@@ -12,43 +12,26 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
-
#
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
# Create Lean project directory and initialize a new Lean project with Mathlib4
|
|
30
|
-
RUN mkdir -p /lean4 && cd /lean4 && \
|
|
31
|
-
/root/.elan/bin/lake new my_project && \
|
|
32
|
-
cd my_project && \
|
|
33
|
-
echo 'leanprover/lean4:v4.12.0' > lean-toolchain && \
|
|
34
|
-
echo 'require mathlib from git "https://github.com/leanprover-community/mathlib4" @ "v4.12.0"' >> lakefile.lean
|
|
35
|
-
|
|
36
|
-
# Download and cache Mathlib4 to avoid recompiling, then build the project
|
|
37
|
-
RUN cd /lean4/my_project && \
|
|
38
|
-
/root/.elan/bin/lake exe cache get && \
|
|
39
|
-
/root/.elan/bin/lake build
|
|
40
|
-
|
|
41
|
-
# Set environment variables to include Lean project path
|
|
42
|
-
ENV LEAN_PATH="/lean4/my_project"
|
|
43
|
-
ENV PATH="/lean4/my_project:$PATH"
|
|
15
|
+
# UWSGI_CHEAPER sets the number of initial uWSGI worker processes
|
|
16
|
+
# UWSGI_PROCESSES sets the maximum number of uWSGI worker processes
|
|
17
|
+
ARG UWSGI_CHEAPER=5
|
|
18
|
+
ARG UWSGI_PROCESSES=10
|
|
19
|
+
|
|
20
|
+
# Use the base image with Python 3.13
|
|
21
|
+
FROM python:3.13-slim-bookworm
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
RUN apt update && \
|
|
25
|
+
apt upgrade && \
|
|
26
|
+
apt install -y --no-install-recommends libexpat1 && \
|
|
27
|
+
apt clean && \
|
|
28
|
+
rm -rf /var/lib/apt/lists/*
|
|
44
29
|
|
|
45
30
|
# Set up application code and install Python dependencies
|
|
46
31
|
COPY sandbox.requirements.txt /app/requirements.txt
|
|
47
32
|
RUN pip install --no-cache-dir -r /app/requirements.txt
|
|
48
33
|
COPY local_sandbox_server.py /app/main.py
|
|
49
|
-
|
|
50
|
-
# Set the working directory to /app
|
|
51
|
-
WORKDIR /app
|
|
34
|
+
RUN mkdir /workspace
|
|
52
35
|
|
|
53
36
|
# Set Flask app environment variables and ports
|
|
54
37
|
ARG UWSGI_CHEAPER
|
|
@@ -58,3 +41,7 @@ ARG UWSGI_PROCESSES
|
|
|
58
41
|
ENV UWSGI_PROCESSES=$UWSGI_PROCESSES
|
|
59
42
|
|
|
60
43
|
ENV LISTEN_PORT=6000
|
|
44
|
+
EXPOSE 6000
|
|
45
|
+
|
|
46
|
+
WORKDIR /app
|
|
47
|
+
CMD uwsgi --http 0.0.0.0:${LISTEN_PORT} --master -p ${UWSGI_PROCESSES} --force-cwd /workspace -w main:app
|
|
@@ -19,7 +19,11 @@
|
|
|
19
19
|
|
|
20
20
|
DOCKER_COMMAND=${DOCKER_COMMAND:-"docker"}
|
|
21
21
|
SANDBOX_NAME=${1:-'local-sandbox'}
|
|
22
|
-
|
|
22
|
+
|
|
23
|
+
# UWSGI_CHEAPER sets the number of initial uWSGI worker processes
|
|
24
|
+
# UWSGI_PROCESSES sets the maximum number of uWSGI worker processes
|
|
25
|
+
UWSGI_CHEAPER=${UWSGI_CHEAPER:-5}
|
|
26
|
+
UWSGI_PROCESSES=${UWSGI_PROCESSES:-10}
|
|
23
27
|
|
|
24
28
|
# Get the output_data directory path for mounting
|
|
25
29
|
# Priority: command line argument > environment variable > default path (current directory)
|
|
@@ -37,14 +41,16 @@ fi
|
|
|
37
41
|
# Check if the Docker image already exists
|
|
38
42
|
if ! ${DOCKER_COMMAND} images ${SANDBOX_NAME} | grep -q "${SANDBOX_NAME}"; then
|
|
39
43
|
echo "Docker image not found locally. Building ${SANDBOX_NAME}..."
|
|
40
|
-
${DOCKER_COMMAND} build --tag=${SANDBOX_NAME}
|
|
44
|
+
${DOCKER_COMMAND} build --tag=${SANDBOX_NAME} \
|
|
45
|
+
--build-arg="UWSGI_PROCESSES=${UWSGI_PROCESSES}" \
|
|
46
|
+
--build-arg="UWSGI_CHEAPER=${UWSGI_CHEAPER}" \
|
|
47
|
+
-f Dockerfile.sandbox .
|
|
41
48
|
else
|
|
42
49
|
echo "Using existing Docker image: ${SANDBOX_NAME}"
|
|
43
50
|
fi
|
|
44
51
|
|
|
45
52
|
# Mount the output_data directory directly so files created in container appear in the local directory
|
|
46
|
-
${DOCKER_COMMAND} run --rm --name=local-sandbox \
|
|
53
|
+
${DOCKER_COMMAND} run --rm -ti --name=local-sandbox \
|
|
47
54
|
--network=host \
|
|
48
55
|
-v "${OUTPUT_DATA_PATH}:/workspace" \
|
|
49
|
-
-w /workspace \
|
|
50
56
|
${SANDBOX_NAME}
|
nat/utils/__init__.py
CHANGED
|
@@ -29,7 +29,8 @@ async def run_workflow(*,
|
|
|
29
29
|
config: "Config | None" = None,
|
|
30
30
|
config_file: "StrPath | None" = None,
|
|
31
31
|
prompt: str,
|
|
32
|
-
to_type: type[_T] = str
|
|
32
|
+
to_type: type[_T] = str,
|
|
33
|
+
session_kwargs: dict[str, typing.Any] | None = None) -> _T:
|
|
33
34
|
"""
|
|
34
35
|
Wrapper to run a workflow given either a config or a config file path and a prompt, returning the result in the
|
|
35
36
|
type specified by the `to_type`.
|
|
@@ -66,7 +67,10 @@ async def run_workflow(*,
|
|
|
66
67
|
|
|
67
68
|
config = load_config(config_file)
|
|
68
69
|
|
|
70
|
+
session_kwargs = session_kwargs or {}
|
|
71
|
+
|
|
69
72
|
async with WorkflowBuilder.from_config(config=config) as workflow_builder:
|
|
70
|
-
|
|
71
|
-
async with
|
|
72
|
-
|
|
73
|
+
session_manager = SessionManager(await workflow_builder.build())
|
|
74
|
+
async with session_manager.session(**session_kwargs) as session:
|
|
75
|
+
async with session.run(prompt) as runner:
|
|
76
|
+
return await runner.result(to_type=to_type)
|
|
@@ -148,9 +148,7 @@ nat/data_models/step_adaptor.py,sha256=1qn56wB_nIYBM5IjN4ftsltCAkqaJd3Sqe5v0TRR4
|
|
|
148
148
|
nat/data_models/streaming.py,sha256=sSqJqLqb70qyw69_4R9QC2RMbRw7UjTLPdo3FYBUGxE,1159
|
|
149
149
|
nat/data_models/swe_bench_model.py,sha256=uZs-hLFuT1B5CiPFwFg1PHinDW8PHne8TBzu7tHFv_k,1718
|
|
150
150
|
nat/data_models/telemetry_exporter.py,sha256=P7kqxIQnFVuvo_UFpH9QSB8fACy_0U2Uzkw_IfWXagE,998
|
|
151
|
-
nat/data_models/temperature_mixin.py,sha256=LlpfWrWtDrPJfSKfNx5E0P3p5SNGZli7ACRRpmO0QqA,1628
|
|
152
151
|
nat/data_models/thinking_mixin.py,sha256=VRDUJZ8XP_Vv0gW2FRZUf8O9-kVgNEdZCEZ8oEmHyMk,3335
|
|
153
|
-
nat/data_models/top_p_mixin.py,sha256=mu0DLnCAiwNzpSFR8FOW4kQBUpodSrvUR4MsLrNtbgA,1599
|
|
154
152
|
nat/data_models/ttc_strategy.py,sha256=tAkKWcyEBmBOOYtHMtQTgeCbHxFTk5SEkmFunNVnfyE,1114
|
|
155
153
|
nat/embedder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
156
154
|
nat/embedder/azure_openai_embedder.py,sha256=yMHOA6lWZl15Pvd9Gpp_rHy5q2qmBiRjbFesFBGuC_U,2441
|
|
@@ -167,7 +165,7 @@ nat/eval/runtime_event_subscriber.py,sha256=9MVgZLKvpnThHINaPbszPYDWnJ61sntS09YZ
|
|
|
167
165
|
nat/eval/usage_stats.py,sha256=r8Zr3bIy2qXU1BgVKXr_4mr7bG0hte3BPNQLSgZvg8M,1376
|
|
168
166
|
nat/eval/dataset_handler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
169
167
|
nat/eval/dataset_handler/dataset_downloader.py,sha256=ZaNohbRSvoHPWIj0C_FqyJnhQKoTBk_uZF7ijjfk6vE,4641
|
|
170
|
-
nat/eval/dataset_handler/dataset_filter.py,sha256=
|
|
168
|
+
nat/eval/dataset_handler/dataset_filter.py,sha256=8ul9j8XvMJG28k2NoiTzGhN6l8pq58Y1T22uuW_m1rk,3397
|
|
171
169
|
nat/eval/dataset_handler/dataset_handler.py,sha256=my28rKvQiiRN_h2TJz6fdKeMOjP3LC3_e2aJNnPPYhE,18159
|
|
172
170
|
nat/eval/evaluator/__init__.py,sha256=GUJrgGtpvyMUCjUBvR3faAdv-tZzbU9W-izgx9aMEQg,680
|
|
173
171
|
nat/eval/evaluator/base_evaluator.py,sha256=5WaVGhCGzkynCJyQdxRv7CtqLoUpr6B4O8tilP_gb3g,3232
|
|
@@ -271,11 +269,11 @@ nat/front_ends/mcp/tool_converter.py,sha256=IOHb8UoW_TVvRoiML2yi6nlbx13KgcmUsuYO
|
|
|
271
269
|
nat/front_ends/simple_base/__init__.py,sha256=Xs1JQ16L9btwreh4pdGKwskffAw1YFO48jKrU4ib_7c,685
|
|
272
270
|
nat/front_ends/simple_base/simple_front_end_plugin_base.py,sha256=py_yA9XAw-yHfK5cQJLM8ElnubEEM2ac8M0bvz-ScWs,1801
|
|
273
271
|
nat/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
274
|
-
nat/llm/aws_bedrock_llm.py,sha256=
|
|
275
|
-
nat/llm/azure_openai_llm.py,sha256=
|
|
276
|
-
nat/llm/litellm_llm.py,sha256=
|
|
277
|
-
nat/llm/nim_llm.py,sha256=
|
|
278
|
-
nat/llm/openai_llm.py,sha256=
|
|
272
|
+
nat/llm/aws_bedrock_llm.py,sha256=PooVYBdM_wesYD-uoZRblHoWt247O3_9Y-cGBSHDK0s,3653
|
|
273
|
+
nat/llm/azure_openai_llm.py,sha256=3Z90Jmq4kYwAIzlHGyT8aXNjL8UEGqnNxzk5Xf9HWsI,3371
|
|
274
|
+
nat/llm/litellm_llm.py,sha256=_EBDUfGSIZXs-O1adQ3MPI7qMtwHJpDPd7awJNA3c6A,3608
|
|
275
|
+
nat/llm/nim_llm.py,sha256=FWOvoU_QcuRdZID63xEGw_mii9OtAUMQ6lvvlvNy2FY,3232
|
|
276
|
+
nat/llm/openai_llm.py,sha256=MpQ0AIMjoxJeU7-JRLigAL6gtigUaSB77s-ZAD3uUzo,3114
|
|
279
277
|
nat/llm/register.py,sha256=7xDYdK4w4opAwIjzDM5x7moJXT3QeEGaGGc_nDfY0i4,1090
|
|
280
278
|
nat/llm/utils/__init__.py,sha256=GUJrgGtpvyMUCjUBvR3faAdv-tZzbU9W-izgx9aMEQg,680
|
|
281
279
|
nat/llm/utils/env_config_value.py,sha256=kBVsv0pEokIAfDQx5omR7_FevFv_5fTPswcbnvhVT2c,3548
|
|
@@ -427,21 +425,20 @@ nat/tool/retriever.py,sha256=FP5JL1vCQNrqaKz4F1up-osjxEPhxPFOyaScrgByc34,3877
|
|
|
427
425
|
nat/tool/server_tools.py,sha256=rQLipwRv8lAyU-gohky2JoVDxWQTUTSttNWjhu7lcHU,3194
|
|
428
426
|
nat/tool/code_execution/README.md,sha256=sl3YX4As95HX61XqTXOGnUcHBV1lla-OeuTnLI4qgng,4019
|
|
429
427
|
nat/tool/code_execution/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
430
|
-
nat/tool/code_execution/code_sandbox.py,sha256=
|
|
428
|
+
nat/tool/code_execution/code_sandbox.py,sha256=evu0n9oTc_MrrQuYbo1TP0sjnjLniRgCsOubO5G5WT4,10090
|
|
431
429
|
nat/tool/code_execution/register.py,sha256=zPFzYqaQhH2B3K8iildVYY_7RKgpoRNKdAo00KmBLQI,3322
|
|
432
|
-
nat/tool/code_execution/test_code_execution_sandbox.py,sha256=iyK9awJs6ST8fc_S3lNPye6It0DxSNp1UIrAUKPJ1h4,14776
|
|
433
430
|
nat/tool/code_execution/utils.py,sha256=__W-T1kaphFKYSc2AydQW8lCdvD7zAccarvs7XVFTtI,4194
|
|
434
431
|
nat/tool/code_execution/local_sandbox/.gitignore,sha256=BrV-H5osDtwwIx0eieoexolpnaJvc2DQLV15j95Qtyg,19
|
|
435
|
-
nat/tool/code_execution/local_sandbox/Dockerfile.sandbox,sha256=
|
|
432
|
+
nat/tool/code_execution/local_sandbox/Dockerfile.sandbox,sha256=mIbGFhhY-9ARQVK-tGmx-ft68wFIpB_RvM69V37OEAY,1557
|
|
436
433
|
nat/tool/code_execution/local_sandbox/__init__.py,sha256=k25Kqr_PrH4s0YCfzVzC9vaBAiu8yI428C4HX-qUcqA,615
|
|
437
|
-
nat/tool/code_execution/local_sandbox/local_sandbox_server.py,sha256=
|
|
438
|
-
nat/tool/code_execution/local_sandbox/sandbox.requirements.txt,sha256=
|
|
439
|
-
nat/tool/code_execution/local_sandbox/start_local_sandbox.sh,sha256=
|
|
434
|
+
nat/tool/code_execution/local_sandbox/local_sandbox_server.py,sha256=VGLY2ASmSSYNacGNdMSiedclwTCILNHs6PX45LxQ4Wc,7040
|
|
435
|
+
nat/tool/code_execution/local_sandbox/sandbox.requirements.txt,sha256=DL4dVNC8G-PMvr_x2C5AhwmeOpSKynpy4FNHWtWWtG4,69
|
|
436
|
+
nat/tool/code_execution/local_sandbox/start_local_sandbox.sh,sha256=BejvFn0uqI-BdBh3HqGWu7CngeJObiUFw-vAVQZA5VE,2260
|
|
440
437
|
nat/tool/memory_tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
441
438
|
nat/tool/memory_tools/add_memory_tool.py,sha256=N400PPvI37NUCMh5KcuoAL8khK8ecUQyfenahfjzbHQ,3368
|
|
442
439
|
nat/tool/memory_tools/delete_memory_tool.py,sha256=zMllkpC0of9qFPNuG9vkVOoydRblOViCQf0uSbqz0sE,2461
|
|
443
440
|
nat/tool/memory_tools/get_memory_tool.py,sha256=fcW6QE7bMZrpNK62et3sTw_QZ8cV9lXfEuDsm1-05bE,2768
|
|
444
|
-
nat/utils/__init__.py,sha256=
|
|
441
|
+
nat/utils/__init__.py,sha256=v6KZnvq8gT1VK-P_g5zHgoq9lhD4wvGwqCyb67aBkTo,2859
|
|
445
442
|
nat/utils/callable_utils.py,sha256=EIao6NhHRFEoBqYRC7aWoFqhlr2LeFT0XK-ac0coF9E,2475
|
|
446
443
|
nat/utils/debugging_utils.py,sha256=6M4JhbHDNDnfmSRGmHvT5IgEeWSHBore3VngdE_PMqc,1332
|
|
447
444
|
nat/utils/decorators.py,sha256=AoMip9zmqrZm5wovZQytNvzFfIlS3PQxSYcgYeoLhxA,8240
|
|
@@ -475,10 +472,10 @@ nat/utils/reactive/base/observer_base.py,sha256=6BiQfx26EMumotJ3KoVcdmFBYR_fnAss
|
|
|
475
472
|
nat/utils/reactive/base/subject_base.py,sha256=UQOxlkZTIeeyYmG5qLtDpNf_63Y7p-doEeUA08_R8ME,2521
|
|
476
473
|
nat/utils/settings/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
477
474
|
nat/utils/settings/global_settings.py,sha256=9JaO6pxKT_Pjw6rxJRsRlFCXdVKCl_xUKU2QHZQWWNM,7294
|
|
478
|
-
nvidia_nat-1.4.
|
|
479
|
-
nvidia_nat-1.4.
|
|
480
|
-
nvidia_nat-1.4.
|
|
481
|
-
nvidia_nat-1.4.
|
|
482
|
-
nvidia_nat-1.4.
|
|
483
|
-
nvidia_nat-1.4.
|
|
484
|
-
nvidia_nat-1.4.
|
|
475
|
+
nvidia_nat-1.4.0a20251107.dist-info/licenses/LICENSE-3rd-party.txt,sha256=fOk5jMmCX9YoKWyYzTtfgl-SUy477audFC5hNY4oP7Q,284609
|
|
476
|
+
nvidia_nat-1.4.0a20251107.dist-info/licenses/LICENSE.md,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
|
|
477
|
+
nvidia_nat-1.4.0a20251107.dist-info/METADATA,sha256=XV37UjKy841cyPFRaYIBrhQdZnmu_XcVyf0jEGhsoH8,10248
|
|
478
|
+
nvidia_nat-1.4.0a20251107.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
479
|
+
nvidia_nat-1.4.0a20251107.dist-info/entry_points.txt,sha256=4jCqjyETMpyoWbCBf4GalZU8I_wbstpzwQNezdAVbbo,698
|
|
480
|
+
nvidia_nat-1.4.0a20251107.dist-info/top_level.txt,sha256=lgJWLkigiVZuZ_O1nxVnD_ziYBwgpE2OStdaCduMEGc,8
|
|
481
|
+
nvidia_nat-1.4.0a20251107.dist-info/RECORD,,
|
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
-
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
-
#
|
|
4
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
-
# you may not use this file except in compliance with the License.
|
|
6
|
-
# You may obtain a copy of the License at
|
|
7
|
-
#
|
|
8
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
-
#
|
|
10
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
-
# See the License for the specific language governing permissions and
|
|
14
|
-
# limitations under the License.
|
|
15
|
-
|
|
16
|
-
import re
|
|
17
|
-
|
|
18
|
-
from pydantic import BaseModel
|
|
19
|
-
|
|
20
|
-
from nat.data_models.gated_field_mixin import GatedFieldMixin
|
|
21
|
-
from nat.data_models.optimizable import OptimizableField
|
|
22
|
-
from nat.data_models.optimizable import SearchSpace
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
class TemperatureMixin(
|
|
26
|
-
BaseModel,
|
|
27
|
-
GatedFieldMixin,
|
|
28
|
-
field_name="temperature",
|
|
29
|
-
default_if_supported=0.0,
|
|
30
|
-
keys=("model_name", "model", "azure_deployment"),
|
|
31
|
-
unsupported=(re.compile(r"gpt-?5", re.IGNORECASE), ),
|
|
32
|
-
):
|
|
33
|
-
"""
|
|
34
|
-
Mixin class for temperature configuration. Unsupported on models like gpt-5.
|
|
35
|
-
|
|
36
|
-
Attributes:
|
|
37
|
-
temperature: Sampling temperature in [0, 1]. Defaults to 0.0 when supported on the model.
|
|
38
|
-
"""
|
|
39
|
-
temperature: float | None = OptimizableField(
|
|
40
|
-
default=None,
|
|
41
|
-
ge=0.0,
|
|
42
|
-
le=1.0,
|
|
43
|
-
description="Sampling temperature in [0, 1]. Defaults to 0.0 when supported on the model.",
|
|
44
|
-
space=SearchSpace(high=0.9, low=0.1, step=0.2))
|
nat/data_models/top_p_mixin.py
DELETED
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
-
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
-
#
|
|
4
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
-
# you may not use this file except in compliance with the License.
|
|
6
|
-
# You may obtain a copy of the License at
|
|
7
|
-
#
|
|
8
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
-
#
|
|
10
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
-
# See the License for the specific language governing permissions and
|
|
14
|
-
# limitations under the License.
|
|
15
|
-
|
|
16
|
-
import re
|
|
17
|
-
|
|
18
|
-
from pydantic import BaseModel
|
|
19
|
-
|
|
20
|
-
from nat.data_models.gated_field_mixin import GatedFieldMixin
|
|
21
|
-
from nat.data_models.optimizable import OptimizableField
|
|
22
|
-
from nat.data_models.optimizable import SearchSpace
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
class TopPMixin(
|
|
26
|
-
BaseModel,
|
|
27
|
-
GatedFieldMixin,
|
|
28
|
-
field_name="top_p",
|
|
29
|
-
default_if_supported=1.0,
|
|
30
|
-
keys=("model_name", "model", "azure_deployment"),
|
|
31
|
-
unsupported=(re.compile(r"gpt-?5", re.IGNORECASE), ),
|
|
32
|
-
):
|
|
33
|
-
"""
|
|
34
|
-
Mixin class for top-p configuration. Unsupported on models like gpt-5.
|
|
35
|
-
|
|
36
|
-
Attributes:
|
|
37
|
-
top_p: Top-p for distribution sampling. Defaults to 1.0 when supported on the model.
|
|
38
|
-
"""
|
|
39
|
-
top_p: float | None = OptimizableField(
|
|
40
|
-
default=None,
|
|
41
|
-
ge=0.0,
|
|
42
|
-
le=1.0,
|
|
43
|
-
description="Top-p for distribution sampling. Defaults to 1.0 when supported on the model.",
|
|
44
|
-
space=SearchSpace(high=1.0, low=0.5, step=0.1))
|
|
@@ -1,414 +0,0 @@
|
|
|
1
|
-
# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
-
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
-
#
|
|
4
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
-
# you may not use this file except in compliance with the License.
|
|
6
|
-
# You may obtain a copy of the License at
|
|
7
|
-
#
|
|
8
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
-
#
|
|
10
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
-
# See the License for the specific language governing permissions and
|
|
14
|
-
# limitations under the License.
|
|
15
|
-
"""
|
|
16
|
-
Test suite for Code Execution Sandbox using pytest.
|
|
17
|
-
|
|
18
|
-
This module provides comprehensive testing for the code execution sandbox service,
|
|
19
|
-
replacing the original bash script with a more maintainable Python implementation.
|
|
20
|
-
"""
|
|
21
|
-
|
|
22
|
-
import os
|
|
23
|
-
from typing import Any
|
|
24
|
-
|
|
25
|
-
import pytest
|
|
26
|
-
import requests
|
|
27
|
-
from requests.exceptions import ConnectionError
|
|
28
|
-
from requests.exceptions import RequestException
|
|
29
|
-
from requests.exceptions import Timeout
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
class TestCodeExecutionSandbox:
|
|
33
|
-
"""Test suite for the Code Execution Sandbox service."""
|
|
34
|
-
|
|
35
|
-
@pytest.fixture(scope="class")
|
|
36
|
-
def sandbox_config(self):
|
|
37
|
-
"""Configuration for sandbox testing."""
|
|
38
|
-
return {
|
|
39
|
-
"url": os.environ.get("SANDBOX_URL", "http://127.0.0.1:6000/execute"),
|
|
40
|
-
"timeout": int(os.environ.get("SANDBOX_TIMEOUT", "30")),
|
|
41
|
-
"connection_timeout": 5
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
@pytest.fixture(scope="class", autouse=True)
|
|
45
|
-
def check_sandbox_running(self, sandbox_config):
|
|
46
|
-
"""Check if sandbox server is running before running tests."""
|
|
47
|
-
try:
|
|
48
|
-
_ = requests.get(sandbox_config["url"], timeout=sandbox_config["connection_timeout"])
|
|
49
|
-
print(f"✓ Sandbox server is running at {sandbox_config['url']}")
|
|
50
|
-
except (ConnectionError, Timeout, RequestException):
|
|
51
|
-
pytest.skip(
|
|
52
|
-
f"Sandbox server is not running at {sandbox_config['url']}. "
|
|
53
|
-
"Please start it with: cd src/nat/tool/code_execution/local_sandbox && ./start_local_sandbox.sh")
|
|
54
|
-
|
|
55
|
-
def execute_code(self, sandbox_config: dict[str, Any], code: str, language: str = "python") -> dict[str, Any]:
|
|
56
|
-
"""
|
|
57
|
-
Execute code in the sandbox and return the response.
|
|
58
|
-
|
|
59
|
-
Args:
|
|
60
|
-
sandbox_config: Configuration dictionary
|
|
61
|
-
code: Code to execute
|
|
62
|
-
language: Programming language (default: python)
|
|
63
|
-
|
|
64
|
-
Returns:
|
|
65
|
-
dictionary containing the response from the sandbox
|
|
66
|
-
"""
|
|
67
|
-
payload = {"generated_code": code, "timeout": sandbox_config["timeout"], "language": language}
|
|
68
|
-
|
|
69
|
-
response = requests.post(
|
|
70
|
-
sandbox_config["url"],
|
|
71
|
-
json=payload,
|
|
72
|
-
timeout=sandbox_config["timeout"] + 5 # Add buffer to request timeout
|
|
73
|
-
)
|
|
74
|
-
|
|
75
|
-
# Ensure we got a response
|
|
76
|
-
response.raise_for_status()
|
|
77
|
-
return response.json()
|
|
78
|
-
|
|
79
|
-
def test_simple_print(self, sandbox_config):
|
|
80
|
-
"""Test simple print statement execution."""
|
|
81
|
-
code = "print('Hello, World!')"
|
|
82
|
-
result = self.execute_code(sandbox_config, code)
|
|
83
|
-
|
|
84
|
-
assert result["process_status"] == "completed"
|
|
85
|
-
assert "Hello, World!" in result["stdout"]
|
|
86
|
-
assert result["stderr"] == ""
|
|
87
|
-
|
|
88
|
-
def test_basic_arithmetic(self, sandbox_config):
|
|
89
|
-
"""Test basic arithmetic operations."""
|
|
90
|
-
code = """
|
|
91
|
-
result = 2 + 3
|
|
92
|
-
print(f'Result: {result}')
|
|
93
|
-
"""
|
|
94
|
-
result = self.execute_code(sandbox_config, code)
|
|
95
|
-
|
|
96
|
-
assert result["process_status"] == "completed"
|
|
97
|
-
assert "Result: 5" in result["stdout"]
|
|
98
|
-
assert result["stderr"] == ""
|
|
99
|
-
|
|
100
|
-
def test_numpy_operations(self, sandbox_config):
|
|
101
|
-
"""Test numpy dependency availability and operations."""
|
|
102
|
-
code = """
|
|
103
|
-
import numpy as np
|
|
104
|
-
arr = np.array([1, 2, 3, 4, 5])
|
|
105
|
-
print(f'Array: {arr}')
|
|
106
|
-
print(f'Mean: {np.mean(arr)}')
|
|
107
|
-
"""
|
|
108
|
-
result = self.execute_code(sandbox_config, code)
|
|
109
|
-
|
|
110
|
-
assert result["process_status"] == "completed"
|
|
111
|
-
assert "Array: [1 2 3 4 5]" in result["stdout"]
|
|
112
|
-
assert "Mean: 3.0" in result["stdout"]
|
|
113
|
-
assert result["stderr"] == ""
|
|
114
|
-
|
|
115
|
-
def test_pandas_operations(self, sandbox_config):
|
|
116
|
-
"""Test pandas dependency availability and operations."""
|
|
117
|
-
code = """
|
|
118
|
-
import pandas as pd
|
|
119
|
-
df = pd.DataFrame({'A': [1, 2, 3], 'B': [4, 5, 6]})
|
|
120
|
-
print(df)
|
|
121
|
-
print(f'Sum of column A: {df["A"].sum()}')
|
|
122
|
-
"""
|
|
123
|
-
result = self.execute_code(sandbox_config, code)
|
|
124
|
-
|
|
125
|
-
assert result["process_status"] == "completed"
|
|
126
|
-
assert "Sum of column A: 6" in result["stdout"]
|
|
127
|
-
assert result["stderr"] == ""
|
|
128
|
-
|
|
129
|
-
def test_plotly_import(self, sandbox_config):
|
|
130
|
-
"""Test plotly dependency availability."""
|
|
131
|
-
code = """
|
|
132
|
-
import plotly.graph_objects as go
|
|
133
|
-
print('Plotly imported successfully')
|
|
134
|
-
fig = go.Figure()
|
|
135
|
-
fig.add_trace(go.Scatter(x=[1, 2, 3], y=[4, 5, 6]))
|
|
136
|
-
print('Plot created successfully')
|
|
137
|
-
"""
|
|
138
|
-
result = self.execute_code(sandbox_config, code)
|
|
139
|
-
|
|
140
|
-
assert result["process_status"] == "completed"
|
|
141
|
-
assert "Plotly imported successfully" in result["stdout"]
|
|
142
|
-
assert "Plot created successfully" in result["stdout"]
|
|
143
|
-
assert result["stderr"] == ""
|
|
144
|
-
|
|
145
|
-
def test_syntax_error_handling(self, sandbox_config):
|
|
146
|
-
"""Test handling of syntax errors."""
|
|
147
|
-
code = """
|
|
148
|
-
print('Hello World'
|
|
149
|
-
# Missing closing parenthesis
|
|
150
|
-
"""
|
|
151
|
-
result = self.execute_code(sandbox_config, code)
|
|
152
|
-
|
|
153
|
-
assert result["process_status"] == "error"
|
|
154
|
-
assert "SyntaxError" in result["stderr"] or "SyntaxError" in result["stdout"]
|
|
155
|
-
|
|
156
|
-
def test_runtime_error_handling(self, sandbox_config):
|
|
157
|
-
"""Test handling of runtime errors."""
|
|
158
|
-
code = """
|
|
159
|
-
x = 1 / 0
|
|
160
|
-
print('This should not print')
|
|
161
|
-
"""
|
|
162
|
-
result = self.execute_code(sandbox_config, code)
|
|
163
|
-
|
|
164
|
-
assert result["process_status"] == "error"
|
|
165
|
-
assert "ZeroDivisionError" in result["stderr"] or "ZeroDivisionError" in result["stdout"]
|
|
166
|
-
|
|
167
|
-
def test_import_error_handling(self, sandbox_config):
|
|
168
|
-
"""Test handling of import errors."""
|
|
169
|
-
code = """
|
|
170
|
-
import nonexistent_module
|
|
171
|
-
print('This should not print')
|
|
172
|
-
"""
|
|
173
|
-
result = self.execute_code(sandbox_config, code)
|
|
174
|
-
|
|
175
|
-
assert result["process_status"] == "error"
|
|
176
|
-
assert "ModuleNotFoundError" in result["stderr"] or "ImportError" in result["stderr"]
|
|
177
|
-
|
|
178
|
-
def test_mixed_output(self, sandbox_config):
|
|
179
|
-
"""Test code that produces both stdout and stderr output."""
|
|
180
|
-
code = """
|
|
181
|
-
import sys
|
|
182
|
-
print('This goes to stdout')
|
|
183
|
-
print('This goes to stderr', file=sys.stderr)
|
|
184
|
-
print('Back to stdout')
|
|
185
|
-
"""
|
|
186
|
-
result = self.execute_code(sandbox_config, code)
|
|
187
|
-
|
|
188
|
-
assert result["process_status"] == "completed"
|
|
189
|
-
assert "This goes to stdout" in result["stdout"]
|
|
190
|
-
assert "Back to stdout" in result["stdout"]
|
|
191
|
-
assert "This goes to stderr" in result["stderr"]
|
|
192
|
-
|
|
193
|
-
def test_long_running_code(self, sandbox_config):
|
|
194
|
-
"""Test code that takes some time to execute but completes within timeout."""
|
|
195
|
-
code = """
|
|
196
|
-
import time
|
|
197
|
-
for i in range(3):
|
|
198
|
-
print(f'Iteration {i}')
|
|
199
|
-
time.sleep(0.5)
|
|
200
|
-
print('Completed')
|
|
201
|
-
"""
|
|
202
|
-
result = self.execute_code(sandbox_config, code)
|
|
203
|
-
|
|
204
|
-
assert result["process_status"] == "completed"
|
|
205
|
-
assert "Iteration 0" in result["stdout"]
|
|
206
|
-
assert "Iteration 1" in result["stdout"]
|
|
207
|
-
assert "Iteration 2" in result["stdout"]
|
|
208
|
-
assert "Completed" in result["stdout"]
|
|
209
|
-
assert result["stderr"] == ""
|
|
210
|
-
|
|
211
|
-
def test_file_operations(self, sandbox_config):
|
|
212
|
-
"""Test basic file operations in the sandbox."""
|
|
213
|
-
code = """
|
|
214
|
-
import os
|
|
215
|
-
print(f'Current directory: {os.getcwd()}')
|
|
216
|
-
with open('test_file.txt', 'w') as f:
|
|
217
|
-
f.write('Hello, World!')
|
|
218
|
-
with open('test_file.txt', 'r') as f:
|
|
219
|
-
content = f.read()
|
|
220
|
-
print(f'File content: {content}')
|
|
221
|
-
os.remove('test_file.txt')
|
|
222
|
-
print('File operations completed')
|
|
223
|
-
"""
|
|
224
|
-
result = self.execute_code(sandbox_config, code)
|
|
225
|
-
|
|
226
|
-
assert result["process_status"] == "completed"
|
|
227
|
-
assert "File content: Hello, World!" in result["stdout"]
|
|
228
|
-
assert "File operations completed" in result["stdout"]
|
|
229
|
-
assert result["stderr"] == ""
|
|
230
|
-
|
|
231
|
-
def test_file_persistence_create(self, sandbox_config):
|
|
232
|
-
"""Test file persistence - create various file types."""
|
|
233
|
-
code = """
|
|
234
|
-
import os
|
|
235
|
-
import pandas as pd
|
|
236
|
-
import numpy as np
|
|
237
|
-
print('Current directory:', os.getcwd())
|
|
238
|
-
print('Directory contents:', os.listdir('.'))
|
|
239
|
-
|
|
240
|
-
# Create a test file
|
|
241
|
-
with open('persistence_test.txt', 'w') as f:
|
|
242
|
-
f.write('Hello from sandbox persistence test!')
|
|
243
|
-
|
|
244
|
-
# Create a CSV file
|
|
245
|
-
df = pd.DataFrame({'A': [1, 2, 3], 'B': [4, 5, 6]})
|
|
246
|
-
df.to_csv('persistence_test.csv', index=False)
|
|
247
|
-
|
|
248
|
-
# Create a numpy array file
|
|
249
|
-
arr = np.array([1, 2, 3, 4, 5])
|
|
250
|
-
np.save('persistence_test.npy', arr)
|
|
251
|
-
|
|
252
|
-
print('Files created:')
|
|
253
|
-
for file in os.listdir('.'):
|
|
254
|
-
if 'persistence_test' in file:
|
|
255
|
-
print(' -', file)
|
|
256
|
-
"""
|
|
257
|
-
result = self.execute_code(sandbox_config, code)
|
|
258
|
-
|
|
259
|
-
assert result["process_status"] == "completed"
|
|
260
|
-
assert "persistence_test.txt" in result["stdout"]
|
|
261
|
-
assert "persistence_test.csv" in result["stdout"]
|
|
262
|
-
assert "persistence_test.npy" in result["stdout"]
|
|
263
|
-
assert result["stderr"] == ""
|
|
264
|
-
|
|
265
|
-
def test_file_persistence_read(self, sandbox_config):
|
|
266
|
-
"""Test file persistence - read back created files."""
|
|
267
|
-
code = """
|
|
268
|
-
import pandas as pd
|
|
269
|
-
import numpy as np
|
|
270
|
-
|
|
271
|
-
# Read back the files we created
|
|
272
|
-
print('=== Reading persistence_test.txt ===')
|
|
273
|
-
with open('persistence_test.txt', 'r') as f:
|
|
274
|
-
content = f.read()
|
|
275
|
-
print(f'Content: {content}')
|
|
276
|
-
|
|
277
|
-
print('\\n=== Reading persistence_test.csv ===')
|
|
278
|
-
df = pd.read_csv('persistence_test.csv')
|
|
279
|
-
print(df)
|
|
280
|
-
print(f'DataFrame shape: {df.shape}')
|
|
281
|
-
|
|
282
|
-
print('\\n=== Reading persistence_test.npy ===')
|
|
283
|
-
arr = np.load('persistence_test.npy')
|
|
284
|
-
print(f'Array: {arr}')
|
|
285
|
-
print(f'Array sum: {np.sum(arr)}')
|
|
286
|
-
|
|
287
|
-
print('\\n=== File persistence test PASSED! ===')
|
|
288
|
-
"""
|
|
289
|
-
result = self.execute_code(sandbox_config, code)
|
|
290
|
-
|
|
291
|
-
assert result["process_status"] == "completed"
|
|
292
|
-
assert "Content: Hello from sandbox persistence test!" in result["stdout"]
|
|
293
|
-
assert "DataFrame shape: (3, 2)" in result["stdout"]
|
|
294
|
-
assert "Array: [1 2 3 4 5]" in result["stdout"]
|
|
295
|
-
assert "Array sum: 15" in result["stdout"]
|
|
296
|
-
assert "File persistence test PASSED!" in result["stdout"]
|
|
297
|
-
assert result["stderr"] == ""
|
|
298
|
-
|
|
299
|
-
def test_json_operations(self, sandbox_config):
|
|
300
|
-
"""Test JSON file operations for persistence."""
|
|
301
|
-
code = """
|
|
302
|
-
import json
|
|
303
|
-
import os
|
|
304
|
-
|
|
305
|
-
# Create a complex JSON file
|
|
306
|
-
data = {
|
|
307
|
-
'test_name': 'sandbox_persistence',
|
|
308
|
-
'timestamp': '2024-07-03',
|
|
309
|
-
'results': {
|
|
310
|
-
'numpy_test': True,
|
|
311
|
-
'pandas_test': True,
|
|
312
|
-
'file_operations': True
|
|
313
|
-
},
|
|
314
|
-
'metrics': [1.5, 2.3, 3.7, 4.1],
|
|
315
|
-
'metadata': {
|
|
316
|
-
'working_dir': os.getcwd(),
|
|
317
|
-
'python_version': '3.x'
|
|
318
|
-
}
|
|
319
|
-
}
|
|
320
|
-
|
|
321
|
-
# Save JSON file
|
|
322
|
-
with open('persistence_test.json', 'w') as f:
|
|
323
|
-
json.dump(data, f, indent=2)
|
|
324
|
-
|
|
325
|
-
# Read it back
|
|
326
|
-
with open('persistence_test.json', 'r') as f:
|
|
327
|
-
loaded_data = json.load(f)
|
|
328
|
-
|
|
329
|
-
print('JSON file created and loaded successfully')
|
|
330
|
-
print(f'Test name: {loaded_data["test_name"]}')
|
|
331
|
-
print(f'Results count: {len(loaded_data["results"])}')
|
|
332
|
-
print(f'Metrics: {loaded_data["metrics"]}')
|
|
333
|
-
print('JSON persistence test completed!')
|
|
334
|
-
"""
|
|
335
|
-
result = self.execute_code(sandbox_config, code)
|
|
336
|
-
|
|
337
|
-
assert result["process_status"] == "completed"
|
|
338
|
-
assert "JSON file created and loaded successfully" in result["stdout"]
|
|
339
|
-
assert "Test name: sandbox_persistence" in result["stdout"]
|
|
340
|
-
assert "Results count: 3" in result["stdout"]
|
|
341
|
-
assert "JSON persistence test completed!" in result["stdout"]
|
|
342
|
-
assert result["stderr"] == ""
|
|
343
|
-
|
|
344
|
-
def test_missing_generated_code_field(self, sandbox_config):
|
|
345
|
-
"""Test request missing the generated_code field."""
|
|
346
|
-
payload = {"timeout": 10, "language": "python"}
|
|
347
|
-
|
|
348
|
-
response = requests.post(sandbox_config["url"], json=payload)
|
|
349
|
-
|
|
350
|
-
# Should return an error status code or error in response
|
|
351
|
-
assert response.status_code != 200 or "error" in response.json()
|
|
352
|
-
|
|
353
|
-
def test_missing_timeout_field(self, sandbox_config):
|
|
354
|
-
"""Test request missing the timeout field."""
|
|
355
|
-
payload = {"generated_code": "print('test')", "language": "python"}
|
|
356
|
-
|
|
357
|
-
response = requests.post(sandbox_config["url"], json=payload)
|
|
358
|
-
|
|
359
|
-
# Should return error for missing timeout field
|
|
360
|
-
result = response.json()
|
|
361
|
-
assert response.status_code == 400 and result["process_status"] == "error"
|
|
362
|
-
|
|
363
|
-
def test_invalid_json(self, sandbox_config):
|
|
364
|
-
"""Test request with invalid JSON."""
|
|
365
|
-
invalid_json = '{"generated_code": "print("test")", "timeout": 10}'
|
|
366
|
-
|
|
367
|
-
response = requests.post(sandbox_config["url"], data=invalid_json, headers={"Content-Type": "application/json"})
|
|
368
|
-
|
|
369
|
-
# Should return error for invalid JSON
|
|
370
|
-
assert response.status_code != 200
|
|
371
|
-
|
|
372
|
-
def test_non_json_request(self, sandbox_config):
|
|
373
|
-
"""Test request with non-JSON content."""
|
|
374
|
-
response = requests.post(sandbox_config["url"], data="This is not JSON", headers={"Content-Type": "text/plain"})
|
|
375
|
-
|
|
376
|
-
# Should return error for non-JSON content
|
|
377
|
-
assert response.status_code != 200
|
|
378
|
-
|
|
379
|
-
def test_timeout_too_low(self, sandbox_config):
|
|
380
|
-
"""Test request with timeout too low."""
|
|
381
|
-
code = """
|
|
382
|
-
import time
|
|
383
|
-
time.sleep(2.0)
|
|
384
|
-
"""
|
|
385
|
-
payload = {"generated_code": code, "timeout": 1, "language": "python"}
|
|
386
|
-
response = requests.post(sandbox_config["url"], json=payload)
|
|
387
|
-
assert response.json()["process_status"] == "timeout"
|
|
388
|
-
assert response.status_code == 200
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
# Pytest configuration and fixtures for command-line options
|
|
392
|
-
def pytest_addoption(parser):
|
|
393
|
-
"""Add custom command-line options for pytest."""
|
|
394
|
-
parser.addoption("--sandbox-url",
|
|
395
|
-
action="store",
|
|
396
|
-
default="http://127.0.0.1:6000/execute",
|
|
397
|
-
help="Sandbox URL for testing")
|
|
398
|
-
parser.addoption("--sandbox-timeout",
|
|
399
|
-
action="store",
|
|
400
|
-
type=int,
|
|
401
|
-
default=30,
|
|
402
|
-
help="Timeout in seconds for sandbox operations")
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
@pytest.fixture(scope="session", autouse=True)
|
|
406
|
-
def setup_environment(request):
|
|
407
|
-
"""Setup environment variables from command-line options."""
|
|
408
|
-
os.environ["SANDBOX_URL"] = request.config.getoption("--sandbox-url", "http://127.0.0.1:6000/execute")
|
|
409
|
-
os.environ["SANDBOX_TIMEOUT"] = str(request.config.getoption("--sandbox-timeout", 30))
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
if __name__ == "__main__":
|
|
413
|
-
# Allow running as a script
|
|
414
|
-
pytest.main([__file__, "-v"])
|
|
File without changes
|
{nvidia_nat-1.4.0a20251105.dist-info → nvidia_nat-1.4.0a20251107.dist-info}/entry_points.txt
RENAMED
|
File without changes
|
|
File without changes
|
{nvidia_nat-1.4.0a20251105.dist-info → nvidia_nat-1.4.0a20251107.dist-info}/licenses/LICENSE.md
RENAMED
|
File without changes
|
|
File without changes
|