fleet-python 0.2.65__py3-none-any.whl → 0.2.66b3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of fleet-python might be problematic. Click here for more details.

fleet/_async/tasks.py CHANGED
@@ -279,17 +279,14 @@ def verifier_from_string(
279
279
  """
280
280
  try:
281
281
  import inspect
282
- import re
283
282
  from .verifiers.verifier import AsyncVerifierFunction
284
283
  from fleet.verifiers.code import TASK_SUCCESSFUL_SCORE, TASK_FAILED_SCORE
285
284
  from fleet.verifiers.db import IgnoreConfig
285
+ from fleet.verifiers.parsing import parse_and_validate_verifier
286
286
 
287
- # Strip @verifier decorator if present to avoid double-wrapping
288
- # Remove lines like: @verifier(key="...")
289
- cleaned_code = re.sub(r'@verifier\([^)]*\)\s*\n', '', verifier_func)
290
- # Also remove the verifier import if present
291
- cleaned_code = re.sub(r'from fleet import.*verifier.*\n', '', cleaned_code)
292
- cleaned_code = re.sub(r'import.*verifier.*\n', '', cleaned_code)
287
+ # Validate the code and extract function name
288
+ # This ensures no arbitrary code execution during import
289
+ func_name = parse_and_validate_verifier(verifier_func)
293
290
 
294
291
  # Create a local namespace for executing the code
295
292
  local_namespace = {
@@ -299,8 +296,9 @@ def verifier_from_string(
299
296
  "Environment": object, # Add Environment type if needed
300
297
  }
301
298
 
302
- # Execute the cleaned verifier code in the namespace
303
- exec(cleaned_code, globals(), local_namespace)
299
+ # Execute the verifier code in the namespace
300
+ # This is now safe because we validated it contains only declarative code
301
+ exec(verifier_func, globals(), local_namespace)
304
302
 
305
303
  # Find the function that was defined (not imported)
306
304
  # Functions defined via exec have co_filename == '<string>'
fleet/tasks.py CHANGED
@@ -272,17 +272,14 @@ def verifier_from_string(
272
272
  """
273
273
  try:
274
274
  import inspect
275
- import re
276
275
  from .verifiers import SyncVerifierFunction
277
276
  from .verifiers.code import TASK_SUCCESSFUL_SCORE, TASK_FAILED_SCORE
278
277
  from .verifiers.db import IgnoreConfig
278
+ from .verifiers.parsing import parse_and_validate_verifier
279
279
 
280
- # Strip @verifier decorator if present to avoid double-wrapping
281
- # Remove lines like: @verifier(key="...")
282
- cleaned_code = re.sub(r'@verifier\([^)]*\)\s*\n', '', verifier_func)
283
- # Also remove the verifier import if present
284
- cleaned_code = re.sub(r'from fleet import.*verifier.*\n', '', cleaned_code)
285
- cleaned_code = re.sub(r'import.*verifier.*\n', '', cleaned_code)
280
+ # Validate the code and extract function name
281
+ # This ensures no arbitrary code execution during import
282
+ func_name = parse_and_validate_verifier(verifier_func)
286
283
 
287
284
  # Create a globals namespace with all required imports
288
285
  exec_globals = globals().copy()
@@ -298,8 +295,9 @@ def verifier_from_string(
298
295
  # Create a local namespace for executing the code
299
296
  local_namespace = {}
300
297
 
301
- # Execute the cleaned verifier code in the namespace
302
- exec(cleaned_code, exec_globals, local_namespace)
298
+ # Execute the verifier code in the namespace
299
+ # This is now safe because we validated it contains only declarative code
300
+ exec(verifier_func, exec_globals, local_namespace)
303
301
 
304
302
  # Find the function that was defined (not imported)
305
303
  # Functions defined via exec have co_filename == '<string>'
@@ -0,0 +1,106 @@
1
+ """Verifier code parsing and validation utilities."""
2
+
3
+ import ast
4
+ from typing import Set
5
+
6
+
7
+ def parse_and_validate_verifier(code: str) -> str:
8
+ """Parse and validate verifier code, returning the first function name.
9
+
10
+ This function ensures that the verifier code only contains safe declarative
11
+ statements and does not execute arbitrary code during import.
12
+
13
+ Args:
14
+ code: Python code string containing the verifier function
15
+
16
+ Returns:
17
+ Name of the first function found in the code
18
+
19
+ Raises:
20
+ ValueError: If code is invalid or contains unsafe statements
21
+ SyntaxError: If code has syntax errors
22
+ """
23
+ try:
24
+ tree = ast.parse(code)
25
+ except SyntaxError as e:
26
+ raise SyntaxError(f"Syntax error in verifier code: {e}")
27
+
28
+ first_function_name = None
29
+
30
+ for node in tree.body:
31
+ # Check for function definitions
32
+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
33
+ # Validate that decorators don't contain function calls
34
+ for decorator in node.decorator_list:
35
+ if _contains_call(decorator):
36
+ raise ValueError(
37
+ f"Line {node.lineno}: Function decorators with function calls "
38
+ f"are not allowed. Decorators execute during import and could "
39
+ f"run arbitrary code."
40
+ )
41
+
42
+ if first_function_name is None:
43
+ first_function_name = node.name
44
+ continue
45
+
46
+ # Allow imports
47
+ if isinstance(node, (ast.Import, ast.ImportFrom)):
48
+ continue
49
+
50
+ # Allow class definitions
51
+ if isinstance(node, ast.ClassDef):
52
+ continue
53
+
54
+ # Allow docstrings and other expression statements (but not calls)
55
+ if isinstance(node, ast.Expr):
56
+ if isinstance(node.value, ast.Constant):
57
+ # Docstring or constant expression - safe
58
+ continue
59
+ else:
60
+ # Check if it's a call or other dangerous expression
61
+ raise ValueError(
62
+ f"Line {node.lineno}: Expression statements that are not "
63
+ f"constants are not allowed at module level. Found: {ast.dump(node.value)}"
64
+ )
65
+
66
+ # Allow variable assignments, but check the value
67
+ if isinstance(node, (ast.Assign, ast.AnnAssign)):
68
+ # Check if the assignment value contains any function calls
69
+ if _contains_call(node.value if isinstance(node, ast.AnnAssign) else node.value):
70
+ raise ValueError(
71
+ f"Line {node.lineno}: Variable assignments with function calls "
72
+ f"are not allowed at module level. This prevents arbitrary code "
73
+ f"execution during import."
74
+ )
75
+ continue
76
+
77
+ # If we get here, it's an unsupported statement type
78
+ raise ValueError(
79
+ f"Line {node.lineno}: Unsupported statement type at module level: "
80
+ f"{node.__class__.__name__}. Only imports, function/class definitions, "
81
+ f"and constant assignments are allowed."
82
+ )
83
+
84
+ if first_function_name is None:
85
+ raise ValueError("No function found in verifier code")
86
+
87
+ return first_function_name
88
+
89
+
90
+ def _contains_call(node: ast.AST) -> bool:
91
+ """Recursively check if an AST node contains any Call nodes.
92
+
93
+ Args:
94
+ node: AST node to check
95
+
96
+ Returns:
97
+ True if the node or any of its children is a Call node
98
+ """
99
+ if isinstance(node, ast.Call):
100
+ return True
101
+
102
+ for child in ast.walk(node):
103
+ if isinstance(child, ast.Call):
104
+ return True
105
+
106
+ return False
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fleet-python
3
- Version: 0.2.65
3
+ Version: 0.2.66b3
4
4
  Summary: Python SDK for Fleet environments
5
5
  Author-email: Fleet AI <nic@fleet.so>
6
6
  License: Apache-2.0
@@ -28,7 +28,7 @@ fleet/config.py,sha256=uY02ZKxVoXqVDta-0IMWaYJeE1CTXF_fA9NI6QUutmU,319
28
28
  fleet/exceptions.py,sha256=fUmPwWhnT8SR97lYsRq0kLHQHKtSh2eJS0VQ2caSzEI,5055
29
29
  fleet/global_client.py,sha256=frrDAFNM2ywN0JHLtlm9qbE1dQpnQJsavJpb7xSR_bU,1072
30
30
  fleet/models.py,sha256=AuSApLRN6aIDTOuJ4mGUyS1K1oLG9Q2AzjIE0Zj61MY,13586
31
- fleet/tasks.py,sha256=7ddJOeTznh1PqVEeyAkYWQ__MhomFhnfjeOt1B1it7I,17526
31
+ fleet/tasks.py,sha256=j14T6aoKDu3NghtQEr0vP-YDW_r8FxdDAazWe-6PQj0,17423
32
32
  fleet/types.py,sha256=L4Y82xICf1tzyCLqhLYUgEoaIIS5h9T05TyFNHSWs3s,652
33
33
  fleet/_async/__init__.py,sha256=5oOTmh16UsPWL2gDKKWkj2j5WGNeUhMzbQFWjX21jsc,8310
34
34
  fleet/_async/base.py,sha256=oisVTQsx0M_yTmyQJc3oij63uKZ97MHz-xYFsWXxQE8,9202
@@ -36,7 +36,7 @@ fleet/_async/client.py,sha256=w_g0aOLfyyK_OojjGVe3TgyQQsWxnybJRFMLOtsb0p8,33027
36
36
  fleet/_async/exceptions.py,sha256=fUmPwWhnT8SR97lYsRq0kLHQHKtSh2eJS0VQ2caSzEI,5055
37
37
  fleet/_async/global_client.py,sha256=4WskpLHbsDEgWW7hXMD09W-brkp4euy8w2ZJ88594rQ,1103
38
38
  fleet/_async/models.py,sha256=6WMN--LJFV-A5L2jW8Y6q7HQfub7qGxGoVkPQhvS_jw,13358
39
- fleet/_async/tasks.py,sha256=h_BerLY19GUjgu9EZypf3Av99CMRkPLOFKVoEXcwwbE,17540
39
+ fleet/_async/tasks.py,sha256=653T8m41i6ea-YCfr5EOsAD7CIi9ystcExMepxhSiA0,17442
40
40
  fleet/_async/env/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
41
41
  fleet/_async/env/client.py,sha256=hUwrKTAkTAbO94_pDaAH8LyvlS9q-7oT7ZRrNo03LNs,1309
42
42
  fleet/_async/instance/__init__.py,sha256=PtmJq8J8bh0SOQ2V55QURz5GJfobozwtQoqhaOk3_tI,515
@@ -67,14 +67,16 @@ fleet/verifiers/code.py,sha256=A1i_UabZspbyj1awzKVQ_HRxgMO3fU7NbkxYyTrp7So,48
67
67
  fleet/verifiers/db.py,sha256=LAh1HambBInH_D9q9E2Z41YNkCOI9JJfpWPFqztjpfQ,27922
68
68
  fleet/verifiers/decorator.py,sha256=nAP3O8szXu7md_kpwpz91hGSUNEVLYjwZQZTkQlV1DM,3260
69
69
  fleet/verifiers/parse.py,sha256=qz9AfJrTbjlg-LU-lE8Ciqi7Yt2a8-cs17FdpjTLhMk,8550
70
+ fleet/verifiers/parsing.py,sha256=EzlfHLogHPC1i5pfsF3ZCBJ8NY3s1uvz7v59CfcuMtI,3713
70
71
  fleet/verifiers/sql_differ.py,sha256=TqTLWyK3uOyLbitT6HYzYEzuSFC39wcyhgk3rcm__k8,6525
71
72
  fleet/verifiers/verifier.py,sha256=_lcxXVm8e0xRrK2gNJy9up7pW1zOkPRY5n5lQ85S8jg,14197
72
- fleet_python-0.2.65.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
73
+ fleet_python-0.2.66b3.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
73
74
  scripts/fix_sync_imports.py,sha256=X9fWLTpiPGkSHsjyQUDepOJkxOqw1DPj7nd8wFlFqLQ,8368
74
75
  scripts/unasync.py,sha256=vWVQxRWX8SRZO5cmzEhpvnG_REhCWXpidIGIpWmEcvI,696
75
76
  tests/__init__.py,sha256=Re1SdyxH8NfyL1kjhi7SQkGP1mYeWB-D6UALqdIMd8I,35
76
77
  tests/test_verifier_from_string.py,sha256=Lxi3TpFHFb-hG4-UhLKZJkqo84ax9YJY8G6beO-1erM,13581
77
- fleet_python-0.2.65.dist-info/METADATA,sha256=8S11v3oWKIXvxy_7pj2nBfj8RUSmKRUX1dDa33iO10Q,3304
78
- fleet_python-0.2.65.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
79
- fleet_python-0.2.65.dist-info/top_level.txt,sha256=qb1zIbtEktyhRFZdqVytwg54l64qtoZL0wjHB4bUg3c,29
80
- fleet_python-0.2.65.dist-info/RECORD,,
78
+ tests/test_verifier_security.py,sha256=AvjWTVV-VlWaysNkHcRz1_UZxjYZYlr4ynM6uy2o9eM,12821
79
+ fleet_python-0.2.66b3.dist-info/METADATA,sha256=U1sVi6oAJw5NxDCEqNSagFSu7zbjAmEgp4lq35VSz0E,3306
80
+ fleet_python-0.2.66b3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
81
+ fleet_python-0.2.66b3.dist-info/top_level.txt,sha256=qb1zIbtEktyhRFZdqVytwg54l64qtoZL0wjHB4bUg3c,29
82
+ fleet_python-0.2.66b3.dist-info/RECORD,,
@@ -0,0 +1,427 @@
1
+ """Security tests for verifier_from_string function.
2
+
3
+ Tests that the verifier parsing and validation properly blocks
4
+ arbitrary code execution during import.
5
+ """
6
+
7
+ import pytest
8
+ from fleet.tasks import verifier_from_string as sync_verifier_from_string
9
+ from fleet._async.tasks import verifier_from_string as async_verifier_from_string
10
+
11
+
12
+ class TestSyncVerifierSecurity:
13
+ """Security tests for sync version of verifier_from_string."""
14
+
15
+ def test_blocks_module_level_subprocess_run(self):
16
+ """Test that module-level subprocess.run() is blocked."""
17
+ code = """
18
+ import subprocess
19
+ subprocess.run(['echo', 'malicious'])
20
+
21
+ def my_verifier(env):
22
+ return 1.0
23
+ """
24
+ with pytest.raises(ValueError, match="Expression statements that are not constants"):
25
+ sync_verifier_from_string(
26
+ verifier_func=code,
27
+ verifier_id="test-verifier",
28
+ verifier_key="test-key",
29
+ sha256="test-sha",
30
+ )
31
+
32
+ def test_blocks_module_level_open(self):
33
+ """Test that module-level open() is blocked."""
34
+ code = """
35
+ open('/etc/passwd', 'r')
36
+
37
+ def my_verifier(env):
38
+ return 1.0
39
+ """
40
+ with pytest.raises(ValueError, match="Expression statements that are not constants"):
41
+ sync_verifier_from_string(
42
+ verifier_func=code,
43
+ verifier_id="test-verifier",
44
+ verifier_key="test-key",
45
+ sha256="test-sha",
46
+ )
47
+
48
+ def test_blocks_assignment_with_subprocess_call(self):
49
+ """Test that variable assignment with subprocess call is blocked."""
50
+ code = """
51
+ import subprocess
52
+ result = subprocess.run(['echo', 'malicious'])
53
+
54
+ def my_verifier(env):
55
+ return 1.0
56
+ """
57
+ with pytest.raises(ValueError, match="Variable assignments with function calls"):
58
+ sync_verifier_from_string(
59
+ verifier_func=code,
60
+ verifier_id="test-verifier",
61
+ verifier_key="test-key",
62
+ sha256="test-sha",
63
+ )
64
+
65
+ def test_blocks_assignment_with_open_call(self):
66
+ """Test that variable assignment with open() is blocked."""
67
+ code = """
68
+ file_handle = open('/etc/passwd', 'r')
69
+
70
+ def my_verifier(env):
71
+ return 1.0
72
+ """
73
+ with pytest.raises(ValueError, match="Variable assignments with function calls"):
74
+ sync_verifier_from_string(
75
+ verifier_func=code,
76
+ verifier_id="test-verifier",
77
+ verifier_key="test-key",
78
+ sha256="test-sha",
79
+ )
80
+
81
+ def test_blocks_assignment_with_any_function_call(self):
82
+ """Test that variable assignment with any function call is blocked."""
83
+ code = """
84
+ import os
85
+ path = os.getcwd()
86
+
87
+ def my_verifier(env):
88
+ return 1.0
89
+ """
90
+ with pytest.raises(ValueError, match="Variable assignments with function calls"):
91
+ sync_verifier_from_string(
92
+ verifier_func=code,
93
+ verifier_id="test-verifier",
94
+ verifier_key="test-key",
95
+ sha256="test-sha",
96
+ )
97
+
98
+ def test_allows_constant_assignment(self):
99
+ """Test that constant variable assignments are allowed."""
100
+ code = """
101
+ CONSTANT_VALUE = 42
102
+ ANOTHER_CONSTANT = "test"
103
+ PI = 3.14159
104
+
105
+ def my_verifier(env):
106
+ return CONSTANT_VALUE
107
+ """
108
+ # Should not raise
109
+ verifier = sync_verifier_from_string(
110
+ verifier_func=code,
111
+ verifier_id="test-verifier",
112
+ verifier_key="test-key",
113
+ sha256="test-sha",
114
+ )
115
+ assert verifier is not None
116
+
117
+ def test_allows_list_dict_constant_assignment(self):
118
+ """Test that list/dict constant assignments are allowed."""
119
+ code = """
120
+ MY_LIST = [1, 2, 3]
121
+ MY_DICT = {"key": "value"}
122
+ MY_TUPLE = (1, 2, 3)
123
+
124
+ def my_verifier(env):
125
+ return 1.0
126
+ """
127
+ # Should not raise
128
+ verifier = sync_verifier_from_string(
129
+ verifier_func=code,
130
+ verifier_id="test-verifier",
131
+ verifier_key="test-key",
132
+ sha256="test-sha",
133
+ )
134
+ assert verifier is not None
135
+
136
+ def test_allows_valid_imports(self):
137
+ """Test that imports are allowed."""
138
+ code = """
139
+ import json
140
+ import os
141
+ from typing import Dict
142
+
143
+ def my_verifier(env):
144
+ return 1.0
145
+ """
146
+ # Should not raise
147
+ verifier = sync_verifier_from_string(
148
+ verifier_func=code,
149
+ verifier_id="test-verifier",
150
+ verifier_key="test-key",
151
+ sha256="test-sha",
152
+ )
153
+ assert verifier is not None
154
+
155
+ def test_allows_class_definitions(self):
156
+ """Test that class definitions are allowed."""
157
+ code = """
158
+ class MyHelper:
159
+ def __init__(self):
160
+ self.value = 42
161
+
162
+ def get_value(self):
163
+ return self.value
164
+
165
+ def my_verifier(env):
166
+ helper = MyHelper()
167
+ return helper.get_value()
168
+ """
169
+ # Should not raise
170
+ verifier = sync_verifier_from_string(
171
+ verifier_func=code,
172
+ verifier_id="test-verifier",
173
+ verifier_key="test-key",
174
+ sha256="test-sha",
175
+ )
176
+ assert verifier is not None
177
+
178
+ def test_allows_multiple_functions(self):
179
+ """Test that multiple function definitions are allowed."""
180
+ code = """
181
+ def helper_function(x):
182
+ return x * 2
183
+
184
+ def my_verifier(env):
185
+ return helper_function(0.5)
186
+ """
187
+ # Should not raise
188
+ verifier = sync_verifier_from_string(
189
+ verifier_func=code,
190
+ verifier_id="test-verifier",
191
+ verifier_key="test-key",
192
+ sha256="test-sha",
193
+ )
194
+ assert verifier is not None
195
+
196
+ def test_extracts_first_function_name(self):
197
+ """Test that the first function name is correctly extracted."""
198
+ code = """
199
+ def first_function(env):
200
+ return 1.0
201
+
202
+ def second_function(env):
203
+ return 0.5
204
+ """
205
+ verifier = sync_verifier_from_string(
206
+ verifier_func=code,
207
+ verifier_id="test-verifier",
208
+ verifier_key="test-key",
209
+ sha256="test-sha",
210
+ )
211
+ # The first function should be used
212
+ assert verifier.func.__name__ == "first_function"
213
+
214
+ def test_error_message_includes_line_number(self):
215
+ """Test that error messages include helpful line numbers."""
216
+ code = """
217
+ import subprocess
218
+
219
+ subprocess.run(['echo', 'test'])
220
+
221
+ def my_verifier(env):
222
+ return 1.0
223
+ """
224
+ with pytest.raises(ValueError, match=r"Line \d+"):
225
+ sync_verifier_from_string(
226
+ verifier_func=code,
227
+ verifier_id="test-verifier",
228
+ verifier_key="test-key",
229
+ sha256="test-sha",
230
+ )
231
+
232
+ def test_blocks_nested_function_call_in_list(self):
233
+ """Test that function calls nested in list assignments are blocked."""
234
+ code = """
235
+ import os
236
+ MY_LIST = [1, 2, os.getcwd()]
237
+
238
+ def my_verifier(env):
239
+ return 1.0
240
+ """
241
+ with pytest.raises(ValueError, match="Variable assignments with function calls"):
242
+ sync_verifier_from_string(
243
+ verifier_func=code,
244
+ verifier_id="test-verifier",
245
+ verifier_key="test-key",
246
+ sha256="test-sha",
247
+ )
248
+
249
+ def test_blocks_nested_function_call_in_dict(self):
250
+ """Test that function calls nested in dict assignments are blocked."""
251
+ code = """
252
+ import os
253
+ MY_DICT = {"cwd": os.getcwd()}
254
+
255
+ def my_verifier(env):
256
+ return 1.0
257
+ """
258
+ with pytest.raises(ValueError, match="Variable assignments with function calls"):
259
+ sync_verifier_from_string(
260
+ verifier_func=code,
261
+ verifier_id="test-verifier",
262
+ verifier_key="test-key",
263
+ sha256="test-sha",
264
+ )
265
+
266
+ def test_allows_docstrings(self):
267
+ """Test that module-level docstrings are allowed."""
268
+ code = '''
269
+ """This is a module docstring."""
270
+
271
+ def my_verifier(env):
272
+ """This is a function docstring."""
273
+ return 1.0
274
+ '''
275
+ # Should not raise
276
+ verifier = sync_verifier_from_string(
277
+ verifier_func=code,
278
+ verifier_id="test-verifier",
279
+ verifier_key="test-key",
280
+ sha256="test-sha",
281
+ )
282
+ assert verifier is not None
283
+
284
+ def test_function_with_decorator_extracts_correct_name(self):
285
+ """Test that decorators don't affect function name extraction."""
286
+ code = """
287
+ def some_decorator(func):
288
+ return func
289
+
290
+ @some_decorator
291
+ def my_actual_function(env):
292
+ return 1.0
293
+ """
294
+ verifier = sync_verifier_from_string(
295
+ verifier_func=code,
296
+ verifier_id="test-verifier",
297
+ verifier_key="test-key",
298
+ sha256="test-sha",
299
+ )
300
+ # Should extract 'some_decorator' (first function) or 'my_actual_function'
301
+ # depending on order, but NOT the decorator name itself
302
+ assert verifier.func.__name__ in ["some_decorator", "my_actual_function"]
303
+
304
+ def test_blocks_decorator_with_function_call(self):
305
+ """Test that decorators with function calls are blocked."""
306
+ code = """
307
+ import subprocess
308
+
309
+ @subprocess.run(['echo', 'bad'])
310
+ def my_verifier(env):
311
+ return 1.0
312
+ """
313
+ # Decorators execute during import, so calls in decorators are dangerous
314
+ with pytest.raises(ValueError, match="Function decorators with function calls"):
315
+ sync_verifier_from_string(
316
+ verifier_func=code,
317
+ verifier_id="test-verifier",
318
+ verifier_key="test-key",
319
+ sha256="test-sha",
320
+ )
321
+
322
+ def test_allows_simple_decorator_reference(self):
323
+ """Test that simple decorator references (no calls) are allowed."""
324
+ code = """
325
+ def my_decorator(func):
326
+ return func
327
+
328
+ @my_decorator
329
+ def my_verifier(env):
330
+ return 1.0
331
+ """
332
+ # Simple decorator reference (no call) should be allowed
333
+ verifier = sync_verifier_from_string(
334
+ verifier_func=code,
335
+ verifier_id="test-verifier",
336
+ verifier_key="test-key",
337
+ sha256="test-sha",
338
+ )
339
+ assert verifier is not None
340
+
341
+
342
+ class TestAsyncVerifierSecurity:
343
+ """Security tests for async version of verifier_from_string."""
344
+
345
+ def test_blocks_module_level_subprocess_run(self):
346
+ """Test that module-level subprocess.run() is blocked."""
347
+ code = """
348
+ import subprocess
349
+ subprocess.run(['echo', 'malicious'])
350
+
351
+ async def my_async_verifier(env):
352
+ return 1.0
353
+ """
354
+ with pytest.raises(ValueError, match="Expression statements that are not constants"):
355
+ async_verifier_from_string(
356
+ verifier_func=code,
357
+ verifier_id="test-verifier",
358
+ verifier_key="test-key",
359
+ sha256="test-sha",
360
+ )
361
+
362
+ def test_blocks_assignment_with_function_call(self):
363
+ """Test that variable assignment with function call is blocked."""
364
+ code = """
365
+ import subprocess
366
+ result = subprocess.run(['echo', 'malicious'])
367
+
368
+ async def my_async_verifier(env):
369
+ return 1.0
370
+ """
371
+ with pytest.raises(ValueError, match="Variable assignments with function calls"):
372
+ async_verifier_from_string(
373
+ verifier_func=code,
374
+ verifier_id="test-verifier",
375
+ verifier_key="test-key",
376
+ sha256="test-sha",
377
+ )
378
+
379
+ def test_allows_constant_assignment(self):
380
+ """Test that constant variable assignments are allowed."""
381
+ code = """
382
+ CONSTANT_VALUE = 42
383
+
384
+ async def my_async_verifier(env):
385
+ return CONSTANT_VALUE
386
+ """
387
+ # Should not raise
388
+ verifier = async_verifier_from_string(
389
+ verifier_func=code,
390
+ verifier_id="test-verifier",
391
+ verifier_key="test-key",
392
+ sha256="test-sha",
393
+ )
394
+ assert verifier is not None
395
+
396
+ def test_allows_async_function_definitions(self):
397
+ """Test that async function definitions are recognized."""
398
+ code = """
399
+ async def my_async_verifier(env):
400
+ return 1.0
401
+ """
402
+ # Should not raise
403
+ verifier = async_verifier_from_string(
404
+ verifier_func=code,
405
+ verifier_id="test-verifier",
406
+ verifier_key="test-key",
407
+ sha256="test-sha",
408
+ )
409
+ assert verifier is not None
410
+
411
+ def test_extracts_first_async_function_name(self):
412
+ """Test that the first async function name is correctly extracted."""
413
+ code = """
414
+ async def first_async_function(env):
415
+ return 1.0
416
+
417
+ async def second_async_function(env):
418
+ return 0.5
419
+ """
420
+ verifier = async_verifier_from_string(
421
+ verifier_func=code,
422
+ verifier_id="test-verifier",
423
+ verifier_key="test-key",
424
+ sha256="test-sha",
425
+ )
426
+ # The first function should be used
427
+ assert verifier.func.__name__ == "first_async_function"