fleet-python 0.2.25__py3-none-any.whl → 0.2.27__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of fleet-python might be problematic. Click here for more details.
- examples/example_task.py +10 -6
- examples/example_tasks.py +34 -0
- fleet/_async/client.py +199 -2
- fleet/_async/models.py +16 -0
- fleet/_async/tasks.py +58 -1
- fleet/_async/verifiers/__init__.py +1 -1
- fleet/_async/verifiers/verifier.py +61 -19
- fleet/client.py +205 -2
- fleet/instance/client.py +1 -1
- fleet/models.py +16 -0
- fleet/verifiers/__init__.py +2 -2
- fleet/verifiers/parse.py +65 -1
- fleet/verifiers/verifier.py +63 -20
- {fleet_python-0.2.25.dist-info → fleet_python-0.2.27.dist-info}/METADATA +5 -4
- {fleet_python-0.2.25.dist-info → fleet_python-0.2.27.dist-info}/RECORD +19 -19
- scripts/fix_sync_imports.py +44 -7
- fleet/tasks.py +0 -44
- {fleet_python-0.2.25.dist-info → fleet_python-0.2.27.dist-info}/WHEEL +0 -0
- {fleet_python-0.2.25.dist-info → fleet_python-0.2.27.dist-info}/licenses/LICENSE +0 -0
- {fleet_python-0.2.25.dist-info → fleet_python-0.2.27.dist-info}/top_level.txt +0 -0
fleet/client.py
CHANGED
|
@@ -17,9 +17,10 @@
|
|
|
17
17
|
import base64
|
|
18
18
|
import cloudpickle
|
|
19
19
|
import httpx
|
|
20
|
+
import json
|
|
20
21
|
import logging
|
|
21
22
|
import os
|
|
22
|
-
from typing import List, Optional, Dict
|
|
23
|
+
from typing import List, Optional, Dict, TYPE_CHECKING
|
|
23
24
|
|
|
24
25
|
from .base import EnvironmentBase, SyncWrapper
|
|
25
26
|
from .models import (
|
|
@@ -30,8 +31,13 @@ from .models import (
|
|
|
30
31
|
VerifiersExecuteResponse,
|
|
31
32
|
TaskListResponse,
|
|
32
33
|
AccountResponse,
|
|
34
|
+
TaskRequest,
|
|
33
35
|
)
|
|
34
36
|
from .tasks import Task
|
|
37
|
+
from .verifiers.parse import extract_function_name, convert_new_to_old_verifier
|
|
38
|
+
|
|
39
|
+
if TYPE_CHECKING:
|
|
40
|
+
from .verifiers import SyncVerifierFunction
|
|
35
41
|
|
|
36
42
|
from .instance import (
|
|
37
43
|
InstanceClient,
|
|
@@ -260,18 +266,112 @@ class Fleet:
|
|
|
260
266
|
# Transform TaskResponse objects to Task objects
|
|
261
267
|
tasks = []
|
|
262
268
|
for task_response in task_list_response.tasks:
|
|
269
|
+
# Create verifier function if verifier data is present
|
|
270
|
+
verifier = None
|
|
271
|
+
verifier_func = task_response.verifier_func
|
|
272
|
+
|
|
273
|
+
if task_response.verifier:
|
|
274
|
+
# Use verifier code from the embedded verifier object
|
|
275
|
+
verifier_func = task_response.verifier.code
|
|
276
|
+
|
|
277
|
+
# Create VerifierFunction from the embedded data
|
|
278
|
+
try:
|
|
279
|
+
verifier = self._create_verifier_from_data(
|
|
280
|
+
verifier_id=task_response.verifier.verifier_id,
|
|
281
|
+
verifier_key=task_response.verifier.key,
|
|
282
|
+
verifier_code=task_response.verifier.code,
|
|
283
|
+
verifier_sha=task_response.verifier.sha256
|
|
284
|
+
)
|
|
285
|
+
except Exception as e:
|
|
286
|
+
logger.warning(f"Failed to create verifier {task_response.verifier.key}: {e}")
|
|
287
|
+
|
|
263
288
|
task = Task(
|
|
264
289
|
key=task_response.key,
|
|
265
290
|
prompt=task_response.prompt,
|
|
266
291
|
env_id=task_response.environment_id, # Map environment_id -> env_id
|
|
267
292
|
created_at=task_response.created_at,
|
|
268
|
-
|
|
293
|
+
version=task_response.version,
|
|
294
|
+
env_variables=task_response.env_variables or {},
|
|
295
|
+
verifier_func=verifier_func, # Set verifier code
|
|
296
|
+
verifier=verifier, # Use created verifier or None
|
|
269
297
|
metadata={} # Default empty metadata
|
|
270
298
|
)
|
|
271
299
|
tasks.append(task)
|
|
272
300
|
|
|
273
301
|
return tasks
|
|
274
302
|
|
|
303
|
+
def export_tasks(self, env_key: Optional[str] = None, filename: Optional[str] = None):
|
|
304
|
+
"""Export tasks for the authenticated team, optionally filtered by environment.
|
|
305
|
+
|
|
306
|
+
Args:
|
|
307
|
+
env_key: Optional environment key to filter tasks by
|
|
308
|
+
filename: Optional filename to write tasks to. If not provided, defaults to 'tasks.json' or 'tasks_{env_key}.json'
|
|
309
|
+
|
|
310
|
+
Returns:
|
|
311
|
+
str: Path to the exported file if tasks were written, None if no tasks found
|
|
312
|
+
"""
|
|
313
|
+
tasks = self.load_tasks(env_key)
|
|
314
|
+
if tasks:
|
|
315
|
+
# Generate filename if not provided
|
|
316
|
+
if filename is None:
|
|
317
|
+
if env_key:
|
|
318
|
+
filename = f"tasks_{env_key}.json"
|
|
319
|
+
else:
|
|
320
|
+
filename = "tasks.json"
|
|
321
|
+
|
|
322
|
+
# Convert tasks to serializable format
|
|
323
|
+
tasks_data = []
|
|
324
|
+
for task in tasks:
|
|
325
|
+
task_dict = task.model_dump()
|
|
326
|
+
# Remove non-serializable verifier object, keep verifier_func (code string)
|
|
327
|
+
if 'verifier' in task_dict:
|
|
328
|
+
task_dict.pop('verifier')
|
|
329
|
+
tasks_data.append(task_dict)
|
|
330
|
+
|
|
331
|
+
# Write to JSON file
|
|
332
|
+
with open(filename, 'w', encoding='utf-8') as f:
|
|
333
|
+
json.dump(tasks_data, f, indent=2, default=str)
|
|
334
|
+
|
|
335
|
+
logger.info(f"Exported {len(tasks)} tasks to {filename}")
|
|
336
|
+
return filename
|
|
337
|
+
else:
|
|
338
|
+
logger.info("No tasks found to export")
|
|
339
|
+
return None
|
|
340
|
+
|
|
341
|
+
def import_tasks(self, filename: str):
|
|
342
|
+
"""Import tasks from a JSON file.
|
|
343
|
+
|
|
344
|
+
Args:
|
|
345
|
+
filename: Path to the JSON file of Task objects to import
|
|
346
|
+
|
|
347
|
+
Returns:
|
|
348
|
+
List[Task] containing imported Task objects
|
|
349
|
+
"""
|
|
350
|
+
with open(filename, 'r', encoding='utf-8') as f:
|
|
351
|
+
tasks_data = json.load(f)
|
|
352
|
+
|
|
353
|
+
# Create tasks from the loaded data
|
|
354
|
+
tasks = []
|
|
355
|
+
for task_data in tasks_data:
|
|
356
|
+
task = Task(**task_data)
|
|
357
|
+
tasks.append(task)
|
|
358
|
+
|
|
359
|
+
for task in tasks:
|
|
360
|
+
payload = TaskRequest(
|
|
361
|
+
key=task.key,
|
|
362
|
+
prompt=task.prompt,
|
|
363
|
+
environment_id=task.env_id,
|
|
364
|
+
verifier_func=task.verifier_func,
|
|
365
|
+
version=task.version or None,
|
|
366
|
+
env_variables=task.env_variables or {},
|
|
367
|
+
)
|
|
368
|
+
try:
|
|
369
|
+
response = self.client.request("POST", "/v1/tasks", json=payload.model_dump())
|
|
370
|
+
except Exception as e:
|
|
371
|
+
logger.error(f"Failed to import task {task.key}: {e}")
|
|
372
|
+
continue
|
|
373
|
+
|
|
374
|
+
|
|
275
375
|
def account(self) -> AccountResponse:
|
|
276
376
|
"""Get account information including instance limits and usage.
|
|
277
377
|
|
|
@@ -280,6 +380,109 @@ class Fleet:
|
|
|
280
380
|
"""
|
|
281
381
|
response = self.client.request("GET", "/v1/account")
|
|
282
382
|
return AccountResponse(**response.json())
|
|
383
|
+
|
|
384
|
+
def _create_verifier_from_data(
|
|
385
|
+
self,
|
|
386
|
+
verifier_id: str,
|
|
387
|
+
verifier_key: str,
|
|
388
|
+
verifier_code: str,
|
|
389
|
+
verifier_sha: str
|
|
390
|
+
) -> "SyncVerifierFunction":
|
|
391
|
+
"""Create an AsyncVerifierFunction from verifier data.
|
|
392
|
+
|
|
393
|
+
Args:
|
|
394
|
+
verifier_id: The verifier ID
|
|
395
|
+
verifier_key: The verifier key
|
|
396
|
+
verifier_code: The verifier code
|
|
397
|
+
verifier_sha: The verifier SHA256
|
|
398
|
+
|
|
399
|
+
Returns:
|
|
400
|
+
AsyncVerifierFunction created from the verifier code
|
|
401
|
+
"""
|
|
402
|
+
from .verifiers.verifier import SyncVerifierFunction
|
|
403
|
+
|
|
404
|
+
# Convert async verifier code to sync
|
|
405
|
+
if 'async def' in verifier_code:
|
|
406
|
+
verifier_code = verifier_code.replace('async def', 'def')
|
|
407
|
+
if 'await ' in verifier_code:
|
|
408
|
+
verifier_code = verifier_code.replace('await ', '')
|
|
409
|
+
|
|
410
|
+
# Check if this is a new format verifier (has before/after parameters)
|
|
411
|
+
if 'before: DatabaseSnapshot' in verifier_code and 'after: DatabaseSnapshot' in verifier_code:
|
|
412
|
+
# Convert new format to old format
|
|
413
|
+
verifier_code = convert_new_to_old_verifier(verifier_code)
|
|
414
|
+
# Update function name since wrapper adds _wrapper suffix
|
|
415
|
+
original_name = extract_function_name(verifier_code.split('\n')[0])
|
|
416
|
+
if original_name and original_name.endswith('_wrapper'):
|
|
417
|
+
function_name = original_name
|
|
418
|
+
else:
|
|
419
|
+
function_name = extract_function_name(verifier_code)
|
|
420
|
+
else:
|
|
421
|
+
# Extract function name from code
|
|
422
|
+
function_name = extract_function_name(verifier_code)
|
|
423
|
+
|
|
424
|
+
if not function_name:
|
|
425
|
+
raise ValueError(f"Could not extract function name from verifier {verifier_id}")
|
|
426
|
+
|
|
427
|
+
# Create a function object from the code
|
|
428
|
+
# Import necessary classes for the namespace
|
|
429
|
+
from .verifiers.db import IgnoreConfig, DatabaseSnapshot
|
|
430
|
+
|
|
431
|
+
# Create a namespace for the function
|
|
432
|
+
namespace = {
|
|
433
|
+
'__builtins__': __builtins__,
|
|
434
|
+
'Environment': object, # Placeholder, will be provided at runtime
|
|
435
|
+
'IgnoreConfig': IgnoreConfig,
|
|
436
|
+
'DatabaseSnapshot': DatabaseSnapshot,
|
|
437
|
+
'TASK_FAILED_SCORE': 0,
|
|
438
|
+
'TASK_SUCCESSFUL_SCORE': 1,
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
# Execute the code to define the function
|
|
442
|
+
exec(verifier_code, namespace)
|
|
443
|
+
|
|
444
|
+
# Get the function object
|
|
445
|
+
if function_name not in namespace:
|
|
446
|
+
raise ValueError(f"Function {function_name} not found in verifier code")
|
|
447
|
+
|
|
448
|
+
func = namespace[function_name]
|
|
449
|
+
|
|
450
|
+
# Create and return AsyncVerifierFunction with SHA if available
|
|
451
|
+
# Since the function was created via exec, we need to provide the raw code
|
|
452
|
+
verifier_func = SyncVerifierFunction(
|
|
453
|
+
func=func,
|
|
454
|
+
key=verifier_key,
|
|
455
|
+
extra_requirements=[],
|
|
456
|
+
verifier_id=verifier_id,
|
|
457
|
+
sha256=verifier_sha, # Pass SHA directly to constructor
|
|
458
|
+
raw_code=verifier_code # Provide raw code since func won't have extractable source
|
|
459
|
+
)
|
|
460
|
+
|
|
461
|
+
# Store the original verifier code for reference
|
|
462
|
+
verifier_func._verifier_code = verifier_code
|
|
463
|
+
|
|
464
|
+
return verifier_func
|
|
465
|
+
|
|
466
|
+
def _load_verifier(self, verifier_id: str) -> "SyncVerifierFunction":
|
|
467
|
+
"""Load a verifier by ID and create an AsyncVerifierFunction.
|
|
468
|
+
|
|
469
|
+
Args:
|
|
470
|
+
verifier_id: The verifier ID to fetch
|
|
471
|
+
|
|
472
|
+
Returns:
|
|
473
|
+
AsyncVerifierFunction created from the verifier code
|
|
474
|
+
"""
|
|
475
|
+
# Fetch verifier from API
|
|
476
|
+
response = self.client.request("GET", f"/v1/verifier/{verifier_id}")
|
|
477
|
+
verifier_data = response.json()
|
|
478
|
+
|
|
479
|
+
# Use the common method to create verifier
|
|
480
|
+
return self._create_verifier_from_data(
|
|
481
|
+
verifier_id=verifier_id,
|
|
482
|
+
verifier_key=verifier_data["key"],
|
|
483
|
+
verifier_code=verifier_data["code"],
|
|
484
|
+
verifier_sha=verifier_data.get("sha256", "")
|
|
485
|
+
)
|
|
283
486
|
|
|
284
487
|
|
|
285
488
|
# Shared
|
fleet/instance/client.py
CHANGED
|
@@ -11,7 +11,7 @@ from ..resources.sqlite import SQLiteResource
|
|
|
11
11
|
from ..resources.browser import BrowserResource
|
|
12
12
|
from ..resources.base import Resource
|
|
13
13
|
|
|
14
|
-
from
|
|
14
|
+
from fleet.verifiers import DatabaseSnapshot
|
|
15
15
|
from fleet.verifiers.parse import convert_verifier_string, extract_function_name
|
|
16
16
|
|
|
17
17
|
from ..exceptions import FleetEnvironmentError
|
fleet/models.py
CHANGED
|
@@ -152,6 +152,18 @@ class TaskRequest(BaseModel):
|
|
|
152
152
|
prompt: str = Field(..., title='Prompt')
|
|
153
153
|
environment_id: str = Field(..., title='Environment Id')
|
|
154
154
|
verifier_id: Optional[str] = Field(None, title='Verifier Id')
|
|
155
|
+
version: Optional[str] = Field(None, title='Version')
|
|
156
|
+
env_variables: Optional[Dict[str, Any]] = Field(None, title='Env Variables')
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
class VerifierData(BaseModel):
|
|
160
|
+
verifier_id: str = Field(..., title='Verifier Id')
|
|
161
|
+
key: str = Field(..., title='Key')
|
|
162
|
+
version: int = Field(..., title='Version')
|
|
163
|
+
sha256: str = Field(..., title='Sha256')
|
|
164
|
+
code: str = Field(..., title='Code')
|
|
165
|
+
comment: Optional[str] = Field(None, title='Comment')
|
|
166
|
+
created_at: str = Field(..., title='Created At')
|
|
155
167
|
|
|
156
168
|
|
|
157
169
|
class TaskResponse(BaseModel):
|
|
@@ -161,6 +173,10 @@ class TaskResponse(BaseModel):
|
|
|
161
173
|
environment_id: str = Field(..., title='Environment Id')
|
|
162
174
|
created_at: str = Field(..., title='Created At')
|
|
163
175
|
verifier_id: Optional[str] = Field(None, title='Verifier Id')
|
|
176
|
+
verifier_func: Optional[str] = Field(None, title='Verifier Func')
|
|
177
|
+
version: Optional[str] = Field(None, title='Version')
|
|
178
|
+
env_variables: Optional[Dict[str, Any]] = Field(None, title='Env Variables')
|
|
179
|
+
verifier: Optional[VerifierData] = Field(None, title='Verifier')
|
|
164
180
|
|
|
165
181
|
|
|
166
182
|
class ValidationError(BaseModel):
|
fleet/verifiers/__init__.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"""Fleet verifiers module - database snapshot validation utilities and verifier decorator."""
|
|
2
2
|
|
|
3
|
-
from
|
|
4
|
-
from
|
|
3
|
+
from .db import DatabaseSnapshot, IgnoreConfig, SnapshotDiff
|
|
4
|
+
from .code import TASK_SUCCESSFUL_SCORE, TASK_FAILED_SCORE
|
|
5
5
|
from .decorator import (
|
|
6
6
|
verifier,
|
|
7
7
|
SyncVerifierFunction,
|
fleet/verifiers/parse.py
CHANGED
|
@@ -140,4 +140,68 @@ def convert_verifier_string(verifier_str: str) -> str:
|
|
|
140
140
|
# Replace TASK_FAILED_SCORE with 0 in the function string
|
|
141
141
|
new_func = new_func.replace('TASK_FAILED_SCORE', '0')
|
|
142
142
|
|
|
143
|
-
return new_func
|
|
143
|
+
return new_func
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def convert_new_to_old_verifier(verifier_str: str) -> str:
|
|
147
|
+
"""
|
|
148
|
+
Convert a verifier function from the new format (before/after: DatabaseSnapshot)
|
|
149
|
+
to the old format (env: Environment).
|
|
150
|
+
|
|
151
|
+
This is the inverse of convert_verifier_string.
|
|
152
|
+
|
|
153
|
+
Args:
|
|
154
|
+
verifier_str: The new format verifier function as a string
|
|
155
|
+
|
|
156
|
+
Returns:
|
|
157
|
+
The converted verifier function string that accepts env
|
|
158
|
+
"""
|
|
159
|
+
# Extract function name, parameters, docstring, and body
|
|
160
|
+
# Pattern for new format with flexible whitespace and multiline support
|
|
161
|
+
func_pattern = r'def\s+(\w+)\s*\(\s*before\s*:\s*DatabaseSnapshot\s*,?\s*after\s*:\s*DatabaseSnapshot\s*,?\s*transcript\s*:\s*str\s*\|\s*None\s*=\s*None\s*,?\s*\)\s*->\s*int:\s*((?:\s*""".*?"""\s*)?)(.*)'
|
|
162
|
+
|
|
163
|
+
# Try multiline pattern that's more flexible
|
|
164
|
+
func_pattern_multiline = r'def\s+(\w+)\s*\(\s*\n?\s*before\s*:\s*DatabaseSnapshot\s*,?\s*\n?\s*after\s*:\s*DatabaseSnapshot\s*,?\s*\n?\s*transcript\s*:\s*str\s*\|\s*None\s*=\s*None\s*,?\s*\n?\s*\)\s*->\s*int:\s*\n?((?:\s*""".*?"""\s*)?)(.*)'
|
|
165
|
+
|
|
166
|
+
match = re.match(func_pattern_multiline, verifier_str.strip(), re.DOTALL | re.MULTILINE)
|
|
167
|
+
|
|
168
|
+
if not match:
|
|
169
|
+
# Even more flexible pattern
|
|
170
|
+
func_pattern_flexible = r'def\s+(\w+)\s*\([^)]*\)\s*->\s*int:\s*\n?((?:\s*""".*?"""\s*)?)(.*)'
|
|
171
|
+
match = re.match(func_pattern_flexible, verifier_str.strip(), re.DOTALL)
|
|
172
|
+
|
|
173
|
+
if not match:
|
|
174
|
+
raise ValueError("Could not parse new format verifier function")
|
|
175
|
+
|
|
176
|
+
func_name = match.group(1)
|
|
177
|
+
docstring = match.group(2).strip()
|
|
178
|
+
body = match.group(3)
|
|
179
|
+
|
|
180
|
+
# Indent the original function body
|
|
181
|
+
indented_verifier = '\n'.join(' ' + line if line.strip() else line for line in verifier_str.splitlines())
|
|
182
|
+
|
|
183
|
+
# Build the wrapper function
|
|
184
|
+
wrapper_func = f'''def {func_name}_wrapper(env, *args, **kwargs) -> float:
|
|
185
|
+
"""Wrapper to adapt new format verifier to old format."""
|
|
186
|
+
# Import required modules
|
|
187
|
+
from .verifiers.db import DatabaseSnapshot, IgnoreConfig
|
|
188
|
+
|
|
189
|
+
# Constants
|
|
190
|
+
TASK_SUCCESSFUL_SCORE = 1
|
|
191
|
+
TASK_FAILED_SCORE = 0
|
|
192
|
+
|
|
193
|
+
# Extract before and after from env
|
|
194
|
+
before = env.db("seed")
|
|
195
|
+
after = env.db("current")
|
|
196
|
+
|
|
197
|
+
# Get transcript from kwargs if provided
|
|
198
|
+
transcript = kwargs.get('transcript', kwargs.get('final_answer', None))
|
|
199
|
+
|
|
200
|
+
# Define the inner function
|
|
201
|
+
{indented_verifier}
|
|
202
|
+
|
|
203
|
+
# Call the inner function and convert result
|
|
204
|
+
result = {func_name}(before, after, transcript)
|
|
205
|
+
return float(result)'''
|
|
206
|
+
|
|
207
|
+
return wrapper_func
|
fleet/verifiers/verifier.py
CHANGED
|
@@ -11,6 +11,7 @@ import functools
|
|
|
11
11
|
import uuid
|
|
12
12
|
import logging
|
|
13
13
|
import hashlib
|
|
14
|
+
import inspect
|
|
14
15
|
from typing import Any, Callable, Dict, Optional, List, TypeVar, Set
|
|
15
16
|
|
|
16
17
|
from .bundler import FunctionBundler
|
|
@@ -38,16 +39,19 @@ class SyncVerifierFunction:
|
|
|
38
39
|
func: F,
|
|
39
40
|
key: str,
|
|
40
41
|
extra_requirements: Optional[List[str]] = None,
|
|
41
|
-
verifier_id: Optional[str] = None
|
|
42
|
+
verifier_id: Optional[str] = None,
|
|
43
|
+
sha256: Optional[str] = None,
|
|
44
|
+
raw_code: Optional[str] = None
|
|
42
45
|
):
|
|
43
46
|
self.func = func
|
|
44
47
|
self.key = key
|
|
45
48
|
self.verifier_id = verifier_id or str(uuid.uuid4())
|
|
46
49
|
self.extra_requirements = extra_requirements or []
|
|
47
50
|
self._bundler = FunctionBundler()
|
|
48
|
-
self._bundle_sha: Optional[str] =
|
|
51
|
+
self._bundle_sha: Optional[str] = sha256 # Use provided SHA if available
|
|
49
52
|
self._bundle_data: Optional[bytes] = None # Cached bundle data
|
|
50
|
-
self.
|
|
53
|
+
self._raw_code: Optional[str] = raw_code # Store raw code if provided
|
|
54
|
+
self._is_async = inspect.iscoroutinefunction(func)
|
|
51
55
|
|
|
52
56
|
# Copy function metadata
|
|
53
57
|
functools.update_wrapper(self, func)
|
|
@@ -55,14 +59,40 @@ class SyncVerifierFunction:
|
|
|
55
59
|
def _get_or_create_bundle(self) -> tuple[bytes, str]:
|
|
56
60
|
"""Get or create bundle data and return (bundle_data, sha)."""
|
|
57
61
|
if self._bundle_data is None or self._bundle_sha is None:
|
|
58
|
-
#
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
62
|
+
# If we have raw code, create a bundle from it
|
|
63
|
+
if self._raw_code:
|
|
64
|
+
import io
|
|
65
|
+
import zipfile
|
|
66
|
+
|
|
67
|
+
# Create zip bundle directly (matching bundler format)
|
|
68
|
+
zip_buffer = io.BytesIO()
|
|
69
|
+
with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zf:
|
|
70
|
+
# Add requirements.txt
|
|
71
|
+
requirements = self.extra_requirements or []
|
|
72
|
+
if "fleet-python" not in requirements:
|
|
73
|
+
requirements.append("fleet-python")
|
|
74
|
+
req_content = "\n".join(requirements)
|
|
75
|
+
zf.writestr("requirements.txt", req_content)
|
|
76
|
+
|
|
77
|
+
# Add verifier.py with the raw code
|
|
78
|
+
zf.writestr("verifier.py", self._raw_code)
|
|
79
|
+
|
|
80
|
+
self._bundle_data = zip_buffer.getvalue()
|
|
81
|
+
self._bundle_sha = _get_bundle_sha(self._bundle_data)
|
|
82
|
+
logger.debug(f"Created bundle from raw code for {self.key} with SHA: {self._bundle_sha}")
|
|
83
|
+
else:
|
|
84
|
+
# Try to create bundle from function source
|
|
85
|
+
try:
|
|
86
|
+
self._bundle_data = self._bundler.create_bundle(
|
|
87
|
+
self.func,
|
|
88
|
+
self.extra_requirements,
|
|
89
|
+
self.verifier_id
|
|
90
|
+
)
|
|
91
|
+
self._bundle_sha = _get_bundle_sha(self._bundle_data)
|
|
92
|
+
logger.debug(f"Created bundle for {self.key} with SHA: {self._bundle_sha}")
|
|
93
|
+
except OSError as e:
|
|
94
|
+
# Can't create bundle - no source and no raw code
|
|
95
|
+
raise OSError(f"Cannot create bundle for {self.key}: {e}")
|
|
66
96
|
|
|
67
97
|
return self._bundle_data, self._bundle_sha
|
|
68
98
|
|
|
@@ -70,6 +100,11 @@ class SyncVerifierFunction:
|
|
|
70
100
|
"""Check if bundle needs to be uploaded and return (sha, needs_upload)."""
|
|
71
101
|
bundle_data, bundle_sha = self._get_or_create_bundle()
|
|
72
102
|
|
|
103
|
+
# If bundle_data is empty, we're using server-side bundle
|
|
104
|
+
if not bundle_data:
|
|
105
|
+
logger.debug(f"Using server-side bundle {bundle_sha[:8]}...")
|
|
106
|
+
return bundle_sha, False # No upload needed, server has it
|
|
107
|
+
|
|
73
108
|
# 1. Check local process cache first
|
|
74
109
|
if bundle_sha in _uploaded_bundle_shas:
|
|
75
110
|
logger.debug(f"Bundle {bundle_sha[:8]}... found in local cache")
|
|
@@ -121,12 +156,13 @@ class SyncVerifierFunction:
|
|
|
121
156
|
|
|
122
157
|
def remote(self, env: SyncEnv, *args, **kwargs) -> float:
|
|
123
158
|
"""Remote execution of the verifier function with SHA-based bundle caching."""
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
159
|
+
# Async verifiers are now supported by the backend
|
|
160
|
+
# if self._is_async:
|
|
161
|
+
# raise NotImplementedError(
|
|
162
|
+
# f"Async verifier '{self.key}' cannot be executed remotely. "
|
|
163
|
+
# "The remote execution environment only supports synchronous functions. "
|
|
164
|
+
# "Please provide a synchronous version of your verifier."
|
|
165
|
+
# )
|
|
130
166
|
|
|
131
167
|
args_array = list(args)
|
|
132
168
|
args_array.append({"env": env.instance_id})
|
|
@@ -162,11 +198,12 @@ class SyncVerifierFunction:
|
|
|
162
198
|
bundle_data, _ = self._get_or_create_bundle()
|
|
163
199
|
|
|
164
200
|
response = env.execute_verifier_remote(
|
|
165
|
-
bundle_data=bundle_data, #
|
|
201
|
+
bundle_data=bundle_data or b'', # Empty if using server-side bundle
|
|
166
202
|
bundle_sha=bundle_sha,
|
|
167
203
|
key=self.key,
|
|
168
204
|
function_name=self.func.__name__,
|
|
169
205
|
args=args,
|
|
206
|
+
args_array=args_array,
|
|
170
207
|
kwargs=kwargs,
|
|
171
208
|
needs_upload=False # Don't upload, just execute
|
|
172
209
|
)
|
|
@@ -248,7 +285,9 @@ Remote traceback:
|
|
|
248
285
|
|
|
249
286
|
def verifier(
|
|
250
287
|
key: Optional[str] = None,
|
|
251
|
-
extra_requirements: Optional[List[str]] = None
|
|
288
|
+
extra_requirements: Optional[List[str]] = None,
|
|
289
|
+
sha256: Optional[str] = None,
|
|
290
|
+
raw_code: Optional[str] = None
|
|
252
291
|
) -> Callable[[F], SyncVerifierFunction]:
|
|
253
292
|
"""
|
|
254
293
|
Decorator to create a verifier function with env-first pattern.
|
|
@@ -260,6 +299,8 @@ def verifier(
|
|
|
260
299
|
Args:
|
|
261
300
|
key: Optional key for the verifier. Defaults to function name.
|
|
262
301
|
extra_requirements: Additional PyPI packages needed by the verifier.
|
|
302
|
+
sha256: Optional SHA256 hash of existing server-side bundle to use.
|
|
303
|
+
raw_code: Optional raw code to use as bundle (bypasses source extraction).
|
|
263
304
|
|
|
264
305
|
Example:
|
|
265
306
|
# Synchronous verifier (works locally and remotely)
|
|
@@ -297,7 +338,9 @@ def verifier(
|
|
|
297
338
|
func,
|
|
298
339
|
verifier_key,
|
|
299
340
|
extra_requirements,
|
|
300
|
-
verifier_uuid
|
|
341
|
+
verifier_uuid,
|
|
342
|
+
sha256,
|
|
343
|
+
raw_code
|
|
301
344
|
)
|
|
302
345
|
|
|
303
346
|
return decorator
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fleet-python
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.27
|
|
4
4
|
Summary: Python SDK for Fleet environments
|
|
5
5
|
Author-email: Fleet AI <nic@fleet.so>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -66,6 +66,7 @@ export FLEET_API_KEY="sk_your_key_here"
|
|
|
66
66
|
|
|
67
67
|
```python
|
|
68
68
|
import fleet as flt
|
|
69
|
+
import datetime
|
|
69
70
|
|
|
70
71
|
# Create environment by key
|
|
71
72
|
env = flt.env.make("fira")
|
|
@@ -73,11 +74,11 @@ env = flt.env.make("fira")
|
|
|
73
74
|
# Reset environment with seed and options
|
|
74
75
|
env.reset(
|
|
75
76
|
seed=42,
|
|
76
|
-
timestamp=datetime.now()
|
|
77
|
+
timestamp=int(datetime.datetime.now().timestamp())
|
|
77
78
|
)
|
|
78
79
|
|
|
79
|
-
# Access environment state ('
|
|
80
|
-
sql = env.state("sqlite://
|
|
80
|
+
# Access environment state ('current' is the resource id for a sqlite database)
|
|
81
|
+
sql = env.state("sqlite://current")
|
|
81
82
|
sql.exec("UPDATE customers SET status = 'active' WHERE id = 123")
|
|
82
83
|
|
|
83
84
|
# Clean up
|
|
@@ -6,7 +6,8 @@ examples/example_client.py,sha256=70HKEhz_Gb79YcvKQauCPdS08AAwjo9unt2dh1jN_Oo,10
|
|
|
6
6
|
examples/example_mcp_anthropic.py,sha256=iyO0OCem5SWEtJQGan6pBuGfN3LZXeeIN9DdMHw9f9M,2542
|
|
7
7
|
examples/example_mcp_openai.py,sha256=5BQzMWrNdtBrBCNcUYgU6qB9Aabc10UTMKvaFb4NerI,492
|
|
8
8
|
examples/example_sync.py,sha256=l8l-QOGTMnijiSK0mt5aofzXqbv-kKU3WCow4QS-7yg,976
|
|
9
|
-
examples/example_task.py,sha256=
|
|
9
|
+
examples/example_task.py,sha256=W6Cuhtb06FWv2N972KjdVBdbNzkHwLBEZBTAeoHrweQ,7014
|
|
10
|
+
examples/example_tasks.py,sha256=tZMtT7wNzv6jb4VYoEKeR0eX-dJhwajTtoBN-os6-I4,776
|
|
10
11
|
examples/example_verifier.py,sha256=PfGphI6gnfhSM_SRb4KyhHYddJmapk2hhRrSIaxAtr4,2025
|
|
11
12
|
examples/gemini_example.py,sha256=8mDXGGCaodyK6uXgpWhxi-DQ5OA-GFW12Gfwh0b3EDY,16177
|
|
12
13
|
examples/json_tasks_example.py,sha256=3ub2LLiC6hXpVEH1175QxCmfCD3Blfo3yoG85uV5CS8,5334
|
|
@@ -17,18 +18,17 @@ examples/query_builder_example.py,sha256=Q3lUBETHpu1aS2FXAO79ADYqCxOjMMMZNgCcFVa
|
|
|
17
18
|
examples/quickstart.py,sha256=1VT39IRRhemsJgxi0O0gprdpcw7HB4pYO97GAYagIcg,3788
|
|
18
19
|
fleet/__init__.py,sha256=2WHAk_ZRIxDs1uxrWf-sSeokiK4f_nDVHJJ1_VUFSPA,2306
|
|
19
20
|
fleet/base.py,sha256=0yYuMN0lBkrfTTZBt5NQp5112xWgziuWEk4GuHJB1wE,9189
|
|
20
|
-
fleet/client.py,sha256=
|
|
21
|
+
fleet/client.py,sha256=Nb4m6AX7uG6k0hdNqyBzAcEdYu1kJLicUnzvTAE_u8Y,20242
|
|
21
22
|
fleet/config.py,sha256=rqR-y2TbZS-VYACaqrg-PUe0y0UDbR1ZNU1KGJZBwNQ,272
|
|
22
23
|
fleet/exceptions.py,sha256=fUmPwWhnT8SR97lYsRq0kLHQHKtSh2eJS0VQ2caSzEI,5055
|
|
23
|
-
fleet/models.py,sha256=
|
|
24
|
-
fleet/tasks.py,sha256=OScp0tHIbCmOpmCFzHATBmA7WiJMlehP3rvEfeMAoPk,1648
|
|
24
|
+
fleet/models.py,sha256=6SqoWdTIygF3jriZW53a-jQ1JZoPatuWZ4JQ1qIY5MY,12453
|
|
25
25
|
fleet/types.py,sha256=eXeI8BFmiU5hln0PVvJbUZs7BSjl6wSqAtN9zaJT6yY,652
|
|
26
26
|
fleet/_async/__init__.py,sha256=AJWCnuo7XKja4yBb8fK2wX7ntciLXQrpzdRHwjTRP6M,62
|
|
27
27
|
fleet/_async/base.py,sha256=s0rYOtXsMJeitOvpa-Oh8ciLV226p_TIPp3fplzWvV4,9209
|
|
28
|
-
fleet/_async/client.py,sha256=
|
|
28
|
+
fleet/_async/client.py,sha256=O2LB8FyJ4rGzAuE8nAeycYZAACQGPPENMKS7YDhTOmA,20388
|
|
29
29
|
fleet/_async/exceptions.py,sha256=fUmPwWhnT8SR97lYsRq0kLHQHKtSh2eJS0VQ2caSzEI,5055
|
|
30
|
-
fleet/_async/models.py,sha256=
|
|
31
|
-
fleet/_async/tasks.py,sha256=
|
|
30
|
+
fleet/_async/models.py,sha256=6SqoWdTIygF3jriZW53a-jQ1JZoPatuWZ4JQ1qIY5MY,12453
|
|
31
|
+
fleet/_async/tasks.py,sha256=CJDt3dhg_88bHUYVqp2WcVenh6Y41wvp_iMaRiJeFDs,4069
|
|
32
32
|
fleet/_async/env/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
33
33
|
fleet/_async/env/client.py,sha256=MR2Ju7AudGWmztTVUPPut-ys3yqD7D00D_jsxZaRybA,858
|
|
34
34
|
fleet/_async/instance/__init__.py,sha256=PtmJq8J8bh0SOQ2V55QURz5GJfobozwtQoqhaOk3_tI,515
|
|
@@ -38,32 +38,32 @@ fleet/_async/resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3
|
|
|
38
38
|
fleet/_async/resources/base.py,sha256=ZFx2I9L7px-Z8LvbPUZJtbtajpqUn1yS1NFbZZutP9Q,668
|
|
39
39
|
fleet/_async/resources/browser.py,sha256=oldoSiymJ1lJkADhpUG81ViOBDNyppX1jSoEwe9-W94,1369
|
|
40
40
|
fleet/_async/resources/sqlite.py,sha256=DvDLRI5dJ7_v4WkHw-Zday1M_FQUdzAqUCy9FtfOY8w,26636
|
|
41
|
-
fleet/_async/verifiers/__init__.py,sha256=
|
|
41
|
+
fleet/_async/verifiers/__init__.py,sha256=7dtLOFgirYQmEIiDMcx02-ZqR7yqrfjOiqBBuXBz914,466
|
|
42
42
|
fleet/_async/verifiers/bundler.py,sha256=A4yR3wBOcVZYFAv87CD58QlJn6L4QXeilrasnVm8n74,26185
|
|
43
|
-
fleet/_async/verifiers/verifier.py,sha256
|
|
43
|
+
fleet/_async/verifiers/verifier.py,sha256=vS2Nv-tLgDa_1xhMHdkw0297uNMzzSlRPsXe4zIJsqw,14528
|
|
44
44
|
fleet/env/__init__.py,sha256=Wt6mO2dQnOdzdjOCvtiIP49wpSrmS7OaZ_OeHNrzqsY,595
|
|
45
45
|
fleet/env/client.py,sha256=2FzA7qfR-B7tVdCNa4uQk10g8zXV-9MNjAqx2D5vQ7c,710
|
|
46
46
|
fleet/instance/__init__.py,sha256=-Anms7Vw_FO8VBIvwnAnq4rQjwl_XNfAS-i7bypHMow,478
|
|
47
47
|
fleet/instance/base.py,sha256=OYqzBwZFfTX9wlBGSG5gljqj98NbiJeKIfFJ3uj5I4s,1587
|
|
48
|
-
fleet/instance/client.py,sha256=
|
|
48
|
+
fleet/instance/client.py,sha256=0gNXWRKNSXUqVTf6E5VUFO2gEZrBTF87QnvpLr5xaD8,5895
|
|
49
49
|
fleet/instance/models.py,sha256=ZTiue0YOuhuwX8jYfJAoCzGfqjLqqXRLqK1LVFhq6rQ,4183
|
|
50
50
|
fleet/resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
51
51
|
fleet/resources/base.py,sha256=203gD54NP1IvjuSqFo-f7FvrkhtjChggtzrxJK7xf2E,667
|
|
52
52
|
fleet/resources/browser.py,sha256=hRNM0YMsVQUAraZGNi_B-KXxLpuddy4ntoEDFSw7czU,1295
|
|
53
53
|
fleet/resources/mcp.py,sha256=lDMyY4pMC3Khwp2Wycc6Ds6KeLAUOgHkDnmztuAnXm8,1872
|
|
54
54
|
fleet/resources/sqlite.py,sha256=FeAykW7T0Pol7ckDLCtcjJDHDTI7y9ilWp7fnw4tuM8,26266
|
|
55
|
-
fleet/verifiers/__init__.py,sha256=
|
|
55
|
+
fleet/verifiers/__init__.py,sha256=qfjAuF_QKu7rPD8yNtBcGMaAUiyCoECDma3MHgSTUuQ,459
|
|
56
56
|
fleet/verifiers/bundler.py,sha256=A4yR3wBOcVZYFAv87CD58QlJn6L4QXeilrasnVm8n74,26185
|
|
57
57
|
fleet/verifiers/code.py,sha256=EOi6ES8Zdzlm9iybRFaJmz9t2W4Ulo2wrCdbEBqxzbc,47
|
|
58
58
|
fleet/verifiers/db.py,sha256=tssmvJjDHuBIy8qlL_P5-UdmEFUw2DZcqLsWZ8ot3Xw,27766
|
|
59
59
|
fleet/verifiers/decorator.py,sha256=Q-KHhicnIYFwX7FX_VZguzNfu8ZslqNUeWxcS2CwNVY,3386
|
|
60
|
-
fleet/verifiers/parse.py,sha256=
|
|
60
|
+
fleet/verifiers/parse.py,sha256=W4cBnto-XfPNrRwsYs7PFR3QJoI5UfWyB-QWsm1jUEg,7844
|
|
61
61
|
fleet/verifiers/sql_differ.py,sha256=dmiGCFXVMEMbAX519OjhVqgA8ZvhnvdmC1BVpL7QCF0,6490
|
|
62
|
-
fleet/verifiers/verifier.py,sha256=
|
|
63
|
-
fleet_python-0.2.
|
|
64
|
-
scripts/fix_sync_imports.py,sha256=
|
|
62
|
+
fleet/verifiers/verifier.py,sha256=7yiqko332lX0pgjbyAQ6KbaGm_V8TX864ceH-4fvvW0,14470
|
|
63
|
+
fleet_python-0.2.27.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
64
|
+
scripts/fix_sync_imports.py,sha256=m2OHz1lCAdDVIRhPvn8grNXpRaPo1b99dRMYI7TiEes,7117
|
|
65
65
|
scripts/unasync.py,sha256=--Fmaae47o-dZ1HYgX1c3Nvi-rMjcFymTRlJcWWnmpw,725
|
|
66
|
-
fleet_python-0.2.
|
|
67
|
-
fleet_python-0.2.
|
|
68
|
-
fleet_python-0.2.
|
|
69
|
-
fleet_python-0.2.
|
|
66
|
+
fleet_python-0.2.27.dist-info/METADATA,sha256=LS8LDwtM3SwLodzsypJJRxoaag9-9pP5cMvYD5yUkas,3347
|
|
67
|
+
fleet_python-0.2.27.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
68
|
+
fleet_python-0.2.27.dist-info/top_level.txt,sha256=_3DSmTohvSDf3AIP_BYfGzhwO1ECFwuzg83X-wHCx3Y,23
|
|
69
|
+
fleet_python-0.2.27.dist-info/RECORD,,
|