fleet-python 0.2.72b2__py3-none-any.whl → 0.2.73__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- examples/fetch_tasks.py +230 -0
- examples/iterate_verifiers.py +519 -0
- fleet/__init__.py +1 -1
- fleet/_async/__init__.py +1 -1
- fleet/_async/base.py +7 -3
- fleet/_async/client.py +16 -1
- fleet/_async/verifiers/verifier.py +5 -0
- fleet/base.py +7 -3
- fleet/client.py +16 -1
- fleet/tasks.py +4 -1
- fleet/verifiers/verifier.py +5 -0
- {fleet_python-0.2.72b2.dist-info → fleet_python-0.2.73.dist-info}/METADATA +1 -1
- {fleet_python-0.2.72b2.dist-info → fleet_python-0.2.73.dist-info}/RECORD +16 -14
- {fleet_python-0.2.72b2.dist-info → fleet_python-0.2.73.dist-info}/WHEEL +0 -0
- {fleet_python-0.2.72b2.dist-info → fleet_python-0.2.73.dist-info}/licenses/LICENSE +0 -0
- {fleet_python-0.2.72b2.dist-info → fleet_python-0.2.73.dist-info}/top_level.txt +0 -0
examples/fetch_tasks.py
ADDED
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import asyncio
|
|
3
|
+
import json
|
|
4
|
+
import sys
|
|
5
|
+
from typing import List, Dict, Any, Optional, Tuple
|
|
6
|
+
import fleet
|
|
7
|
+
from dotenv import load_dotenv
|
|
8
|
+
|
|
9
|
+
load_dotenv()
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
async def fetch_task(
|
|
13
|
+
task_key: str, semaphore: asyncio.Semaphore
|
|
14
|
+
) -> Tuple[str, Optional[Dict[str, Any]], Optional[str]]:
|
|
15
|
+
"""
|
|
16
|
+
Fetch a single task from the Fleet API.
|
|
17
|
+
|
|
18
|
+
Args:
|
|
19
|
+
task_key: Task key to fetch
|
|
20
|
+
semaphore: Semaphore to limit concurrent requests
|
|
21
|
+
|
|
22
|
+
Returns:
|
|
23
|
+
Tuple of (task_key, task_data_dict, error_message)
|
|
24
|
+
"""
|
|
25
|
+
async with semaphore:
|
|
26
|
+
try:
|
|
27
|
+
# Use load_tasks with keys parameter to get Task objects
|
|
28
|
+
tasks = await fleet.load_tasks_async(keys=[task_key])
|
|
29
|
+
if tasks:
|
|
30
|
+
task = tasks[0]
|
|
31
|
+
# Convert to dict using model_dump() like export_tasks.py does
|
|
32
|
+
return task_key, task.model_dump(), None
|
|
33
|
+
else:
|
|
34
|
+
return task_key, None, "Task not found"
|
|
35
|
+
except Exception as e:
|
|
36
|
+
error_msg = f"{type(e).__name__}: {str(e)}"
|
|
37
|
+
return task_key, None, error_msg
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
async def fetch_tasks_batch(
|
|
41
|
+
task_keys: List[str], max_concurrent: int = 20
|
|
42
|
+
) -> Dict[str, Dict[str, Any]]:
|
|
43
|
+
"""
|
|
44
|
+
Fetch multiple tasks concurrently from the Fleet API.
|
|
45
|
+
|
|
46
|
+
Args:
|
|
47
|
+
task_keys: List of task keys to fetch
|
|
48
|
+
max_concurrent: Maximum number of concurrent requests
|
|
49
|
+
|
|
50
|
+
Returns:
|
|
51
|
+
Dictionary mapping task_key to task data
|
|
52
|
+
"""
|
|
53
|
+
print(f"\nFetching {len(task_keys)} task(s) from Fleet API...")
|
|
54
|
+
print(f"Max concurrent requests: {max_concurrent}")
|
|
55
|
+
|
|
56
|
+
semaphore = asyncio.Semaphore(max_concurrent)
|
|
57
|
+
|
|
58
|
+
# Fetch all tasks concurrently
|
|
59
|
+
results = await asyncio.gather(
|
|
60
|
+
*[fetch_task(key, semaphore) for key in task_keys],
|
|
61
|
+
return_exceptions=True,
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
# Process results
|
|
65
|
+
fetched_tasks = {}
|
|
66
|
+
errors = []
|
|
67
|
+
|
|
68
|
+
for result in results:
|
|
69
|
+
if isinstance(result, Exception):
|
|
70
|
+
errors.append(f"Unexpected error: {result}")
|
|
71
|
+
continue
|
|
72
|
+
|
|
73
|
+
task_key, task_data, error = result
|
|
74
|
+
|
|
75
|
+
if error:
|
|
76
|
+
errors.append(f"{task_key}: {error}")
|
|
77
|
+
print(f" ✗ {task_key}: {error}")
|
|
78
|
+
elif task_data:
|
|
79
|
+
# Task data is already a dict from model_dump()
|
|
80
|
+
fetched_tasks[task_key] = task_data
|
|
81
|
+
print(f" ✓ {task_key}")
|
|
82
|
+
|
|
83
|
+
print(f"\n✓ Successfully fetched {len(fetched_tasks)} task(s)")
|
|
84
|
+
|
|
85
|
+
if errors:
|
|
86
|
+
print(f"\n⚠ {len(errors)} error(s) occurred:")
|
|
87
|
+
for error in errors[:10]: # Show first 10
|
|
88
|
+
print(f" - {error}")
|
|
89
|
+
if len(errors) > 10:
|
|
90
|
+
print(f" ... and {len(errors) - 10} more")
|
|
91
|
+
|
|
92
|
+
return fetched_tasks
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
async def main():
|
|
96
|
+
parser = argparse.ArgumentParser(
|
|
97
|
+
description="Fetch tasks from Fleet API and update JSON file",
|
|
98
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
99
|
+
epilog="""
|
|
100
|
+
Examples:
|
|
101
|
+
# Fetch tasks and update in-place
|
|
102
|
+
%(prog)s tasks.json
|
|
103
|
+
|
|
104
|
+
# Fetch tasks and save to a new file
|
|
105
|
+
%(prog)s tasks.json --output updated_tasks.json
|
|
106
|
+
|
|
107
|
+
# Limit concurrent requests
|
|
108
|
+
%(prog)s tasks.json --max-concurrent 10
|
|
109
|
+
""",
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
parser.add_argument("json_file", help="Path to JSON file containing tasks")
|
|
113
|
+
parser.add_argument(
|
|
114
|
+
"--output",
|
|
115
|
+
"-o",
|
|
116
|
+
help="Path to output JSON file (defaults to overwriting input)",
|
|
117
|
+
default=None,
|
|
118
|
+
)
|
|
119
|
+
parser.add_argument(
|
|
120
|
+
"--max-concurrent",
|
|
121
|
+
"-c",
|
|
122
|
+
type=int,
|
|
123
|
+
default=20,
|
|
124
|
+
help="Maximum number of concurrent API requests (default: 20)",
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
args = parser.parse_args()
|
|
128
|
+
|
|
129
|
+
# Load JSON file
|
|
130
|
+
print(f"Reading tasks from: {args.json_file}")
|
|
131
|
+
try:
|
|
132
|
+
with open(args.json_file, "r", encoding="utf-8") as f:
|
|
133
|
+
tasks_data = json.load(f)
|
|
134
|
+
except FileNotFoundError:
|
|
135
|
+
print(f"✗ Error: File '{args.json_file}' not found")
|
|
136
|
+
sys.exit(1)
|
|
137
|
+
except json.JSONDecodeError as e:
|
|
138
|
+
print(f"✗ Error: Invalid JSON in '{args.json_file}': {e}")
|
|
139
|
+
sys.exit(1)
|
|
140
|
+
|
|
141
|
+
if not isinstance(tasks_data, list):
|
|
142
|
+
print("✗ Error: JSON file must contain an array of tasks")
|
|
143
|
+
sys.exit(1)
|
|
144
|
+
|
|
145
|
+
print(f"Found {len(tasks_data)} task(s) in file")
|
|
146
|
+
|
|
147
|
+
# Extract task keys
|
|
148
|
+
task_keys = []
|
|
149
|
+
missing_keys = []
|
|
150
|
+
|
|
151
|
+
for i, task in enumerate(tasks_data):
|
|
152
|
+
task_key = task.get("key") or task.get("id")
|
|
153
|
+
if task_key:
|
|
154
|
+
task_keys.append(task_key)
|
|
155
|
+
else:
|
|
156
|
+
missing_keys.append(f"Task at index {i}")
|
|
157
|
+
|
|
158
|
+
if missing_keys:
|
|
159
|
+
print(f"\n⚠ Warning: {len(missing_keys)} task(s) missing key/id:")
|
|
160
|
+
for key in missing_keys[:10]:
|
|
161
|
+
print(f" - {key}")
|
|
162
|
+
if len(missing_keys) > 10:
|
|
163
|
+
print(f" ... and {len(missing_keys) - 10} more")
|
|
164
|
+
|
|
165
|
+
if not task_keys:
|
|
166
|
+
print("\n✗ Error: No valid task keys found in JSON file")
|
|
167
|
+
sys.exit(1)
|
|
168
|
+
|
|
169
|
+
print(f"\nExtracted {len(task_keys)} task key(s)")
|
|
170
|
+
|
|
171
|
+
# Get account info
|
|
172
|
+
account = await fleet.env.account_async()
|
|
173
|
+
print(f"Fetching from team: {account.team_name}")
|
|
174
|
+
|
|
175
|
+
# Fetch tasks from API
|
|
176
|
+
fetched_tasks = await fetch_tasks_batch(task_keys, args.max_concurrent)
|
|
177
|
+
|
|
178
|
+
if not fetched_tasks:
|
|
179
|
+
print("\n✗ Error: No tasks were successfully fetched")
|
|
180
|
+
sys.exit(1)
|
|
181
|
+
|
|
182
|
+
# Update tasks in the original data
|
|
183
|
+
updated_count = 0
|
|
184
|
+
not_found = []
|
|
185
|
+
|
|
186
|
+
print("\nUpdating task data...")
|
|
187
|
+
for i, task in enumerate(tasks_data):
|
|
188
|
+
task_key = task.get("key") or task.get("id")
|
|
189
|
+
|
|
190
|
+
if task_key in fetched_tasks:
|
|
191
|
+
# Replace entire task with fetched data
|
|
192
|
+
tasks_data[i] = fetched_tasks[task_key]
|
|
193
|
+
updated_count += 1
|
|
194
|
+
else:
|
|
195
|
+
not_found.append(task_key or f"index {i}")
|
|
196
|
+
|
|
197
|
+
print(f"✓ Updated {updated_count} task(s)")
|
|
198
|
+
|
|
199
|
+
if not_found:
|
|
200
|
+
print(f"\n⚠ Warning: {len(not_found)} task(s) not fetched:")
|
|
201
|
+
for key in not_found[:10]:
|
|
202
|
+
print(f" - {key}")
|
|
203
|
+
if len(not_found) > 10:
|
|
204
|
+
print(f" ... and {len(not_found) - 10} more")
|
|
205
|
+
|
|
206
|
+
# Write output
|
|
207
|
+
output_file = args.output or args.json_file
|
|
208
|
+
print(f"\nWriting updated tasks to: {output_file}")
|
|
209
|
+
|
|
210
|
+
try:
|
|
211
|
+
with open(output_file, "w", encoding="utf-8") as f:
|
|
212
|
+
json.dump(tasks_data, f, indent=2, ensure_ascii=False)
|
|
213
|
+
print(f"✓ Successfully wrote {len(tasks_data)} task(s) to '{output_file}'")
|
|
214
|
+
except Exception as e:
|
|
215
|
+
print(f"✗ Error writing output file: {e}")
|
|
216
|
+
sys.exit(1)
|
|
217
|
+
|
|
218
|
+
# Summary
|
|
219
|
+
print("\n" + "=" * 60)
|
|
220
|
+
print("Summary:")
|
|
221
|
+
print(f" Total tasks in file: {len(tasks_data)}")
|
|
222
|
+
print(f" Successfully fetched: {len(fetched_tasks)}")
|
|
223
|
+
print(f" Updated in file: {updated_count}")
|
|
224
|
+
print(f" Failed to fetch: {len(task_keys) - len(fetched_tasks)}")
|
|
225
|
+
print("=" * 60)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
if __name__ == "__main__":
|
|
229
|
+
asyncio.run(main())
|
|
230
|
+
|
|
@@ -0,0 +1,519 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import json
|
|
3
|
+
import re
|
|
4
|
+
import sys
|
|
5
|
+
from typing import Dict, Tuple, Optional
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def extract_function_info(function_code: str) -> Optional[Tuple[str, bool]]:
|
|
9
|
+
"""
|
|
10
|
+
Extract function name and async status from Python function code.
|
|
11
|
+
Handles both regular functions (def) and async functions (async def).
|
|
12
|
+
|
|
13
|
+
Args:
|
|
14
|
+
function_code: Python function code as a string
|
|
15
|
+
|
|
16
|
+
Returns:
|
|
17
|
+
A tuple of (function_name, is_async) if found, None otherwise
|
|
18
|
+
"""
|
|
19
|
+
# Normalize escaped newlines and strip common Markdown code fences
|
|
20
|
+
code = function_code.replace("\\n", "\n").strip()
|
|
21
|
+
if "```" in code:
|
|
22
|
+
# Extract the first fenced block if present
|
|
23
|
+
fence_blocks = re.findall(r"```[a-zA-Z0-9_+-]*\n([\s\S]*?)\n```", code)
|
|
24
|
+
if fence_blocks:
|
|
25
|
+
code = fence_blocks[0].strip()
|
|
26
|
+
|
|
27
|
+
# Remove leading decorators (keep them for regex but allow preceding lines)
|
|
28
|
+
# Robust regex: allow optional decorators and whitespace before the def
|
|
29
|
+
pattern = r"^\s*(?:@[\w\.\n+() ,]*\n\s*)*(async\s+)?def\s+([A-Za-z_]\w*)\s*\("
|
|
30
|
+
|
|
31
|
+
match = re.search(pattern, code, flags=re.MULTILINE)
|
|
32
|
+
if match:
|
|
33
|
+
is_async = match.group(1) is not None
|
|
34
|
+
function_name = match.group(2)
|
|
35
|
+
return (function_name, is_async)
|
|
36
|
+
|
|
37
|
+
# Fallback: search anywhere (not anchored) for a def signature
|
|
38
|
+
fallback = r"(async\s+)?def\s+([A-Za-z_]\w*)\s*\("
|
|
39
|
+
match = re.search(fallback, code)
|
|
40
|
+
if match:
|
|
41
|
+
is_async = match.group(1) is not None
|
|
42
|
+
function_name = match.group(2)
|
|
43
|
+
return (function_name, is_async)
|
|
44
|
+
|
|
45
|
+
return None
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def clean_verifier_code(code: str) -> str:
|
|
49
|
+
"""
|
|
50
|
+
Clean verifier code by removing markdown code fences and normalizing whitespace.
|
|
51
|
+
|
|
52
|
+
Args:
|
|
53
|
+
code: Raw verifier code string
|
|
54
|
+
|
|
55
|
+
Returns:
|
|
56
|
+
Cleaned code string
|
|
57
|
+
"""
|
|
58
|
+
# Normalize escaped newlines
|
|
59
|
+
code = code.replace("\\n", "\n").strip()
|
|
60
|
+
|
|
61
|
+
# Remove markdown code fences if present
|
|
62
|
+
if "```" in code:
|
|
63
|
+
fence_blocks = re.findall(r"```[a-zA-Z0-9_+-]*\n([\s\S]*?)\n```", code)
|
|
64
|
+
if fence_blocks:
|
|
65
|
+
code = fence_blocks[0].strip()
|
|
66
|
+
|
|
67
|
+
return code
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def extract_verifiers_to_file(json_path: str, py_path: str) -> None:
|
|
71
|
+
"""
|
|
72
|
+
Extract verifiers from JSON file and write them to a Python file with decorators.
|
|
73
|
+
|
|
74
|
+
Args:
|
|
75
|
+
json_path: Path to input JSON file
|
|
76
|
+
py_path: Path to output Python file
|
|
77
|
+
"""
|
|
78
|
+
print(f"Reading tasks from: {json_path}")
|
|
79
|
+
|
|
80
|
+
# Load JSON file
|
|
81
|
+
try:
|
|
82
|
+
with open(json_path, "r", encoding="utf-8") as f:
|
|
83
|
+
tasks = json.load(f)
|
|
84
|
+
except FileNotFoundError:
|
|
85
|
+
print(f"✗ Error: File '{json_path}' not found")
|
|
86
|
+
sys.exit(1)
|
|
87
|
+
except json.JSONDecodeError as e:
|
|
88
|
+
print(f"✗ Error: Invalid JSON in '{json_path}': {e}")
|
|
89
|
+
sys.exit(1)
|
|
90
|
+
|
|
91
|
+
if not isinstance(tasks, list):
|
|
92
|
+
print("✗ Error: JSON file must contain an array of tasks")
|
|
93
|
+
sys.exit(1)
|
|
94
|
+
|
|
95
|
+
print(f"Found {len(tasks)} task(s)")
|
|
96
|
+
|
|
97
|
+
# Extract verifiers
|
|
98
|
+
verifiers = []
|
|
99
|
+
missing_verifier = []
|
|
100
|
+
duplicate_keys = set()
|
|
101
|
+
seen_keys = set()
|
|
102
|
+
|
|
103
|
+
for i, task in enumerate(tasks):
|
|
104
|
+
task_key = task.get("key") or task.get("id")
|
|
105
|
+
if not task_key:
|
|
106
|
+
print(f"⚠ Warning: Task at index {i} has no key or id, skipping")
|
|
107
|
+
continue
|
|
108
|
+
|
|
109
|
+
# Check for duplicate keys
|
|
110
|
+
if task_key in seen_keys:
|
|
111
|
+
duplicate_keys.add(task_key)
|
|
112
|
+
print(f"⚠ Warning: Duplicate task key '{task_key}' found")
|
|
113
|
+
seen_keys.add(task_key)
|
|
114
|
+
|
|
115
|
+
# Get verifier code from multiple possible locations
|
|
116
|
+
verifier_code = (
|
|
117
|
+
task.get("verifier_func")
|
|
118
|
+
or task.get("verifier_code")
|
|
119
|
+
or task.get("metadata", {}).get("verifier_code")
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
if not verifier_code:
|
|
123
|
+
missing_verifier.append(task_key)
|
|
124
|
+
continue
|
|
125
|
+
|
|
126
|
+
# Clean the code
|
|
127
|
+
cleaned_code = clean_verifier_code(verifier_code)
|
|
128
|
+
|
|
129
|
+
# Extract function info
|
|
130
|
+
func_info = extract_function_info(cleaned_code)
|
|
131
|
+
if not func_info:
|
|
132
|
+
print(
|
|
133
|
+
f"⚠ Warning: Could not extract function name from verifier for task '{task_key}'"
|
|
134
|
+
)
|
|
135
|
+
continue
|
|
136
|
+
|
|
137
|
+
function_name, is_async = func_info
|
|
138
|
+
|
|
139
|
+
verifiers.append(
|
|
140
|
+
{
|
|
141
|
+
"task_key": task_key,
|
|
142
|
+
"function_name": function_name,
|
|
143
|
+
"is_async": is_async,
|
|
144
|
+
"code": cleaned_code,
|
|
145
|
+
}
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
if missing_verifier:
|
|
149
|
+
print(f"\n⚠ Warning: {len(missing_verifier)} task(s) missing verifier code:")
|
|
150
|
+
for key in missing_verifier[:10]: # Show first 10
|
|
151
|
+
print(f" - {key}")
|
|
152
|
+
if len(missing_verifier) > 10:
|
|
153
|
+
print(f" ... and {len(missing_verifier) - 10} more")
|
|
154
|
+
|
|
155
|
+
if duplicate_keys:
|
|
156
|
+
print(f"\n⚠ Warning: {len(duplicate_keys)} duplicate task key(s) found:")
|
|
157
|
+
for key in list(duplicate_keys)[:10]:
|
|
158
|
+
print(f" - {key}")
|
|
159
|
+
|
|
160
|
+
print(f"\n✓ Extracted {len(verifiers)} verifier(s)")
|
|
161
|
+
|
|
162
|
+
# Count async vs sync
|
|
163
|
+
async_count = sum(1 for v in verifiers if v["is_async"])
|
|
164
|
+
sync_count = len(verifiers) - async_count
|
|
165
|
+
print(f" - {async_count} async verifier(s)")
|
|
166
|
+
print(f" - {sync_count} sync verifier(s)")
|
|
167
|
+
|
|
168
|
+
# Write to Python file
|
|
169
|
+
print(f"\nWriting verifiers to: {py_path}")
|
|
170
|
+
|
|
171
|
+
with open(py_path, "w", encoding="utf-8") as f:
|
|
172
|
+
# Write header
|
|
173
|
+
f.write('"""Auto-generated verifiers file.\n\n')
|
|
174
|
+
f.write(f"Extracted from: {json_path}\n")
|
|
175
|
+
f.write(f"Total verifiers: {len(verifiers)}\n")
|
|
176
|
+
f.write(f" - Async: {async_count}\n")
|
|
177
|
+
f.write(f" - Sync: {sync_count}\n")
|
|
178
|
+
f.write('"""\n\n')
|
|
179
|
+
|
|
180
|
+
# Write imports
|
|
181
|
+
f.write("# Import verifier decorators and dependencies\n")
|
|
182
|
+
f.write("from fleet import (\n")
|
|
183
|
+
f.write(" verifier,\n")
|
|
184
|
+
f.write(" AsyncEnv,\n")
|
|
185
|
+
f.write(" SyncEnv,\n")
|
|
186
|
+
f.write(" SyncEnv as Environment,\n")
|
|
187
|
+
f.write(" AsyncEnv as AsyncEnvironment,\n")
|
|
188
|
+
f.write(" IgnoreConfig,\n")
|
|
189
|
+
f.write(" TASK_FAILED_SCORE,\n")
|
|
190
|
+
f.write(" TASK_SUCCESSFUL_SCORE,\n")
|
|
191
|
+
f.write(")\n")
|
|
192
|
+
f.write("from fleet.verifiers.verifier import verifier as verifier_sync\n")
|
|
193
|
+
f.write("\n")
|
|
194
|
+
f.write("# Standard library imports used in verifiers\n")
|
|
195
|
+
f.write("import json\n")
|
|
196
|
+
f.write("import re\n")
|
|
197
|
+
f.write("import string\n")
|
|
198
|
+
f.write("from typing import Any\n")
|
|
199
|
+
f.write("\n")
|
|
200
|
+
f.write("# Helper functions available in verifier namespace\n")
|
|
201
|
+
f.write(
|
|
202
|
+
'_TRANSLATOR = str.maketrans(string.punctuation, " " * len(string.punctuation))\n'
|
|
203
|
+
)
|
|
204
|
+
f.write("\n")
|
|
205
|
+
f.write("def _normalize_text(value: str) -> str:\n")
|
|
206
|
+
f.write(" text = value.lower().translate(_TRANSLATOR)\n")
|
|
207
|
+
f.write(' return "".join(text.split())\n')
|
|
208
|
+
f.write("\n")
|
|
209
|
+
f.write("def _stringify_content(content: Any) -> str:\n")
|
|
210
|
+
f.write(" if isinstance(content, (dict, list)):\n")
|
|
211
|
+
f.write(" return json.dumps(content, sort_keys=True)\n")
|
|
212
|
+
f.write(" return str(content)\n")
|
|
213
|
+
f.write("\n")
|
|
214
|
+
f.write("def normalized_contains(target: str, blob: Any) -> bool:\n")
|
|
215
|
+
f.write(" normalized_target = _normalize_text(target)\n")
|
|
216
|
+
f.write(" normalized_blob = _normalize_text(_stringify_content(blob))\n")
|
|
217
|
+
f.write(" return normalized_target in normalized_blob\n")
|
|
218
|
+
f.write("\n")
|
|
219
|
+
f.write("def extract_numbers(text: str) -> list:\n")
|
|
220
|
+
f.write(" cleaned_text = text.replace(',', '')\n")
|
|
221
|
+
f.write(" pattern = r'-?\\d+\\.?\\d*'\n")
|
|
222
|
+
f.write(" matches = re.findall(pattern, cleaned_text)\n")
|
|
223
|
+
f.write(" return [float(num) for num in matches]\n")
|
|
224
|
+
f.write("\n")
|
|
225
|
+
f.write("def contains_number(text: str, target_number) -> bool:\n")
|
|
226
|
+
f.write(" numbers = extract_numbers(text)\n")
|
|
227
|
+
f.write(" try:\n")
|
|
228
|
+
f.write(" if isinstance(target_number, str):\n")
|
|
229
|
+
f.write(" target_number = target_number.replace(',', '')\n")
|
|
230
|
+
f.write(" target = float(target_number)\n")
|
|
231
|
+
f.write(" except (ValueError, AttributeError):\n")
|
|
232
|
+
f.write(" return False\n")
|
|
233
|
+
f.write(" return target in numbers\n")
|
|
234
|
+
f.write("\n")
|
|
235
|
+
f.write("# " + "=" * 78 + "\n")
|
|
236
|
+
f.write("# VERIFIERS\n")
|
|
237
|
+
f.write("# " + "=" * 78 + "\n\n")
|
|
238
|
+
|
|
239
|
+
# Write each verifier
|
|
240
|
+
for i, ver in enumerate(verifiers):
|
|
241
|
+
# Write separator comment
|
|
242
|
+
if i > 0:
|
|
243
|
+
f.write("\n" + "# " + "-" * 78 + "\n\n")
|
|
244
|
+
|
|
245
|
+
# Write task key comment
|
|
246
|
+
f.write(f"# Task: {ver['task_key']}\n")
|
|
247
|
+
f.write(
|
|
248
|
+
f"# Function: {ver['function_name']} ({'async' if ver['is_async'] else 'sync'})\n"
|
|
249
|
+
)
|
|
250
|
+
|
|
251
|
+
# Write decorator - use verifier for async, verifier_sync for sync
|
|
252
|
+
decorator_name = "verifier" if ver["is_async"] else "verifier_sync"
|
|
253
|
+
f.write(f'@{decorator_name}(key="{ver["task_key"]}")\n')
|
|
254
|
+
|
|
255
|
+
# Write function code
|
|
256
|
+
f.write(ver["code"])
|
|
257
|
+
f.write("\n")
|
|
258
|
+
|
|
259
|
+
print(f"✓ Successfully wrote {len(verifiers)} verifier(s) to '{py_path}'")
|
|
260
|
+
print("\nNext steps:")
|
|
261
|
+
print(f" 1. Edit the verifiers in '{py_path}'")
|
|
262
|
+
print(f" 2. Run: python {sys.argv[0]} apply {json_path} {py_path}")
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def parse_verifiers_from_file(python_path: str) -> Dict[str, str]:
|
|
266
|
+
"""
|
|
267
|
+
Parse verifiers from a Python file and extract them by task key.
|
|
268
|
+
|
|
269
|
+
Args:
|
|
270
|
+
python_path: Path to Python file containing verifiers
|
|
271
|
+
|
|
272
|
+
Returns:
|
|
273
|
+
Dictionary mapping task_key to verifier code
|
|
274
|
+
"""
|
|
275
|
+
print(f"Reading verifiers from: {python_path}")
|
|
276
|
+
|
|
277
|
+
try:
|
|
278
|
+
with open(python_path, "r", encoding="utf-8") as f:
|
|
279
|
+
content = f.read()
|
|
280
|
+
except FileNotFoundError:
|
|
281
|
+
print(f"✗ Error: File '{python_path}' not found")
|
|
282
|
+
sys.exit(1)
|
|
283
|
+
|
|
284
|
+
# Split content by the separator comments to get individual verifier sections
|
|
285
|
+
# The separator is "# ------------------------------------------------------------------------------"
|
|
286
|
+
# Each section starts with "# Task: <key>"
|
|
287
|
+
|
|
288
|
+
verifiers = {}
|
|
289
|
+
|
|
290
|
+
# Split by "# Task: " markers to find each verifier block
|
|
291
|
+
task_blocks = re.split(r"\n# Task: ", content)
|
|
292
|
+
|
|
293
|
+
for block in task_blocks[1:]: # Skip the first block (header)
|
|
294
|
+
# Extract task key from the first line
|
|
295
|
+
lines = block.split("\n")
|
|
296
|
+
if not lines:
|
|
297
|
+
continue
|
|
298
|
+
|
|
299
|
+
# First line should be the task key
|
|
300
|
+
task_key = lines[0].strip()
|
|
301
|
+
|
|
302
|
+
# Find the @verifier or @verifier_sync decorator to extract the key parameter
|
|
303
|
+
verifier_match = re.search(
|
|
304
|
+
r'@verifier(?:_sync)?\(key=["\']([^"\']+)["\']\s*(?:,\s*[^)]+)?\)', block
|
|
305
|
+
)
|
|
306
|
+
if verifier_match:
|
|
307
|
+
task_key = verifier_match.group(1)
|
|
308
|
+
|
|
309
|
+
# Find the function definition (async def or def)
|
|
310
|
+
# Extract from the function start until we hit the separator or end
|
|
311
|
+
func_pattern = r"((async\s+)?def\s+\w+.*?)(?=\n# -+\n|\n# Task:|\Z)"
|
|
312
|
+
func_match = re.search(func_pattern, block, re.DOTALL)
|
|
313
|
+
|
|
314
|
+
if func_match:
|
|
315
|
+
function_code = func_match.group(1).strip()
|
|
316
|
+
verifiers[task_key] = function_code
|
|
317
|
+
|
|
318
|
+
# If the above approach didn't work, try a direct pattern match
|
|
319
|
+
if not verifiers:
|
|
320
|
+
# Pattern to match @verifier or @verifier_sync decorator with key and the following function
|
|
321
|
+
# Look for the decorator, then capture everything until we hit a dedented line or separator
|
|
322
|
+
pattern = r'@verifier(?:_sync)?\(key=["\']([^"\']+)["\']\s*(?:,\s*[^)]+)?\)\s*\n((?:async\s+)?def\s+[^\n]+:(?:\n(?: |\t).*)*(?:\n(?: |\t).*)*)'
|
|
323
|
+
|
|
324
|
+
matches = re.findall(pattern, content, re.MULTILINE)
|
|
325
|
+
|
|
326
|
+
for task_key, function_code in matches:
|
|
327
|
+
verifiers[task_key] = function_code.strip()
|
|
328
|
+
|
|
329
|
+
print(f"✓ Found {len(verifiers)} verifier(s)")
|
|
330
|
+
|
|
331
|
+
# Analyze async vs sync
|
|
332
|
+
async_count = 0
|
|
333
|
+
sync_count = 0
|
|
334
|
+
for code in verifiers.values():
|
|
335
|
+
func_info = extract_function_info(code)
|
|
336
|
+
if func_info:
|
|
337
|
+
_, is_async = func_info
|
|
338
|
+
if is_async:
|
|
339
|
+
async_count += 1
|
|
340
|
+
else:
|
|
341
|
+
sync_count += 1
|
|
342
|
+
|
|
343
|
+
print(f" - {async_count} async verifier(s)")
|
|
344
|
+
print(f" - {sync_count} sync verifier(s)")
|
|
345
|
+
|
|
346
|
+
return verifiers
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def apply_verifiers_to_json(json_path: str, python_path: str) -> None:
|
|
350
|
+
"""
|
|
351
|
+
Apply verifiers from Python file back into JSON task file (updates in-place).
|
|
352
|
+
|
|
353
|
+
Args:
|
|
354
|
+
json_path: Path to JSON file to update
|
|
355
|
+
python_path: Path to Python file with verifiers
|
|
356
|
+
"""
|
|
357
|
+
# Parse verifiers from Python file
|
|
358
|
+
verifiers = parse_verifiers_from_file(python_path)
|
|
359
|
+
|
|
360
|
+
# Load JSON file
|
|
361
|
+
print(f"\nReading tasks from: {json_path}")
|
|
362
|
+
try:
|
|
363
|
+
with open(json_path, "r", encoding="utf-8") as f:
|
|
364
|
+
tasks = json.load(f)
|
|
365
|
+
except FileNotFoundError:
|
|
366
|
+
print(f"✗ Error: File '{json_path}' not found")
|
|
367
|
+
sys.exit(1)
|
|
368
|
+
except json.JSONDecodeError as e:
|
|
369
|
+
print(f"✗ Error: Invalid JSON in '{json_path}': {e}")
|
|
370
|
+
sys.exit(1)
|
|
371
|
+
|
|
372
|
+
if not isinstance(tasks, list):
|
|
373
|
+
print("✗ Error: JSON file must contain an array of tasks")
|
|
374
|
+
sys.exit(1)
|
|
375
|
+
|
|
376
|
+
print(f"Found {len(tasks)} task(s)")
|
|
377
|
+
|
|
378
|
+
# Update tasks with new verifiers (only if changed)
|
|
379
|
+
updated_count = 0
|
|
380
|
+
updated_keys = []
|
|
381
|
+
not_found = []
|
|
382
|
+
|
|
383
|
+
for task in tasks:
|
|
384
|
+
task_key = task.get("key") or task.get("id")
|
|
385
|
+
if not task_key:
|
|
386
|
+
continue
|
|
387
|
+
|
|
388
|
+
if task_key in verifiers:
|
|
389
|
+
new_code = verifiers[task_key]
|
|
390
|
+
old_code = task.get("verifier_func", "").strip()
|
|
391
|
+
|
|
392
|
+
# Only update if the code actually changed
|
|
393
|
+
if old_code != new_code.strip():
|
|
394
|
+
# Update verifier_func with new code
|
|
395
|
+
task["verifier_func"] = new_code
|
|
396
|
+
|
|
397
|
+
# Also update metadata if it exists
|
|
398
|
+
if "metadata" in task and isinstance(task["metadata"], dict):
|
|
399
|
+
task["metadata"]["verifier_code"] = new_code
|
|
400
|
+
|
|
401
|
+
# Clear verifier_id and verifier_sha to force re-upload
|
|
402
|
+
task["verifier_id"] = None
|
|
403
|
+
task["verifier_sha"] = None
|
|
404
|
+
|
|
405
|
+
updated_count += 1
|
|
406
|
+
updated_keys.append(task_key)
|
|
407
|
+
else:
|
|
408
|
+
not_found.append(task_key)
|
|
409
|
+
|
|
410
|
+
print(f"\n✓ Updated {updated_count} task(s) with new verifiers")
|
|
411
|
+
|
|
412
|
+
if updated_keys:
|
|
413
|
+
print("\nUpdated task keys:")
|
|
414
|
+
for key in updated_keys:
|
|
415
|
+
print(f" - {key}")
|
|
416
|
+
|
|
417
|
+
if not_found:
|
|
418
|
+
print(f"\n⚠ Warning: {len(not_found)} task(s) not found in Python file:")
|
|
419
|
+
for key in not_found[:10]:
|
|
420
|
+
print(f" - {key}")
|
|
421
|
+
if len(not_found) > 10:
|
|
422
|
+
print(f" ... and {len(not_found) - 10} more")
|
|
423
|
+
|
|
424
|
+
# Write output back to the same JSON file
|
|
425
|
+
print(f"\nWriting updated tasks to: {json_path}")
|
|
426
|
+
|
|
427
|
+
try:
|
|
428
|
+
with open(json_path, "w", encoding="utf-8") as f:
|
|
429
|
+
json.dump(tasks, f, indent=2, ensure_ascii=False)
|
|
430
|
+
print(f"✓ Successfully updated {len(tasks)} task(s) in '{json_path}'")
|
|
431
|
+
except Exception as e:
|
|
432
|
+
print(f"✗ Error writing JSON file: {e}")
|
|
433
|
+
sys.exit(1)
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def validate_verifiers_file(python_path: str) -> None:
|
|
437
|
+
"""
|
|
438
|
+
Validate that a Python verifiers file can be parsed correctly.
|
|
439
|
+
|
|
440
|
+
Args:
|
|
441
|
+
python_path: Path to Python file with verifiers
|
|
442
|
+
"""
|
|
443
|
+
verifiers = parse_verifiers_from_file(python_path)
|
|
444
|
+
|
|
445
|
+
print("\nValidating verifiers...")
|
|
446
|
+
errors = []
|
|
447
|
+
|
|
448
|
+
for task_key, code in verifiers.items():
|
|
449
|
+
func_info = extract_function_info(code)
|
|
450
|
+
if not func_info:
|
|
451
|
+
errors.append(f" - {task_key}: Could not extract function info")
|
|
452
|
+
else:
|
|
453
|
+
function_name, is_async = func_info
|
|
454
|
+
print(
|
|
455
|
+
f" ✓ {task_key}: {function_name} ({'async' if is_async else 'sync'})"
|
|
456
|
+
)
|
|
457
|
+
|
|
458
|
+
if errors:
|
|
459
|
+
print(f"\n✗ Found {len(errors)} error(s):")
|
|
460
|
+
for error in errors:
|
|
461
|
+
print(error)
|
|
462
|
+
sys.exit(1)
|
|
463
|
+
else:
|
|
464
|
+
print(f"\n✓ All {len(verifiers)} verifier(s) are valid!")
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def main():
|
|
468
|
+
parser = argparse.ArgumentParser(
|
|
469
|
+
description="Iterate on verifier code from JSON task files",
|
|
470
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
471
|
+
epilog="""
|
|
472
|
+
Examples:
|
|
473
|
+
# Extract verifiers from JSON to Python file
|
|
474
|
+
%(prog)s extract xai-day-10-batch.json verifiers.py
|
|
475
|
+
|
|
476
|
+
# Edit verifiers.py...
|
|
477
|
+
|
|
478
|
+
# Apply changes back to JSON file (updates in-place)
|
|
479
|
+
%(prog)s apply xai-day-10-batch.json verifiers.py
|
|
480
|
+
|
|
481
|
+
# Validate verifiers file
|
|
482
|
+
%(prog)s validate verifiers.py
|
|
483
|
+
""",
|
|
484
|
+
)
|
|
485
|
+
|
|
486
|
+
subparsers = parser.add_subparsers(dest="command", help="Command to run")
|
|
487
|
+
subparsers.required = True
|
|
488
|
+
|
|
489
|
+
# Extract command
|
|
490
|
+
extract_parser = subparsers.add_parser(
|
|
491
|
+
"extract", help="Extract verifiers from JSON to Python file"
|
|
492
|
+
)
|
|
493
|
+
extract_parser.add_argument("json_file", help="Path to JSON file containing tasks")
|
|
494
|
+
extract_parser.add_argument("py_file", help="Path to output Python file")
|
|
495
|
+
|
|
496
|
+
# Apply command
|
|
497
|
+
apply_parser = subparsers.add_parser(
|
|
498
|
+
"apply", help="Apply verifiers from Python file back to JSON (updates in-place)"
|
|
499
|
+
)
|
|
500
|
+
apply_parser.add_argument("json_file", help="Path to JSON file to update")
|
|
501
|
+
apply_parser.add_argument("py_file", help="Path to Python file with verifiers")
|
|
502
|
+
|
|
503
|
+
# Validate command
|
|
504
|
+
validate_parser = subparsers.add_parser("validate", help="Validate verifiers file")
|
|
505
|
+
validate_parser.add_argument("py_file", help="Path to Python file with verifiers")
|
|
506
|
+
|
|
507
|
+
args = parser.parse_args()
|
|
508
|
+
|
|
509
|
+
# Execute command
|
|
510
|
+
if args.command == "extract":
|
|
511
|
+
extract_verifiers_to_file(args.json_file, args.py_file)
|
|
512
|
+
elif args.command == "apply":
|
|
513
|
+
apply_verifiers_to_json(args.json_file, args.py_file)
|
|
514
|
+
elif args.command == "validate":
|
|
515
|
+
validate_verifiers_file(args.py_file)
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
if __name__ == "__main__":
|
|
519
|
+
main()
|
fleet/__init__.py
CHANGED
fleet/_async/__init__.py
CHANGED
fleet/_async/base.py
CHANGED
|
@@ -2,6 +2,7 @@ import httpx
|
|
|
2
2
|
from typing import Dict, Any, Optional
|
|
3
3
|
import json
|
|
4
4
|
import logging
|
|
5
|
+
import time
|
|
5
6
|
import uuid
|
|
6
7
|
|
|
7
8
|
from ..models import InstanceResponse
|
|
@@ -25,7 +26,7 @@ from .exceptions import (
|
|
|
25
26
|
try:
|
|
26
27
|
from .. import __version__
|
|
27
28
|
except ImportError:
|
|
28
|
-
__version__ = "0.2.
|
|
29
|
+
__version__ = "0.2.72"
|
|
29
30
|
|
|
30
31
|
logger = logging.getLogger(__name__)
|
|
31
32
|
|
|
@@ -51,11 +52,14 @@ class BaseWrapper:
|
|
|
51
52
|
"X-Fleet-SDK-Version": __version__,
|
|
52
53
|
}
|
|
53
54
|
headers["Authorization"] = f"Bearer {self.api_key}"
|
|
54
|
-
|
|
55
|
+
|
|
55
56
|
# Add request ID for idempotency (persists across retries)
|
|
56
57
|
if request_id:
|
|
57
58
|
headers["X-Request-ID"] = request_id
|
|
58
|
-
|
|
59
|
+
|
|
60
|
+
# Add timestamp for all requests
|
|
61
|
+
headers["X-Request-Timestamp"] = str(int(time.time() * 1000))
|
|
62
|
+
|
|
59
63
|
return headers
|
|
60
64
|
|
|
61
65
|
|
fleet/_async/client.py
CHANGED
|
@@ -172,6 +172,7 @@ class AsyncEnv(EnvironmentBase):
|
|
|
172
172
|
kwargs: dict,
|
|
173
173
|
timeout: Optional[int] = 30,
|
|
174
174
|
needs_upload: bool = True,
|
|
175
|
+
verifier_runtime_version: Optional[str] = None,
|
|
175
176
|
) -> VerifiersExecuteResponse:
|
|
176
177
|
return await _execute_verifier_remote(
|
|
177
178
|
self._load_client,
|
|
@@ -184,6 +185,7 @@ class AsyncEnv(EnvironmentBase):
|
|
|
184
185
|
kwargs,
|
|
185
186
|
timeout,
|
|
186
187
|
needs_upload,
|
|
188
|
+
verifier_runtime_version,
|
|
187
189
|
)
|
|
188
190
|
|
|
189
191
|
def __getstate__(self):
|
|
@@ -611,6 +613,11 @@ class AsyncFleet:
|
|
|
611
613
|
if not verifier_id:
|
|
612
614
|
verifier_id = task_json.get("key", task_json.get("id"))
|
|
613
615
|
|
|
616
|
+
# Extract verifier_runtime_version from metadata if present
|
|
617
|
+
verifier_runtime_version = None
|
|
618
|
+
if "metadata" in task_json and isinstance(task_json["metadata"], dict):
|
|
619
|
+
verifier_runtime_version = task_json["metadata"].get("verifier_runtime_version")
|
|
620
|
+
|
|
614
621
|
try:
|
|
615
622
|
if verifier_id and verifier_code:
|
|
616
623
|
verifier = await self._create_verifier_from_data(
|
|
@@ -618,6 +625,7 @@ class AsyncFleet:
|
|
|
618
625
|
verifier_key=task_json.get("key", task_json.get("id")),
|
|
619
626
|
verifier_code=verifier_code,
|
|
620
627
|
verifier_sha=verifier_sha,
|
|
628
|
+
verifier_runtime_version=verifier_runtime_version,
|
|
621
629
|
)
|
|
622
630
|
except Exception as e:
|
|
623
631
|
error_msg = f"Failed to create verifier {task_json.get('key', task_json.get('id'))}: {e}"
|
|
@@ -641,6 +649,7 @@ class AsyncFleet:
|
|
|
641
649
|
verifier=verifier, # Use created verifier or None
|
|
642
650
|
verifier_id=verifier_id, # Set verifier_id so _rebuild_verifier works
|
|
643
651
|
verifier_sha=verifier_sha, # Set verifier_sha
|
|
652
|
+
verifier_runtime_version=verifier_runtime_version, # Set verifier_runtime_version
|
|
644
653
|
metadata=task_json.get("metadata", {}), # Default empty metadata
|
|
645
654
|
output_json_schema=task_json.get("output_json_schema"), # JSON schema for output
|
|
646
655
|
)
|
|
@@ -999,7 +1008,7 @@ class AsyncFleet:
|
|
|
999
1008
|
return TaskResponse(**response.json())
|
|
1000
1009
|
|
|
1001
1010
|
async def _create_verifier_from_data(
|
|
1002
|
-
self, verifier_id: str, verifier_key: str, verifier_code: str, verifier_sha: str
|
|
1011
|
+
self, verifier_id: str, verifier_key: str, verifier_code: str, verifier_sha: str, verifier_runtime_version: Optional[str] = None
|
|
1003
1012
|
) -> "AsyncVerifierFunction":
|
|
1004
1013
|
"""Create an AsyncVerifierFunction from verifier data.
|
|
1005
1014
|
|
|
@@ -1020,6 +1029,7 @@ class AsyncFleet:
|
|
|
1020
1029
|
verifier_id=verifier_id,
|
|
1021
1030
|
verifier_key=verifier_key,
|
|
1022
1031
|
sha256=verifier_sha,
|
|
1032
|
+
verifier_runtime_version=verifier_runtime_version or "",
|
|
1023
1033
|
)
|
|
1024
1034
|
|
|
1025
1035
|
# Store the original verifier code for reference
|
|
@@ -1104,6 +1114,7 @@ async def _execute_verifier_remote(
|
|
|
1104
1114
|
kwargs: dict,
|
|
1105
1115
|
timeout: Optional[int] = 30,
|
|
1106
1116
|
needs_upload: bool = True,
|
|
1117
|
+
verifier_runtime_version: Optional[str] = None,
|
|
1107
1118
|
) -> VerifiersExecuteResponse:
|
|
1108
1119
|
# Pickle args and kwargs together
|
|
1109
1120
|
# The first arg should be None as a placeholder for env
|
|
@@ -1127,6 +1138,10 @@ async def _execute_verifier_remote(
|
|
|
1127
1138
|
bundle_b64 = base64.b64encode(bundle_data).decode("utf-8")
|
|
1128
1139
|
request_data["bundle"] = bundle_b64
|
|
1129
1140
|
|
|
1141
|
+
# Add verifier_runtime_version if present
|
|
1142
|
+
if verifier_runtime_version:
|
|
1143
|
+
request_data["verifier_runtime_version"] = verifier_runtime_version
|
|
1144
|
+
|
|
1130
1145
|
# Debug logging
|
|
1131
1146
|
# logger.debug(
|
|
1132
1147
|
# f"Sending verifier execute request: key={key}, sha256={bundle_sha[:8]}..., function_name={function_name}"
|
|
@@ -42,6 +42,7 @@ class AsyncVerifierFunction:
|
|
|
42
42
|
verifier_id: Optional[str] = None,
|
|
43
43
|
sha256: Optional[str] = None,
|
|
44
44
|
raw_code: Optional[str] = None,
|
|
45
|
+
verifier_runtime_version: Optional[str] = None,
|
|
45
46
|
):
|
|
46
47
|
self.func = func
|
|
47
48
|
self.key = key
|
|
@@ -52,6 +53,7 @@ class AsyncVerifierFunction:
|
|
|
52
53
|
self._bundle_data: Optional[bytes] = None # Cached bundle data
|
|
53
54
|
self._raw_code: Optional[str] = raw_code # Store raw code if provided
|
|
54
55
|
self._is_async = asyncio.iscoroutinefunction(func)
|
|
56
|
+
self.verifier_runtime_version = verifier_runtime_version
|
|
55
57
|
|
|
56
58
|
# Copy function metadata
|
|
57
59
|
functools.update_wrapper(self, func)
|
|
@@ -251,6 +253,7 @@ Remote traceback:
|
|
|
251
253
|
args_array=args_array,
|
|
252
254
|
kwargs=kwargs,
|
|
253
255
|
needs_upload=True,
|
|
256
|
+
verifier_runtime_version=self.verifier_runtime_version,
|
|
254
257
|
)
|
|
255
258
|
|
|
256
259
|
# logger.debug(f"Bundle {bundle_sha[:8]}... uploaded successfully")
|
|
@@ -267,6 +270,7 @@ Remote traceback:
|
|
|
267
270
|
args_array=args_array,
|
|
268
271
|
kwargs=kwargs,
|
|
269
272
|
needs_upload=False,
|
|
273
|
+
verifier_runtime_version=self.verifier_runtime_version,
|
|
270
274
|
)
|
|
271
275
|
|
|
272
276
|
return response
|
|
@@ -287,6 +291,7 @@ Remote traceback:
|
|
|
287
291
|
args_array=args_array,
|
|
288
292
|
kwargs=kwargs,
|
|
289
293
|
needs_upload=True,
|
|
294
|
+
verifier_runtime_version=self.verifier_runtime_version,
|
|
290
295
|
)
|
|
291
296
|
return response
|
|
292
297
|
else:
|
fleet/base.py
CHANGED
|
@@ -2,6 +2,7 @@ import httpx
|
|
|
2
2
|
from typing import Dict, Any, Optional
|
|
3
3
|
import json
|
|
4
4
|
import logging
|
|
5
|
+
import time
|
|
5
6
|
import uuid
|
|
6
7
|
|
|
7
8
|
from .models import InstanceResponse
|
|
@@ -25,7 +26,7 @@ from .exceptions import (
|
|
|
25
26
|
try:
|
|
26
27
|
from . import __version__
|
|
27
28
|
except ImportError:
|
|
28
|
-
__version__ = "0.2.
|
|
29
|
+
__version__ = "0.2.72"
|
|
29
30
|
|
|
30
31
|
logger = logging.getLogger(__name__)
|
|
31
32
|
|
|
@@ -51,11 +52,14 @@ class BaseWrapper:
|
|
|
51
52
|
"X-Fleet-SDK-Version": __version__,
|
|
52
53
|
}
|
|
53
54
|
headers["Authorization"] = f"Bearer {self.api_key}"
|
|
54
|
-
|
|
55
|
+
|
|
55
56
|
# Add request ID for idempotency (persists across retries)
|
|
56
57
|
if request_id:
|
|
57
58
|
headers["X-Request-ID"] = request_id
|
|
58
|
-
|
|
59
|
+
|
|
60
|
+
# Add timestamp for all requests
|
|
61
|
+
headers["X-Request-Timestamp"] = str(int(time.time() * 1000))
|
|
62
|
+
|
|
59
63
|
return headers
|
|
60
64
|
|
|
61
65
|
|
fleet/client.py
CHANGED
|
@@ -181,6 +181,7 @@ class SyncEnv(EnvironmentBase):
|
|
|
181
181
|
kwargs: dict,
|
|
182
182
|
timeout: Optional[int] = 30,
|
|
183
183
|
needs_upload: bool = True,
|
|
184
|
+
verifier_runtime_version: Optional[str] = None,
|
|
184
185
|
) -> VerifiersExecuteResponse:
|
|
185
186
|
return _execute_verifier_remote(
|
|
186
187
|
self._load_client,
|
|
@@ -193,6 +194,7 @@ class SyncEnv(EnvironmentBase):
|
|
|
193
194
|
kwargs,
|
|
194
195
|
timeout,
|
|
195
196
|
needs_upload,
|
|
197
|
+
verifier_runtime_version,
|
|
196
198
|
)
|
|
197
199
|
|
|
198
200
|
def __getstate__(self):
|
|
@@ -620,6 +622,11 @@ class Fleet:
|
|
|
620
622
|
if not verifier_id:
|
|
621
623
|
verifier_id = task_json.get("key", task_json.get("id"))
|
|
622
624
|
|
|
625
|
+
# Extract verifier_runtime_version from metadata if present
|
|
626
|
+
verifier_runtime_version = None
|
|
627
|
+
if "metadata" in task_json and isinstance(task_json["metadata"], dict):
|
|
628
|
+
verifier_runtime_version = task_json["metadata"].get("verifier_runtime_version")
|
|
629
|
+
|
|
623
630
|
try:
|
|
624
631
|
if verifier_id and verifier_code:
|
|
625
632
|
verifier = self._create_verifier_from_data(
|
|
@@ -627,6 +634,7 @@ class Fleet:
|
|
|
627
634
|
verifier_key=task_json.get("key", task_json.get("id")),
|
|
628
635
|
verifier_code=verifier_code,
|
|
629
636
|
verifier_sha=verifier_sha,
|
|
637
|
+
verifier_runtime_version=verifier_runtime_version,
|
|
630
638
|
)
|
|
631
639
|
except Exception as e:
|
|
632
640
|
error_msg = f"Failed to create verifier {task_json.get('key', task_json.get('id'))}: {e}"
|
|
@@ -650,6 +658,7 @@ class Fleet:
|
|
|
650
658
|
verifier=verifier, # Use created verifier or None
|
|
651
659
|
verifier_id=verifier_id, # Set verifier_id so _rebuild_verifier works
|
|
652
660
|
verifier_sha=verifier_sha, # Set verifier_sha
|
|
661
|
+
verifier_runtime_version=verifier_runtime_version, # Set verifier_runtime_version
|
|
653
662
|
metadata=task_json.get("metadata", {}), # Default empty metadata
|
|
654
663
|
output_json_schema=task_json.get("output_json_schema"), # JSON schema for output
|
|
655
664
|
)
|
|
@@ -1007,7 +1016,7 @@ class Fleet:
|
|
|
1007
1016
|
return TaskResponse(**response.json())
|
|
1008
1017
|
|
|
1009
1018
|
def _create_verifier_from_data(
|
|
1010
|
-
self, verifier_id: str, verifier_key: str, verifier_code: str, verifier_sha: str
|
|
1019
|
+
self, verifier_id: str, verifier_key: str, verifier_code: str, verifier_sha: str, verifier_runtime_version: Optional[str] = None
|
|
1011
1020
|
) -> "SyncVerifierFunction":
|
|
1012
1021
|
"""Create an AsyncVerifierFunction from verifier data.
|
|
1013
1022
|
|
|
@@ -1029,6 +1038,7 @@ class Fleet:
|
|
|
1029
1038
|
verifier_id=verifier_id,
|
|
1030
1039
|
verifier_key=verifier_key,
|
|
1031
1040
|
sha256=verifier_sha,
|
|
1041
|
+
verifier_runtime_version=verifier_runtime_version or "",
|
|
1032
1042
|
)
|
|
1033
1043
|
|
|
1034
1044
|
# Store the original verifier code for reference
|
|
@@ -1113,6 +1123,7 @@ def _execute_verifier_remote(
|
|
|
1113
1123
|
kwargs: dict,
|
|
1114
1124
|
timeout: Optional[int] = 30,
|
|
1115
1125
|
needs_upload: bool = True,
|
|
1126
|
+
verifier_runtime_version: Optional[str] = None,
|
|
1116
1127
|
) -> VerifiersExecuteResponse:
|
|
1117
1128
|
# Pickle args and kwargs together
|
|
1118
1129
|
# The first arg should be None as a placeholder for env
|
|
@@ -1136,6 +1147,10 @@ def _execute_verifier_remote(
|
|
|
1136
1147
|
bundle_b64 = base64.b64encode(bundle_data).decode("utf-8")
|
|
1137
1148
|
request_data["bundle"] = bundle_b64
|
|
1138
1149
|
|
|
1150
|
+
# Add verifier_runtime_version if present
|
|
1151
|
+
if verifier_runtime_version:
|
|
1152
|
+
request_data["verifier_runtime_version"] = verifier_runtime_version
|
|
1153
|
+
|
|
1139
1154
|
# Debug logging
|
|
1140
1155
|
# logger.debug(
|
|
1141
1156
|
# f"Sending verifier execute request: key={key}, sha256={bundle_sha[:8]}..., function_name={function_name}"
|
fleet/tasks.py
CHANGED
|
@@ -36,6 +36,7 @@ class Task(BaseModel):
|
|
|
36
36
|
)
|
|
37
37
|
verifier_id: Optional[str] = Field(None, description="Verifier identifier")
|
|
38
38
|
verifier_sha: Optional[str] = Field(None, description="Verifier SHA256 hash")
|
|
39
|
+
verifier_runtime_version: Optional[str] = Field(None, description="Verifier runtime version")
|
|
39
40
|
metadata: Optional[Dict[str, Any]] = Field(
|
|
40
41
|
default_factory=dict, description="Additional task metadata"
|
|
41
42
|
)
|
|
@@ -199,6 +200,7 @@ class Task(BaseModel):
|
|
|
199
200
|
verifier_id=verifier_id,
|
|
200
201
|
verifier_key=self.key,
|
|
201
202
|
sha256=self.verifier_sha or "",
|
|
203
|
+
verifier_runtime_version=self.verifier_runtime_version or "",
|
|
202
204
|
)
|
|
203
205
|
self.verifier = verifier
|
|
204
206
|
|
|
@@ -273,7 +275,7 @@ class Task(BaseModel):
|
|
|
273
275
|
|
|
274
276
|
|
|
275
277
|
def verifier_from_string(
|
|
276
|
-
verifier_func: str, verifier_id: str, verifier_key: str, sha256: str = ""
|
|
278
|
+
verifier_func: str, verifier_id: str, verifier_key: str, sha256: str = "", verifier_runtime_version: str = ""
|
|
277
279
|
) -> "VerifierFunction":
|
|
278
280
|
"""Create a verifier function from string code.
|
|
279
281
|
|
|
@@ -380,6 +382,7 @@ def verifier_from_string(
|
|
|
380
382
|
verifier_id=verifier_id,
|
|
381
383
|
sha256=sha256,
|
|
382
384
|
raw_code=verifier_func,
|
|
385
|
+
verifier_runtime_version=verifier_runtime_version if verifier_runtime_version else None,
|
|
383
386
|
)
|
|
384
387
|
|
|
385
388
|
# Store additional metadata
|
fleet/verifiers/verifier.py
CHANGED
|
@@ -53,6 +53,7 @@ class SyncVerifierFunction:
|
|
|
53
53
|
verifier_id: Optional[str] = None,
|
|
54
54
|
sha256: Optional[str] = None,
|
|
55
55
|
raw_code: Optional[str] = None,
|
|
56
|
+
verifier_runtime_version: Optional[str] = None,
|
|
56
57
|
):
|
|
57
58
|
self.func = func
|
|
58
59
|
self.key = key
|
|
@@ -63,6 +64,7 @@ class SyncVerifierFunction:
|
|
|
63
64
|
self._bundle_data: Optional[bytes] = None # Cached bundle data
|
|
64
65
|
self._raw_code: Optional[str] = raw_code # Store raw code if provided
|
|
65
66
|
self._is_async = inspect.iscoroutinefunction(func)
|
|
67
|
+
self.verifier_runtime_version = verifier_runtime_version
|
|
66
68
|
|
|
67
69
|
# Copy function metadata
|
|
68
70
|
functools.update_wrapper(self, func)
|
|
@@ -262,6 +264,7 @@ Remote traceback:
|
|
|
262
264
|
args_array=args_array,
|
|
263
265
|
kwargs=kwargs,
|
|
264
266
|
needs_upload=True,
|
|
267
|
+
verifier_runtime_version=self.verifier_runtime_version,
|
|
265
268
|
)
|
|
266
269
|
|
|
267
270
|
# logger.debug(f"Bundle {bundle_sha[:8]}... uploaded successfully")
|
|
@@ -279,6 +282,7 @@ Remote traceback:
|
|
|
279
282
|
args_array=args_array,
|
|
280
283
|
kwargs=kwargs,
|
|
281
284
|
needs_upload=False,
|
|
285
|
+
verifier_runtime_version=self.verifier_runtime_version,
|
|
282
286
|
)
|
|
283
287
|
return response
|
|
284
288
|
|
|
@@ -298,6 +302,7 @@ Remote traceback:
|
|
|
298
302
|
args_array=args_array,
|
|
299
303
|
kwargs=kwargs,
|
|
300
304
|
needs_upload=True,
|
|
305
|
+
verifier_runtime_version=self.verifier_runtime_version,
|
|
301
306
|
)
|
|
302
307
|
return response
|
|
303
308
|
else:
|
|
@@ -12,8 +12,10 @@ examples/example_task.py,sha256=dhG6STAkNsTdHs9cO1RFH9WfuvRmq5bRC211hTeFrk8,7088
|
|
|
12
12
|
examples/example_tasks.py,sha256=xTL8UWVAuolSX6swskfrAcmDrLIzn45dJ7YPWCwoEBU,514
|
|
13
13
|
examples/example_verifier.py,sha256=0vwNITIG3m4CkSPwIxNXcGx9TqrxEsCGqK2A8keKZMM,2392
|
|
14
14
|
examples/export_tasks.py,sha256=Sk2Id4i5tZeVs4ZFQoAn-ABFZv7ewyv4pqtZ-Z-tIEI,3574
|
|
15
|
+
examples/fetch_tasks.py,sha256=3A4F0OSUTwUddTOoDNlI-CSxeOnpl3E3zHBh0Qsw9A8,6865
|
|
15
16
|
examples/gemini_example.py,sha256=qj9WDazQTYNiRHNeUg9Tjkp33lJMwbx8gDfpFe1sDQo,16180
|
|
16
17
|
examples/import_tasks.py,sha256=McF5MbHenPZ7HTjQfNoHHevot0EBILMSsWtroicVT30,11619
|
|
18
|
+
examples/iterate_verifiers.py,sha256=vYinKK3JJwJdk8j5BkES6rN6Thjc04xfeft1WKDQnvw,18174
|
|
17
19
|
examples/json_tasks_example.py,sha256=CYPESGGtOo0fmsDdLidujTfsE4QlJHw7rOhyVqPJ_Ls,5329
|
|
18
20
|
examples/nova_act_example.py,sha256=rH23Lp74Okf0rn8ynMdWjK2aviEf5NLPH4k_53Pyxho,831
|
|
19
21
|
examples/openai_example.py,sha256=dEWERrTEP5xBiGkLkQjBQGd2NqoxX6gcW6XteBPsWFQ,8231
|
|
@@ -21,18 +23,18 @@ examples/openai_simple_example.py,sha256=HmiufucrAZne7tHq9uoEsDWlEhjNC265bQAyIGB
|
|
|
21
23
|
examples/query_builder_example.py,sha256=-cOMfWGNifYfYEt_Ds73XpwATZvFDL6F4KTkVxdMjzg,3951
|
|
22
24
|
examples/quickstart.py,sha256=1VT39IRRhemsJgxi0O0gprdpcw7HB4pYO97GAYagIcg,3788
|
|
23
25
|
examples/test_cdp_logging.py,sha256=AkCwQCgOTQEI8w3v0knWK_4eXMph7L9x07wj9yIYM10,2836
|
|
24
|
-
fleet/__init__.py,sha256=
|
|
25
|
-
fleet/base.py,sha256=
|
|
26
|
-
fleet/client.py,sha256=
|
|
26
|
+
fleet/__init__.py,sha256=g3iCjVyltxf8RE1h-7UKeGGpLuQW-mjdAG9zQ65qvS0,4218
|
|
27
|
+
fleet/base.py,sha256=rn-Jpu37cw_ogMRaW6FWwDh_L6-807CJnBSkiJ_Ggxw,9609
|
|
28
|
+
fleet/client.py,sha256=9NjDhcBA1iEqKvdahhgGN1p4OX-S8opWPyb0j7H662A,44205
|
|
27
29
|
fleet/config.py,sha256=n_wh9Sahu3gGE7nHJ7kqNFUH1qDiBtF4bgZq9MvIBMU,319
|
|
28
30
|
fleet/exceptions.py,sha256=fUmPwWhnT8SR97lYsRq0kLHQHKtSh2eJS0VQ2caSzEI,5055
|
|
29
31
|
fleet/global_client.py,sha256=frrDAFNM2ywN0JHLtlm9qbE1dQpnQJsavJpb7xSR_bU,1072
|
|
30
32
|
fleet/models.py,sha256=7ZRUVBT0_6tiJkotWLTj52oVaeDfP4OsjjkWJxhf8CA,15409
|
|
31
|
-
fleet/tasks.py,sha256=
|
|
33
|
+
fleet/tasks.py,sha256=7PqxpG_CRYJLiEFewpRPdiFU30Trc9B6UAAqKtF3iy4,20719
|
|
32
34
|
fleet/types.py,sha256=L4Y82xICf1tzyCLqhLYUgEoaIIS5h9T05TyFNHSWs3s,652
|
|
33
|
-
fleet/_async/__init__.py,sha256=
|
|
34
|
-
fleet/_async/base.py,sha256=
|
|
35
|
-
fleet/_async/client.py,sha256=
|
|
35
|
+
fleet/_async/__init__.py,sha256=80CAiLFSyLyjdWNIdUNxdvTl0kqWH83505DuDCzFJzY,9101
|
|
36
|
+
fleet/_async/base.py,sha256=tTn7AD4xH-9ZuwuGPmeH6dgQVP8r7abB90LykLTVGn0,9630
|
|
37
|
+
fleet/_async/client.py,sha256=eBB6c3NI3XY0S2Q-yj_QdN2zXdZsGXJgp9ROVpG6EY0,44266
|
|
36
38
|
fleet/_async/exceptions.py,sha256=fUmPwWhnT8SR97lYsRq0kLHQHKtSh2eJS0VQ2caSzEI,5055
|
|
37
39
|
fleet/_async/global_client.py,sha256=4WskpLHbsDEgWW7hXMD09W-brkp4euy8w2ZJ88594rQ,1103
|
|
38
40
|
fleet/_async/models.py,sha256=-3xv2QyoHsvYcWmdKLf9Z93md8XB17DBeJCxdRCB3bo,13571
|
|
@@ -49,7 +51,7 @@ fleet/_async/resources/mcp.py,sha256=TLEsLiFhfVfZFs0Fu_uDPm-h4FPdvqgQblYqs-PTHhc
|
|
|
49
51
|
fleet/_async/resources/sqlite.py,sha256=smzDpSyjSswKnFEZhO-tbonMrItHTKl-9E-CRu2t7sM,51881
|
|
50
52
|
fleet/_async/verifiers/__init__.py,sha256=1WTlCNq4tIFbbXaQu5Bf2WppZq0A8suhtZbxMTSOwxI,465
|
|
51
53
|
fleet/_async/verifiers/bundler.py,sha256=9aWWXFsovBPcndE06IATn5jaeli5fRORAYeenF9heN0,26264
|
|
52
|
-
fleet/_async/verifiers/verifier.py,sha256=
|
|
54
|
+
fleet/_async/verifiers/verifier.py,sha256=iSa-rO-E1R3IQTFS9Z7jbQvQVtsDkilITQP9IIQU2JA,14556
|
|
53
55
|
fleet/env/__init__.py,sha256=BVPZ4AYTznL6AYNrVmjr1yLF16qBcT1U56jWwF7AJ5o,964
|
|
54
56
|
fleet/env/client.py,sha256=N9oF2hGSNmfVacTnyQhIEocoN3BA5fMa-arbpVeNi-E,3221
|
|
55
57
|
fleet/instance/__init__.py,sha256=CyWUkbGAK-DBPw4DC4AnCW-MqqheGhZMA5QSRVu-ws4,479
|
|
@@ -68,8 +70,8 @@ fleet/verifiers/db.py,sha256=LAh1HambBInH_D9q9E2Z41YNkCOI9JJfpWPFqztjpfQ,27922
|
|
|
68
70
|
fleet/verifiers/decorator.py,sha256=RuTjjDijbicNfMSjA7HcTpKueEki5dzNOdTuHS7UoZs,3262
|
|
69
71
|
fleet/verifiers/parse.py,sha256=qz9AfJrTbjlg-LU-lE8Ciqi7Yt2a8-cs17FdpjTLhMk,8550
|
|
70
72
|
fleet/verifiers/sql_differ.py,sha256=TqTLWyK3uOyLbitT6HYzYEzuSFC39wcyhgk3rcm__k8,6525
|
|
71
|
-
fleet/verifiers/verifier.py,sha256=
|
|
72
|
-
fleet_python-0.2.
|
|
73
|
+
fleet/verifiers/verifier.py,sha256=iqGevW7dSd0J5RdRQjpu-zioy_FYAXnzMfkuB3-QmO0,14601
|
|
74
|
+
fleet_python-0.2.73.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
73
75
|
scripts/fix_sync_imports.py,sha256=X9fWLTpiPGkSHsjyQUDepOJkxOqw1DPj7nd8wFlFqLQ,8368
|
|
74
76
|
scripts/unasync.py,sha256=vWVQxRWX8SRZO5cmzEhpvnG_REhCWXpidIGIpWmEcvI,696
|
|
75
77
|
tests/__init__.py,sha256=Re1SdyxH8NfyL1kjhi7SQkGP1mYeWB-D6UALqdIMd8I,35
|
|
@@ -78,7 +80,7 @@ tests/test_instance_dispatch.py,sha256=CvU4C3LBIqsYZdEsEFfontGjyxAZfVYyXnGwxyIvX
|
|
|
78
80
|
tests/test_sqlite_resource_dual_mode.py,sha256=Mh8jBd-xsIGDYFsOACKKK_5DXMUYlFFS7W-jaY6AjG4,8734
|
|
79
81
|
tests/test_sqlite_shared_memory_behavior.py,sha256=fKx_1BmLS3b8x-9pMgjMycpnaHWY8P-2ZuXEspx6Sbw,4082
|
|
80
82
|
tests/test_verifier_from_string.py,sha256=Lxi3TpFHFb-hG4-UhLKZJkqo84ax9YJY8G6beO-1erM,13581
|
|
81
|
-
fleet_python-0.2.
|
|
82
|
-
fleet_python-0.2.
|
|
83
|
-
fleet_python-0.2.
|
|
84
|
-
fleet_python-0.2.
|
|
83
|
+
fleet_python-0.2.73.dist-info/METADATA,sha256=n9YCaiF3pd1HdNwRxOvBhsLb6PA4DrPy8g1U7Al85d4,3304
|
|
84
|
+
fleet_python-0.2.73.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
85
|
+
fleet_python-0.2.73.dist-info/top_level.txt,sha256=qb1zIbtEktyhRFZdqVytwg54l64qtoZL0wjHB4bUg3c,29
|
|
86
|
+
fleet_python-0.2.73.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|