fleet-python 0.2.48__tar.gz → 0.2.50__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of fleet-python might be problematic. Click here for more details.
- {fleet_python-0.2.48 → fleet_python-0.2.50}/PKG-INFO +1 -1
- {fleet_python-0.2.48 → fleet_python-0.2.50}/examples/example_mcp_anthropic.py +2 -0
- fleet_python-0.2.50/examples/example_tasks.py +24 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/_async/client.py +2 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/_async/env/client.py +6 -1
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/_async/resources/sqlite.py +39 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/_async/tasks.py +11 -2
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/client.py +74 -59
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/env/client.py +6 -1
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/instance/client.py +1 -3
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/models.py +4 -1
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/resources/sqlite.py +38 -9
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/tasks.py +59 -45
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/verifiers/verifier.py +20 -9
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet_python.egg-info/PKG-INFO +1 -1
- {fleet_python-0.2.48 → fleet_python-0.2.50}/pyproject.toml +1 -1
- fleet_python-0.2.48/examples/example_tasks.py +0 -31
- {fleet_python-0.2.48 → fleet_python-0.2.50}/LICENSE +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/README.md +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/examples/diff_example.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/examples/dsl_example.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/examples/example.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/examples/exampleResume.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/examples/example_account.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/examples/example_action_log.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/examples/example_client.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/examples/example_mcp_openai.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/examples/example_sync.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/examples/example_task.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/examples/example_verifier.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/examples/gemini_example.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/examples/json_tasks_example.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/examples/nova_act_example.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/examples/openai_example.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/examples/openai_simple_example.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/examples/query_builder_example.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/examples/quickstart.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/examples/test_cdp_logging.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/__init__.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/_async/__init__.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/_async/base.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/_async/env/__init__.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/_async/exceptions.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/_async/global_client.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/_async/instance/__init__.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/_async/instance/base.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/_async/instance/client.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/_async/models.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/_async/resources/__init__.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/_async/resources/base.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/_async/resources/browser.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/_async/resources/mcp.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/_async/verifiers/__init__.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/_async/verifiers/bundler.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/_async/verifiers/verifier.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/base.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/config.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/env/__init__.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/exceptions.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/global_client.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/instance/__init__.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/instance/base.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/instance/models.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/resources/__init__.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/resources/base.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/resources/browser.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/resources/mcp.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/types.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/verifiers/__init__.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/verifiers/bundler.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/verifiers/code.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/verifiers/db.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/verifiers/decorator.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/verifiers/parse.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet/verifiers/sql_differ.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet_python.egg-info/SOURCES.txt +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet_python.egg-info/dependency_links.txt +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet_python.egg-info/requires.txt +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/fleet_python.egg-info/top_level.txt +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/scripts/fix_sync_imports.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/scripts/unasync.py +0 -0
- {fleet_python-0.2.48 → fleet_python-0.2.50}/setup.cfg +0 -0
|
@@ -15,6 +15,8 @@ async def main():
|
|
|
15
15
|
print("Created environment:", env.urls.app)
|
|
16
16
|
print("MCP URL:", env.mcp.url)
|
|
17
17
|
|
|
18
|
+
asyncio.sleep(5)
|
|
19
|
+
|
|
18
20
|
async with streamablehttp_client(url=env.mcp.url) as streams:
|
|
19
21
|
async with ClientSession(
|
|
20
22
|
read_stream=streams[0], write_stream=streams[1]
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import fleet
|
|
2
|
+
from dotenv import load_dotenv
|
|
3
|
+
|
|
4
|
+
load_dotenv()
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def main():
|
|
8
|
+
account = fleet.env.account()
|
|
9
|
+
print(account)
|
|
10
|
+
|
|
11
|
+
tasks = fleet.load_tasks(team_id="5ca40f9f-9899-4bee-b194-6974138a4f12")
|
|
12
|
+
print(f"Loaded {len(tasks)} tasks")
|
|
13
|
+
|
|
14
|
+
# Save tasks to JSON file
|
|
15
|
+
import json
|
|
16
|
+
|
|
17
|
+
with open(f"{account.team_id}.json", "w") as f:
|
|
18
|
+
json.dump([task.model_dump() for task in tasks], f, indent=2)
|
|
19
|
+
|
|
20
|
+
print(f"Saved {len(tasks)} tasks to saved_tasks.json")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
if __name__ == "__main__":
|
|
24
|
+
main()
|
|
@@ -210,6 +210,7 @@ class AsyncFleet:
|
|
|
210
210
|
data_key: Optional[str] = None,
|
|
211
211
|
region: Optional[str] = None,
|
|
212
212
|
env_variables: Optional[Dict[str, Any]] = None,
|
|
213
|
+
image_type: Optional[str] = None,
|
|
213
214
|
) -> AsyncEnv:
|
|
214
215
|
if ":" in env_key:
|
|
215
216
|
env_key_part, env_version = env_key.split(":", 1)
|
|
@@ -242,6 +243,7 @@ class AsyncFleet:
|
|
|
242
243
|
data_version=data_version,
|
|
243
244
|
region=region,
|
|
244
245
|
env_variables=env_variables,
|
|
246
|
+
image_type=image_type,
|
|
245
247
|
created_from="sdk",
|
|
246
248
|
)
|
|
247
249
|
|
|
@@ -8,9 +8,14 @@ async def make_async(
|
|
|
8
8
|
data_key: Optional[str] = None,
|
|
9
9
|
region: Optional[str] = None,
|
|
10
10
|
env_variables: Optional[Dict[str, Any]] = None,
|
|
11
|
+
image_type: Optional[str] = None,
|
|
11
12
|
) -> AsyncEnv:
|
|
12
13
|
return await AsyncFleet().make(
|
|
13
|
-
env_key,
|
|
14
|
+
env_key,
|
|
15
|
+
data_key=data_key,
|
|
16
|
+
region=region,
|
|
17
|
+
env_variables=env_variables,
|
|
18
|
+
image_type=image_type,
|
|
14
19
|
)
|
|
15
20
|
|
|
16
21
|
|
|
@@ -111,6 +111,21 @@ class AsyncSnapshotQueryBuilder:
|
|
|
111
111
|
qb._conditions.append((column, "=", value))
|
|
112
112
|
return qb
|
|
113
113
|
|
|
114
|
+
def where(
|
|
115
|
+
self,
|
|
116
|
+
conditions: Optional[Dict[str, Any]] = None,
|
|
117
|
+
**kwargs: Any,
|
|
118
|
+
) -> "AsyncSnapshotQueryBuilder":
|
|
119
|
+
qb = self._clone()
|
|
120
|
+
merged: Dict[str, Any] = {}
|
|
121
|
+
if conditions:
|
|
122
|
+
merged.update(conditions)
|
|
123
|
+
if kwargs:
|
|
124
|
+
merged.update(kwargs)
|
|
125
|
+
for column, value in merged.items():
|
|
126
|
+
qb._conditions.append((column, "=", value))
|
|
127
|
+
return qb
|
|
128
|
+
|
|
114
129
|
def limit(self, n: int) -> "AsyncSnapshotQueryBuilder":
|
|
115
130
|
qb = self._clone()
|
|
116
131
|
qb._limit = n
|
|
@@ -304,6 +319,15 @@ class AsyncSnapshotDiff:
|
|
|
304
319
|
self._cached = diff
|
|
305
320
|
return diff
|
|
306
321
|
|
|
322
|
+
@property
|
|
323
|
+
def changes(self) -> Dict[str, Dict[str, Any]]:
|
|
324
|
+
"""Expose cached changes; ensure callers awaited a diff-producing method first."""
|
|
325
|
+
if self._cached is None:
|
|
326
|
+
raise RuntimeError(
|
|
327
|
+
"Diff not collected yet; await an operation like expect_only() first."
|
|
328
|
+
)
|
|
329
|
+
return self._cached
|
|
330
|
+
|
|
307
331
|
async def expect_only(self, allowed_changes: List[Dict[str, Any]]):
|
|
308
332
|
"""Ensure only specified changes occurred."""
|
|
309
333
|
diff = await self._collect()
|
|
@@ -471,6 +495,21 @@ class AsyncQueryBuilder:
|
|
|
471
495
|
def eq(self, column: str, value: Any) -> "AsyncQueryBuilder":
|
|
472
496
|
return self._add_condition(column, "=", value)
|
|
473
497
|
|
|
498
|
+
def where(
|
|
499
|
+
self,
|
|
500
|
+
conditions: Optional[Dict[str, Any]] = None,
|
|
501
|
+
**kwargs: Any,
|
|
502
|
+
) -> "AsyncQueryBuilder":
|
|
503
|
+
qb = self._clone()
|
|
504
|
+
merged: Dict[str, Any] = {}
|
|
505
|
+
if conditions:
|
|
506
|
+
merged.update(conditions)
|
|
507
|
+
if kwargs:
|
|
508
|
+
merged.update(kwargs)
|
|
509
|
+
for column, value in merged.items():
|
|
510
|
+
qb._conditions.append((column, "=", value))
|
|
511
|
+
return qb
|
|
512
|
+
|
|
474
513
|
def neq(self, column: str, value: Any) -> "AsyncQueryBuilder":
|
|
475
514
|
return self._add_condition(column, "!=", value)
|
|
476
515
|
|
|
@@ -5,7 +5,7 @@ from __future__ import annotations
|
|
|
5
5
|
from datetime import datetime
|
|
6
6
|
from typing import Any, Dict, Optional, List, TYPE_CHECKING
|
|
7
7
|
|
|
8
|
-
from pydantic import BaseModel, Field, validator
|
|
8
|
+
from pydantic import BaseModel, Field, validator, field_serializer
|
|
9
9
|
|
|
10
10
|
# Import the shared VerifierFunction type that works for both async and sync
|
|
11
11
|
from fleet.types import VerifierFunction
|
|
@@ -46,6 +46,11 @@ class Task(BaseModel):
|
|
|
46
46
|
"""Set created_at to current time if not provided."""
|
|
47
47
|
return v or datetime.now()
|
|
48
48
|
|
|
49
|
+
@field_serializer("created_at")
|
|
50
|
+
def serialize_created_at(self, dt: Optional[datetime], _info):
|
|
51
|
+
"""Serialize datetime to ISO format string."""
|
|
52
|
+
return dt.isoformat() if dt else None
|
|
53
|
+
|
|
49
54
|
@property
|
|
50
55
|
def env_key(self) -> str:
|
|
51
56
|
"""Get the environment key combining env_id and version."""
|
|
@@ -296,7 +301,11 @@ async def load_tasks(
|
|
|
296
301
|
|
|
297
302
|
client = get_client()
|
|
298
303
|
return await client.load_tasks(
|
|
299
|
-
env_key=env_key,
|
|
304
|
+
env_key=env_key,
|
|
305
|
+
keys=keys,
|
|
306
|
+
version=version,
|
|
307
|
+
team_id=team_id,
|
|
308
|
+
project_key=project_key,
|
|
300
309
|
)
|
|
301
310
|
|
|
302
311
|
|
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
|
|
17
17
|
import base64
|
|
18
18
|
import cloudpickle
|
|
19
|
+
import concurrent.futures
|
|
19
20
|
import httpx
|
|
20
21
|
import json
|
|
21
22
|
import logging
|
|
@@ -209,6 +210,7 @@ class Fleet:
|
|
|
209
210
|
data_key: Optional[str] = None,
|
|
210
211
|
region: Optional[str] = None,
|
|
211
212
|
env_variables: Optional[Dict[str, Any]] = None,
|
|
213
|
+
image_type: Optional[str] = None,
|
|
212
214
|
) -> SyncEnv:
|
|
213
215
|
if ":" in env_key:
|
|
214
216
|
env_key_part, env_version = env_key.split(":", 1)
|
|
@@ -241,6 +243,7 @@ class Fleet:
|
|
|
241
243
|
data_version=data_version,
|
|
242
244
|
region=region,
|
|
243
245
|
env_variables=env_variables,
|
|
246
|
+
image_type=image_type,
|
|
244
247
|
created_from="sdk",
|
|
245
248
|
)
|
|
246
249
|
|
|
@@ -290,9 +293,7 @@ class Fleet:
|
|
|
290
293
|
def execute_verifier_remote(
|
|
291
294
|
self, bundle_data: bytes, args: tuple, kwargs: dict, timeout: Optional[int] = 30
|
|
292
295
|
) -> VerifiersExecuteResponse:
|
|
293
|
-
return _execute_verifier_remote(
|
|
294
|
-
self.client, bundle_data, args, kwargs, timeout
|
|
295
|
-
)
|
|
296
|
+
return _execute_verifier_remote(self.client, bundle_data, args, kwargs, timeout)
|
|
296
297
|
|
|
297
298
|
def delete(self, instance_id: str) -> InstanceResponse:
|
|
298
299
|
return _delete_instance(self.client, instance_id)
|
|
@@ -404,10 +405,9 @@ class Fleet:
|
|
|
404
405
|
response = self.client.request("GET", "/v1/tasks", params=params)
|
|
405
406
|
task_list_response = TaskListResponse(**response.json())
|
|
406
407
|
|
|
407
|
-
# Prepare verifier loading
|
|
408
|
-
|
|
408
|
+
# Prepare verifier loading tasks
|
|
409
|
+
verifier_tasks = []
|
|
409
410
|
task_responses_with_indices = []
|
|
410
|
-
semaphore = asyncio.Semaphore(100) # Limit to 10 concurrent operations
|
|
411
411
|
|
|
412
412
|
for idx, task_response in enumerate(task_list_response.tasks):
|
|
413
413
|
if task_response.verifier:
|
|
@@ -418,61 +418,74 @@ class Fleet:
|
|
|
418
418
|
|
|
419
419
|
def create_verifier_with_fallback(tr, emb_code, is_error):
|
|
420
420
|
"""Create verifier with fallback logic."""
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
421
|
+
if not is_error:
|
|
422
|
+
# Try to create from embedded data
|
|
423
|
+
try:
|
|
424
|
+
return self._create_verifier_from_data(
|
|
425
|
+
verifier_id=tr.verifier.verifier_id,
|
|
426
|
+
verifier_key=tr.verifier.key,
|
|
427
|
+
verifier_code=emb_code,
|
|
428
|
+
verifier_sha=tr.verifier.sha256,
|
|
429
|
+
)
|
|
430
|
+
except Exception as e:
|
|
431
|
+
logger.warning(
|
|
432
|
+
f"Failed to create verifier {tr.verifier.key}: {e}"
|
|
433
|
+
)
|
|
434
|
+
return None
|
|
435
|
+
else:
|
|
436
|
+
# Fallback: try fetching by ID
|
|
437
|
+
try:
|
|
438
|
+
logger.warning(
|
|
439
|
+
f"Embedded verifier code missing for {tr.verifier.key} (NoSuchKey). "
|
|
440
|
+
f"Attempting to refetch by id {tr.verifier.verifier_id}"
|
|
441
|
+
)
|
|
442
|
+
return self._load_verifier(tr.verifier.verifier_id)
|
|
443
|
+
except Exception as e:
|
|
444
|
+
logger.warning(
|
|
445
|
+
f"Refetch by verifier id failed for {tr.verifier.key}: {e}. "
|
|
446
|
+
"Leaving verifier unset."
|
|
447
|
+
)
|
|
448
|
+
return None
|
|
449
|
+
|
|
450
|
+
# Add the task for parallel execution
|
|
451
|
+
verifier_tasks.append(
|
|
452
|
+
(
|
|
453
|
+
create_verifier_with_fallback,
|
|
454
|
+
task_response,
|
|
455
|
+
embedded_code,
|
|
456
|
+
is_embedded_error,
|
|
457
457
|
)
|
|
458
458
|
)
|
|
459
459
|
task_responses_with_indices.append((idx, task_response))
|
|
460
460
|
else:
|
|
461
461
|
# No verifier needed
|
|
462
|
-
|
|
462
|
+
verifier_tasks.append(None)
|
|
463
463
|
task_responses_with_indices.append((idx, task_response))
|
|
464
464
|
|
|
465
|
-
# Execute all verifier loading in parallel
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
465
|
+
# Execute all verifier loading in parallel using ThreadPoolExecutor
|
|
466
|
+
verifier_results = []
|
|
467
|
+
if verifier_tasks:
|
|
468
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
|
|
469
|
+
futures = []
|
|
470
|
+
for task in verifier_tasks:
|
|
471
|
+
if task is not None:
|
|
472
|
+
func, tr, emb_code, is_error = task
|
|
473
|
+
future = executor.submit(func, tr, emb_code, is_error)
|
|
474
|
+
futures.append(future)
|
|
475
|
+
else:
|
|
476
|
+
futures.append(None)
|
|
477
|
+
|
|
478
|
+
# Collect results
|
|
479
|
+
for future in futures:
|
|
480
|
+
if future is None:
|
|
481
|
+
verifier_results.append(None)
|
|
482
|
+
else:
|
|
483
|
+
try:
|
|
484
|
+
result = future.result()
|
|
485
|
+
verifier_results.append(result)
|
|
486
|
+
except Exception as e:
|
|
487
|
+
logger.warning(f"Verifier loading failed: {e}")
|
|
488
|
+
verifier_results.append(None)
|
|
476
489
|
|
|
477
490
|
# Build tasks with results
|
|
478
491
|
tasks = []
|
|
@@ -485,11 +498,7 @@ class Fleet:
|
|
|
485
498
|
|
|
486
499
|
if task_response.verifier:
|
|
487
500
|
# Process verifier result
|
|
488
|
-
if
|
|
489
|
-
logger.warning(
|
|
490
|
-
f"Verifier loading failed for {task_response.key}: {verifier_result}"
|
|
491
|
-
)
|
|
492
|
-
elif verifier_result is not None:
|
|
501
|
+
if verifier_result is not None:
|
|
493
502
|
verifier = verifier_result
|
|
494
503
|
embedded_code = task_response.verifier.code or ""
|
|
495
504
|
is_embedded_error = embedded_code.strip().startswith(
|
|
@@ -575,6 +584,8 @@ class Fleet:
|
|
|
575
584
|
task = Task(**task_data)
|
|
576
585
|
tasks.append(task)
|
|
577
586
|
|
|
587
|
+
responses = []
|
|
588
|
+
|
|
578
589
|
for task in tasks:
|
|
579
590
|
payload = TaskRequest(
|
|
580
591
|
key=task.key,
|
|
@@ -588,10 +599,13 @@ class Fleet:
|
|
|
588
599
|
response = self.client.request(
|
|
589
600
|
"POST", "/v1/tasks", json=payload.model_dump()
|
|
590
601
|
)
|
|
602
|
+
responses.append(response)
|
|
591
603
|
except Exception as e:
|
|
592
604
|
logger.error(f"Failed to import task {task.key}: {e}")
|
|
593
605
|
continue
|
|
594
606
|
|
|
607
|
+
return responses
|
|
608
|
+
|
|
595
609
|
def account(self) -> AccountResponse:
|
|
596
610
|
"""Get account information including instance limits and usage.
|
|
597
611
|
|
|
@@ -638,6 +652,7 @@ class Fleet:
|
|
|
638
652
|
AsyncVerifierFunction created from the verifier code
|
|
639
653
|
"""
|
|
640
654
|
from .tasks import verifier_from_string
|
|
655
|
+
from .verifiers import SyncVerifierFunction
|
|
641
656
|
|
|
642
657
|
# Use verifier_from_string to create the verifier
|
|
643
658
|
verifier_func = verifier_from_string(
|
|
@@ -8,9 +8,14 @@ def make(
|
|
|
8
8
|
data_key: Optional[str] = None,
|
|
9
9
|
region: Optional[str] = None,
|
|
10
10
|
env_variables: Optional[Dict[str, Any]] = None,
|
|
11
|
+
image_type: Optional[str] = None,
|
|
11
12
|
) -> SyncEnv:
|
|
12
13
|
return Fleet().make(
|
|
13
|
-
env_key,
|
|
14
|
+
env_key,
|
|
15
|
+
data_key=data_key,
|
|
16
|
+
region=region,
|
|
17
|
+
env_variables=env_variables,
|
|
18
|
+
image_type=image_type,
|
|
14
19
|
)
|
|
15
20
|
|
|
16
21
|
|
|
@@ -63,9 +63,7 @@ class InstanceClient:
|
|
|
63
63
|
def load(self) -> None:
|
|
64
64
|
self._load_resources()
|
|
65
65
|
|
|
66
|
-
def reset(
|
|
67
|
-
self, reset_request: Optional[ResetRequest] = None
|
|
68
|
-
) -> ResetResponse:
|
|
66
|
+
def reset(self, reset_request: Optional[ResetRequest] = None) -> ResetResponse:
|
|
69
67
|
response = self.client.request(
|
|
70
68
|
"POST", "/reset", json=reset_request.model_dump() if reset_request else None
|
|
71
69
|
)
|
|
@@ -55,7 +55,9 @@ class Instance(BaseModel):
|
|
|
55
55
|
|
|
56
56
|
class InstanceRequest(BaseModel):
|
|
57
57
|
env_key: str = Field(..., title="Env Key")
|
|
58
|
-
|
|
58
|
+
env_version: Optional[str] = Field(None, title="Env Version")
|
|
59
|
+
data_key: Optional[str] = Field(None, title="Data Key")
|
|
60
|
+
data_version: Optional[str] = Field(None, title="Data Version")
|
|
59
61
|
region: Optional[str] = Field("us-west-1", title="Region")
|
|
60
62
|
seed: Optional[int] = Field(None, title="Seed")
|
|
61
63
|
timestamp: Optional[int] = Field(None, title="Timestamp")
|
|
@@ -65,6 +67,7 @@ class InstanceRequest(BaseModel):
|
|
|
65
67
|
task_id: Optional[str] = Field(None, title="Task Id")
|
|
66
68
|
force_pull: Optional[bool] = Field(None, title="Force Pull")
|
|
67
69
|
env_variables: Optional[Dict[str, Any]] = Field(None, title="Env Variables")
|
|
70
|
+
image_type: Optional[str] = Field(None, title="Image Type")
|
|
68
71
|
created_from: Optional[str] = Field(None, title="Created From")
|
|
69
72
|
|
|
70
73
|
|
|
@@ -111,6 +111,21 @@ class SyncSnapshotQueryBuilder:
|
|
|
111
111
|
qb._conditions.append((column, "=", value))
|
|
112
112
|
return qb
|
|
113
113
|
|
|
114
|
+
def where(
|
|
115
|
+
self,
|
|
116
|
+
conditions: Optional[Dict[str, Any]] = None,
|
|
117
|
+
**kwargs: Any,
|
|
118
|
+
) -> "SyncSnapshotQueryBuilder":
|
|
119
|
+
qb = self._clone()
|
|
120
|
+
merged: Dict[str, Any] = {}
|
|
121
|
+
if conditions:
|
|
122
|
+
merged.update(conditions)
|
|
123
|
+
if kwargs:
|
|
124
|
+
merged.update(kwargs)
|
|
125
|
+
for column, value in merged.items():
|
|
126
|
+
qb._conditions.append((column, "=", value))
|
|
127
|
+
return qb
|
|
128
|
+
|
|
114
129
|
def limit(self, n: int) -> "SyncSnapshotQueryBuilder":
|
|
115
130
|
qb = self._clone()
|
|
116
131
|
qb._limit = n
|
|
@@ -304,6 +319,11 @@ class SyncSnapshotDiff:
|
|
|
304
319
|
self._cached = diff
|
|
305
320
|
return diff
|
|
306
321
|
|
|
322
|
+
@property
|
|
323
|
+
def changes(self) -> Dict[str, Dict[str, Any]]:
|
|
324
|
+
"""Expose the computed diff so callers can introspect like the legacy API."""
|
|
325
|
+
return self._collect()
|
|
326
|
+
|
|
307
327
|
def expect_only(self, allowed_changes: List[Dict[str, Any]]):
|
|
308
328
|
"""Ensure only specified changes occurred."""
|
|
309
329
|
diff = self._collect()
|
|
@@ -471,6 +491,21 @@ class SyncQueryBuilder:
|
|
|
471
491
|
def eq(self, column: str, value: Any) -> "SyncQueryBuilder":
|
|
472
492
|
return self._add_condition(column, "=", value)
|
|
473
493
|
|
|
494
|
+
def where(
|
|
495
|
+
self,
|
|
496
|
+
conditions: Optional[Dict[str, Any]] = None,
|
|
497
|
+
**kwargs: Any,
|
|
498
|
+
) -> "SyncQueryBuilder":
|
|
499
|
+
qb = self._clone()
|
|
500
|
+
merged: Dict[str, Any] = {}
|
|
501
|
+
if conditions:
|
|
502
|
+
merged.update(conditions)
|
|
503
|
+
if kwargs:
|
|
504
|
+
merged.update(kwargs)
|
|
505
|
+
for column, value in merged.items():
|
|
506
|
+
qb._conditions.append((column, "=", value))
|
|
507
|
+
return qb
|
|
508
|
+
|
|
474
509
|
def neq(self, column: str, value: Any) -> "SyncQueryBuilder":
|
|
475
510
|
return self._add_condition(column, "!=", value)
|
|
476
511
|
|
|
@@ -651,9 +686,7 @@ class SQLiteResource(Resource):
|
|
|
651
686
|
)
|
|
652
687
|
return DescribeResponse(**response.json())
|
|
653
688
|
|
|
654
|
-
def query(
|
|
655
|
-
self, query: str, args: Optional[List[Any]] = None
|
|
656
|
-
) -> QueryResponse:
|
|
689
|
+
def query(self, query: str, args: Optional[List[Any]] = None) -> QueryResponse:
|
|
657
690
|
return self._query(query, args, read_only=True)
|
|
658
691
|
|
|
659
692
|
def exec(self, query: str, args: Optional[List[Any]] = None) -> QueryResponse:
|
|
@@ -695,12 +728,8 @@ class SQLiteResource(Resource):
|
|
|
695
728
|
AsyncSnapshotDiff: Object containing the differences between the two databases
|
|
696
729
|
"""
|
|
697
730
|
# Create snapshots of both databases
|
|
698
|
-
before_snapshot = self.snapshot(
|
|
699
|
-
|
|
700
|
-
)
|
|
701
|
-
after_snapshot = other.snapshot(
|
|
702
|
-
name=f"after_{datetime.utcnow().isoformat()}"
|
|
703
|
-
)
|
|
731
|
+
before_snapshot = self.snapshot(name=f"before_{datetime.utcnow().isoformat()}")
|
|
732
|
+
after_snapshot = other.snapshot(name=f"after_{datetime.utcnow().isoformat()}")
|
|
704
733
|
|
|
705
734
|
# Return the diff between the snapshots
|
|
706
735
|
return before_snapshot.diff(after_snapshot, ignore_config)
|
|
@@ -2,16 +2,17 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
import asyncio
|
|
5
6
|
from datetime import datetime
|
|
6
7
|
from typing import Any, Dict, Optional, List, TYPE_CHECKING
|
|
7
8
|
|
|
8
|
-
from pydantic import BaseModel, Field, validator
|
|
9
|
+
from pydantic import BaseModel, Field, validator, field_serializer
|
|
9
10
|
|
|
10
11
|
# Import the shared VerifierFunction type that works for both async and sync
|
|
11
12
|
from fleet.types import VerifierFunction
|
|
12
13
|
|
|
13
14
|
if TYPE_CHECKING:
|
|
14
|
-
from fleet.
|
|
15
|
+
from fleet.models import VerifiersExecuteResponse
|
|
15
16
|
|
|
16
17
|
|
|
17
18
|
class Task(BaseModel):
|
|
@@ -29,7 +30,7 @@ class Task(BaseModel):
|
|
|
29
30
|
verifier: Optional[Any] = Field(
|
|
30
31
|
None,
|
|
31
32
|
description="Verifier function with decorator (async or sync)",
|
|
32
|
-
exclude=True,
|
|
33
|
+
exclude=True,
|
|
33
34
|
)
|
|
34
35
|
verifier_id: Optional[str] = Field(None, description="Verifier identifier")
|
|
35
36
|
verifier_sha: Optional[str] = Field(None, description="Verifier SHA256 hash")
|
|
@@ -46,6 +47,11 @@ class Task(BaseModel):
|
|
|
46
47
|
"""Set created_at to current time if not provided."""
|
|
47
48
|
return v or datetime.now()
|
|
48
49
|
|
|
50
|
+
@field_serializer("created_at")
|
|
51
|
+
def serialize_created_at(self, dt: Optional[datetime], _info):
|
|
52
|
+
"""Serialize datetime to ISO format string."""
|
|
53
|
+
return dt.isoformat() if dt else None
|
|
54
|
+
|
|
49
55
|
@property
|
|
50
56
|
def env_key(self) -> str:
|
|
51
57
|
"""Get the environment key combining env_id and version."""
|
|
@@ -75,6 +81,7 @@ class Task(BaseModel):
|
|
|
75
81
|
if self.verifier:
|
|
76
82
|
import inspect
|
|
77
83
|
|
|
84
|
+
# Check if verifier has remote method (for decorated verifiers)
|
|
78
85
|
result = self.verifier.remote(env, *args, **kwargs)
|
|
79
86
|
|
|
80
87
|
# If the result is a coroutine, we need to run it
|
|
@@ -117,32 +124,8 @@ class Task(BaseModel):
|
|
|
117
124
|
else:
|
|
118
125
|
raise ValueError("No verifier function found for this task")
|
|
119
126
|
|
|
120
|
-
def verify_detailed_async(
|
|
121
|
-
self, *args, **kwargs
|
|
122
|
-
) -> "VerifiersExecuteResponse":
|
|
123
|
-
"""Verify the task and return the full execute response model.
|
|
124
|
-
|
|
125
|
-
For async environments, awaits the async verifier.
|
|
126
|
-
Works with both sync and async verifiers in async contexts.
|
|
127
|
-
"""
|
|
128
|
-
# If verifier doesn't exist but verifier_func does, rebuild it
|
|
129
|
-
if not self.verifier and self.verifier_func:
|
|
130
|
-
self._rebuild_verifier()
|
|
131
|
-
|
|
132
|
-
if self.verifier:
|
|
133
|
-
result = self.verifier.remote_with_response(*args, **kwargs)
|
|
134
|
-
# If it's a coroutine, await it
|
|
135
|
-
import inspect
|
|
136
|
-
|
|
137
|
-
if inspect.iscoroutine(result):
|
|
138
|
-
return result
|
|
139
|
-
else:
|
|
140
|
-
return result
|
|
141
|
-
else:
|
|
142
|
-
raise ValueError("No verifier function found for this task")
|
|
143
|
-
|
|
144
127
|
def verify_detailed(self, env, *args, **kwargs) -> "VerifiersExecuteResponse":
|
|
145
|
-
"""Verify the task and return the full execute response model
|
|
128
|
+
"""Verify the task and return the full execute response model.
|
|
146
129
|
|
|
147
130
|
For sync environments, calls the sync verifier directly.
|
|
148
131
|
For async verifiers, automatically runs them with asyncio.run().
|
|
@@ -175,6 +158,23 @@ class Task(BaseModel):
|
|
|
175
158
|
else:
|
|
176
159
|
raise ValueError("No verifier function found for this task")
|
|
177
160
|
|
|
161
|
+
def verify_detailed_async(self, *args, **kwargs) -> "VerifiersExecuteResponse":
|
|
162
|
+
"""Verify the task and return the full execute response model (async version).
|
|
163
|
+
|
|
164
|
+
For async environments, returns a coroutine that when awaited returns the response.
|
|
165
|
+
Works with both sync and async verifiers in async contexts.
|
|
166
|
+
"""
|
|
167
|
+
# If verifier doesn't exist but verifier_func does, rebuild it
|
|
168
|
+
if not self.verifier and self.verifier_func:
|
|
169
|
+
self._rebuild_verifier()
|
|
170
|
+
|
|
171
|
+
if self.verifier:
|
|
172
|
+
result = self.verifier.remote_with_response(*args, **kwargs)
|
|
173
|
+
# Return the result (could be a coroutine or a value)
|
|
174
|
+
return result
|
|
175
|
+
else:
|
|
176
|
+
raise ValueError("No verifier function found for this task")
|
|
177
|
+
|
|
178
178
|
def _rebuild_verifier(self):
|
|
179
179
|
"""Rebuild the verifier from verifier_func string if it exists."""
|
|
180
180
|
if self.verifier_func:
|
|
@@ -217,20 +217,26 @@ def verifier_from_string(
|
|
|
217
217
|
"""
|
|
218
218
|
try:
|
|
219
219
|
import inspect
|
|
220
|
-
from .verifiers
|
|
221
|
-
from
|
|
222
|
-
from
|
|
220
|
+
from .verifiers import SyncVerifierFunction
|
|
221
|
+
from .verifiers.code import TASK_SUCCESSFUL_SCORE, TASK_FAILED_SCORE
|
|
222
|
+
from .verifiers.db import IgnoreConfig
|
|
223
|
+
|
|
224
|
+
# Create a globals namespace with all required imports
|
|
225
|
+
exec_globals = globals().copy()
|
|
226
|
+
exec_globals.update(
|
|
227
|
+
{
|
|
228
|
+
"TASK_SUCCESSFUL_SCORE": TASK_SUCCESSFUL_SCORE,
|
|
229
|
+
"TASK_FAILED_SCORE": TASK_FAILED_SCORE,
|
|
230
|
+
"IgnoreConfig": IgnoreConfig,
|
|
231
|
+
"Environment": object, # Add Environment type if needed
|
|
232
|
+
}
|
|
233
|
+
)
|
|
223
234
|
|
|
224
235
|
# Create a local namespace for executing the code
|
|
225
|
-
local_namespace = {
|
|
226
|
-
"TASK_SUCCESSFUL_SCORE": TASK_SUCCESSFUL_SCORE,
|
|
227
|
-
"TASK_FAILED_SCORE": TASK_FAILED_SCORE,
|
|
228
|
-
"IgnoreConfig": IgnoreConfig,
|
|
229
|
-
"Environment": object, # Add Environment type if needed
|
|
230
|
-
}
|
|
236
|
+
local_namespace = {}
|
|
231
237
|
|
|
232
238
|
# Execute the verifier code in the namespace
|
|
233
|
-
exec(verifier_func,
|
|
239
|
+
exec(verifier_func, exec_globals, local_namespace)
|
|
234
240
|
|
|
235
241
|
# Find the function that was defined
|
|
236
242
|
func_obj = None
|
|
@@ -242,15 +248,19 @@ def verifier_from_string(
|
|
|
242
248
|
if func_obj is None:
|
|
243
249
|
raise ValueError("No function found in verifier code")
|
|
244
250
|
|
|
245
|
-
# Create an
|
|
251
|
+
# Create an SyncVerifierFunction instance with raw code
|
|
246
252
|
verifier_instance = SyncVerifierFunction(
|
|
247
|
-
func_obj,
|
|
248
|
-
verifier_key,
|
|
253
|
+
func=func_obj,
|
|
254
|
+
key=verifier_key,
|
|
249
255
|
verifier_id=verifier_id,
|
|
250
256
|
sha256=sha256,
|
|
251
257
|
raw_code=verifier_func,
|
|
252
258
|
)
|
|
253
259
|
|
|
260
|
+
# Store additional metadata
|
|
261
|
+
verifier_instance._verifier_code = verifier_func
|
|
262
|
+
verifier_instance._sha256 = sha256
|
|
263
|
+
|
|
254
264
|
return verifier_instance
|
|
255
265
|
|
|
256
266
|
except Exception as e:
|
|
@@ -261,7 +271,7 @@ def load_tasks_from_file(filename: str) -> List[Task]:
|
|
|
261
271
|
"""Load tasks from a JSON file.
|
|
262
272
|
|
|
263
273
|
Example:
|
|
264
|
-
tasks =
|
|
274
|
+
tasks = fleet.load_tasks_from_file("my_tasks.json")
|
|
265
275
|
"""
|
|
266
276
|
from .global_client import get_client
|
|
267
277
|
|
|
@@ -294,7 +304,11 @@ def load_tasks(
|
|
|
294
304
|
|
|
295
305
|
client = get_client()
|
|
296
306
|
return client.load_tasks(
|
|
297
|
-
env_key=env_key,
|
|
307
|
+
env_key=env_key,
|
|
308
|
+
keys=keys,
|
|
309
|
+
version=version,
|
|
310
|
+
team_id=team_id,
|
|
311
|
+
project_key=project_key,
|
|
298
312
|
)
|
|
299
313
|
|
|
300
314
|
|
|
@@ -312,8 +326,8 @@ def update_task(
|
|
|
312
326
|
TaskResponse containing the updated task details
|
|
313
327
|
|
|
314
328
|
Examples:
|
|
315
|
-
response =
|
|
316
|
-
response =
|
|
329
|
+
response = fleet.update_task("my-task", prompt="New prompt text")
|
|
330
|
+
response = fleet.update_task("my-task", verifier_code="def verify(env): return True")
|
|
317
331
|
"""
|
|
318
332
|
from .global_client import get_client
|
|
319
333
|
|
|
@@ -12,11 +12,22 @@ import uuid
|
|
|
12
12
|
import logging
|
|
13
13
|
import hashlib
|
|
14
14
|
import inspect
|
|
15
|
-
from typing import
|
|
15
|
+
from typing import (
|
|
16
|
+
Any,
|
|
17
|
+
Callable,
|
|
18
|
+
Dict,
|
|
19
|
+
Optional,
|
|
20
|
+
List,
|
|
21
|
+
TypeVar,
|
|
22
|
+
TYPE_CHECKING,
|
|
23
|
+
Tuple,
|
|
24
|
+
)
|
|
16
25
|
|
|
17
26
|
from .bundler import FunctionBundler
|
|
18
|
-
|
|
19
|
-
|
|
27
|
+
|
|
28
|
+
if TYPE_CHECKING:
|
|
29
|
+
from ..client import SyncEnv
|
|
30
|
+
from ..models import VerifiersExecuteResponse
|
|
20
31
|
|
|
21
32
|
logger = logging.getLogger(__name__)
|
|
22
33
|
|
|
@@ -98,7 +109,7 @@ class SyncVerifierFunction:
|
|
|
98
109
|
|
|
99
110
|
return self._bundle_data, self._bundle_sha
|
|
100
111
|
|
|
101
|
-
def _check_bundle_status(self, env: SyncEnv) -> Tuple[str, bool]:
|
|
112
|
+
def _check_bundle_status(self, env: "SyncEnv") -> Tuple[str, bool]:
|
|
102
113
|
"""Check if bundle needs to be uploaded and return (sha, needs_upload)."""
|
|
103
114
|
bundle_data, bundle_sha = self._get_or_create_bundle()
|
|
104
115
|
|
|
@@ -120,7 +131,7 @@ class SyncVerifierFunction:
|
|
|
120
131
|
logger.info(f"Bundle {bundle_sha[:8]}... needs to be uploaded")
|
|
121
132
|
return bundle_sha, True # Upload needed
|
|
122
133
|
|
|
123
|
-
def __call__(self, env: SyncEnv, *args, **kwargs) -> float:
|
|
134
|
+
def __call__(self, env: "SyncEnv", *args, **kwargs) -> float:
|
|
124
135
|
"""Local execution of the verifier function with env as first parameter."""
|
|
125
136
|
try:
|
|
126
137
|
if self._is_async:
|
|
@@ -151,7 +162,7 @@ class SyncVerifierFunction:
|
|
|
151
162
|
# Return error score 0
|
|
152
163
|
return 0.0
|
|
153
164
|
|
|
154
|
-
def remote(self, env: SyncEnv, *args, **kwargs) -> float:
|
|
165
|
+
def remote(self, env: "SyncEnv", *args, **kwargs) -> float:
|
|
155
166
|
"""Remote execution of the verifier function with SHA-based bundle caching."""
|
|
156
167
|
response = self.remote_with_response(env, *args, **kwargs)
|
|
157
168
|
|
|
@@ -204,7 +215,7 @@ Remote traceback:
|
|
|
204
215
|
except Exception:
|
|
205
216
|
raise RuntimeError(full_message)
|
|
206
217
|
|
|
207
|
-
def _get_env_id(self, env: SyncEnv) -> str:
|
|
218
|
+
def _get_env_id(self, env: "SyncEnv") -> str:
|
|
208
219
|
"""Generate a unique identifier for the environment."""
|
|
209
220
|
# Use instance base URL or similar unique identifier
|
|
210
221
|
if hasattr(env, "instance") and hasattr(env.instance, "base_url"):
|
|
@@ -253,6 +264,7 @@ Remote traceback:
|
|
|
253
264
|
)
|
|
254
265
|
|
|
255
266
|
logger.debug(f"Bundle {bundle_sha[:8]}... uploaded successfully")
|
|
267
|
+
return response
|
|
256
268
|
|
|
257
269
|
else:
|
|
258
270
|
# Bundle already available - execute without upload
|
|
@@ -267,8 +279,7 @@ Remote traceback:
|
|
|
267
279
|
kwargs=kwargs,
|
|
268
280
|
needs_upload=False,
|
|
269
281
|
)
|
|
270
|
-
|
|
271
|
-
return response
|
|
282
|
+
return response
|
|
272
283
|
|
|
273
284
|
except Exception as e:
|
|
274
285
|
# Check if error indicates bundle not found and retry with upload
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
import fleet
|
|
2
|
-
from dotenv import load_dotenv
|
|
3
|
-
|
|
4
|
-
load_dotenv()
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
def main():
|
|
8
|
-
env = fleet.env.make("fira")
|
|
9
|
-
|
|
10
|
-
tasks = fleet.load_tasks(env_key="fira")
|
|
11
|
-
print(f"Loaded {len(tasks)} tasks")
|
|
12
|
-
|
|
13
|
-
for i, task in enumerate(tasks):
|
|
14
|
-
print(f"\nTask {i + 1}:")
|
|
15
|
-
print(f" Key: {task.key}")
|
|
16
|
-
print(f" Prompt: {task.prompt[:80]}...")
|
|
17
|
-
print(f" Verifier: {task.verifier_func[:80]}...")
|
|
18
|
-
|
|
19
|
-
print(f" Verifier: {task.verifier.key}")
|
|
20
|
-
print(" Running verifier...")
|
|
21
|
-
try:
|
|
22
|
-
score = task.verify(env)
|
|
23
|
-
print(f" ✓ Score: {score}")
|
|
24
|
-
except Exception as e:
|
|
25
|
-
print(f" ✗ Error: {type(e).__name__}: {e}")
|
|
26
|
-
|
|
27
|
-
print("-" * 60)
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
if __name__ == "__main__":
|
|
31
|
-
main()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|