fleet-python 0.2.62__py3-none-any.whl → 0.2.63__py3-none-any.whl

examples/import_tasks.py

@@ -2,12 +2,194 @@ import asyncio
 import argparse
 import json
 import sys
+from collections import defaultdict
+from typing import Dict, List, Tuple
 import fleet
+from fleet._async.tasks import Task
 from dotenv import load_dotenv
 
 load_dotenv()
 
 
+async def run_verifier_sanity_check(
+    tasks: List[Task],
+    client: fleet.AsyncFleet,
+) -> Tuple[bool, Dict[str, str]]:
+    """
+    Run sanity check by spinning up instances and running verifiers.
+
+    Args:
+        tasks: List of Task objects to verify
+        client: AsyncFleet client instance
+
+    Returns:
+        Tuple of (all_passed, error_dict) where error_dict maps task_key to error message
+    """
+    print("\n" + "=" * 60)
+    print("Running verifier sanity check...")
+    print("=" * 60)
+
+    # Group tasks by env_key×env_version×data_key×data_version
+    instance_groups = defaultdict(list)
+    for task in tasks:
+        # Build the instance key
+        env_key = task.env_id or ""
+        env_version = task.version or ""
+        data_key = task.data_id or ""
+        data_version = task.data_version or ""
+
+        instance_key = f"{env_key}×{env_version}×{data_key}×{data_version}"
+        instance_groups[instance_key].append(task)
+
+    print(f"\nFound {len(instance_groups)} unique environment/data combinations:")
+    for instance_key, group_tasks in instance_groups.items():
+        print(f"  {instance_key}: {len(group_tasks)} task(s)")
+
+    # Create all instances in parallel
+    print(f"\nCreating {len(instance_groups)} instance(s) in parallel...")
+    instance_map = {}
+
+    async def create_instance(instance_key: str) -> Tuple[str, object]:
+        """Create a single instance."""
+        try:
+            env_key, env_version, data_key, data_version = instance_key.split("×")
+
+            # Build env_key_str and data_key_str
+            if env_version:
+                env_key_str = f"{env_key}:{env_version}"
+            else:
+                env_key_str = env_key
+
+            if data_key and data_version:
+                data_key_str = f"{data_key}:{data_version}"
+            elif data_key:
+                data_key_str = data_key
+            else:
+                data_key_str = None
+
+            print(
+                f"  Creating instance: {env_key_str}"
+                + (f" with data {data_key_str}" if data_key_str else "")
+            )
+            env = await client.make(env_key=env_key_str, data_key=data_key_str)
+            return instance_key, env
+        except Exception as e:
+            print(f"  ✗ Failed to create instance for {instance_key}: {e}")
+            return instance_key, None
+
+    # Create instances concurrently
+    instance_results = await asyncio.gather(
+        *[create_instance(key) for key in instance_groups.keys()],
+        return_exceptions=True,
+    )
+
+    for result in instance_results:
+        if isinstance(result, Exception):
+            print(f"  ✗ Exception creating instance: {result}")
+            return False, {"__instance_creation__": str(result)}
+        instance_key, env = result
+        if env is None:
+            return False, {instance_key: "Failed to create instance"}
+        instance_map[instance_key] = env
+
+    print(f"✓ Created {len(instance_map)} instance(s)")
+
+    # Run all verifiers in parallel with concurrency limit
+    max_concurrent_verifiers = 5  # Limit concurrent verifier executions
+    print(
+        f"\nRunning {len(tasks)} verifier(s) in parallel (max {max_concurrent_verifiers} concurrent)..."
+    )
+    errors = {}
+    semaphore = asyncio.Semaphore(max_concurrent_verifiers)
+
+    async def run_single_verifier(task, instance_key: str) -> Tuple[str, bool, str]:
+        """Run a single verifier and return (task_key, success, error_message)."""
+        async with semaphore:
+            try:
+                env = instance_map[instance_key]
+                task_key = task.key
+
+                # Run the verifier
+                if task.verifier is None:
+                    return task_key, False, "No verifier found"
+
+                result = await task.verify_async(env)
+
+                # For sanity check: we expect verifiers to return 0.0 (TASK_FAILED_SCORE)
+                # since we're running on fresh instances with no task completion.
+                # This confirms the verifier runs without errors.
+                if isinstance(result, float):
+                    if result == 0.0:
+                        print(f"  ✓ {task_key}: {result:.2f} (correctly returns 0.0)")
+                        return task_key, True, ""
+                    else:
+                        print(
+                            f"  ⚠ {task_key}: {result:.2f} (expected 0.0 on fresh instance)"
+                        )
+                        return (
+                            task_key,
+                            False,
+                            f"Expected 0.0 but got {result:.2f} on fresh instance",
+                        )
+                else:
+                    # Non-float result - verifier ran but didn't return expected type
+                    print(f"  ⚠ {task_key}: {result} (expected float 0.0)")
+                    return (
+                        task_key,
+                        False,
+                        f"Expected float 0.0 but got {type(result).__name__}: {result}",
+                    )
+
+            except Exception as e:
+                task_key = task.key
+                error_msg = f"{type(e).__name__}: {str(e)}"
+                print(f"  ✗ {task_key}: {error_msg}")
+                return task_key, False, error_msg
+
+    # Run verifiers concurrently with semaphore
+    verifier_results = await asyncio.gather(
+        *[
+            run_single_verifier(task, instance_key)
+            for instance_key, group_tasks in instance_groups.items()
+            for task in group_tasks
+        ],
+        return_exceptions=True,
+    )
+
+    # Process results
+    for result in verifier_results:
+        if isinstance(result, Exception):
+            print(f"  ✗ Exception running verifier: {result}")
+            errors["__verifier_exception__"] = str(result)
+        else:
+            task_key, success, error_msg = result
+            if not success:
+                errors[task_key] = error_msg
+
+    # Clean up instances
+    print(f"\nCleaning up {len(instance_map)} instance(s)...")
+    cleanup_tasks = [env.close() for env in instance_map.values()]
+    await asyncio.gather(*cleanup_tasks, return_exceptions=True)
+    print("✓ Cleanup complete")
+
+    # Summary
+    passed_count = len(tasks) - len(errors)
+    print("\n" + "=" * 60)
+    print(f"Sanity check complete: {passed_count}/{len(tasks)} passed")
+
+    if errors:
+        print(f"\n✗ {len(errors)} verifier(s) failed:")
+        for task_key, error_msg in list(errors.items())[:10]:
+            print(f"  - {task_key}: {error_msg}")
+        if len(errors) > 10:
+            print(f"  ... and {len(errors) - 10} more")
+        print("\nFix the verifiers and try again.")
+        return False, errors
+    else:
+        print("✓ All verifiers passed!")
+        return True, {}
+
+
 async def main():
     parser = argparse.ArgumentParser(description="Import tasks from a JSON file")
     parser.add_argument("json_file", help="Path to the JSON file containing tasks")
@@ -23,6 +205,11 @@ async def main():
         action="store_true",
         help="Skip confirmation prompt and import automatically",
     )
+    parser.add_argument(
+        "--skip-sanity-check",
+        action="store_true",
+        help="Skip the verifier sanity check (not recommended)",
+    )
 
     args = parser.parse_args()
 
@@ -47,7 +234,7 @@ async def main():
             task_keys.append(task_key)
         else:
             task_keys.append("(no key)")
-        
+
         # Check for verifier_func
         verifier_code = task_data.get("verifier_func") or task_data.get("verifier_code")
         if not verifier_code:
@@ -78,6 +265,30 @@ async def main():
     else:
         print("\nProject key: (none)")
 
+    # Load tasks as Task objects
+    client = fleet.AsyncFleet()
+    tasks = []
+    print("\nLoading tasks...")
+    for task_data in tasks_data:
+        try:
+            task = await client.load_task_from_json(
+                task_data, raise_on_verifier_error=True
+            )
+            tasks.append(task)
+        except Exception as e:
+            task_key = task_data.get("key") or task_data.get("id", "unknown")
+            print(f"✗ Failed to load task {task_key}: {e}")
+            sys.exit(1)
+    print(f"✓ Loaded {len(tasks)} tasks")
+
+    # Run sanity check (unless skipped)
+    if not args.skip_sanity_check:
+        success, errors = await run_verifier_sanity_check(tasks, client)
+        if not success:
+            sys.exit(1)
+    else:
+        print("\n⚠️  Skipping sanity check (--skip-sanity-check)")
+
     # Confirmation prompt (unless --yes flag is provided)
     if not args.yes:
         print("\n" + "=" * 60)

fleet/_async/tasks.py

@@ -274,10 +274,18 @@ def verifier_from_string(
     """
     try:
         import inspect
+        import re
         from .verifiers.verifier import AsyncVerifierFunction
         from fleet.verifiers.code import TASK_SUCCESSFUL_SCORE, TASK_FAILED_SCORE
         from fleet.verifiers.db import IgnoreConfig
 
+        # Strip @verifier decorator if present to avoid double-wrapping
+        # Remove lines like: @verifier(key="...")
+        cleaned_code = re.sub(r'@verifier\([^)]*\)\s*\n', '', verifier_func)
+        # Also remove the verifier import if present
+        cleaned_code = re.sub(r'from fleet import.*verifier.*\n', '', cleaned_code)
+        cleaned_code = re.sub(r'import.*verifier.*\n', '', cleaned_code)
+
         # Create a local namespace for executing the code
         local_namespace = {
             "TASK_SUCCESSFUL_SCORE": TASK_SUCCESSFUL_SCORE,
@@ -286,8 +294,8 @@ def verifier_from_string(
             "Environment": object,  # Add Environment type if needed
         }
 
-        # Execute the verifier code in the namespace
-        exec(verifier_func, globals(), local_namespace)
+        # Execute the cleaned verifier code in the namespace
+        exec(cleaned_code, globals(), local_namespace)
 
         # Find the function that was defined (not imported)
         # Functions defined via exec have co_filename == '<string>'

fleet/tasks.py

@@ -265,10 +265,18 @@ def verifier_from_string(
     """
     try:
         import inspect
+        import re
         from .verifiers import SyncVerifierFunction
         from .verifiers.code import TASK_SUCCESSFUL_SCORE, TASK_FAILED_SCORE
         from .verifiers.db import IgnoreConfig
 
+        # Strip @verifier decorator if present to avoid double-wrapping
+        # Remove lines like: @verifier(key="...")
+        cleaned_code = re.sub(r'@verifier\([^)]*\)\s*\n', '', verifier_func)
+        # Also remove the verifier import if present
+        cleaned_code = re.sub(r'from fleet import.*verifier.*\n', '', cleaned_code)
+        cleaned_code = re.sub(r'import.*verifier.*\n', '', cleaned_code)
+
         # Create a globals namespace with all required imports
         exec_globals = globals().copy()
         exec_globals.update(
@@ -283,8 +291,8 @@ def verifier_from_string(
         # Create a local namespace for executing the code
         local_namespace = {}
 
-        # Execute the verifier code in the namespace
-        exec(verifier_func, exec_globals, local_namespace)
+        # Execute the cleaned verifier code in the namespace
+        exec(cleaned_code, exec_globals, local_namespace)
 
         # Find the function that was defined (not imported)
         # Functions defined via exec have co_filename == '<string>'

{fleet_python-0.2.62.dist-info → fleet_python-0.2.63.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fleet-python
-Version: 0.2.62
+Version: 0.2.63
 Summary: Python SDK for Fleet environments
 Author-email: Fleet AI <nic@fleet.so>
 License: Apache-2.0

{fleet_python-0.2.62.dist-info → fleet_python-0.2.63.dist-info}/RECORD

@@ -13,7 +13,7 @@ examples/example_tasks.py,sha256=xTL8UWVAuolSX6swskfrAcmDrLIzn45dJ7YPWCwoEBU,514
 examples/example_verifier.py,sha256=0vwNITIG3m4CkSPwIxNXcGx9TqrxEsCGqK2A8keKZMM,2392
 examples/export_tasks.py,sha256=cJ_8xND7Q3IOM1JfJPR-DH3aLfHo_KmKJeO-1IVUFrQ,3237
 examples/gemini_example.py,sha256=qj9WDazQTYNiRHNeUg9Tjkp33lJMwbx8gDfpFe1sDQo,16180
-examples/import_tasks.py,sha256=pb60dOXnOn-G1INQxV1wujeHmmY2oWz9Ddj9SbMm1pk,3139
+examples/import_tasks.py,sha256=Duh7T0HUuqsYUZ6LK2AXF3eP0zfSj1izkI5-1p09d9w,11041
 examples/json_tasks_example.py,sha256=CYPESGGtOo0fmsDdLidujTfsE4QlJHw7rOhyVqPJ_Ls,5329
 examples/nova_act_example.py,sha256=rH23Lp74Okf0rn8ynMdWjK2aviEf5NLPH4k_53Pyxho,831
 examples/openai_example.py,sha256=dEWERrTEP5xBiGkLkQjBQGd2NqoxX6gcW6XteBPsWFQ,8231
@@ -28,7 +28,7 @@ fleet/config.py,sha256=uY02ZKxVoXqVDta-0IMWaYJeE1CTXF_fA9NI6QUutmU,319
 fleet/exceptions.py,sha256=fUmPwWhnT8SR97lYsRq0kLHQHKtSh2eJS0VQ2caSzEI,5055
 fleet/global_client.py,sha256=frrDAFNM2ywN0JHLtlm9qbE1dQpnQJsavJpb7xSR_bU,1072
 fleet/models.py,sha256=-8WG10SMkXUljRqD_8pfajCr4PS9qYDVLAZ8RxdJMb0,13392
-fleet/tasks.py,sha256=STR34InS14rMq1DFPsCeAASarsvUqf_ODK_jitT8ORU,16914
+fleet/tasks.py,sha256=PJvq9JAduk88oVv3c9e4P7pwIKpO0Zxym4TFJpQbiSc,17349
 fleet/types.py,sha256=L4Y82xICf1tzyCLqhLYUgEoaIIS5h9T05TyFNHSWs3s,652
 fleet/_async/__init__.py,sha256=Wi8Tjj-Lfnxi4cPOkAxh2lynnpEBNni6mI6Iq80uOro,8054
 fleet/_async/base.py,sha256=oisVTQsx0M_yTmyQJc3oij63uKZ97MHz-xYFsWXxQE8,9202
@@ -36,7 +36,7 @@ fleet/_async/client.py,sha256=Uu9o8LpUce8vOoFxqG0RDe4VtJpu1BJGIFbVecuO71g,32947
 fleet/_async/exceptions.py,sha256=fUmPwWhnT8SR97lYsRq0kLHQHKtSh2eJS0VQ2caSzEI,5055
 fleet/_async/global_client.py,sha256=4WskpLHbsDEgWW7hXMD09W-brkp4euy8w2ZJ88594rQ,1103
 fleet/_async/models.py,sha256=6x3IPYuWz1v6zWjujqgzK2CpR2HB5Rme4LQFLyvUDXE,13164
-fleet/_async/tasks.py,sha256=LiDDIrmT9mIqlUna-I903n4yVpLxbmxUSv4lPdaBcDE,16942
+fleet/_async/tasks.py,sha256=K7xDIPO_ZHga_1ShssIiCuMe8_sx1fWU9a5zQwm_8d0,17377
 fleet/_async/env/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fleet/_async/env/client.py,sha256=C5WG5Ir_McXaFPZNdkQjj0w4V7xMIcw3QyVP5g-3kVk,1237
 fleet/_async/instance/__init__.py,sha256=PtmJq8J8bh0SOQ2V55QURz5GJfobozwtQoqhaOk3_tI,515
@@ -69,12 +69,12 @@ fleet/verifiers/decorator.py,sha256=nAP3O8szXu7md_kpwpz91hGSUNEVLYjwZQZTkQlV1DM,
 fleet/verifiers/parse.py,sha256=qz9AfJrTbjlg-LU-lE8Ciqi7Yt2a8-cs17FdpjTLhMk,8550
 fleet/verifiers/sql_differ.py,sha256=TqTLWyK3uOyLbitT6HYzYEzuSFC39wcyhgk3rcm__k8,6525
 fleet/verifiers/verifier.py,sha256=_lcxXVm8e0xRrK2gNJy9up7pW1zOkPRY5n5lQ85S8jg,14197
-fleet_python-0.2.62.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+fleet_python-0.2.63.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
 scripts/fix_sync_imports.py,sha256=X9fWLTpiPGkSHsjyQUDepOJkxOqw1DPj7nd8wFlFqLQ,8368
 scripts/unasync.py,sha256=vWVQxRWX8SRZO5cmzEhpvnG_REhCWXpidIGIpWmEcvI,696
 tests/__init__.py,sha256=Re1SdyxH8NfyL1kjhi7SQkGP1mYeWB-D6UALqdIMd8I,35
 tests/test_verifier_from_string.py,sha256=Lxi3TpFHFb-hG4-UhLKZJkqo84ax9YJY8G6beO-1erM,13581
-fleet_python-0.2.62.dist-info/METADATA,sha256=pkly98qTBJVHA9jkevvz_v3UKZyGa2-_RFVlbVGLlSg,3304
-fleet_python-0.2.62.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-fleet_python-0.2.62.dist-info/top_level.txt,sha256=qb1zIbtEktyhRFZdqVytwg54l64qtoZL0wjHB4bUg3c,29
-fleet_python-0.2.62.dist-info/RECORD,,
+fleet_python-0.2.63.dist-info/METADATA,sha256=Qe5K9wBiRkdNI6PUnExcW3zQDG_MjpOvVfnpNAB1gJo,3304
+fleet_python-0.2.63.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+fleet_python-0.2.63.dist-info/top_level.txt,sha256=qb1zIbtEktyhRFZdqVytwg54l64qtoZL0wjHB4bUg3c,29
+fleet_python-0.2.63.dist-info/RECORD,,