fleet-python 0.2.28__py3-none-any.whl → 0.2.32__py3-none-any.whl

fleet/tasks.py

@@ -2,10 +2,10 @@
 
 from __future__ import annotations
 
+import asyncio
 import re
 from datetime import datetime
-from typing import Any, Dict, Optional
-from uuid import UUID
+from typing import Any, Dict, Optional, List
 
 from pydantic import BaseModel, Field, validator
 
@@ -13,31 +13,168 @@ from pydantic import BaseModel, Field, validator
 from fleet.types import VerifierFunction
 
 
+def verifier_from_string(
+    verifier_func: str,
+    verifier_id: Optional[str] = None,
+    verifier_key: Optional[str] = None,
+    sha256: Optional[str] = None,
+) -> VerifierFunction:
+    """Create a verifier function from a string of Python code.
+    
+    This function creates either a SyncVerifierFunction or AsyncVerifierFunction
+    based on whether the code contains async function definitions.
+    
+    Args:
+        verifier_func: String containing the verifier function code
+        verifier_id: Optional verifier ID to use
+        verifier_key: Optional verifier key to use (defaults to function name)
+        sha256: Optional SHA256 hash of existing server-side bundle
+        
+    Returns:
+        VerifierFunction: Either SyncVerifierFunction or AsyncVerifierFunction
+        
+    Raises:
+        ValueError: If function name cannot be extracted from the code
+    """
+    from fleet.verifiers.parse import extract_function_name, convert_new_to_old_verifier
+    from fleet.verifiers.verifier import SyncVerifierFunction
+    from fleet._async.verifiers.verifier import AsyncVerifierFunction
+    from fleet.verifiers.db import IgnoreConfig, DatabaseSnapshot
+    
+    # Determine if this is an async verifier
+    is_async = "async def" in verifier_func
+    
+    # Store original code for later
+    original_code = verifier_func
+    
+    # Check if this is a new format verifier (has before/after parameters)
+    if (
+        "before: DatabaseSnapshot" in verifier_func
+        and "after: DatabaseSnapshot" in verifier_func
+    ):
+        # Convert new format to old format
+        verifier_func = convert_new_to_old_verifier(verifier_func)
+        # Update function name since wrapper adds _wrapper suffix
+        original_name = extract_function_name(verifier_func.split("\n")[0])
+        if original_name and original_name.endswith("_wrapper"):
+            function_name = original_name
+        else:
+            function_name = extract_function_name(verifier_func)
+    else:
+        # Extract function name from code
+        function_name = extract_function_name(verifier_func)
+    
+    if not function_name:
+        raise ValueError("Could not extract function name from verifier code")
+    
+    # Create a namespace for the function
+    namespace = {
+        "__builtins__": __builtins__,
+        "Environment": object,  # Placeholder, will be provided at runtime
+        "IgnoreConfig": IgnoreConfig,
+        "DatabaseSnapshot": DatabaseSnapshot,
+        "TASK_FAILED_SCORE": 0,
+        "TASK_SUCCESSFUL_SCORE": 1,
+    }
+    
+    # Execute the code to create the function
+    exec(verifier_func, namespace)
+    
+    # Get the function from the namespace
+    if function_name not in namespace:
+        raise ValueError(f"Function {function_name} not found after execution")
+    
+    func = namespace[function_name]
+    
+    # Use provided key or default to function name
+    key = verifier_key or function_name
+    
+    # Create appropriate verifier function
+    if is_async:
+        # Create AsyncVerifierFunction
+        return AsyncVerifierFunction(
+            func=func,
+            key=key,
+            verifier_id=verifier_id,
+            sha256=sha256,
+            raw_code=original_code,
+            extra_requirements=None,
+        )
+    else:
+        # For sync verifiers, we need to handle the case where the original was async
+        # but got converted during unasync processing
+        if "async def" in original_code and "await " in original_code:
+            # Convert async code to sync for SyncVerifierFunction
+            sync_code = original_code.replace("async def", "def")
+            sync_code = sync_code.replace("await ", "")
+            
+            # Re-execute with sync code
+            namespace = {
+                "__builtins__": __builtins__,
+                "Environment": object,
+                "IgnoreConfig": IgnoreConfig,
+                "DatabaseSnapshot": DatabaseSnapshot,
+                "TASK_FAILED_SCORE": 0,
+                "TASK_SUCCESSFUL_SCORE": 1,
+            }
+            exec(sync_code, namespace)
+            func = namespace[function_name]
+            
+            return SyncVerifierFunction(
+                func=func,
+                key=key,
+                verifier_id=verifier_id,
+                sha256=sha256,
+                raw_code=sync_code,
+                extra_requirements=None,
+            )
+        else:
+            # Already sync code
+            return SyncVerifierFunction(
+                func=func,
+                key=key,
+                verifier_id=verifier_id,
+                sha256=sha256,
+                raw_code=original_code,
+                extra_requirements=None,
+            )
+
+
 class Task(BaseModel):
     """A task model representing a single task in the Fleet system."""
-    
+
     key: str = Field(..., description="Unique task key identifier")
     prompt: str = Field(..., description="Task prompt or instruction")
     env_id: str = Field(..., description="Environment identifier")
-    env_variables: Optional[Dict[str, Any]] = Field(default_factory=dict, description="Environment variables")
+    env_variables: Optional[Dict[str, Any]] = Field(
+        default_factory=dict, description="Environment variables"
+    )
     created_at: Optional[datetime] = Field(None, description="Task creation timestamp")
     version: Optional[str] = Field(None, description="Task version")
     verifier_func: Optional[str] = Field(None, description="Verifier function code")
-    verifier: Optional[Any] = Field(None, description="Verifier function with decorator (async or sync)")
-    metadata: Optional[Dict[str, Any]] = Field(default_factory=dict, description="Additional task metadata")
+    verifier: Optional[Any] = Field(
+        None, description="Verifier function with decorator (async or sync)"
+    )
+    verifier_id: Optional[str] = Field(None, description="Verifier identifier")
+    verifier_sha: Optional[str] = Field(None, description="Verifier SHA256 hash")
+    metadata: Optional[Dict[str, Any]] = Field(
+        default_factory=dict, description="Additional task metadata"
+    )
 
-    @validator('key')
+    @validator("key")
     def validate_key_format(cls, v):
         """Validate key follows kebab-case format."""
-        if not re.match(r'^[a-z0-9]+(-[a-z0-9]+)*$', v):
-            raise ValueError(f'Invalid task key format: {v}. Must follow kebab-case format.')
+        if not re.match(r"^[a-z0-9]+(-[a-z0-9]+)*$", v):
+            raise ValueError(
+                f"Invalid task key format: {v}. Must follow kebab-case format."
+            )
         return v
 
-    @validator('created_at', pre=True, always=True)
+    @validator("created_at", pre=True, always=True)
     def set_created_at(cls, v):
         """Set created_at to current time if not provided."""
         return v or datetime.now()
-    
+
     @property
     def env_key(self) -> str:
         """Get the environment key combining env_id and version."""
@@ -47,26 +184,45 @@ class Task(BaseModel):
 
     class Config:
         """Pydantic model configuration."""
+
         json_encoders = {
             datetime: lambda v: v.isoformat(),
         }
         # Allow arbitrary types for the verifier field
-        arbitrary_types_allowed = True 
+        arbitrary_types_allowed = True
 
     def verify(self, env, *args, **kwargs) -> float:
         """Verify the task using the verifier function (sync version).
-        
+
         For sync environments, calls the sync verifier directly.
         For async verifiers, automatically runs them with asyncio.run().
         """
         if self.verifier:
-            return self.verifier.remote(env, *args, **kwargs)
+            import inspect
+
+            result = self.verifier.remote(env, *args, **kwargs)
+
+            # If the result is a coroutine, we need to run it
+            if inspect.iscoroutine(result):
+                # Check if we're already in an event loop
+                try:
+                    _ = asyncio.get_running_loop()
+                    # We're in an async context, can't use asyncio.run()
+                    raise RuntimeError(
+                        "Cannot run async verifier in sync mode while event loop is running. "
+                        "Use await task.verify_async() instead."
+                    )
+                except RuntimeError:
+                    # No event loop running, safe to use asyncio.run()
+                    return asyncio.run(result)
+            else:
+                return result
         else:
             raise ValueError("No verifier function found for this task")
-    
+
     def verify_async(self, *args, **kwargs) -> float:
         """Verify the task using the verifier function (async version).
-        
+
         For async environments, awaits the async verifier.
         Works with both sync and async verifiers in async contexts.
         """
@@ -74,9 +230,34 @@ class Task(BaseModel):
             result = self.verifier.remote(*args, **kwargs)
             # If it's a coroutine, await it
             import inspect
+
             if inspect.iscoroutine(result):
                 return result
             else:
                 return result
         else:
             raise ValueError("No verifier function found for this task")
+
+    def make_env(self, region: Optional[str] = None):
+        """Create an environment instance for this task's environment.
+
+        Uses the task's env_id (and version if present) to create the env.
+        """
+        if not self.env_id:
+            raise ValueError("Task has no env_id defined")
+        # Deferred import to avoid circular dependencies
+        from .client import Fleet
+
+        return Fleet().make(env_key=self.env_key, region=region)
+
+
+def load_tasks(env_key: Optional[str] = None) -> List[Task]:
+    """Convenience function to load tasks without initializing a client.
+
+    Creates an `AsyncFleet` client under the hood and returns the tasks list.
+    """
+    # Use the global client by default so users can pre-configure it once
+    from .global_client import get_client
+
+    client = get_client()
+    return client.load_tasks(env_key=env_key)

fleet/types.py

@@ -15,4 +15,4 @@ if TYPE_CHECKING:
 
 # Union type to support both async and sync verifiers
 # This definition works for both the async and sync versions of the codebase
-VerifierFunction = Union["SyncVerifierFunction", "AsyncVerifierFunction"] 
+VerifierFunction = Union["SyncVerifierFunction", "AsyncVerifierFunction"]

fleet/verifiers/__init__.py

@@ -9,10 +9,10 @@ from .decorator import (
 
 __all__ = [
     "DatabaseSnapshot",
-    "IgnoreConfig", 
+    "IgnoreConfig",
     "SnapshotDiff",
     "TASK_SUCCESSFUL_SCORE",
     "TASK_FAILED_SCORE",
     "verifier",
     "SyncVerifierFunction",
-]
+]

fleet/verifiers/bundler.py

@@ -544,7 +544,9 @@ class FunctionBundler:
                 # Ensure fleet-python is always included
                 if not requirements:
                     requirements = ["fleet-python"]
-                elif "fleet-python" not in [r.split("==")[0].split(">=")[0] for r in requirements]:
+                elif "fleet-python" not in [
+                    r.split("==")[0].split(">=")[0] for r in requirements
+                ]:
                     requirements.append("fleet-python")
                 requirements_file.write_text("\n".join(sorted(set(requirements))))
 
@@ -663,37 +665,39 @@ class FunctionBundler:
             logger.warning(f"Failed to extract function {function_name}: {e}")
 
         return None
-    
+
     def _get_function_source_without_decorator(self, func: Callable) -> str:
         """Get function source code without the @verifier decorator."""
         source = inspect.getsource(func)
-        lines = source.split('\n')
-        
+        lines = source.split("\n")
+
         # Find where the function definition starts
         func_start = -1
         for i, line in enumerate(lines):
-            if line.strip().startswith('def '):
+            if line.strip().startswith("def "):
                 func_start = i
                 break
-        
+
         if func_start == -1:
             # Couldn't find function definition, return original
             return source
-        
+
         # Return only from the function definition onward
         func_lines = lines[func_start:]
-        
+
         # Remove common indentation
         if func_lines:
             # Find minimum indentation (excluding empty lines)
-            min_indent = float('inf')
+            min_indent = float("inf")
             for line in func_lines:
                 if line.strip():
                     indent = len(line) - len(line.lstrip())
                     min_indent = min(min_indent, indent)
-            
+
             # Remove the common indentation
-            if min_indent < float('inf'):
-                func_lines = [line[min_indent:] if line.strip() else line for line in func_lines]
-        
-        return '\n'.join(func_lines)
+            if min_indent < float("inf"):
+                func_lines = [
+                    line[min_indent:] if line.strip() else line for line in func_lines
+                ]
+
+        return "\n".join(func_lines)

fleet/verifiers/code.py

@@ -1,2 +1,2 @@
 TASK_SUCCESSFUL_SCORE = 1
-TASK_FAILED_SCORE = 0
+TASK_FAILED_SCORE = 0

fleet/verifiers/decorator.py

@@ -13,34 +13,26 @@ import logging
 
 logger = logging.getLogger(__name__)
 
-F = TypeVar('F', bound=Callable[..., Any])
-
-
-
+F = TypeVar("F", bound=Callable[..., Any])
 
 
 class SyncVerifierFunction:
     """Wrapper for a verified function that supports local execution with env-first pattern."""
-    
-    def __init__(
-        self,
-        func: F,
-        key: str,
-        verifier_id: str
-    ):
+
+    def __init__(self, func: F, key: str, verifier_id: str):
         self.func = func
         self.key = key
         self.name = key  # Keep name for backward compatibility
         self.verifier_id = verifier_id
-        
+
         # Copy function metadata
         functools.update_wrapper(self, func)
-    
+
     def __call__(self, env, *args, **kwargs) -> float:
         """Local execution of the verifier function with env as first parameter."""
         try:
             result = self.func(env, *args, **kwargs)
-            
+
             # Handle different return types
             if isinstance(result, (int, float)):
                 # Direct score return
@@ -49,31 +41,33 @@ class SyncVerifierFunction:
                 return float(result["score"])
             else:
                 # Try to extract score from object attributes
-                if hasattr(result, 'score'):
+                if hasattr(result, "score"):
                     return float(result.score)
                 else:
-                    raise ValueError(f"Verifier function must return a score (number). Got {type(result)}")
-                    
+                    raise ValueError(
+                        f"Verifier function must return a score (number). Got {type(result)}"
+                    )
+
         except Exception as e:
             logger.error(f"Error in verifier {self.key}: {e}")
             # Return error score 0
             return 0.0
 
+
 def verifier(
-    key: Optional[str] = None,
-    verifier_id: Optional[str] = None
+    key: Optional[str] = None, verifier_id: Optional[str] = None
 ) -> Callable[[F], SyncVerifierFunction]:
     """
     Decorator to create a verifier function with env-first pattern.
-    
+
     The decorated function must take 'env' as its first parameter, making it explicit
     that verifiers operate within an environment context. This makes verifiers reusable
     across different environments.
-    
+
     Args:
         key: Optional key for the verifier. Defaults to function name.
         verifier_id: Optional unique ID for the verifier. Defaults to generated UUID.
-    
+
     Example:
         @verifier(key="test_database_state")
         def check_user_count(env, expected_count: int) -> float:
@@ -81,23 +75,20 @@ def verifier(
             result = db.query("SELECT COUNT(*) FROM users")
             actual_count = result.rows[0][0]
             return 1.0 if actual_count >= expected_count else 0.0
-        
+
         # Usage with different environments
-        env1 = flt.env.make("fira")
-        env2 = flt.env.make("another_env")
-        
+        env1 = fleet.env.make("fira")
+        env2 = fleet.env.make("another_env")
+
         # Local execution
         result = await check_user_count(env1, 5)
         result = await check_user_count(env2, 5)  # Same verifier, different env
     """
+
     def decorator(func: F) -> SyncVerifierFunction:
         verifier_key = key or func.__name__
         verifier_uuid = verifier_id or str(uuid.uuid4())
-        
-        return SyncVerifierFunction(
-            func,
-            verifier_key,
-            verifier_uuid
-        )
-    
-    return decorator
+
+        return SyncVerifierFunction(func, verifier_key, verifier_uuid)
+
+    return decorator