fleet-python 0.2.29__py3-none-any.whl → 0.2.34__py3-none-any.whl

fleet/tasks.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 import re
 from datetime import datetime
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Optional, List
 from uuid import UUID
 
 from pydantic import BaseModel, Field, validator
@@ -15,31 +15,39 @@ from fleet.types import VerifierFunction
 
 class Task(BaseModel):
     """A task model representing a single task in the Fleet system."""
-    
+
     key: str = Field(..., description="Unique task key identifier")
     prompt: str = Field(..., description="Task prompt or instruction")
     env_id: str = Field(..., description="Environment identifier")
-    env_variables: Optional[Dict[str, Any]] = Field(default_factory=dict, description="Environment variables")
+    env_variables: Optional[Dict[str, Any]] = Field(
+        default_factory=dict, description="Environment variables"
+    )
     created_at: Optional[datetime] = Field(None, description="Task creation timestamp")
     version: Optional[str] = Field(None, description="Task version")
     verifier_func: Optional[str] = Field(None, description="Verifier function code")
-    verifier: Optional[Any] = Field(None, description="Verifier function with decorator (async or sync)")
+    verifier: Optional[Any] = Field(
+        None, description="Verifier function with decorator (async or sync)"
+    )
     verifier_id: Optional[str] = Field(None, description="Verifier identifier")
     verifier_sha: Optional[str] = Field(None, description="Verifier SHA256 hash")
-    metadata: Optional[Dict[str, Any]] = Field(default_factory=dict, description="Additional task metadata")
+    metadata: Optional[Dict[str, Any]] = Field(
+        default_factory=dict, description="Additional task metadata"
+    )
 
-    @validator('key')
+    @validator("key")
     def validate_key_format(cls, v):
         """Validate key follows kebab-case format."""
-        if not re.match(r'^[a-z0-9]+(-[a-z0-9]+)*$', v):
-            raise ValueError(f'Invalid task key format: {v}. Must follow kebab-case format.')
+        if not re.match(r"^[a-z0-9]+(-[a-z0-9]+)*$", v):
+            raise ValueError(
+                f"Invalid task key format: {v}. Must follow kebab-case format."
+            )
         return v
 
-    @validator('created_at', pre=True, always=True)
+    @validator("created_at", pre=True, always=True)
     def set_created_at(cls, v):
         """Set created_at to current time if not provided."""
         return v or datetime.now()
-    
+
     @property
     def env_key(self) -> str:
         """Get the environment key combining env_id and version."""
@@ -49,26 +57,45 @@ class Task(BaseModel):
 
     class Config:
         """Pydantic model configuration."""
+
         json_encoders = {
             datetime: lambda v: v.isoformat(),
         }
         # Allow arbitrary types for the verifier field
-        arbitrary_types_allowed = True 
+        arbitrary_types_allowed = True
 
     def verify(self, env, *args, **kwargs) -> float:
         """Verify the task using the verifier function (sync version).
-        
+
         For sync environments, calls the sync verifier directly.
         For async verifiers, automatically runs them with asyncio.run().
         """
         if self.verifier:
-            return self.verifier.remote(env, *args, **kwargs)
+            import inspect
+
+            result = self.verifier.remote(env, *args, **kwargs)
+
+            # If the result is a coroutine, we need to run it
+            if inspect.iscoroutine(result):
+                # Check if we're already in an event loop
+                try:
+                    loop = asyncio.get_running_loop()
+                    # We're in an async context, can't use asyncio.run()
+                    raise RuntimeError(
+                        "Cannot run async verifier in sync mode while event loop is running. "
+                        "Use await task.verify_async() instead."
+                    )
+                except RuntimeError:
+                    # No event loop running, safe to use asyncio.run()
+                    return asyncio.run(result)
+            else:
+                return result
         else:
             raise ValueError("No verifier function found for this task")
-    
+
     def verify_async(self, *args, **kwargs) -> float:
         """Verify the task using the verifier function (async version).
-        
+
         For async environments, awaits the async verifier.
         Works with both sync and async verifiers in async contexts.
         """
@@ -76,9 +103,53 @@ class Task(BaseModel):
             result = self.verifier.remote(*args, **kwargs)
             # If it's a coroutine, await it
             import inspect
+
             if inspect.iscoroutine(result):
                 return result
             else:
                 return result
         else:
             raise ValueError("No verifier function found for this task")
+
+    def make_env(self, region: Optional[str] = None):
+        """Create an environment instance for this task's environment.
+
+        Uses the task's env_id (and version if present) to create the env.
+        """
+        if not self.env_id:
+            raise ValueError("Task has no env_id defined")
+        # Deferred import to avoid circular dependencies
+        from .client import Fleet
+
+        return Fleet().make(env_key=self.env_key, region=region)
+
+
+def load_tasks(
+    env_key: Optional[str] = None,
+    keys: Optional[List[str]] = None,
+    version: Optional[str] = None,
+    team_id: Optional[str] = None
+) -> List[Task]:
+    """Convenience function to load tasks with optional filtering.
+
+    Args:
+        env_key: Optional environment key to filter tasks by
+        keys: Optional list of task keys to filter by
+        version: Optional version to filter tasks by
+        team_id: Optional team_id to filter by (admin only)
+
+    Examples:
+        tasks = await fleet.load_tasks(env_key="fira")
+        tasks = await fleet.load_tasks(keys=["task1", "task2"])
+        tasks = await fleet.load_tasks(env_key="fira", version="v1.0")
+    """
+    # Use the global client by default so users can pre-configure it once
+    from .global_client import get_client
+
+    client = get_client()
+    return client.load_tasks(
+        env_key=env_key, 
+        keys=keys, 
+        version=version, 
+        team_id=team_id
+    )

fleet/types.py

@@ -15,4 +15,4 @@ if TYPE_CHECKING:
 
 # Union type to support both async and sync verifiers
 # This definition works for both the async and sync versions of the codebase
-VerifierFunction = Union["SyncVerifierFunction", "AsyncVerifierFunction"] 
+VerifierFunction = Union["SyncVerifierFunction", "AsyncVerifierFunction"]

fleet/verifiers/__init__.py

@@ -9,10 +9,10 @@ from .decorator import (
 
 __all__ = [
     "DatabaseSnapshot",
-    "IgnoreConfig", 
+    "IgnoreConfig",
     "SnapshotDiff",
     "TASK_SUCCESSFUL_SCORE",
     "TASK_FAILED_SCORE",
     "verifier",
     "SyncVerifierFunction",
-]
+]

fleet/verifiers/bundler.py

@@ -544,7 +544,9 @@ class FunctionBundler:
                 # Ensure fleet-python is always included
                 if not requirements:
                     requirements = ["fleet-python"]
-                elif "fleet-python" not in [r.split("==")[0].split(">=")[0] for r in requirements]:
+                elif "fleet-python" not in [
+                    r.split("==")[0].split(">=")[0] for r in requirements
+                ]:
                     requirements.append("fleet-python")
                 requirements_file.write_text("\n".join(sorted(set(requirements))))
 
@@ -663,37 +665,39 @@ class FunctionBundler:
             logger.warning(f"Failed to extract function {function_name}: {e}")
 
         return None
-    
+
     def _get_function_source_without_decorator(self, func: Callable) -> str:
         """Get function source code without the @verifier decorator."""
         source = inspect.getsource(func)
-        lines = source.split('\n')
-        
+        lines = source.split("\n")
+
         # Find where the function definition starts
         func_start = -1
         for i, line in enumerate(lines):
-            if line.strip().startswith('def '):
+            if line.strip().startswith("def "):
                 func_start = i
                 break
-        
+
         if func_start == -1:
             # Couldn't find function definition, return original
             return source
-        
+
         # Return only from the function definition onward
         func_lines = lines[func_start:]
-        
+
         # Remove common indentation
         if func_lines:
             # Find minimum indentation (excluding empty lines)
-            min_indent = float('inf')
+            min_indent = float("inf")
             for line in func_lines:
                 if line.strip():
                     indent = len(line) - len(line.lstrip())
                     min_indent = min(min_indent, indent)
-            
+
             # Remove the common indentation
-            if min_indent < float('inf'):
-                func_lines = [line[min_indent:] if line.strip() else line for line in func_lines]
-        
-        return '\n'.join(func_lines)
+            if min_indent < float("inf"):
+                func_lines = [
+                    line[min_indent:] if line.strip() else line for line in func_lines
+                ]
+
+        return "\n".join(func_lines)

fleet/verifiers/code.py

@@ -1,2 +1,2 @@
 TASK_SUCCESSFUL_SCORE = 1
-TASK_FAILED_SCORE = 0
+TASK_FAILED_SCORE = 0

fleet/verifiers/decorator.py

@@ -13,34 +13,26 @@ import logging
 
 logger = logging.getLogger(__name__)
 
-F = TypeVar('F', bound=Callable[..., Any])
-
-
-
+F = TypeVar("F", bound=Callable[..., Any])
 
 
 class SyncVerifierFunction:
     """Wrapper for a verified function that supports local execution with env-first pattern."""
-    
-    def __init__(
-        self,
-        func: F,
-        key: str,
-        verifier_id: str
-    ):
+
+    def __init__(self, func: F, key: str, verifier_id: str):
         self.func = func
         self.key = key
         self.name = key  # Keep name for backward compatibility
         self.verifier_id = verifier_id
-        
+
         # Copy function metadata
         functools.update_wrapper(self, func)
-    
+
     def __call__(self, env, *args, **kwargs) -> float:
         """Local execution of the verifier function with env as first parameter."""
         try:
             result = self.func(env, *args, **kwargs)
-            
+
             # Handle different return types
             if isinstance(result, (int, float)):
                 # Direct score return
@@ -49,31 +41,33 @@ class SyncVerifierFunction:
                 return float(result["score"])
             else:
                 # Try to extract score from object attributes
-                if hasattr(result, 'score'):
+                if hasattr(result, "score"):
                     return float(result.score)
                 else:
-                    raise ValueError(f"Verifier function must return a score (number). Got {type(result)}")
-                    
+                    raise ValueError(
+                        f"Verifier function must return a score (number). Got {type(result)}"
+                    )
+
         except Exception as e:
             logger.error(f"Error in verifier {self.key}: {e}")
             # Return error score 0
             return 0.0
 
+
 def verifier(
-    key: Optional[str] = None,
-    verifier_id: Optional[str] = None
+    key: Optional[str] = None, verifier_id: Optional[str] = None
 ) -> Callable[[F], SyncVerifierFunction]:
     """
     Decorator to create a verifier function with env-first pattern.
-    
+
     The decorated function must take 'env' as its first parameter, making it explicit
     that verifiers operate within an environment context. This makes verifiers reusable
     across different environments.
-    
+
     Args:
         key: Optional key for the verifier. Defaults to function name.
         verifier_id: Optional unique ID for the verifier. Defaults to generated UUID.
-    
+
     Example:
         @verifier(key="test_database_state")
         def check_user_count(env, expected_count: int) -> float:
@@ -81,23 +75,20 @@ def verifier(
             result = db.query("SELECT COUNT(*) FROM users")
             actual_count = result.rows[0][0]
             return 1.0 if actual_count >= expected_count else 0.0
-        
+
         # Usage with different environments
-        env1 = flt.env.make("fira")
-        env2 = flt.env.make("another_env")
-        
+        env1 = fleet.env.make("fira")
+        env2 = fleet.env.make("another_env")
+
         # Local execution
         result = await check_user_count(env1, 5)
         result = await check_user_count(env2, 5)  # Same verifier, different env
     """
+
     def decorator(func: F) -> SyncVerifierFunction:
         verifier_key = key or func.__name__
         verifier_uuid = verifier_id or str(uuid.uuid4())
-        
-        return SyncVerifierFunction(
-            func,
-            verifier_key,
-            verifier_uuid
-        )
-    
-    return decorator
+
+        return SyncVerifierFunction(func, verifier_key, verifier_uuid)
+
+    return decorator

fleet/verifiers/parse.py

@@ -4,68 +4,86 @@ import re
 def extract_function_name(function_code: str) -> str | None:
     """
     Extract function name from Python function code.
-    
+
     Handles both regular functions (def) and async functions (async def).
-    
+
     Args:
         function_code: Python function code as a string
-        
+
     Returns:
         The function name if found, None otherwise
     """
-    # Pattern to match both def and async def functions
-    # Handles various formatting styles and type annotations
-    pattern = r'(?:async\s+)?def\s+(\w+)\s*\('
-    
-    match = re.search(pattern, function_code)
+    # Normalize escaped newlines and strip common Markdown code fences
+    code = function_code.replace("\\n", "\n").strip()
+
+    if "```" in code:
+        # Extract the first fenced block if present
+        fence_blocks = re.findall(r"```[a-zA-Z0-9_+-]*\n([\s\S]*?)\n```", code)
+        if fence_blocks:
+            code = fence_blocks[0].strip()
+
+    # Remove leading decorators (keep them for regex but allow preceding lines)
+    # Robust regex: allow optional decorators and whitespace before the def
+    pattern = r"^\s*(?:@[\w\.\n+() ,]*\n\s*)*(?:async\s+)?def\s+([A-Za-z_]\w*)\s*\("
+    match = re.search(pattern, code, flags=re.MULTILINE)
     if match:
         return match.group(1)
-    
+
+    # Fallback: search anywhere (not anchored) for a def signature
+    fallback = r"(?:async\s+)?def\s+([A-Za-z_]\w*)\s*\("
+    match = re.search(fallback, code)
+    if match:
+        return match.group(1)
+
     return None
 
 
 def convert_verifier_string(verifier_str: str) -> str:
     """
-    Convert a verifier function string from the old format (env: Environment) 
+    Convert a verifier function string from the old format (env: Environment)
     to the new format (before: DatabaseSnapshot, after: DatabaseSnapshot).
-    
+
     Args:
         verifier_str: The original verifier function as a string
-        
+
     Returns:
         The converted verifier function string
     """
     # First, handle escaped newlines in the input
-    verifier_str = verifier_str.replace('\\n', '\n')
-    
+    verifier_str = verifier_str.replace("\\n", "\n")
+
     # Extract function name, docstring, and body
     # More flexible pattern that accepts both int and float return types
     func_pattern = r'def\s+(\w+)\s*\(\s*env(?:\s*:\s*Environment)?\s*,?\s*final_answer(?:\s*:\s*str\s*\|\s*None)?\s*(?:=\s*None)?\s*\)\s*->\s*(?:float|int):\s*\n((?:\s*""".*?"""\s*\n)?)(.*)'
     match = re.match(func_pattern, verifier_str.strip(), re.DOTALL)
-    
+
     if not match:
         # Try with multiline pattern
         func_pattern_multiline = r'def\s+(\w+)\s*\(\s*\n?\s*env(?:\s*:\s*Environment)?\s*,?\s*\n?\s*final_answer(?:\s*:\s*str\s*\|\s*None)?\s*(?:=\s*None)?\s*\n?\s*\)\s*->\s*(?:float|int):\s*\n((?:\s*""".*?"""\s*\n)?)(.*)'
         match = re.match(func_pattern_multiline, verifier_str.strip(), re.DOTALL)
-        
+
         if not match:
-            raise ValueError("Could not parse verifier function. Expected format: def function_name(env: Environment, final_answer: str | None = None) -> float/int:")
-    
+            raise ValueError(
+                "Could not parse verifier function. Expected format: def function_name(env: Environment, final_answer: str | None = None) -> float/int:"
+            )
+
     func_name = match.group(1)
     docstring = match.group(2).strip()
     body = match.group(3)
-    
+
     # Find all unique env.db() calls
     db_calls = re.findall(r'env\.db\("(\w+)"\)', body)
-    unique_db_names = list(dict.fromkeys(db_calls))  # Remove duplicates while preserving order
-    
+    unique_db_names = list(
+        dict.fromkeys(db_calls)
+    )  # Remove duplicates while preserving order
+
     # Build the new function
-    new_func = f'''def {func_name}(
+    new_func = f"""def {func_name}(
     before: DatabaseSnapshot, after: DatabaseSnapshot, transcript: str | None = None
 ) -> int:
     class Environment:
-        def db(self, name: str) -> DatabaseSnapshot:'''
-    
+        def db(self, name: str) -> DatabaseSnapshot:"""
+
     # Build the db method based on found database names
     if unique_db_names:
         conditions = []
@@ -73,28 +91,32 @@ def convert_verifier_string(verifier_str: str) -> str:
             if db_name == "seed":
                 conditions.append('before if name == "seed"')
             elif db_name == "current":
-                conditions.append('after')
+                conditions.append("after")
             else:
                 # Handle other database names if needed
                 conditions.append(f'None  # Handle "{db_name}"')
-        
-        if len(conditions) == 2 and "seed" in unique_db_names and "current" in unique_db_names:
-            new_func += f'''
-            return before if name == "seed" else after'''
+
+        if (
+            len(conditions) == 2
+            and "seed" in unique_db_names
+            and "current" in unique_db_names
+        ):
+            new_func += f"""
+            return before if name == "seed" else after"""
         else:
             # More complex mapping if needed
-            new_func += f'''
+            new_func += f"""
             if name == "seed":
                 return before
             elif name == "current":
                 return after
             else:
-                raise ValueError(f"Unknown database name: {{name}}")'''
+                raise ValueError(f"Unknown database name: {{name}}")"""
     else:
-        new_func += '''
-            return before if name == "seed" else after'''
-    
-    new_func += '''
+        new_func += """
+            return before if name == "seed" else after"""
+
+    new_func += """
 
         @property
         def instance(self):
@@ -103,42 +125,44 @@ def convert_verifier_string(verifier_str: str) -> str:
         def load(self):
             pass
 
-    def verifier(env: Environment, final_answer: str | None = None) -> float:'''
-    
+    def verifier(env: Environment, final_answer: str | None = None) -> float:"""
+
     if docstring:
-        new_func += f'\n        {docstring}'
-    
+        new_func += f"\n        {docstring}"
+
     # First, find the minimum indentation in the body (excluding empty lines)
     body_lines = body.splitlines()
-    min_indent = float('inf')
+    min_indent = float("inf")
     for line in body_lines:
         if line.strip():  # Non-empty line
             indent_len = len(line) - len(line.lstrip())
             min_indent = min(min_indent, indent_len)
-    
+
     # If we didn't find any non-empty lines, set min_indent to 0
-    if min_indent == float('inf'):
+    if min_indent == float("inf"):
         min_indent = 0
-    
+
     # Now strip the minimum indentation and re-indent to 8 spaces
     if body_lines:
         indented_lines = []
         for line in body_lines:
             if line.strip():  # Non-empty line
                 # Remove the minimum indentation and add 8 spaces
-                stripped_line = line[min_indent:] if len(line) > min_indent else line.lstrip()
-                indented_lines.append('        ' + stripped_line)
+                stripped_line = (
+                    line[min_indent:] if len(line) > min_indent else line.lstrip()
+                )
+                indented_lines.append("        " + stripped_line)
             else:  # Empty line
-                indented_lines.append('')
-        
-        indented_body = '\n'.join(indented_lines)
-        new_func += f'\n{indented_body}'
-    
+                indented_lines.append("")
+
+        indented_body = "\n".join(indented_lines)
+        new_func += f"\n{indented_body}"
+
     # Add the return statement
-    new_func += '\n\n    return verifier(Environment(), transcript)'
+    new_func += "\n\n    return verifier(Environment(), transcript)"
 
     # Replace TASK_FAILED_SCORE with 0 in the function string
-    new_func = new_func.replace('TASK_FAILED_SCORE', '0')
+    new_func = new_func.replace("TASK_FAILED_SCORE", "0")
 
     return new_func
 
@@ -147,39 +171,45 @@ def convert_new_to_old_verifier(verifier_str: str) -> str:
     """
     Convert a verifier function from the new format (before/after: DatabaseSnapshot)
     to the old format (env: Environment).
-    
+
     This is the inverse of convert_verifier_string.
-    
+
     Args:
         verifier_str: The new format verifier function as a string
-        
+
     Returns:
         The converted verifier function string that accepts env
-    """   
+    """
     # Extract function name, parameters, docstring, and body
     # Pattern for new format with flexible whitespace and multiline support
     func_pattern = r'def\s+(\w+)\s*\(\s*before\s*:\s*DatabaseSnapshot\s*,?\s*after\s*:\s*DatabaseSnapshot\s*,?\s*transcript\s*:\s*str\s*\|\s*None\s*=\s*None\s*,?\s*\)\s*->\s*int:\s*((?:\s*""".*?"""\s*)?)(.*)'
-    
+
     # Try multiline pattern that's more flexible
     func_pattern_multiline = r'def\s+(\w+)\s*\(\s*\n?\s*before\s*:\s*DatabaseSnapshot\s*,?\s*\n?\s*after\s*:\s*DatabaseSnapshot\s*,?\s*\n?\s*transcript\s*:\s*str\s*\|\s*None\s*=\s*None\s*,?\s*\n?\s*\)\s*->\s*int:\s*\n?((?:\s*""".*?"""\s*)?)(.*)'
-    
-    match = re.match(func_pattern_multiline, verifier_str.strip(), re.DOTALL | re.MULTILINE)
-    
+
+    match = re.match(
+        func_pattern_multiline, verifier_str.strip(), re.DOTALL | re.MULTILINE
+    )
+
     if not match:
         # Even more flexible pattern
-        func_pattern_flexible = r'def\s+(\w+)\s*\([^)]*\)\s*->\s*int:\s*\n?((?:\s*""".*?"""\s*)?)(.*)'
+        func_pattern_flexible = (
+            r'def\s+(\w+)\s*\([^)]*\)\s*->\s*int:\s*\n?((?:\s*""".*?"""\s*)?)(.*)'
+        )
         match = re.match(func_pattern_flexible, verifier_str.strip(), re.DOTALL)
-        
+
         if not match:
             raise ValueError("Could not parse new format verifier function")
-    
+
     func_name = match.group(1)
     docstring = match.group(2).strip()
     body = match.group(3)
-    
+
     # Indent the original function body
-    indented_verifier = '\n'.join('    ' + line if line.strip() else line for line in verifier_str.splitlines())
-    
+    indented_verifier = "\n".join(
+        "    " + line if line.strip() else line for line in verifier_str.splitlines()
+    )
+
     # Build the wrapper function
     wrapper_func = f'''def {func_name}_wrapper(env, *args, **kwargs) -> float:
     """Wrapper to adapt new format verifier to old format."""
@@ -203,5 +233,5 @@ def convert_new_to_old_verifier(verifier_str: str) -> str:
     # Call the inner function and convert result
     result = {func_name}(before, after, transcript)
     return float(result)'''
-    
-    return wrapper_func
+
+    return wrapper_func