fleet-python 0.2.28__py3-none-any.whl → 0.2.32__py3-none-any.whl

fleet/verifiers/parse.py

@@ -4,68 +4,86 @@ import re
 def extract_function_name(function_code: str) -> str | None:
     """
     Extract function name from Python function code.
-    
+
     Handles both regular functions (def) and async functions (async def).
-    
+
     Args:
         function_code: Python function code as a string
-        
+
     Returns:
         The function name if found, None otherwise
     """
-    # Pattern to match both def and async def functions
-    # Handles various formatting styles and type annotations
-    pattern = r'(?:async\s+)?def\s+(\w+)\s*\('
-    
-    match = re.search(pattern, function_code)
+    # Normalize escaped newlines and strip common Markdown code fences
+    code = function_code.replace("\\n", "\n").strip()
+
+    if "```" in code:
+        # Extract the first fenced block if present
+        fence_blocks = re.findall(r"```[a-zA-Z0-9_+-]*\n([\s\S]*?)\n```", code)
+        if fence_blocks:
+            code = fence_blocks[0].strip()
+
+    # Remove leading decorators (keep them for regex but allow preceding lines)
+    # Robust regex: allow optional decorators and whitespace before the def
+    pattern = r"^\s*(?:@[\w\.\n+() ,]*\n\s*)*(?:async\s+)?def\s+([A-Za-z_]\w*)\s*\("
+    match = re.search(pattern, code, flags=re.MULTILINE)
     if match:
         return match.group(1)
-    
+
+    # Fallback: search anywhere (not anchored) for a def signature
+    fallback = r"(?:async\s+)?def\s+([A-Za-z_]\w*)\s*\("
+    match = re.search(fallback, code)
+    if match:
+        return match.group(1)
+
     return None
 
 
 def convert_verifier_string(verifier_str: str) -> str:
     """
-    Convert a verifier function string from the old format (env: Environment) 
+    Convert a verifier function string from the old format (env: Environment)
     to the new format (before: DatabaseSnapshot, after: DatabaseSnapshot).
-    
+
     Args:
         verifier_str: The original verifier function as a string
-        
+
     Returns:
         The converted verifier function string
     """
     # First, handle escaped newlines in the input
-    verifier_str = verifier_str.replace('\\n', '\n')
-    
+    verifier_str = verifier_str.replace("\\n", "\n")
+
     # Extract function name, docstring, and body
     # More flexible pattern that accepts both int and float return types
     func_pattern = r'def\s+(\w+)\s*\(\s*env(?:\s*:\s*Environment)?\s*,?\s*final_answer(?:\s*:\s*str\s*\|\s*None)?\s*(?:=\s*None)?\s*\)\s*->\s*(?:float|int):\s*\n((?:\s*""".*?"""\s*\n)?)(.*)'
     match = re.match(func_pattern, verifier_str.strip(), re.DOTALL)
-    
+
     if not match:
         # Try with multiline pattern
         func_pattern_multiline = r'def\s+(\w+)\s*\(\s*\n?\s*env(?:\s*:\s*Environment)?\s*,?\s*\n?\s*final_answer(?:\s*:\s*str\s*\|\s*None)?\s*(?:=\s*None)?\s*\n?\s*\)\s*->\s*(?:float|int):\s*\n((?:\s*""".*?"""\s*\n)?)(.*)'
         match = re.match(func_pattern_multiline, verifier_str.strip(), re.DOTALL)
-        
+
         if not match:
-            raise ValueError("Could not parse verifier function. Expected format: def function_name(env: Environment, final_answer: str | None = None) -> float/int:")
-    
+            raise ValueError(
+                "Could not parse verifier function. Expected format: def function_name(env: Environment, final_answer: str | None = None) -> float/int:"
+            )
+
     func_name = match.group(1)
     docstring = match.group(2).strip()
     body = match.group(3)
-    
+
     # Find all unique env.db() calls
     db_calls = re.findall(r'env\.db\("(\w+)"\)', body)
-    unique_db_names = list(dict.fromkeys(db_calls))  # Remove duplicates while preserving order
-    
+    unique_db_names = list(
+        dict.fromkeys(db_calls)
+    )  # Remove duplicates while preserving order
+
     # Build the new function
-    new_func = f'''def {func_name}(
+    new_func = f"""def {func_name}(
     before: DatabaseSnapshot, after: DatabaseSnapshot, transcript: str | None = None
 ) -> int:
     class Environment:
-        def db(self, name: str) -> DatabaseSnapshot:'''
-    
+        def db(self, name: str) -> DatabaseSnapshot:"""
+
     # Build the db method based on found database names
     if unique_db_names:
         conditions = []
@@ -73,28 +91,32 @@ def convert_verifier_string(verifier_str: str) -> str:
             if db_name == "seed":
                 conditions.append('before if name == "seed"')
             elif db_name == "current":
-                conditions.append('after')
+                conditions.append("after")
             else:
                 # Handle other database names if needed
                 conditions.append(f'None  # Handle "{db_name}"')
-        
-        if len(conditions) == 2 and "seed" in unique_db_names and "current" in unique_db_names:
-            new_func += f'''
-            return before if name == "seed" else after'''
+
+        if (
+            len(conditions) == 2
+            and "seed" in unique_db_names
+            and "current" in unique_db_names
+        ):
+            new_func += f"""
+            return before if name == "seed" else after"""
         else:
             # More complex mapping if needed
-            new_func += f'''
+            new_func += f"""
             if name == "seed":
                 return before
             elif name == "current":
                 return after
             else:
-                raise ValueError(f"Unknown database name: {{name}}")'''
+                raise ValueError(f"Unknown database name: {{name}}")"""
     else:
-        new_func += '''
-            return before if name == "seed" else after'''
-    
-    new_func += '''
+        new_func += """
+            return before if name == "seed" else after"""
+
+    new_func += """
 
         @property
         def instance(self):
@@ -103,42 +125,44 @@ def convert_verifier_string(verifier_str: str) -> str:
         def load(self):
             pass
 
-    def verifier(env: Environment, final_answer: str | None = None) -> float:'''
-    
+    def verifier(env: Environment, final_answer: str | None = None) -> float:"""
+
     if docstring:
-        new_func += f'\n        {docstring}'
-    
+        new_func += f"\n        {docstring}"
+
     # First, find the minimum indentation in the body (excluding empty lines)
     body_lines = body.splitlines()
-    min_indent = float('inf')
+    min_indent = float("inf")
     for line in body_lines:
         if line.strip():  # Non-empty line
             indent_len = len(line) - len(line.lstrip())
             min_indent = min(min_indent, indent_len)
-    
+
     # If we didn't find any non-empty lines, set min_indent to 0
-    if min_indent == float('inf'):
+    if min_indent == float("inf"):
         min_indent = 0
-    
+
     # Now strip the minimum indentation and re-indent to 8 spaces
     if body_lines:
         indented_lines = []
         for line in body_lines:
             if line.strip():  # Non-empty line
                 # Remove the minimum indentation and add 8 spaces
-                stripped_line = line[min_indent:] if len(line) > min_indent else line.lstrip()
-                indented_lines.append('        ' + stripped_line)
+                stripped_line = (
+                    line[min_indent:] if len(line) > min_indent else line.lstrip()
+                )
+                indented_lines.append("        " + stripped_line)
             else:  # Empty line
-                indented_lines.append('')
-        
-        indented_body = '\n'.join(indented_lines)
-        new_func += f'\n{indented_body}'
-    
+                indented_lines.append("")
+
+        indented_body = "\n".join(indented_lines)
+        new_func += f"\n{indented_body}"
+
     # Add the return statement
-    new_func += '\n\n    return verifier(Environment(), transcript)'
+    new_func += "\n\n    return verifier(Environment(), transcript)"
 
     # Replace TASK_FAILED_SCORE with 0 in the function string
-    new_func = new_func.replace('TASK_FAILED_SCORE', '0')
+    new_func = new_func.replace("TASK_FAILED_SCORE", "0")
 
     return new_func
 
@@ -147,39 +171,45 @@ def convert_new_to_old_verifier(verifier_str: str) -> str:
     """
     Convert a verifier function from the new format (before/after: DatabaseSnapshot)
     to the old format (env: Environment).
-    
+
     This is the inverse of convert_verifier_string.
-    
+
     Args:
         verifier_str: The new format verifier function as a string
-        
+
     Returns:
         The converted verifier function string that accepts env
-    """   
+    """
     # Extract function name, parameters, docstring, and body
     # Pattern for new format with flexible whitespace and multiline support
     func_pattern = r'def\s+(\w+)\s*\(\s*before\s*:\s*DatabaseSnapshot\s*,?\s*after\s*:\s*DatabaseSnapshot\s*,?\s*transcript\s*:\s*str\s*\|\s*None\s*=\s*None\s*,?\s*\)\s*->\s*int:\s*((?:\s*""".*?"""\s*)?)(.*)'
-    
+
     # Try multiline pattern that's more flexible
     func_pattern_multiline = r'def\s+(\w+)\s*\(\s*\n?\s*before\s*:\s*DatabaseSnapshot\s*,?\s*\n?\s*after\s*:\s*DatabaseSnapshot\s*,?\s*\n?\s*transcript\s*:\s*str\s*\|\s*None\s*=\s*None\s*,?\s*\n?\s*\)\s*->\s*int:\s*\n?((?:\s*""".*?"""\s*)?)(.*)'
-    
-    match = re.match(func_pattern_multiline, verifier_str.strip(), re.DOTALL | re.MULTILINE)
-    
+
+    match = re.match(
+        func_pattern_multiline, verifier_str.strip(), re.DOTALL | re.MULTILINE
+    )
+
     if not match:
         # Even more flexible pattern
-        func_pattern_flexible = r'def\s+(\w+)\s*\([^)]*\)\s*->\s*int:\s*\n?((?:\s*""".*?"""\s*)?)(.*)'
+        func_pattern_flexible = (
+            r'def\s+(\w+)\s*\([^)]*\)\s*->\s*int:\s*\n?((?:\s*""".*?"""\s*)?)(.*)'
+        )
         match = re.match(func_pattern_flexible, verifier_str.strip(), re.DOTALL)
-        
+
         if not match:
             raise ValueError("Could not parse new format verifier function")
-    
+
     func_name = match.group(1)
     docstring = match.group(2).strip()
     body = match.group(3)
-    
+
     # Indent the original function body
-    indented_verifier = '\n'.join('    ' + line if line.strip() else line for line in verifier_str.splitlines())
-    
+    indented_verifier = "\n".join(
+        "    " + line if line.strip() else line for line in verifier_str.splitlines()
+    )
+
     # Build the wrapper function
     wrapper_func = f'''def {func_name}_wrapper(env, *args, **kwargs) -> float:
     """Wrapper to adapt new format verifier to old format."""
@@ -203,5 +233,5 @@ def convert_new_to_old_verifier(verifier_str: str) -> str:
     # Call the inner function and convert result
     result = {func_name}(before, after, transcript)
     return float(result)'''
-    
-    return wrapper_func
+
+    return wrapper_func

fleet/verifiers/verifier.py

@@ -19,7 +19,7 @@ from ..client import SyncEnv
 
 logger = logging.getLogger(__name__)
 
-F = TypeVar('F', bound=Callable[..., Any])
+F = TypeVar("F", bound=Callable[..., Any])
 
 # Global cache to track which bundle SHAs have been uploaded to S3
 _uploaded_bundle_shas: Set[str] = set()
@@ -33,7 +33,7 @@ def _get_bundle_sha(bundle_data: bytes) -> str:
 
 class SyncVerifierFunction:
     """Wrapper for a verified function that supports local execution with env-first pattern."""
-    
+
     def __init__(
         self,
         func: F,
@@ -41,7 +41,7 @@ class SyncVerifierFunction:
         extra_requirements: Optional[List[str]] = None,
         verifier_id: Optional[str] = None,
         sha256: Optional[str] = None,
-        raw_code: Optional[str] = None
+        raw_code: Optional[str] = None,
     ):
         self.func = func
         self.key = key
@@ -52,10 +52,10 @@ class SyncVerifierFunction:
         self._bundle_data: Optional[bytes] = None  # Cached bundle data
         self._raw_code: Optional[str] = raw_code  # Store raw code if provided
         self._is_async = inspect.iscoroutinefunction(func)
-        
+
         # Copy function metadata
         functools.update_wrapper(self, func)
-    
+
     def _get_or_create_bundle(self) -> tuple[bytes, str]:
         """Get or create bundle data and return (bundle_data, sha)."""
         if self._bundle_data is None or self._bundle_sha is None:
@@ -63,68 +63,72 @@ class SyncVerifierFunction:
             if self._raw_code:
                 import io
                 import zipfile
-                
+
                 # Create zip bundle directly (matching bundler format)
                 zip_buffer = io.BytesIO()
-                with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zf:
+                with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf:
                     # Add requirements.txt
                     requirements = self.extra_requirements or []
                     if "fleet-python" not in requirements:
                         requirements.append("fleet-python")
                     req_content = "\n".join(requirements)
                     zf.writestr("requirements.txt", req_content)
-                    
+
                     # Add verifier.py with the raw code
                     zf.writestr("verifier.py", self._raw_code)
-                
+
                 self._bundle_data = zip_buffer.getvalue()
                 self._bundle_sha = _get_bundle_sha(self._bundle_data)
-                logger.debug(f"Created bundle from raw code for {self.key} with SHA: {self._bundle_sha}")
+                logger.debug(
+                    f"Created bundle from raw code for {self.key} with SHA: {self._bundle_sha}"
+                )
             else:
                 # Try to create bundle from function source
                 try:
                     self._bundle_data = self._bundler.create_bundle(
-                        self.func, 
-                        self.extra_requirements,
-                        self.verifier_id
+                        self.func, self.extra_requirements, self.verifier_id
                     )
                     self._bundle_sha = _get_bundle_sha(self._bundle_data)
-                    logger.debug(f"Created bundle for {self.key} with SHA: {self._bundle_sha}")
+                    logger.debug(
+                        f"Created bundle for {self.key} with SHA: {self._bundle_sha}"
+                    )
                 except OSError as e:
                     # Can't create bundle - no source and no raw code
                     raise OSError(f"Cannot create bundle for {self.key}: {e}")
-        
+
         return self._bundle_data, self._bundle_sha
-    
+
     def _check_bundle_status(self, env: SyncEnv) -> tuple[str, bool]:
         """Check if bundle needs to be uploaded and return (sha, needs_upload)."""
         bundle_data, bundle_sha = self._get_or_create_bundle()
-        
+
         # If bundle_data is empty, we're using server-side bundle
         if not bundle_data:
             logger.debug(f"Using server-side bundle {bundle_sha[:8]}...")
             return bundle_sha, False  # No upload needed, server has it
-        
+
         # 1. Check local process cache first
         if bundle_sha in _uploaded_bundle_shas:
             logger.debug(f"Bundle {bundle_sha[:8]}... found in local cache")
             return bundle_sha, False  # Already uploaded, no upload needed
-        
+
         # 2. Check if bundle exists on server (pseudocode)
         # TODO: Add endpoint to check if bundle SHA exists in S3
         try:
             exists = env.check_bundle_exists(bundle_sha)
             if exists.success:
-                logger.info(f"Bundle {bundle_sha[:8]}... found on server, updating cache")
+                logger.info(
+                    f"Bundle {bundle_sha[:8]}... found on server, updating cache"
+                )
                 _uploaded_bundle_shas.add(bundle_sha)
                 return bundle_sha, False  # Found on server, no upload needed
         except Exception as e:
             logger.warning(f"Failed to check bundle existence: {e}")
-        
+
         # 3. Bundle not found locally or on server - upload needed
         logger.info(f"Bundle {bundle_sha[:8]}... needs to be uploaded")
         return bundle_sha, True  # Upload needed
-    
+
     def __call__(self, env: SyncEnv, *args, **kwargs) -> float:
         """Local execution of the verifier function with env as first parameter."""
         try:
@@ -134,7 +138,7 @@ class SyncVerifierFunction:
             else:
                 # For sync functions, call directly
                 result = self.func(env, *args, **kwargs)
-            
+
             # Handle different return types
             if isinstance(result, (int, float)):
                 # Direct score return
@@ -144,39 +148,34 @@ class SyncVerifierFunction:
                 return result
             else:
                 # Try to extract score from object attributes
-                if hasattr(result, 'score'):
+                if hasattr(result, "score"):
                     return float(result.score)
                 else:
-                    raise ValueError(f"Verifier function must return a score (number). Got {type(result)}")
-                    
+                    raise ValueError(
+                        f"Verifier function must return a score (number). Got {type(result)}"
+                    )
+
         except Exception as e:
             logger.error(f"Error in verifier {self.key}: {e}")
             # Return error score 0
             return 0.0
-    
+
     def remote(self, env: SyncEnv, *args, **kwargs) -> float:
         """Remote execution of the verifier function with SHA-based bundle caching."""
-        # Async verifiers are now supported by the backend
-        # if self._is_async:
-        #     raise NotImplementedError(
-        #         f"Async verifier '{self.key}' cannot be executed remotely. "
-        #         "The remote execution environment only supports synchronous functions. "
-        #         "Please provide a synchronous version of your verifier."
-        #     )
-        
+
         args_array = list(args)
         args_array.append({"env": env.instance_id})
         args = tuple(args_array)
-        
+
         try:
             # Check if bundle needs to be uploaded
             bundle_sha, needs_upload = self._check_bundle_status(env)
-            
+
             if needs_upload:
                 # Need to upload bundle to S3
                 logger.info(f"Uploading bundle {bundle_sha[:8]}... for {self.key}")
                 bundle_data, _ = self._get_or_create_bundle()
-                
+
                 response = env.execute_verifier_remote(
                     bundle_data=bundle_data,
                     bundle_sha=bundle_sha,
@@ -185,42 +184,46 @@ class SyncVerifierFunction:
                     args=args,
                     args_array=args_array,
                     kwargs=kwargs,
-                    needs_upload=True
+                    needs_upload=True,
                 )
-                
+
                 # Mark as uploaded after successful execution
                 _uploaded_bundle_shas.add(bundle_sha)
                 logger.debug(f"Registered bundle {bundle_sha[:8]}... as uploaded")
-                
+
             else:
                 # Bundle already available - execute without upload
-                logger.info(f"Executing cached bundle {bundle_sha[:8]}... for {self.key}")
+                logger.info(
+                    f"Executing cached bundle {bundle_sha[:8]}... for {self.key}"
+                )
                 bundle_data, _ = self._get_or_create_bundle()
-                
+
                 response = env.execute_verifier_remote(
-                    bundle_data=bundle_data or b'',  # Empty if using server-side bundle
+                    bundle_data=bundle_data or b"",  # Empty if using server-side bundle
                     bundle_sha=bundle_sha,
                     key=self.key,
                     function_name=self.func.__name__,
                     args=args,
                     args_array=args_array,
                     kwargs=kwargs,
-                    needs_upload=False  # Don't upload, just execute
+                    needs_upload=False,  # Don't upload, just execute
                 )
-            
+
             # Handle response
+            if response.stdout:
+                print(response.stdout)
             if response.success:
                 return self._process_result(response.result)
             else:
                 self._raise_remote_error(response.error)
-                
+
         except Exception as e:
             logger.error(f"Remote execution failed for {self.key}: {e}")
             # If it's an HTTP error, try to get more details
-            if hasattr(e, 'response') and hasattr(e.response, 'text'):
+            if hasattr(e, "response") and hasattr(e.response, "text"):
                 logger.error(f"Server response: {e.response.text}")
             raise
-    
+
     def _process_result(self, result: Any) -> float:
         """Process remote execution result, handling different return types."""
         # Handle different return types like local execution
@@ -230,7 +233,7 @@ class SyncVerifierFunction:
             return float(result["score"])
         else:
             # Try to extract score from object attributes
-            if hasattr(result, 'score'):
+            if hasattr(result, "score"):
                 return float(result.score)
             else:
                 # Best effort conversion
@@ -239,13 +242,13 @@ class SyncVerifierFunction:
                 except (ValueError, TypeError):
                     logger.warning(f"Could not convert result to float: {result}")
                     return 0.0
-    
+
     def _raise_remote_error(self, error_info: Dict[str, Any]):
         """Reconstruct remote error as local exception."""
         error_type = error_info.get("type", "RuntimeError")
         message = error_info.get("message", "Remote execution failed")
         traceback_str = error_info.get("traceback", "")
-        
+
         # Create a rich error message
         full_message = f"""
 Remote verifier execution failed:
@@ -254,32 +257,32 @@ Remote verifier execution failed:
 Remote traceback:
 {traceback_str}
         """.strip()
-        
+
         # Try to raise the original exception type
         try:
             exception_class = getattr(__builtins__, error_type, RuntimeError)
             raise exception_class(full_message)
         except:
             raise RuntimeError(full_message)
-    
+
     def _get_env_id(self, env: SyncEnv) -> str:
         """Generate a unique identifier for the environment."""
         # Use instance base URL or similar unique identifier
-        if hasattr(env, 'instance') and hasattr(env.instance, 'base_url'):
+        if hasattr(env, "instance") and hasattr(env.instance, "base_url"):
             return f"{env.instance.base_url}"
         else:
             # Fallback to object id (less ideal but works)
             return str(id(env))
-    
+
     def _is_bundle_not_found_error(self, error: Exception) -> bool:
         """Check if the error indicates the bundle was not found on the server."""
         # Check for common "bundle not found" error patterns
         error_msg = str(error).lower()
         return (
-            "bundle not found" in error_msg or
-            "verifier not found" in error_msg or
-            "404" in error_msg or
-            "not found" in error_msg
+            "bundle not found" in error_msg
+            or "verifier not found" in error_msg
+            or "404" in error_msg
+            or "not found" in error_msg
         )
 
 
@@ -287,21 +290,21 @@ def verifier(
     key: Optional[str] = None,
     extra_requirements: Optional[List[str]] = None,
     sha256: Optional[str] = None,
-    raw_code: Optional[str] = None
+    raw_code: Optional[str] = None,
 ) -> Callable[[F], SyncVerifierFunction]:
     """
     Decorator to create a verifier function with env-first pattern.
-    
+
     The decorated function must take 'env' as its first parameter, making it explicit
     that verifiers operate within an environment context. This makes verifiers reusable
     across different environments.
-    
+
     Args:
         key: Optional key for the verifier. Defaults to function name.
         extra_requirements: Additional PyPI packages needed by the verifier.
         sha256: Optional SHA256 hash of existing server-side bundle to use.
         raw_code: Optional raw code to use as bundle (bypasses source extraction).
-    
+
     Example:
         # Synchronous verifier (works locally and remotely)
         @verifier(key="check_user_count")
@@ -310,7 +313,7 @@ def verifier(
             result = db.query("SELECT COUNT(*) FROM users")
             actual_count = result.rows[0][0]
             return 1.0 if actual_count >= expected_count else 0.0
-        
+
         # Async verifier (only works locally)
         @verifier(key="check_user_async")
         async def check_user_async(env, expected_count: int) -> float:
@@ -318,29 +321,25 @@ def verifier(
             result = await db.query("SELECT COUNT(*) FROM users")
             actual_count = result.rows[0][0]
             return 1.0 if actual_count >= expected_count else 0.0
-        
+
         # Usage
-        env = await flt.env.make_async("fira")
-        
+        env = await fleet.env.make_async("fira")
+
         # Local execution
         result = await check_user_count(env, 5)        # sync verifier
         result = await check_user_async(env, 5)       # async verifier
-        
+
         # Remote execution
         result = await check_user_count.remote(env, 5) # sync verifier works
         # await check_user_async.remote(env, 5)        # raises NotImplementedError
     """
+
     def decorator(func: F) -> SyncVerifierFunction:
         verifier_key = key or func.__name__
         verifier_uuid = str(uuid.uuid4())
-        
+
         return SyncVerifierFunction(
-            func,
-            verifier_key,
-            extra_requirements,
-            verifier_uuid,
-            sha256,
-            raw_code
+            func, verifier_key, extra_requirements, verifier_uuid, sha256, raw_code
         )
-    
-    return decorator
+
+    return decorator