fleet-python 0.2.25__py3-none-any.whl → 0.2.27__py3-none-any.whl

examples/example_task.py

@@ -4,8 +4,7 @@
 This example shows how to create a simple task with the @verifier decorator
 that can be verified in a Jira environment.
 
-Note: The remote execution environment only supports synchronous verifiers.
-For async environments, we need separate sync and async verifiers.
+Both sync and async verifiers are now supported for remote execution.
 """
 
 import os
@@ -97,7 +96,7 @@ async def create_bug_issue_async(
 async def main():
     """Run the task example."""
     print("=== Fleet Task Example with Jira ===\n")
-    print("Note: Remote execution only supports synchronous verifiers.\n")
+    print("Note: Both sync and async verifiers are now supported for remote execution.\n")
 
     # Create task using the async verifier for local execution
     task = Task(
@@ -151,12 +150,17 @@ async def main():
             print(f"  ✗ Remote execution failed: {e}")
         print()
 
-        # Demonstrate that async verifiers can't run remotely
-        print("Attempting remote execution with async verifier...")
+        # Test async verifier remote execution
+        print("Testing remote execution with async verifier...")
         try:
-            await create_bug_issue_async.remote(env)
+            result = await create_bug_issue_async.remote(env, project_key="SCRUM", issue_title="Login button not working")
+            print(f"  ✓ Async remote check result: {result}")
         except NotImplementedError as e:
             print(f"  ✓ Expected error: {e}")
+        except Exception as e:
+            print(f"  ✗ Unexpected error: {e}")
+            import traceback
+            traceback.print_exc()
         print()
 
         # Create the issue

examples/example_tasks.py

@@ -0,0 +1,34 @@
+import fleet as flt
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+client = flt.Fleet()
+
+
+def main():
+    env = flt.env.make("fira")
+
+    tasks = client.load_tasks(env_key="fira")
+    print(f"Loaded {len(tasks)} tasks")
+
+    for i, task in enumerate(tasks):
+        print(f"\nTask {i + 1}:")
+        print(f"  Key: {task.key}")
+        print(f"  Prompt: {task.prompt[:80]}...")
+        print(f"  Verifier: {task.verifier_func[:80]}...")
+
+        print(f"  Verifier: {task.verifier.key}")
+        print("  Running verifier...")
+        try:
+            score = task.verify(env)
+            print(f"  ✓ Score: {score}")
+        except Exception as e:
+            print(f"  ✗ Error: {type(e).__name__}: {e}")
+
+        print("-" * 60)
+
+
+if __name__ == "__main__":
+    main()

fleet/_async/client.py

@@ -17,9 +17,10 @@
 import base64
 import cloudpickle
 import httpx
+import json
 import logging
 import os
-from typing import List, Optional, Dict
+from typing import List, Optional, Dict, TYPE_CHECKING
 
 from .base import EnvironmentBase, AsyncWrapper
 from ..models import (
@@ -30,8 +31,13 @@ from ..models import (
     VerifiersExecuteResponse,
     TaskListResponse,
     AccountResponse,
+    TaskRequest,
 )
 from .tasks import Task
+from ..verifiers.parse import extract_function_name, convert_new_to_old_verifier
+
+if TYPE_CHECKING:
+    from .verifiers import AsyncVerifierFunction
 
 from .instance import (
     AsyncInstanceClient,
@@ -260,18 +266,112 @@ class AsyncFleet:
         # Transform TaskResponse objects to Task objects
         tasks = []
         for task_response in task_list_response.tasks:
+            # Create verifier function if verifier data is present
+            verifier = None
+            verifier_func = task_response.verifier_func
+            
+            if task_response.verifier:
+                # Use verifier code from the embedded verifier object
+                verifier_func = task_response.verifier.code
+                
+                # Create VerifierFunction from the embedded data
+                try:
+                    verifier = await self._create_verifier_from_data(
+                        verifier_id=task_response.verifier.verifier_id,
+                        verifier_key=task_response.verifier.key,
+                        verifier_code=task_response.verifier.code,
+                        verifier_sha=task_response.verifier.sha256
+                    )
+                except Exception as e:
+                    logger.warning(f"Failed to create verifier {task_response.verifier.key}: {e}")
+            
             task = Task(
                 key=task_response.key,
                 prompt=task_response.prompt,
                 env_id=task_response.environment_id,  # Map environment_id -> env_id
                 created_at=task_response.created_at,
-                verifier=None,  # Keep blank for now as requested
+                version=task_response.version,
+                env_variables=task_response.env_variables or {},
+                verifier_func=verifier_func,  # Set verifier code
+                verifier=verifier,  # Use created verifier or None
                 metadata={}  # Default empty metadata
             )
             tasks.append(task)
         
         return tasks
 
+    async def export_tasks(self, env_key: Optional[str] = None, filename: Optional[str] = None):
+        """Export tasks for the authenticated team, optionally filtered by environment.
+        
+        Args:
+            env_key: Optional environment key to filter tasks by
+            filename: Optional filename to write tasks to. If not provided, defaults to 'tasks.json' or 'tasks_{env_key}.json'
+            
+        Returns:
+            str: Path to the exported file if tasks were written, None if no tasks found
+        """
+        tasks = await self.load_tasks(env_key)
+        if tasks:
+            # Generate filename if not provided
+            if filename is None:
+                if env_key:
+                    filename = f"tasks_{env_key}.json"
+                else:
+                    filename = "tasks.json"
+            
+            # Convert tasks to serializable format
+            tasks_data = []
+            for task in tasks:
+                task_dict = task.model_dump()
+                # Remove non-serializable verifier object, keep verifier_func (code string)
+                if 'verifier' in task_dict:
+                    task_dict.pop('verifier')
+                tasks_data.append(task_dict)
+            
+            # Write to JSON file
+            with open(filename, 'w', encoding='utf-8') as f:
+                json.dump(tasks_data, f, indent=2, default=str)
+            
+            logger.info(f"Exported {len(tasks)} tasks to {filename}")
+            return filename
+        else:
+            logger.info("No tasks found to export")
+            return None
+            
+    async def import_tasks(self, filename: str):
+        """Import tasks from a JSON file.
+        
+        Args:
+            filename: Path to the JSON file of Task objects to import
+            
+        Returns:
+            List[Task] containing imported Task objects
+        """
+        with open(filename, 'r', encoding='utf-8') as f:
+            tasks_data = json.load(f)
+            
+        # Create tasks from the loaded data
+        tasks = []
+        for task_data in tasks_data:
+            task = Task(**task_data)
+            tasks.append(task)
+
+        for task in tasks:
+            payload = TaskRequest(
+                key=task.key,
+                prompt=task.prompt,
+                environment_id=task.env_id,
+                verifier_func=task.verifier_func,
+                version=task.version or None,
+                env_variables=task.env_variables or {},
+            )
+            try:
+                response = await self.client.request("POST", "/v1/tasks", json=payload.model_dump())
+            except Exception as e:
+                logger.error(f"Failed to import task {task.key}: {e}")
+                continue
+
+
     async def account(self) -> AccountResponse:
         """Get account information including instance limits and usage.
 
@@ -280,6 +380,103 @@ class AsyncFleet:
         """
         response = await self.client.request("GET", "/v1/account")
         return AccountResponse(**response.json())
+    
+    async def _create_verifier_from_data(
+        self, 
+        verifier_id: str,
+        verifier_key: str,
+        verifier_code: str,
+        verifier_sha: str
+    ) -> "AsyncVerifierFunction":
+        """Create an AsyncVerifierFunction from verifier data.
+        
+        Args:
+            verifier_id: The verifier ID
+            verifier_key: The verifier key
+            verifier_code: The verifier code
+            verifier_sha: The verifier SHA256
+            
+        Returns:
+            AsyncVerifierFunction created from the verifier code
+        """
+        from .verifiers.verifier import AsyncVerifierFunction
+        
+        # Check if this is a new format verifier (has before/after parameters)
+        if 'before: DatabaseSnapshot' in verifier_code and 'after: DatabaseSnapshot' in verifier_code:
+            # Convert new format to old format
+            verifier_code = convert_new_to_old_verifier(verifier_code)
+            # Update function name since wrapper adds _wrapper suffix
+            original_name = extract_function_name(verifier_code.split('\n')[0])
+            if original_name and original_name.endswith('_wrapper'):
+                function_name = original_name
+            else:
+                function_name = extract_function_name(verifier_code)
+        else:
+            # Extract function name from code
+            function_name = extract_function_name(verifier_code)
+        
+        if not function_name:
+            raise ValueError(f"Could not extract function name from verifier {verifier_id}")
+        
+        # Create a function object from the code
+        # Import necessary classes for the namespace
+        from ..verifiers.db import IgnoreConfig, DatabaseSnapshot
+        
+        # Create a namespace for the function
+        namespace = {
+            '__builtins__': __builtins__,
+            'Environment': object,  # Placeholder, will be provided at runtime
+            'IgnoreConfig': IgnoreConfig,
+            'DatabaseSnapshot': DatabaseSnapshot,
+            'TASK_FAILED_SCORE': 0,
+            'TASK_SUCCESSFUL_SCORE': 1,
+        }
+        
+        # Execute the code to define the function
+        exec(verifier_code, namespace)
+        
+        # Get the function object
+        if function_name not in namespace:
+            raise ValueError(f"Function {function_name} not found in verifier code")
+        
+        func = namespace[function_name]
+        
+        # Create and return AsyncVerifierFunction with SHA if available
+        # Since the function was created via exec, we need to provide the raw code
+        verifier_func = AsyncVerifierFunction(
+            func=func,
+            key=verifier_key,
+            extra_requirements=[],
+            verifier_id=verifier_id,
+            sha256=verifier_sha,  # Pass SHA directly to constructor
+            raw_code=verifier_code  # Provide raw code since func won't have extractable source
+        )
+        
+        # Store the original verifier code for reference
+        verifier_func._verifier_code = verifier_code
+        
+        return verifier_func
+    
+    async def _load_verifier(self, verifier_id: str) -> "AsyncVerifierFunction":
+        """Load a verifier by ID and create an AsyncVerifierFunction.
+        
+        Args:
+            verifier_id: The verifier ID to fetch
+            
+        Returns:
+            AsyncVerifierFunction created from the verifier code
+        """
+        # Fetch verifier from API
+        response = await self.client.request("GET", f"/v1/verifier/{verifier_id}")
+        verifier_data = response.json()
+        
+        # Use the common method to create verifier
+        return await self._create_verifier_from_data(
+            verifier_id=verifier_id,
+            verifier_key=verifier_data["key"],
+            verifier_code=verifier_data["code"],
+            verifier_sha=verifier_data.get("sha256", "")
+        )
 
 
 # Shared

fleet/_async/models.py

@@ -152,6 +152,18 @@ class TaskRequest(BaseModel):
     prompt: str = Field(..., title='Prompt')
     environment_id: str = Field(..., title='Environment Id')
     verifier_id: Optional[str] = Field(None, title='Verifier Id')
+    version: Optional[str] = Field(None, title='Version')
+    env_variables: Optional[Dict[str, Any]] = Field(None, title='Env Variables')
+
+
+class VerifierData(BaseModel):
+    verifier_id: str = Field(..., title='Verifier Id')
+    key: str = Field(..., title='Key')
+    version: int = Field(..., title='Version')
+    sha256: str = Field(..., title='Sha256')
+    code: str = Field(..., title='Code')
+    comment: Optional[str] = Field(None, title='Comment')
+    created_at: str = Field(..., title='Created At')
 
 
 class TaskResponse(BaseModel):
@@ -161,6 +173,10 @@ class TaskResponse(BaseModel):
     environment_id: str = Field(..., title='Environment Id')
     created_at: str = Field(..., title='Created At')
     verifier_id: Optional[str] = Field(None, title='Verifier Id')
+    verifier_func: Optional[str] = Field(None, title='Verifier Func')
+    version: Optional[str] = Field(None, title='Version')
+    env_variables: Optional[Dict[str, Any]] = Field(None, title='Env Variables')
+    verifier: Optional[VerifierData] = Field(None, title='Verifier')
 
 
 class ValidationError(BaseModel):

fleet/_async/tasks.py

@@ -19,7 +19,10 @@ class Task(BaseModel):
     key: str = Field(..., description="Unique task key identifier")
     prompt: str = Field(..., description="Task prompt or instruction")
     env_id: str = Field(..., description="Environment identifier")
+    env_variables: Optional[Dict[str, Any]] = Field(default_factory=dict, description="Environment variables")
     created_at: Optional[datetime] = Field(None, description="Task creation timestamp")
+    version: Optional[str] = Field(None, description="Task version")
+    verifier_func: Optional[str] = Field(None, description="Verifier function code")
     verifier: Optional[Any] = Field(None, description="Verifier function with decorator (async or sync)")
     metadata: Optional[Dict[str, Any]] = Field(default_factory=dict, description="Additional task metadata")
 
@@ -34,6 +37,13 @@ class Task(BaseModel):
     def set_created_at(cls, v):
         """Set created_at to current time if not provided."""
         return v or datetime.now()
+    
+    @property
+    def env_key(self) -> str:
+        """Get the environment key combining env_id and version."""
+        if self.version:
+            return f"{self.env_id}:{self.version}"
+        return self.env_id
 
     class Config:
         """Pydantic model configuration."""
@@ -41,4 +51,51 @@ class Task(BaseModel):
             datetime: lambda v: v.isoformat(),
         }
         # Allow arbitrary types for the verifier field
-        arbitrary_types_allowed = True 
+        arbitrary_types_allowed = True 
+
+    def verify(self, env, *args, **kwargs) -> float:
+        """Verify the task using the verifier function (sync version).
+        
+        For sync environments, calls the sync verifier directly.
+        For async verifiers, automatically runs them with asyncio.run().
+        """
+        if self.verifier:
+            import asyncio
+            import inspect
+            
+            result = self.verifier.remote(env, *args, **kwargs)
+            
+            # If the result is a coroutine, we need to run it
+            if inspect.iscoroutine(result):
+                # Check if we're already in an event loop
+                try:
+                    loop = asyncio.get_running_loop()
+                    # We're in an async context, can't use asyncio.run()
+                    raise RuntimeError(
+                        "Cannot run async verifier in sync mode while event loop is running. "
+                        "Use await task.verify_async() instead."
+                    )
+                except RuntimeError:
+                    # No event loop running, safe to use asyncio.run()
+                    return asyncio.run(result)
+            else:
+                return result
+        else:
+            raise ValueError("No verifier function found for this task")
+    
+    async def verify_async(self, *args, **kwargs) -> float:
+        """Verify the task using the verifier function (async version).
+        
+        For async environments, awaits the async verifier.
+        Works with both sync and async verifiers in async contexts.
+        """
+        if self.verifier:
+            result = self.verifier.remote(*args, **kwargs)
+            # If it's a coroutine, await it
+            import inspect
+            if inspect.iscoroutine(result):
+                return await result
+            else:
+                return result
+        else:
+            raise ValueError("No verifier function found for this task")

fleet/_async/verifiers/__init__.py

@@ -1,7 +1,7 @@
 """Fleet verifiers module - database snapshot validation utilities and verifier decorator."""
 
 from fleet.verifiers.db import DatabaseSnapshot, IgnoreConfig, SnapshotDiff
-from fleet.verifiers.code import TASK_SUCCESSFUL_SCORE
+from fleet.verifiers.code import TASK_SUCCESSFUL_SCORE, TASK_FAILED_SCORE
 from .verifier import (
     verifier,
     AsyncVerifierFunction,

fleet/_async/verifiers/verifier.py

@@ -39,15 +39,18 @@ class AsyncVerifierFunction:
         func: F,
         key: str,
         extra_requirements: Optional[List[str]] = None,
-        verifier_id: Optional[str] = None
+        verifier_id: Optional[str] = None,
+        sha256: Optional[str] = None,
+        raw_code: Optional[str] = None
     ):
         self.func = func
         self.key = key
         self.verifier_id = verifier_id or str(uuid.uuid4())
         self.extra_requirements = extra_requirements or []
         self._bundler = FunctionBundler()
-        self._bundle_sha: Optional[str] = None  # Cached bundle SHA
+        self._bundle_sha: Optional[str] = sha256  # Use provided SHA if available
         self._bundle_data: Optional[bytes] = None  # Cached bundle data
+        self._raw_code: Optional[str] = raw_code  # Store raw code if provided
         self._is_async = asyncio.iscoroutinefunction(func)
         
         # Copy function metadata
@@ -56,14 +59,40 @@ class AsyncVerifierFunction:
     def _get_or_create_bundle(self) -> tuple[bytes, str]:
         """Get or create bundle data and return (bundle_data, sha)."""
         if self._bundle_data is None or self._bundle_sha is None:
-            # Create bundle and cache it
-            self._bundle_data = self._bundler.create_bundle(
-                self.func, 
-                self.extra_requirements,
-                self.verifier_id
-            )
-            self._bundle_sha = _get_bundle_sha(self._bundle_data)
-            logger.debug(f"Created bundle for {self.key} with SHA: {self._bundle_sha}")
+            # If we have raw code, create a bundle from it
+            if self._raw_code:
+                import io
+                import zipfile
+                
+                # Create zip bundle directly (matching bundler format)
+                zip_buffer = io.BytesIO()
+                with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zf:
+                    # Add requirements.txt
+                    requirements = self.extra_requirements or []
+                    if "fleet-python" not in requirements:
+                        requirements.append("fleet-python")
+                    req_content = "\n".join(requirements)
+                    zf.writestr("requirements.txt", req_content)
+                    
+                    # Add verifier.py with the raw code
+                    zf.writestr("verifier.py", self._raw_code)
+                
+                self._bundle_data = zip_buffer.getvalue()
+                self._bundle_sha = _get_bundle_sha(self._bundle_data)
+                logger.debug(f"Created bundle from raw code for {self.key} with SHA: {self._bundle_sha}")
+            else:
+                # Try to create bundle from function source
+                try:
+                    self._bundle_data = self._bundler.create_bundle(
+                        self.func, 
+                        self.extra_requirements,
+                        self.verifier_id
+                    )
+                    self._bundle_sha = _get_bundle_sha(self._bundle_data)
+                    logger.debug(f"Created bundle for {self.key} with SHA: {self._bundle_sha}")
+                except OSError as e:
+                    # Can't create bundle - no source and no raw code
+                    raise OSError(f"Cannot create bundle for {self.key}: {e}")
         
         return self._bundle_data, self._bundle_sha
     
@@ -71,6 +100,11 @@ class AsyncVerifierFunction:
         """Check if bundle needs to be uploaded and return (sha, needs_upload)."""
         bundle_data, bundle_sha = self._get_or_create_bundle()
         
+        # If bundle_data is empty, we're using server-side bundle
+        if not bundle_data:
+            logger.debug(f"Using server-side bundle {bundle_sha[:8]}...")
+            return bundle_sha, False  # No upload needed, server has it
+        
         # 1. Check local process cache first
         if bundle_sha in _uploaded_bundle_shas:
             logger.debug(f"Bundle {bundle_sha[:8]}... found in local cache")
@@ -122,12 +156,13 @@ class AsyncVerifierFunction:
     
     async def remote(self, env: AsyncEnv, *args, **kwargs) -> float:
         """Remote execution of the verifier function with SHA-based bundle caching."""
-        if self._is_async:
-            raise NotImplementedError(
-                f"Async verifier '{self.key}' cannot be executed remotely. "
-                "The remote execution environment only supports synchronous functions. "
-                "Please provide a synchronous version of your verifier."
-            )
+        # Async verifiers are now supported by the backend
+        # if self._is_async:
+        #     raise NotImplementedError(
+        #         f"Async verifier '{self.key}' cannot be executed remotely. "
+        #         "The remote execution environment only supports synchronous functions. "
+        #         "Please provide a synchronous version of your verifier."
+        #     )
         
         args_array = list(args)
         args_array.append({"env": env.instance_id})
@@ -163,11 +198,12 @@ class AsyncVerifierFunction:
                 bundle_data, _ = self._get_or_create_bundle()
                 
                 response = await env.execute_verifier_remote(
-                    bundle_data=bundle_data,  # Still need bundle_data for local caching
+                    bundle_data=bundle_data or b'',  # Empty if using server-side bundle
                     bundle_sha=bundle_sha,
                     key=self.key,
                     function_name=self.func.__name__,
                     args=args,
+                    args_array=args_array,
                     kwargs=kwargs,
                     needs_upload=False  # Don't upload, just execute
                 )
@@ -249,7 +285,9 @@ Remote traceback:
 
 def verifier(
     key: Optional[str] = None,
-    extra_requirements: Optional[List[str]] = None
+    extra_requirements: Optional[List[str]] = None,
+    sha256: Optional[str] = None,
+    raw_code: Optional[str] = None
 ) -> Callable[[F], AsyncVerifierFunction]:
     """
     Decorator to create a verifier function with env-first pattern.
@@ -261,6 +299,8 @@ def verifier(
     Args:
         key: Optional key for the verifier. Defaults to function name.
         extra_requirements: Additional PyPI packages needed by the verifier.
+        sha256: Optional SHA256 hash of existing server-side bundle to use.
+        raw_code: Optional raw code to use as bundle (bypasses source extraction).
     
     Example:
         # Synchronous verifier (works locally and remotely)
@@ -298,7 +338,9 @@ def verifier(
             func,
             verifier_key,
             extra_requirements,
-            verifier_uuid
+            verifier_uuid,
+            sha256,
+            raw_code
         )
     
     return decorator