fleet-python 0.2.43__py3-none-any.whl → 0.2.45__py3-none-any.whl

fleet/client.py

@@ -130,7 +130,7 @@ class SyncEnv(EnvironmentBase):
         return self.instance.verify(validator)
 
     def verify_raw(
-        self, function_code: str, function_name: str | None = None
+        self, function_code: str, function_name: Optional[str] = None
     ) -> ExecuteFunctionResponse:
         return self.instance.verify_raw(function_code, function_name)
 
@@ -302,10 +302,19 @@ class Fleet:
 
         return self.load_task_array_from_string(tasks_data)
 
-    def load_task_array_from_string(self, serialized_tasks: List[Dict]) -> List[Task]:
+    def load_task_array_from_string(self, serialized_tasks: str) -> List[Task]:
         tasks = []
 
-        json_tasks = json.loads(serialized_tasks)
+        parsed_data = json.loads(serialized_tasks)
+        if isinstance(parsed_data, list):
+            json_tasks = parsed_data
+        elif isinstance(parsed_data, dict) and "tasks" in parsed_data:
+            json_tasks = parsed_data["tasks"]
+        else:
+            raise ValueError(
+                "Invalid JSON structure: expected array or object with 'tasks' key"
+            )
+
         for json_task in json_tasks:
             parsed_task = self.load_task_from_json(json_task)
             tasks.append(parsed_task)
@@ -316,25 +325,47 @@ class Fleet:
         return self.load_task_from_json(task_json)
 
     def load_task_from_json(self, task_json: Dict) -> Task:
+        verifier = None
+        verifier_code = task_json.get("verifier_func") or task_json.get("verifier_code")
+
+        # Try to find verifier_id in multiple locations
+        verifier_id = task_json.get("verifier_id")
+        if (
+            not verifier_id
+            and "metadata" in task_json
+            and isinstance(task_json["metadata"], dict)
+        ):
+            verifier_metadata = task_json["metadata"].get("verifier", {})
+            if isinstance(verifier_metadata, dict):
+                verifier_id = verifier_metadata.get("verifier_id")
+
+        # If no verifier_id found, use the task key/id as fallback
+        if not verifier_id:
+            verifier_id = task_json.get("key", task_json.get("id"))
+
         try:
-            if "verifier_id" in task_json and task_json["verifier_id"]:
+            if verifier_id and verifier_code:
                 verifier = self._create_verifier_from_data(
-                    verifier_id=task_json["verifier_id"],
-                    verifier_key=task_json["key"],
-                    verifier_code=task_json["verifier_func"],
+                    verifier_id=verifier_id,
+                    verifier_key=task_json.get("key", task_json.get("id")),
+                    verifier_code=verifier_code,
                     verifier_sha=task_json.get("verifier_sha", ""),
                 )
         except Exception as e:
-            logger.warning(f"Failed to create verifier {task_json['key']}: {e}")
+            logger.warning(
+                f"Failed to create verifier {task_json.get('key', task_json.get('id'))}: {e}"
+            )
 
         task = Task(
-            key=task_json["key"],
+            key=task_json.get("key", task_json.get("id")),
             prompt=task_json["prompt"],
-            env_id=task_json["env_id"],  # Use env_id from the data
-            created_at=task_json["created_at"],
+            env_id=task_json.get(
+                "env_id", task_json.get("env_key")
+            ),  # Use env_id or fallback to env_key
+            created_at=task_json.get("created_at"),
             version=task_json.get("version"),
             env_variables=task_json.get("env_variables", {}),
-            verifier_func=task_json.get("verifier_func"),  # Set verifier code
+            verifier_func=verifier_code,  # Set verifier code
             verifier=verifier,  # Use created verifier or None
             metadata=task_json.get("metadata", {}),  # Default empty metadata
         )
@@ -636,7 +667,7 @@ class Fleet:
             AsyncVerifierFunction created from the verifier code
         """
         # Fetch verifier from API
-        response = self.client.request("GET", f"/v1/verifier/{verifier_id}")
+        response = self.client.request("GET", f"/v1/verifiers/{verifier_id}")
         verifier_data = response.json()
 
         # Use the common method to create verifier

fleet/instance/client.py

@@ -63,9 +63,7 @@ class InstanceClient:
     def load(self) -> None:
         self._load_resources()
 
-    def reset(
-        self, reset_request: Optional[ResetRequest] = None
-    ) -> ResetResponse:
+    def reset(self, reset_request: Optional[ResetRequest] = None) -> ResetResponse:
         response = self.client.request(
             "POST", "/reset", json=reset_request.model_dump() if reset_request else None
         )
@@ -108,7 +106,7 @@ class InstanceClient:
         return self.verify_raw(function_code, function_name)
 
     def verify_raw(
-        self, function_code: str, function_name: str | None = None
+        self, function_code: str, function_name: Optional[str] = None
     ) -> ExecuteFunctionResponse:
         try:
             function_code = convert_verifier_string(function_code)

fleet/resources/sqlite.py

@@ -1,4 +1,4 @@
-from typing import Any, List, Optional
+from typing import Any, List, Optional, Dict, Tuple
 from ..instance.models import Resource as ResourceModel
 from ..instance.models import DescribeResponse, QueryRequest, QueryResponse
 from .base import Resource
@@ -25,12 +25,12 @@ from fleet.verifiers.db import (
 class SyncDatabaseSnapshot:
     """Async database snapshot that fetches data through API and stores locally for diffing."""
 
-    def __init__(self, resource: "SQLiteResource", name: str | None = None):
+    def __init__(self, resource: "SQLiteResource", name: Optional[str] = None):
         self.resource = resource
         self.name = name or f"snapshot_{datetime.utcnow().isoformat()}"
         self.created_at = datetime.utcnow()
-        self._data: dict[str, list[dict[str, Any]]] = {}
-        self._schemas: dict[str, list[str]] = {}
+        self._data: Dict[str, List[Dict[str, Any]]] = {}
+        self._schemas: Dict[str, List[str]] = {}
         self._fetched = False
 
     def _ensure_fetched(self):
@@ -69,7 +69,7 @@ class SyncDatabaseSnapshot:
 
         self._fetched = True
 
-    def tables(self) -> list[str]:
+    def tables(self) -> List[str]:
         """Get list of all tables in the snapshot."""
         self._ensure_fetched()
         return list(self._data.keys())
@@ -81,7 +81,7 @@ class SyncDatabaseSnapshot:
     def diff(
         self,
         other: "SyncDatabaseSnapshot",
-        ignore_config: IgnoreConfig | None = None,
+        ignore_config: Optional[IgnoreConfig] = None,
     ) -> "SyncSnapshotDiff":
         """Compare this snapshot with another."""
         self._ensure_fetched()
@@ -95,13 +95,13 @@ class SyncSnapshotQueryBuilder:
     def __init__(self, snapshot: SyncDatabaseSnapshot, table: str):
         self._snapshot = snapshot
         self._table = table
-        self._select_cols: list[str] = ["*"]
-        self._conditions: list[tuple[str, str, Any]] = []
-        self._limit: int | None = None
-        self._order_by: str | None = None
+        self._select_cols: List[str] = ["*"]
+        self._conditions: List[Tuple[str, str, Any]] = []
+        self._limit: Optional[int] = None
+        self._order_by: Optional[str] = None
         self._order_desc: bool = False
 
-    def _get_data(self) -> list[dict[str, Any]]:
+    def _get_data(self) -> List[Dict[str, Any]]:
         """Get table data from snapshot."""
         self._snapshot._ensure_fetched()
         return self._snapshot._data.get(self._table, [])
@@ -122,11 +122,11 @@ class SyncSnapshotQueryBuilder:
         qb._order_desc = desc
         return qb
 
-    def first(self) -> dict[str, Any] | None:
+    def first(self) -> Optional[Dict[str, Any]]:
         rows = self.all()
         return rows[0] if rows else None
 
-    def all(self) -> list[dict[str, Any]]:
+    def all(self) -> List[Dict[str, Any]]:
         data = self._get_data()
 
         # Apply filters
@@ -185,14 +185,14 @@ class SyncSnapshotDiff:
         self,
         before: SyncDatabaseSnapshot,
         after: SyncDatabaseSnapshot,
-        ignore_config: IgnoreConfig | None = None,
+        ignore_config: Optional[IgnoreConfig] = None,
     ):
         self.before = before
         self.after = after
         self.ignore_config = ignore_config or IgnoreConfig()
-        self._cached: dict[str, Any] | None = None
+        self._cached: Optional[Dict[str, Any]] = None
 
-    def _get_primary_key_columns(self, table: str) -> list[str]:
+    def _get_primary_key_columns(self, table: str) -> List[str]:
         """Get primary key columns for a table."""
         # Try to get from schema
         schema_response = self.after.resource.query(f"PRAGMA table_info({table})")
@@ -222,7 +222,7 @@ class SyncSnapshotDiff:
             return self._cached
 
         all_tables = set(self.before.tables()) | set(self.after.tables())
-        diff: dict[str, dict[str, Any]] = {}
+        diff: Dict[str, Dict[str, Any]] = {}
 
         for tbl in all_tables:
             if self.ignore_config.should_ignore_table(tbl):
@@ -236,7 +236,7 @@ class SyncSnapshotDiff:
             after_data = self.after._data.get(tbl, [])
 
             # Create indexes by primary key
-            def make_key(row: dict, pk_cols: list[str]) -> Any:
+            def make_key(row: dict, pk_cols: List[str]) -> Any:
                 if len(pk_cols) == 1:
                     return row.get(pk_cols[0])
                 return tuple(row.get(col) for col in pk_cols)
@@ -304,12 +304,12 @@ class SyncSnapshotDiff:
         self._cached = diff
         return diff
 
-    def expect_only(self, allowed_changes: list[dict[str, Any]]):
+    def expect_only(self, allowed_changes: List[Dict[str, Any]]):
         """Ensure only specified changes occurred."""
         diff = self._collect()
 
         def _is_change_allowed(
-            table: str, row_id: Any, field: str | None, after_value: Any
+            table: str, row_id: Any, field: Optional[str], after_value: Any
         ) -> bool:
             """Check if a change is in the allowed list using semantic comparison."""
             for allowed in allowed_changes:
@@ -440,11 +440,11 @@ class SyncQueryBuilder:
     def __init__(self, resource: "SQLiteResource", table: str):
         self._resource = resource
         self._table = table
-        self._select_cols: list[str] = ["*"]
-        self._conditions: list[tuple[str, str, Any]] = []
-        self._joins: list[tuple[str, dict[str, str]]] = []
-        self._limit: int | None = None
-        self._order_by: str | None = None
+        self._select_cols: List[str] = ["*"]
+        self._conditions: List[Tuple[str, str, Any]] = []
+        self._joins: List[Tuple[str, Dict[str, str]]] = []
+        self._limit: Optional[int] = None
+        self._order_by: Optional[str] = None
 
     # Column projection / limiting / ordering
     def select(self, *columns: str) -> "SyncQueryBuilder":
@@ -486,12 +486,12 @@ class SyncQueryBuilder:
     def lte(self, column: str, value: Any) -> "SyncQueryBuilder":
         return self._add_condition(column, "<=", value)
 
-    def in_(self, column: str, values: list[Any]) -> "SyncQueryBuilder":
+    def in_(self, column: str, values: List[Any]) -> "SyncQueryBuilder":
         qb = self._clone()
         qb._conditions.append((column, "IN", tuple(values)))
         return qb
 
-    def not_in(self, column: str, values: list[Any]) -> "SyncQueryBuilder":
+    def not_in(self, column: str, values: List[Any]) -> "SyncQueryBuilder":
         qb = self._clone()
         qb._conditions.append((column, "NOT IN", tuple(values)))
         return qb
@@ -508,16 +508,16 @@ class SyncQueryBuilder:
         return qb
 
     # JOIN
-    def join(self, other_table: str, on: dict[str, str]) -> "SyncQueryBuilder":
+    def join(self, other_table: str, on: Dict[str, str]) -> "SyncQueryBuilder":
         qb = self._clone()
         qb._joins.append((other_table, on))
         return qb
 
     # Compile to SQL
-    def _compile(self) -> tuple[str, list[Any]]:
+    def _compile(self) -> Tuple[str, List[Any]]:
         cols = ", ".join(self._select_cols)
         sql = [f"SELECT {cols} FROM {self._table}"]
-        params: list[Any] = []
+        params: List[Any] = []
 
         # Joins
         for tbl, onmap in self._joins:
@@ -558,11 +558,11 @@ class SyncQueryBuilder:
             return row_dict.get("__cnt__", 0)
         return 0
 
-    def first(self) -> dict[str, Any] | None:
+    def first(self) -> Optional[Dict[str, Any]]:
         rows = self.limit(1).all()
         return rows[0] if rows else None
 
-    def all(self) -> list[dict[str, Any]]:
+    def all(self) -> List[Dict[str, Any]]:
         sql, params = self._compile()
         response = self._resource.query(sql, params)
         if not response.rows:
@@ -651,9 +651,7 @@ class SQLiteResource(Resource):
         )
         return DescribeResponse(**response.json())
 
-    def query(
-        self, query: str, args: Optional[List[Any]] = None
-    ) -> QueryResponse:
+    def query(self, query: str, args: Optional[List[Any]] = None) -> QueryResponse:
         return self._query(query, args, read_only=True)
 
     def exec(self, query: str, args: Optional[List[Any]] = None) -> QueryResponse:
@@ -674,7 +672,7 @@ class SQLiteResource(Resource):
         """Create a query builder for the specified table."""
         return SyncQueryBuilder(self, table_name)
 
-    def snapshot(self, name: str | None = None) -> SyncDatabaseSnapshot:
+    def snapshot(self, name: Optional[str] = None) -> SyncDatabaseSnapshot:
         """Create a snapshot of the current database state."""
         snapshot = SyncDatabaseSnapshot(self, name)
         snapshot._ensure_fetched()
@@ -683,7 +681,7 @@ class SQLiteResource(Resource):
     def diff(
         self,
         other: "SQLiteResource",
-        ignore_config: IgnoreConfig | None = None,
+        ignore_config: Optional[IgnoreConfig] = None,
     ) -> SyncSnapshotDiff:
         """Compare this database with another AsyncSQLiteResource.
 
@@ -695,12 +693,8 @@ class SQLiteResource(Resource):
             AsyncSnapshotDiff: Object containing the differences between the two databases
         """
         # Create snapshots of both databases
-        before_snapshot = self.snapshot(
-            name=f"before_{datetime.utcnow().isoformat()}"
-        )
-        after_snapshot = other.snapshot(
-            name=f"after_{datetime.utcnow().isoformat()}"
-        )
+        before_snapshot = self.snapshot(name=f"before_{datetime.utcnow().isoformat()}")
+        after_snapshot = other.snapshot(name=f"after_{datetime.utcnow().isoformat()}")
 
         # Return the diff between the snapshots
         return before_snapshot.diff(after_snapshot, ignore_config)

fleet/tasks.py

@@ -2,11 +2,9 @@
 
 from __future__ import annotations
 
-import re
 import asyncio
 from datetime import datetime
 from typing import Any, Dict, Optional, List
-from uuid import UUID
 
 from pydantic import BaseModel, Field, validator
 
@@ -27,7 +25,9 @@ class Task(BaseModel):
     version: Optional[str] = Field(None, description="Task version")
     verifier_func: Optional[str] = Field(None, description="Verifier function code")
     verifier: Optional[Any] = Field(
-        None, description="Verifier function with decorator (async or sync)"
+        None,
+        description="Verifier function with decorator (async or sync)",
+        exclude=True,
     )
     verifier_id: Optional[str] = Field(None, description="Verifier identifier")
     verifier_sha: Optional[str] = Field(None, description="Verifier SHA256 hash")
@@ -47,7 +47,7 @@ class Task(BaseModel):
     @property
     def env_key(self) -> str:
         """Get the environment key combining env_id and version."""
-        if self.version and self.version != "None":
+        if self.version and self.version != "None" and ":" not in self.env_id:
             return f"{self.env_id}:{self.version}"
         return self.env_id
 
@@ -66,21 +66,21 @@ class Task(BaseModel):
         For sync environments, calls the sync verifier directly.
         For async verifiers, automatically runs them with asyncio.run().
         """
+        # If verifier doesn't exist but verifier_func does, rebuild it
+        if not self.verifier and self.verifier_func:
+            self._rebuild_verifier()
+
         if self.verifier:
             import inspect
 
             # Check if verifier has remote method (for decorated verifiers)
-            if hasattr(self.verifier, "remote"):
-                result = self.verifier.remote(env, *args, **kwargs)
-            else:
-                # For verifiers created from string, call directly
-                result = self.verifier(env, *args, **kwargs)
+            result = self.verifier.remote(env, *args, **kwargs)
 
             # If the result is a coroutine, we need to run it
             if inspect.iscoroutine(result):
                 # Check if we're already in an event loop
                 try:
-                    loop = asyncio.get_running_loop()
+                    asyncio.get_running_loop()
                     # We're in an async context, can't use asyncio.run()
                     raise RuntimeError(
                         "Cannot run async verifier in sync mode while event loop is running. "
@@ -100,6 +100,10 @@ class Task(BaseModel):
         For async environments, awaits the async verifier.
         Works with both sync and async verifiers in async contexts.
         """
+        # If verifier doesn't exist but verifier_func does, rebuild it
+        if not self.verifier and self.verifier_func:
+            self._rebuild_verifier()
+
         if self.verifier:
             result = self.verifier.remote(*args, **kwargs)
             # If it's a coroutine, await it
@@ -112,6 +116,19 @@ class Task(BaseModel):
         else:
             raise ValueError("No verifier function found for this task")
 
+    def _rebuild_verifier(self):
+        """Rebuild the verifier from verifier_func string if it exists."""
+        if self.verifier_func:
+            # Use the same logic as in verifier_from_string
+            verifier_id = self.verifier_id or self.key
+            verifier = verifier_from_string(
+                verifier_func=self.verifier_func,
+                verifier_id=verifier_id,
+                verifier_key=self.key,
+                sha256=self.verifier_sha or "",
+            )
+            self.verifier = verifier
+
     def make_env(self, region: Optional[str] = None):
         """Create an environment instance for this task's environment.
 
@@ -141,7 +158,7 @@ def verifier_from_string(
     """
     try:
         import inspect
-        from .verifiers import verifier, SyncVerifierFunction
+        from .verifiers import SyncVerifierFunction
         from .verifiers.code import TASK_SUCCESSFUL_SCORE, TASK_FAILED_SCORE
         from .verifiers.db import IgnoreConfig
 
@@ -172,36 +189,13 @@ def verifier_from_string(
         if func_obj is None:
             raise ValueError("No function found in verifier code")
 
-        # Create a wrapper function that provides the necessary globals
-        def wrapped_verifier(env, *args, **kwargs):
-            # Set up globals for the function execution
-            func_globals = (
-                func_obj.__globals__.copy() if hasattr(func_obj, "__globals__") else {}
-            )
-            func_globals.update(
-                {
-                    "TASK_SUCCESSFUL_SCORE": TASK_SUCCESSFUL_SCORE,
-                    "TASK_FAILED_SCORE": TASK_FAILED_SCORE,
-                    "IgnoreConfig": IgnoreConfig,
-                }
-            )
-
-            # Create a new function with the updated globals
-            import types
-
-            new_func = types.FunctionType(
-                func_obj.__code__,
-                func_globals,
-                func_obj.__name__,
-                func_obj.__defaults__,
-                func_obj.__closure__,
-            )
-
-            return new_func(env, *args, **kwargs)
-
-        # Create an AsyncVerifierFunction instance with the wrapped function
+        # Create an SyncVerifierFunction instance with raw code
         verifier_instance = SyncVerifierFunction(
-            wrapped_verifier, verifier_key, verifier_id
+            func=func_obj,
+            key=verifier_key,
+            verifier_id=verifier_id,
+            sha256=sha256,
+            raw_code=verifier_func,
         )
 
         # Store additional metadata
@@ -214,6 +208,18 @@ def verifier_from_string(
         raise ValueError(f"Failed to create verifier from string: {e}")
 
 
+def load_tasks_from_file(filename: str) -> List[Task]:
+    """Load tasks from a JSON file.
+
+    Example:
+        tasks = fleet.load_tasks_from_file("my_tasks.json")
+    """
+    from .global_client import get_client
+
+    client = get_client()
+    return client.load_tasks_from_file(filename)
+
+
 def load_tasks(
     env_key: Optional[str] = None,
     keys: Optional[List[str]] = None,
@@ -260,7 +266,6 @@ def update_task(
         response = fleet.update_task("my-task", verifier_code="def verify(env): return True")
     """
     from .global_client import get_client
-    from .models import TaskResponse
 
     client = get_client()
     return client.update_task(

fleet/verifiers/__init__.py

@@ -2,7 +2,7 @@
 
 from .db import DatabaseSnapshot, IgnoreConfig, SnapshotDiff
 from .code import TASK_SUCCESSFUL_SCORE, TASK_FAILED_SCORE
-from .decorator import (
+from .verifier import (
     verifier,
     SyncVerifierFunction,
 )