fleet-python 0.2.19__tar.gz → 0.2.21__tar.gz

{fleet_python-0.2.19 → fleet_python-0.2.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fleet-python
-Version: 0.2.19
+Version: 0.2.21
 Summary: Python SDK for Fleet environments
 Author-email: Fleet AI <nic@fleet.so>
 License: Apache-2.0

{fleet_python-0.2.19 → fleet_python-0.2.21}/examples/dsl_example.py

@@ -68,7 +68,7 @@ def validate_new_deal_creation(
 async def main():
     # Create a new instance
     print("Creating new Hubspot instance...")
-    env = await flt.env.make_async("hubspot:v1.2.7")
+    env = await flt.env.make_async("hubspot")
     print(f"New Instance: {env.instance_id}")
 
     try:
@@ -85,6 +85,7 @@ async def main():
         print(f"Message: {response.message}")
 
         # Get the database resource
+        await env.instance.load()
         db = env.db()
         
         # Take a snapshot before insertion

{fleet_python-0.2.19 → fleet_python-0.2.21}/fleet/__init__.py

@@ -34,6 +34,7 @@ from .verifiers import (
     DatabaseSnapshot,
     IgnoreConfig,
     SnapshotDiff,
+    TASK_FAILED_SCORE,
     TASK_SUCCESSFUL_SCORE,
 )
 
@@ -80,6 +81,7 @@ __all__ = [
     "DatabaseSnapshot",
     "IgnoreConfig", 
     "SnapshotDiff",
+    "TASK_FAILED_SCORE",
     "TASK_SUCCESSFUL_SCORE",
     # Environment module
     "env",

{fleet_python-0.2.19 → fleet_python-0.2.21}/fleet/_async/client.py

@@ -109,7 +109,7 @@ class AsyncEnv(EnvironmentBase):
         return await self.instance.verify(validator)
 
     async def verify_raw(
-        self, function_code: str, function_name: str
+        self, function_code: str, function_name: str | None = None
     ) -> ExecuteFunctionResponse:
         return await self.instance.verify_raw(function_code, function_name)
 
@@ -152,12 +152,14 @@ class AsyncEnv(EnvironmentBase):
 class AsyncFleet:
     def __init__(
         self,
-        api_key: Optional[str] = os.getenv("FLEET_API_KEY"),
+        api_key: Optional[str] = None,
         base_url: Optional[str] = None,
         httpx_client: Optional[httpx.AsyncClient] = None,
         max_retries: int = DEFAULT_MAX_RETRIES,
         timeout: float = DEFAULT_TIMEOUT,
     ):
+        if api_key is None:
+            api_key = os.getenv("FLEET_API_KEY")
         self._httpx_client = httpx_client or default_httpx_client(max_retries, timeout)
         self.client = AsyncWrapper(
             api_key=api_key,
@@ -182,7 +184,7 @@ class AsyncFleet:
     ) -> AsyncEnv:
         if ":" in env_key:
             env_key_part, version = env_key.split(":", 1)
-            if not version.startswith("v"):
+            if not version.startswith("v") and len(version) != 0 and version[0].isdigit():
                 version = f"v{version}"
         else:
             env_key_part = env_key

{fleet_python-0.2.19 → fleet_python-0.2.21}/fleet/_async/instance/client.py

@@ -12,6 +12,7 @@ from ..resources.browser import AsyncBrowserResource
 from ..resources.base import Resource
 
 from fleet.verifiers import DatabaseSnapshot
+from fleet.verifiers.parse import convert_verifier_string, extract_function_name
 
 from ..exceptions import FleetEnvironmentError
 from ...config import DEFAULT_MAX_RETRIES, DEFAULT_TIMEOUT
@@ -107,8 +108,16 @@ class AsyncInstanceClient:
         return await self.verify_raw(function_code, function_name)
 
     async def verify_raw(
-        self, function_code: str, function_name: str
+        self, function_code: str, function_name: str | None = None
     ) -> ExecuteFunctionResponse:
+        try:
+            function_code = convert_verifier_string(function_code)
+        except:
+            pass
+
+        if function_name is None:
+            function_name = extract_function_name(function_code)
+
         response = await self.client.request(
             "POST",
             "/execute_verifier_function",

{fleet_python-0.2.19 → fleet_python-0.2.21}/fleet/client.py

@@ -227,7 +227,7 @@ class Environment(EnvironmentBase):
         return self.instance.verify(validator)
 
     def verify_raw(
-        self, function_code: str, function_name: str
+        self, function_code: str, function_name: str | None = None
     ) -> ExecuteFunctionResponse:
         return self.instance.verify_raw(function_code, function_name)
 
@@ -384,12 +384,14 @@ class Environment(EnvironmentBase):
 class Fleet:
     def __init__(
         self,
-        api_key: Optional[str] = os.getenv("FLEET_API_KEY"),
+        api_key: Optional[str] = None,
         base_url: Optional[str] = None,
         httpx_client: Optional[httpx.Client] = None,
         max_retries: int = DEFAULT_MAX_RETRIES,
         timeout: float = DEFAULT_TIMEOUT,
     ):
+        if api_key is None:
+            api_key = os.getenv("FLEET_API_KEY")
         self._httpx_client = httpx_client or default_httpx_client(max_retries, timeout)
         self.client = SyncWrapper(
             api_key=api_key,
@@ -417,7 +419,7 @@ class Fleet:
     ) -> Environment:
         if ":" in env_key:
             env_key_part, version = env_key.split(":", 1)
-            if not version.startswith("v"):
+            if not version.startswith("v") and len(version) != 0 and version[0].isdigit():
                 version = f"v{version}"
         else:
             env_key_part = env_key

{fleet_python-0.2.19 → fleet_python-0.2.21}/fleet/instance/client.py

@@ -7,6 +7,8 @@ import time
 import logging
 from urllib.parse import urlparse
 
+from fleet.verifiers.parse import convert_verifier_string, extract_function_name
+
 from ..resources.sqlite import SQLiteResource
 from ..resources.browser import BrowserResource
 from ..resources.base import Resource
@@ -116,8 +118,17 @@ class InstanceClient:
         return self.verify_raw(function_code, function_name)
 
     def verify_raw(
-        self, function_code: str, function_name: str
+        self, function_code: str, function_name: str | None = None
     ) -> ExecuteFunctionResponse:
+        try:
+            function_code = convert_verifier_string(function_code)
+        except:
+            pass
+
+        # Extract function name if not provided
+        if function_name is None:
+            function_name = extract_function_name(function_code)
+
         response = self.client.request(
             "POST",
             "/execute_verifier_function",

{fleet_python-0.2.19 → fleet_python-0.2.21}/fleet/resources/sqlite.py

@@ -379,6 +379,9 @@ class SyncSnapshotDiff:
         
     def _expect_only_targeted(self, allowed_changes: list[dict[str, Any]]):
         """Optimized version that only queries specific rows mentioned in allowed_changes."""
+        import concurrent.futures
+        from threading import Lock
+        
         # Group allowed changes by table
         changes_by_table: dict[str, list[dict[str, Any]]] = {}
         for change in allowed_changes:
@@ -387,19 +390,12 @@ class SyncSnapshotDiff:
                 changes_by_table[table] = []
             changes_by_table[table].append(change)
         
-        # For each table with allowed changes, query only those specific rows
-        for table, table_changes in changes_by_table.items():
-            if self.ignore_config.should_ignore_table(table):
-                continue
-                
-            # Get primary key columns
-            pk_columns = self._get_primary_key_columns(table)
-            
-            # Extract unique PKs to check
-            pks_to_check = {change["pk"] for change in table_changes}
-            
-            # Query only these specific rows from both snapshots
-            for pk in pks_to_check:
+        errors = []
+        errors_lock = Lock()
+        
+        # Function to check a single row
+        def check_row(table: str, pk: Any, table_changes: list[dict[str, Any]], pk_columns: list[str]):
+            try:
                 # Build WHERE clause for this PK
                 where_sql = self._build_pk_where_clause(pk_columns, pk)
                 
@@ -423,38 +419,99 @@ class SyncSnapshotDiff:
                         if not _values_equivalent(before_val, after_val):
                             # Check if this change is allowed
                             if not self._is_field_change_allowed(table_changes, pk, field, after_val):
-                                raise AssertionError(
+                                error_msg = (
                                     f"Unexpected change in table '{table}', "
                                     f"row {pk}, field '{field}': "
                                     f"{repr(before_val)} -> {repr(after_val)}"
                                 )
+                                with errors_lock:
+                                    errors.append(AssertionError(error_msg))
+                                return  # Stop checking this row
                 elif not before_row and after_row:
                     # Added row
                     if not self._is_row_change_allowed(table_changes, pk, "__added__"):
-                        raise AssertionError(f"Unexpected row added in table '{table}': {pk}")
+                        error_msg = f"Unexpected row added in table '{table}': {pk}"
+                        with errors_lock:
+                            errors.append(AssertionError(error_msg))
                 elif before_row and not after_row:
                     # Removed row
                     if not self._is_row_change_allowed(table_changes, pk, "__removed__"):
-                        raise AssertionError(f"Unexpected row removed from table '{table}': {pk}")
+                        error_msg = f"Unexpected row removed from table '{table}': {pk}"
+                        with errors_lock:
+                            errors.append(AssertionError(error_msg))
+            except Exception as e:
+                with errors_lock:
+                    errors.append(e)
         
-        # Now check tables not mentioned in allowed_changes to ensure no changes
-        all_tables = set(self.before.tables()) | set(self.after.tables())
-        for table in all_tables:
-            if table in changes_by_table or self.ignore_config.should_ignore_table(table):
+        # Prepare all row checks
+        row_checks = []
+        for table, table_changes in changes_by_table.items():
+            if self.ignore_config.should_ignore_table(table):
                 continue
+                
+            # Get primary key columns once per table
+            pk_columns = self._get_primary_key_columns(table)
             
-            # For tables with no allowed changes, just check row counts
-            before_count_response = self.before.resource.query(f"SELECT COUNT(*) FROM {table}")
-            before_count = before_count_response.rows[0][0] if before_count_response.rows else 0
-            
-            after_count_response = self.after.resource.query(f"SELECT COUNT(*) FROM {table}")
-            after_count = after_count_response.rows[0][0] if after_count_response.rows else 0
+            # Extract unique PKs to check
+            pks_to_check = {change["pk"] for change in table_changes}
             
-            if before_count != after_count:
-                raise AssertionError(
-                    f"Unexpected change in table '{table}': "
-                    f"row count changed from {before_count} to {after_count}"
-                )
+            for pk in pks_to_check:
+                row_checks.append((table, pk, table_changes, pk_columns))
+        
+        # Execute row checks in parallel
+        if row_checks:
+            with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
+                futures = [
+                    executor.submit(check_row, table, pk, table_changes, pk_columns)
+                    for table, pk, table_changes, pk_columns in row_checks
+                ]
+                concurrent.futures.wait(futures)
+        
+        # Check for errors from row checks
+        if errors:
+            raise errors[0]
+        
+        # Now check tables not mentioned in allowed_changes to ensure no changes
+        all_tables = set(self.before.tables()) | set(self.after.tables())
+        tables_to_verify = []
+        
+        for table in all_tables:
+            if table not in changes_by_table and not self.ignore_config.should_ignore_table(table):
+                tables_to_verify.append(table)
+        
+        # Function to verify no changes in a table
+        def verify_no_changes(table: str):
+            try:
+                # For tables with no allowed changes, just check row counts
+                before_count_response = self.before.resource.query(f"SELECT COUNT(*) FROM {table}")
+                before_count = before_count_response.rows[0][0] if before_count_response.rows else 0
+                
+                after_count_response = self.after.resource.query(f"SELECT COUNT(*) FROM {table}")
+                after_count = after_count_response.rows[0][0] if after_count_response.rows else 0
+                
+                if before_count != after_count:
+                    error_msg = (
+                        f"Unexpected change in table '{table}': "
+                        f"row count changed from {before_count} to {after_count}"
+                    )
+                    with errors_lock:
+                        errors.append(AssertionError(error_msg))
+            except Exception as e:
+                with errors_lock:
+                    errors.append(e)
+        
+        # Execute table verification in parallel
+        if tables_to_verify:
+            with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
+                futures = [
+                    executor.submit(verify_no_changes, table)
+                    for table in tables_to_verify
+                ]
+                concurrent.futures.wait(futures)
+        
+        # Final error check
+        if errors:
+            raise errors[0]
         
         return self
         
@@ -500,6 +557,9 @@ class SyncSnapshotDiff:
     def _expect_no_changes(self):
         """Efficiently verify that no changes occurred between snapshots using row counts."""
         try:
+            import concurrent.futures
+            from threading import Lock
+            
             # Get all tables from both snapshots
             before_tables = set(self.before.tables())
             after_tables = set(self.after.tables())
@@ -516,41 +576,84 @@ class SyncSnapshotDiff:
                 if not self.ignore_config.should_ignore_table(table):
                     raise AssertionError(f"Unexpected table removed: {table}")
             
-            # For each table, compare row counts
+            # Prepare tables to check
+            tables_to_check = []
             all_tables = before_tables | after_tables
             for table in all_tables:
-                if self.ignore_config.should_ignore_table(table):
-                    continue
+                if not self.ignore_config.should_ignore_table(table):
+                    tables_to_check.append(table)
+            
+            # If no tables to check, we're done
+            if not tables_to_check:
+                return self
+            
+            # Use ThreadPoolExecutor to parallelize count queries
+            # We use threads instead of processes since the queries are I/O bound
+            errors = []
+            errors_lock = Lock()
+            tables_needing_verification = []
+            verification_lock = Lock()
+            
+            def check_table_counts(table: str):
+                """Check row counts for a single table."""
+                try:
+                    # Get row counts from both snapshots
+                    before_count = 0
+                    after_count = 0
                     
-                # Get row counts from both snapshots
-                before_count = 0
-                after_count = 0
-                
-                if table in before_tables:
-                    before_count_response = self.before.resource.query(f"SELECT COUNT(*) FROM {table}")
-                    before_count = before_count_response.rows[0][0] if before_count_response.rows else 0
+                    if table in before_tables:
+                        before_count_response = self.before.resource.query(f"SELECT COUNT(*) FROM {table}")
+                        before_count = before_count_response.rows[0][0] if before_count_response.rows else 0
+                        
+                    if table in after_tables:
+                        after_count_response = self.after.resource.query(f"SELECT COUNT(*) FROM {table}")
+                        after_count = after_count_response.rows[0][0] if after_count_response.rows else 0
                     
-                if table in after_tables:
-                    after_count_response = self.after.resource.query(f"SELECT COUNT(*) FROM {table}")
-                    after_count = after_count_response.rows[0][0] if after_count_response.rows else 0
+                    if before_count != after_count:
+                        error_msg = (
+                            f"Unexpected change in table '{table}': "
+                            f"row count changed from {before_count} to {after_count}"
+                        )
+                        with errors_lock:
+                            errors.append(AssertionError(error_msg))
+                    elif before_count > 0 and before_count <= 1000:
+                        # Mark for detailed verification
+                        with verification_lock:
+                            tables_needing_verification.append(table)
+                            
+                except Exception as e:
+                    with errors_lock:
+                        errors.append(e)
+            
+            # Execute count checks in parallel
+            with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
+                futures = [executor.submit(check_table_counts, table) for table in tables_to_check]
+                concurrent.futures.wait(futures)
+            
+            # Check if any errors occurred during count checking
+            if errors:
+                # Raise the first error
+                raise errors[0]
+            
+            # Now verify small tables for data changes (also in parallel)
+            if tables_needing_verification:
+                verification_errors = []
                 
-                if before_count != after_count:
-                    raise AssertionError(
-                        f"Unexpected change in table '{table}': "
-                        f"row count changed from {before_count} to {after_count}"
-                    )
-                    
-                # If counts match but there could be modifications, we need to check further
-                # For now, we'll do a more detailed check only if counts are small
-                # to avoid performance issues
-                if before_count > 0 and before_count <= 1000:  # Threshold for detailed check
-                    # Do a quick hash check of the data
-                    self._verify_table_unchanged(table)
-                elif before_count > 1000:
-                    # For large tables, we could sample or use checksums
-                    # For now, we'll trust that count matching means no changes
-                    # This is a reasonable assumption for expect_only([])
-                    pass
+                def verify_table(table: str):
+                    """Verify a single table's data hasn't changed."""
+                    try:
+                        self._verify_table_unchanged(table)
+                    except AssertionError as e:
+                        with errors_lock:
+                            verification_errors.append(e)
+                
+                with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
+                    futures = [executor.submit(verify_table, table) for table in tables_needing_verification]
+                    concurrent.futures.wait(futures)
+                
+                # Check if any errors occurred during verification
+                if verification_errors:
+                    raise verification_errors[0]
                     
             return self
             

{fleet_python-0.2.19 → fleet_python-0.2.21}/fleet/verifiers/__init__.py

@@ -1,7 +1,7 @@
 """Fleet verifiers module - database snapshot validation utilities and verifier decorator."""
 
 from fleet.verifiers.db import DatabaseSnapshot, IgnoreConfig, SnapshotDiff
-from fleet.verifiers.code import TASK_SUCCESSFUL_SCORE
+from fleet.verifiers.code import TASK_SUCCESSFUL_SCORE, TASK_FAILED_SCORE
 from .decorator import (
     verifier,
     SyncVerifierFunction,
@@ -12,6 +12,7 @@ __all__ = [
     "IgnoreConfig", 
     "SnapshotDiff",
     "TASK_SUCCESSFUL_SCORE",
+    "TASK_FAILED_SCORE",
     "verifier",
     "SyncVerifierFunction",
 ]

fleet_python-0.2.21/fleet/verifiers/parse.py

@@ -0,0 +1,143 @@
+import re
+
+
+def extract_function_name(function_code: str) -> str | None:
+    """
+    Extract function name from Python function code.
+    
+    Handles both regular functions (def) and async functions (async def).
+    
+    Args:
+        function_code: Python function code as a string
+        
+    Returns:
+        The function name if found, None otherwise
+    """
+    # Pattern to match both def and async def functions
+    # Handles various formatting styles and type annotations
+    pattern = r'(?:async\s+)?def\s+(\w+)\s*\('
+    
+    match = re.search(pattern, function_code)
+    if match:
+        return match.group(1)
+    
+    return None
+
+
+def convert_verifier_string(verifier_str: str) -> str:
+    """
+    Convert a verifier function string from the old format (env: Environment) 
+    to the new format (before: DatabaseSnapshot, after: DatabaseSnapshot).
+    
+    Args:
+        verifier_str: The original verifier function as a string
+        
+    Returns:
+        The converted verifier function string
+    """
+    # First, handle escaped newlines in the input
+    verifier_str = verifier_str.replace('\\n', '\n')
+    
+    # Extract function name, docstring, and body
+    # More flexible pattern that accepts both int and float return types
+    func_pattern = r'def\s+(\w+)\s*\(\s*env(?:\s*:\s*Environment)?\s*,?\s*final_answer(?:\s*:\s*str\s*\|\s*None)?\s*(?:=\s*None)?\s*\)\s*->\s*(?:float|int):\s*\n((?:\s*""".*?"""\s*\n)?)(.*)'
+    match = re.match(func_pattern, verifier_str.strip(), re.DOTALL)
+    
+    if not match:
+        # Try with multiline pattern
+        func_pattern_multiline = r'def\s+(\w+)\s*\(\s*\n?\s*env(?:\s*:\s*Environment)?\s*,?\s*\n?\s*final_answer(?:\s*:\s*str\s*\|\s*None)?\s*(?:=\s*None)?\s*\n?\s*\)\s*->\s*(?:float|int):\s*\n((?:\s*""".*?"""\s*\n)?)(.*)'
+        match = re.match(func_pattern_multiline, verifier_str.strip(), re.DOTALL)
+        
+        if not match:
+            raise ValueError("Could not parse verifier function. Expected format: def function_name(env: Environment, final_answer: str | None = None) -> float/int:")
+    
+    func_name = match.group(1)
+    docstring = match.group(2).strip()
+    body = match.group(3)
+    
+    # Find all unique env.db() calls
+    db_calls = re.findall(r'env\.db\("(\w+)"\)', body)
+    unique_db_names = list(dict.fromkeys(db_calls))  # Remove duplicates while preserving order
+    
+    # Build the new function
+    new_func = f'''def {func_name}(
+    before: DatabaseSnapshot, after: DatabaseSnapshot, transcript: str | None = None
+) -> int:
+    class Environment:
+        def db(self, name: str) -> DatabaseSnapshot:'''
+    
+    # Build the db method based on found database names
+    if unique_db_names:
+        conditions = []
+        for db_name in unique_db_names:
+            if db_name == "seed":
+                conditions.append('before if name == "seed"')
+            elif db_name == "current":
+                conditions.append('after')
+            else:
+                # Handle other database names if needed
+                conditions.append(f'None  # Handle "{db_name}"')
+        
+        if len(conditions) == 2 and "seed" in unique_db_names and "current" in unique_db_names:
+            new_func += f'''
+            return before if name == "seed" else after'''
+        else:
+            # More complex mapping if needed
+            new_func += f'''
+            if name == "seed":
+                return before
+            elif name == "current":
+                return after
+            else:
+                raise ValueError(f"Unknown database name: {{name}}")'''
+    else:
+        new_func += '''
+            return before if name == "seed" else after'''
+    
+    new_func += '''
+
+        @property
+        def instance(self):
+            return self
+        
+        def load(self):
+            pass
+
+    def verifier(env: Environment, final_answer: str | None = None) -> float:'''
+    
+    if docstring:
+        new_func += f'\n        {docstring}'
+    
+    # First, find the minimum indentation in the body (excluding empty lines)
+    body_lines = body.splitlines()
+    min_indent = float('inf')
+    for line in body_lines:
+        if line.strip():  # Non-empty line
+            indent_len = len(line) - len(line.lstrip())
+            min_indent = min(min_indent, indent_len)
+    
+    # If we didn't find any non-empty lines, set min_indent to 0
+    if min_indent == float('inf'):
+        min_indent = 0
+    
+    # Now strip the minimum indentation and re-indent to 8 spaces
+    if body_lines:
+        indented_lines = []
+        for line in body_lines:
+            if line.strip():  # Non-empty line
+                # Remove the minimum indentation and add 8 spaces
+                stripped_line = line[min_indent:] if len(line) > min_indent else line.lstrip()
+                indented_lines.append('        ' + stripped_line)
+            else:  # Empty line
+                indented_lines.append('')
+        
+        indented_body = '\n'.join(indented_lines)
+        new_func += f'\n{indented_body}'
+    
+    # Add the return statement
+    new_func += '\n\n    return verifier(Environment(), transcript)'
+
+    # Replace TASK_FAILED_SCORE with 0 in the function string
+    new_func = new_func.replace('TASK_FAILED_SCORE', '0')
+
+    return new_func

{fleet_python-0.2.19 → fleet_python-0.2.21}/fleet_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fleet-python
-Version: 0.2.19
+Version: 0.2.21
 Summary: Python SDK for Fleet environments
 Author-email: Fleet AI <nic@fleet.so>
 License: Apache-2.0

{fleet_python-0.2.19 → fleet_python-0.2.21}/fleet_python.egg-info/SOURCES.txt

@@ -59,6 +59,7 @@ fleet/verifiers/bundler.py
 fleet/verifiers/code.py
 fleet/verifiers/db.py
 fleet/verifiers/decorator.py
+fleet/verifiers/parse.py
 fleet/verifiers/sql_differ.py
 fleet/verifiers/verifier.py
 fleet_python.egg-info/PKG-INFO

{fleet_python-0.2.19 → fleet_python-0.2.21}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "fleet-python"
-version = "0.2.19"
+version = "0.2.21"
 description = "Python SDK for Fleet environments"
 authors = [
     {name = "Fleet AI", email = "nic@fleet.so"},