eval-protocol 0.3.10.dev1__tar.gz → 0.3.11__tar.gz

{eval_protocol-0.3.10.dev1/eval_protocol.egg-info → eval_protocol-0.3.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.3.10.dev1
+Version: 0.3.11
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT
@@ -29,7 +29,7 @@ Requires-Dist: pytest>=6.0.0
 Requires-Dist: pytest-asyncio>=0.21.0
 Requires-Dist: peewee>=3.18.2
 Requires-Dist: backoff>=2.2.0
-Requires-Dist: fireworks-ai==1.0.0a22
+Requires-Dist: fireworks-ai==1.0.0a20
 Requires-Dist: questionary>=2.0.0
 Requires-Dist: toml>=0.10.0
 Requires-Dist: loguru>=0.6.0

{eval_protocol-0.3.10.dev1 → eval_protocol-0.3.11}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2026-01-13T15:54:22-0800",
+ "date": "2026-01-13T17:18:11-0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "3314becfcdf35f771c41988a24f38dcb91593203",
- "version": "0.3.10.dev.1"
+ "full-revisionid": "6702c557e88f2d256fd820770e6ab6b32db72701",
+ "version": "0.3.11"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.3.10.dev1 → eval_protocol-0.3.11}/eval_protocol/auth.py

@@ -3,30 +3,9 @@ import os
 from typing import Optional
 
 import requests
-from dotenv import find_dotenv, load_dotenv
 
 logger = logging.getLogger(__name__)
 
-# --- Load .env files ---
-# Attempt to load .env.dev first, then .env as a fallback.
-# This happens when the module is imported.
-# We use override=False (default) so that existing environment variables
-# (e.g., set in the shell) are NOT overridden by .env files.
-_ENV_DEV_PATH = find_dotenv(filename=".env.dev", raise_error_if_not_found=False, usecwd=True)
-if _ENV_DEV_PATH:
-    load_dotenv(dotenv_path=_ENV_DEV_PATH, override=False)
-    logger.debug(f"eval_protocol.auth: Loaded environment variables from: {_ENV_DEV_PATH}")
-else:
-    _ENV_PATH = find_dotenv(filename=".env", raise_error_if_not_found=False, usecwd=True)
-    if _ENV_PATH:
-        load_dotenv(dotenv_path=_ENV_PATH, override=False)
-        logger.debug(f"eval_protocol.auth: Loaded environment variables from: {_ENV_PATH}")
-    else:
-        logger.debug(
-            "eval_protocol.auth: No .env.dev or .env file found. Relying on shell/existing environment variables."
-        )
-# --- End .env loading ---
-
 
 def get_fireworks_api_key() -> Optional[str]:
     """
@@ -94,8 +73,6 @@ def verify_api_key_and_get_account_id(
     Args:
         api_key: Optional explicit API key. When None, resolves via get_fireworks_api_key().
         api_base: Optional explicit API base. When None, resolves via get_fireworks_api_base().
-            If api_base is api.fireworks.ai, it is used directly. Otherwise, defaults to
-            dev.api.fireworks.ai for the verification call.
 
     Returns:
         The resolved account id if verification succeeds and the header is present; otherwise None.
@@ -104,12 +81,7 @@ def verify_api_key_and_get_account_id(
         resolved_key = api_key or get_fireworks_api_key()
         if not resolved_key:
             return None
-        provided_base = api_base or get_fireworks_api_base()
-        # Use api.fireworks.ai if explicitly provided, otherwise fall back to dev
-        if "api.fireworks.ai" in provided_base:
-            resolved_base = provided_base
-        else:
-            resolved_base = "https://dev.api.fireworks.ai"
+        resolved_base = api_base or get_fireworks_api_base()
 
         from .common_utils import get_user_agent
 

{eval_protocol-0.3.10.dev1 → eval_protocol-0.3.11}/eval_protocol/cli.py

@@ -81,12 +81,13 @@ def _configure_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParse
         "--env-file",
         help="Path to .env file containing secrets to upload (default: .env in current directory)",
     )
+    upload_parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Overwrite existing evaluator with the same ID",
+    )
 
     # Auto-generate flags from SDK Fireworks().evaluators.create() signature
-    # Note: We use Fireworks() directly here instead of create_fireworks_client()
-    # because we only need the method signature for introspection, not a fully
-    # authenticated client. create_fireworks_client() would trigger an HTTP request
-    # to verify the API key, causing delays even for --help invocations.
     create_evaluator_fn = Fireworks().evaluators.create
 
     upload_skip_fields = {
@@ -136,6 +137,7 @@ def _configure_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParse
 
     rft_parser.add_argument("--yes", "-y", action="store_true", help="Non-interactive mode")
     rft_parser.add_argument("--dry-run", action="store_true", help="Print planned SDK call without sending")
+    rft_parser.add_argument("--force", action="store_true", help="Overwrite existing evaluator with the same ID")
     rft_parser.add_argument("--skip-validation", action="store_true", help="Skip local dataset/evaluator validation")
     rft_parser.add_argument(
         "--ignore-docker",
@@ -196,10 +198,6 @@ def _configure_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParse
         "loss_config.method": "RL loss method for underlying trainers. One of {grpo,dapo}.",
     }
 
-    # Note: We use Fireworks() directly here instead of create_fireworks_client()
-    # because we only need the method signature for introspection, not a fully
-    # authenticated client. create_fireworks_client() would trigger an HTTP request
-    # to verify the API key, causing delays even for --help invocations.
     create_rft_job_fn = Fireworks().reinforcement_fine_tuning_jobs.create
 
     add_args_from_callable_signature(

{eval_protocol-0.3.10.dev1 → eval_protocol-0.3.11}/eval_protocol/cli_commands/create_rft.py

@@ -7,18 +7,19 @@ import sys
 import time
 from typing import Any, Callable, Dict, Optional
 import inspect
+import requests
 import tempfile
 from pydantic import ValidationError
 
 from ..auth import get_fireworks_api_base, get_fireworks_api_key
-from ..fireworks_client import create_fireworks_client
-from ..common_utils import load_jsonl
+from ..common_utils import get_user_agent, load_jsonl
 from ..fireworks_rft import (
     create_dataset_from_jsonl,
     detect_dataset_builder,
     materialize_dataset_via_builder,
 )
 from ..models import EvaluationRow
+from .upload import upload_command
 from .utils import (
     _build_entry_point,
     _build_trimmed_dataset_id,
@@ -34,6 +35,8 @@ from .utils import (
 )
 from .local_test import run_evaluator_test
 
+from fireworks import Fireworks
+
 
 def _extract_dataset_adapter(
     test_file_path: str, test_func_name: str
@@ -220,68 +223,64 @@ def _extract_jsonl_from_input_dataset(test_file_path: str, test_func_name: str)
         return None
 
 
-def _poll_evaluator_version_status(
-    evaluator_id: str,
-    version_id: str,
-    api_key: str,
-    api_base: str,
-    timeout_minutes: int = 10,
+def _poll_evaluator_status(
+    evaluator_resource_name: str, api_key: str, api_base: str, timeout_minutes: int = 10
 ) -> bool:
     """
-    Poll a specific evaluator version status until it becomes ACTIVE or times out.
-
-    Uses the Fireworks SDK to get the specified version of the evaluator and checks
-    its build state.
+    Poll evaluator status until it becomes ACTIVE or times out.
 
     Args:
-        evaluator_id: The evaluator ID (not full resource name)
-        version_id: The specific version ID to poll
+        evaluator_resource_name: Full evaluator resource name (e.g., accounts/xxx/evaluators/yyy)
         api_key: Fireworks API key
         api_base: Fireworks API base URL
         timeout_minutes: Maximum time to wait in minutes
 
     Returns:
-        True if evaluator version becomes ACTIVE, False if timeout or BUILD_FAILED
+        True if evaluator becomes ACTIVE, False if timeout or BUILD_FAILED
     """
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+        "User-Agent": get_user_agent(),
+    }
+
+    check_url = f"{api_base}/v1/{evaluator_resource_name}"
     timeout_seconds = timeout_minutes * 60
     poll_interval = 10  # seconds
     start_time = time.time()
 
-    print(
-        f"Polling evaluator version '{version_id}' status (timeout: {timeout_minutes}m, interval: {poll_interval}s)..."
-    )
-
-    client = create_fireworks_client(api_key=api_key, base_url=api_base)
+    print(f"Polling evaluator status (timeout: {timeout_minutes}m, interval: {poll_interval}s)...")
 
     while time.time() - start_time < timeout_seconds:
         try:
-            version = client.evaluator_versions.get(version_id, evaluator_id=evaluator_id)
-            state = version.state or "STATE_UNSPECIFIED"
-            status_msg = ""
-            if version.status and version.status.message:
-                status_msg = version.status.message
+            response = requests.get(check_url, headers=headers, timeout=30)
+            response.raise_for_status()
+
+            evaluator_data = response.json()
+            state = evaluator_data.get("state", "STATE_UNSPECIFIED")
+            status = evaluator_data.get("status", "")
 
             if state == "ACTIVE":
-                print("✅ Evaluator version is ACTIVE and ready!")
+                print("✅ Evaluator is ACTIVE and ready!")
                 return True
             elif state == "BUILD_FAILED":
-                print(f"❌ Evaluator version build failed. Status: {status_msg}")
+                print(f"❌ Evaluator build failed. Status: {status}")
                 return False
             elif state == "BUILDING":
                 elapsed_minutes = (time.time() - start_time) / 60
-                print(f"⏳ Evaluator version is still building... ({elapsed_minutes:.1f}m elapsed)")
+                print(f"⏳ Evaluator is still building... ({elapsed_minutes:.1f}m elapsed)")
             else:
-                print(f"⏳ Evaluator version state: {state}, status: {status_msg}")
+                print(f"⏳ Evaluator state: {state}, status: {status}")
 
-        except Exception as e:
-            print(f"Warning: Failed to check evaluator version status: {e}")
+        except requests.exceptions.RequestException as e:
+            print(f"Warning: Failed to check evaluator status: {e}")
 
         # Wait before next poll
         time.sleep(poll_interval)
 
     # Timeout reached
     elapsed_minutes = (time.time() - start_time) / 60
-    print(f"⏰ Timeout after {elapsed_minutes:.1f}m - evaluator version is not yet ACTIVE")
+    print(f"⏰ Timeout after {elapsed_minutes:.1f}m - evaluator is not yet ACTIVE")
     return False
 
 
@@ -566,16 +565,42 @@ def _upload_dataset(
 def _upload_and_ensure_evaluator(
     project_root: str,
     evaluator_id: str,
+    evaluator_resource_name: str,
     api_key: str,
     api_base: str,
+    force: bool,
 ) -> bool:
-    """Upload evaluator and ensure its version becomes ACTIVE.
-
-    Creates/updates the evaluator and uploads the code, then polls the specific
-    version until it becomes ACTIVE.
-    """
-    from eval_protocol.evaluation import create_evaluation
+    """Ensure the evaluator exists and is ACTIVE, uploading it if needed."""
+    # Optional short-circuit: if evaluator already exists and not forcing, skip upload path
+    if not force:
+        try:
+            headers = {
+                "Authorization": f"Bearer {api_key}",
+                "Content-Type": "application/json",
+                "User-Agent": get_user_agent(),
+            }
+            resp = requests.get(f"{api_base}/v1/{evaluator_resource_name}", headers=headers, timeout=10)
+            if resp.ok:
+                state = resp.json().get("state", "STATE_UNSPECIFIED")
+                print(f"✓ Evaluator exists (state: {state}). Skipping upload (use --force to overwrite).")
+                # Poll for ACTIVE before proceeding
+                print(f"Waiting for evaluator '{evaluator_id}' to become ACTIVE...")
+                if not _poll_evaluator_status(
+                    evaluator_resource_name=evaluator_resource_name,
+                    api_key=api_key,
+                    api_base=api_base,
+                    timeout_minutes=10,
+                ):
+                    dashboard_url = _build_evaluator_dashboard_url(evaluator_id)
+                    print("\n❌ Evaluator is not ready within the timeout period.")
+                    print(f"📊 Please check the evaluator status at: {dashboard_url}")
+                    print("   Wait for it to become ACTIVE, then run 'eval-protocol create rft' again.")
+                    return False
+                return True
+        except requests.exceptions.RequestException:
+            pass
 
+    # Ensure evaluator exists by invoking the upload flow programmatically
     try:
         tests = _discover_tests(project_root)
         selected_entry: Optional[str] = None
@@ -592,37 +617,43 @@ def _upload_and_ensure_evaluator(
             )
             return False
 
-        print(f"\nUploading evaluator '{evaluator_id}'...")
-        result, version_id = create_evaluation(
-            evaluator_id=evaluator_id,
-            display_name=evaluator_id,
-            description=f"Evaluator for {evaluator_id}",
-            entry_point=selected_entry,
+        upload_args = argparse.Namespace(
+            path=project_root,
+            entry=selected_entry,
+            id=evaluator_id,
+            display_name=None,
+            description=None,
+            force=force,  # Pass through the --force flag
+            yes=True,
+            env_file=None,  # Add the new env_file parameter
         )
 
-        if not version_id:
-            print("Warning: Evaluator created but version upload failed.")
-            return False
+        if force:
+            print(f"🔄 Force flag enabled - will overwrite existing evaluator '{evaluator_id}'")
 
-        print(f"✓ Uploaded evaluator: {evaluator_id} (version: {version_id})")
+        rc = upload_command(upload_args)
+        if rc == 0:
+            print(f"✓ Uploaded/ensured evaluator: {evaluator_id}")
 
-        # Poll for the specific evaluator version status
-        print(f"Waiting for evaluator '{evaluator_id}' version '{version_id}' to become ACTIVE...")
-        is_active = _poll_evaluator_version_status(
-            evaluator_id=evaluator_id,
-            version_id=version_id,
-            api_key=api_key,
-            api_base=api_base,
-            timeout_minutes=10,
-        )
+            # Poll for evaluator status
+            print(f"Waiting for evaluator '{evaluator_id}' to become ACTIVE...")
+            is_active = _poll_evaluator_status(
+                evaluator_resource_name=evaluator_resource_name,
+                api_key=api_key,
+                api_base=api_base,
+                timeout_minutes=10,
+            )
 
-        if not is_active:
-            dashboard_url = _build_evaluator_dashboard_url(evaluator_id)
-            print("\n❌ Evaluator version is not ready within the timeout period.")
-            print(f"📊 Please check the evaluator status at: {dashboard_url}")
-            print("   Wait for it to become ACTIVE, then run 'eval-protocol create rft' again.")
+            if not is_active:
+                dashboard_url = _build_evaluator_dashboard_url(evaluator_id)
+                print("\n❌ Evaluator is not ready within the timeout period.")
+                print(f"📊 Please check the evaluator status at: {dashboard_url}")
+                print("   Wait for it to become ACTIVE, then run 'eval-protocol create rft' again.")
+                return False
+            return True
+        else:
+            print("Warning: Evaluator upload did not complete successfully; proceeding to RFT creation.")
             return False
-        return True
     except Exception as e:
         print(f"Warning: Failed to upload evaluator automatically: {e}")
         return False
@@ -641,7 +672,7 @@ def _create_rft_job(
 ) -> int:
     """Build and submit the RFT job request (via Fireworks SDK)."""
 
-    signature = inspect.signature(create_fireworks_client().reinforcement_fine_tuning_jobs.create)
+    signature = inspect.signature(Fireworks().reinforcement_fine_tuning_jobs.create)
 
     # Build top-level SDK kwargs
     sdk_kwargs: Dict[str, Any] = {
@@ -680,7 +711,7 @@ def _create_rft_job(
         return 0
 
     try:
-        fw: Fireworks = create_fireworks_client(api_key=api_key, base_url=api_base)
+        fw: Fireworks = Fireworks(api_key=api_key, base_url=api_base)
         job: ReinforcementFineTuningJob = fw.reinforcement_fine_tuning_jobs.create(account_id=account_id, **sdk_kwargs)
         job_name = job.name
         print(f"\n✅ Created Reinforcement Fine-tuning Job: {job_name}")
@@ -708,6 +739,7 @@ def create_rft_command(args) -> int:
     evaluator_arg: Optional[str] = getattr(args, "evaluator", None)
     non_interactive: bool = bool(getattr(args, "yes", False))
     dry_run: bool = bool(getattr(args, "dry_run", False))
+    force: bool = bool(getattr(args, "force", False))
     skip_validation: bool = bool(getattr(args, "skip_validation", False))
     ignore_docker: bool = bool(getattr(args, "ignore_docker", False))
     docker_build_extra: str = getattr(args, "docker_build_extra", "") or ""
@@ -778,12 +810,14 @@ def create_rft_command(args) -> int:
     if not dataset_id or not dataset_resource:
         return 1
 
-    # 5) Ensure evaluator exists and its latest version is ACTIVE (upload + poll if needed)
+    # 5) Ensure evaluator exists and is ACTIVE (upload + poll if needed)
     if not _upload_and_ensure_evaluator(
         project_root=project_root,
         evaluator_id=evaluator_id,
+        evaluator_resource_name=evaluator_resource_name,
         api_key=api_key,
         api_base=api_base,
+        force=force,
     ):
         return 1
 

{eval_protocol-0.3.10.dev1 → eval_protocol-0.3.11}/eval_protocol/cli_commands/upload.py

@@ -289,6 +289,7 @@ def upload_command(args: argparse.Namespace) -> int:
     base_id = getattr(args, "id", None)
     display_name = getattr(args, "display_name", None)
     description = getattr(args, "description", None)
+    force = bool(getattr(args, "force", False))
     env_file = getattr(args, "env_file", None)
 
     # Load secrets from .env file and ensure they're available on Fireworks
@@ -377,18 +378,17 @@ def upload_command(args: argparse.Namespace) -> int:
 
         print(f"\nUploading evaluator '{evaluator_id}' for {qualname.split('.')[-1]}...")
         try:
-            result, version_id = create_evaluation(
+            result = create_evaluation(
                 evaluator_id=evaluator_id,
                 display_name=display_name or evaluator_id,
                 description=description or f"Evaluator for {qualname}",
+                force=force,
                 entry_point=entry_point,
             )
             name = result.get("name", evaluator_id) if isinstance(result, dict) else evaluator_id
 
             # Print success message with Fireworks dashboard link
             print(f"\n✅ Successfully uploaded evaluator: {evaluator_id}")
-            if version_id:
-                print(f"   Version: {version_id}")
             print("📊 View in Fireworks Dashboard:")
             dashboard_url = _build_evaluator_dashboard_url(evaluator_id)
             print(f"   {dashboard_url}\n")

{eval_protocol-0.3.10.dev1 → eval_protocol-0.3.11}/eval_protocol/evaluation.py

@@ -4,15 +4,14 @@ import time
 from typing import List, Optional
 
 import fireworks
-from fireworks.types import EvaluatorVersionParam
 import requests
+from fireworks import Fireworks
 
 from eval_protocol.auth import (
     get_fireworks_account_id,
     get_fireworks_api_key,
     verify_api_key_and_get_account_id,
 )
-from eval_protocol.fireworks_client import create_fireworks_client
 from eval_protocol.get_pep440_version import get_pep440_version
 
 logger = logging.getLogger(__name__)
@@ -154,7 +153,7 @@ class Evaluator:
         logger.info(f"Created {output_path} ({size_bytes:,} bytes)")
         return size_bytes
 
-    def create(self, evaluator_id, display_name=None, description=None):
+    def create(self, evaluator_id, display_name=None, description=None, force=False):
         auth_token = self.api_key or get_fireworks_api_key()
         account_id = self.account_id or get_fireworks_account_id()
         if not account_id and auth_token:
@@ -164,11 +163,7 @@ class Evaluator:
             logger.error("Authentication error: API credentials appear to be invalid or incomplete.")
             raise ValueError("Invalid or missing API credentials.")
 
-        client = create_fireworks_client(
-            api_key=auth_token,
-            base_url=self.api_base,
-            account_id=account_id,
-        )
+        client = Fireworks(api_key=auth_token, base_url=self.api_base, account_id=account_id)
 
         self.display_name = display_name or evaluator_id
         self.description = description or f"Evaluator created from {evaluator_id}"
@@ -202,20 +197,28 @@ class Evaluator:
         logger.info(f"Creating evaluator '{evaluator_id}' for account '{account_id}'...")
 
         try:
-            # Try to create evaluator using SDK
-            try:
-                result = client.evaluators.create(
-                    evaluator_id=evaluator_id,
-                    evaluator=evaluator_params,
-                )
-                logger.info(f"Successfully created evaluator '{evaluator_id}'")
-            except fireworks.APIStatusError as create_error:
-                if create_error.status_code == 409:
-                    # Evaluator already exists, get the existing one and proceed to create a new version
-                    logger.info(f"Evaluator '{evaluator_id}' already exists, creating new version...")
-                    result = client.evaluators.get(evaluator_id=evaluator_id)
-                else:
-                    raise
+            if force:
+                try:
+                    logger.info("Checking if evaluator exists")
+                    existing_evaluator = client.evaluators.get(evaluator_id=evaluator_id)
+                    if existing_evaluator:
+                        logger.info(f"Evaluator '{evaluator_id}' already exists, deleting and recreating...")
+                        try:
+                            client.evaluators.delete(evaluator_id=evaluator_id)
+                            logger.info(f"Successfully deleted evaluator '{evaluator_id}'")
+                        except fireworks.NotFoundError:
+                            logger.info(f"Evaluator '{evaluator_id}' not found, creating...")
+                        except fireworks.APIError as e:
+                            logger.warning(f"Error deleting evaluator: {str(e)}")
+                except fireworks.NotFoundError:
+                    logger.info(f"Evaluator '{evaluator_id}' does not exist, creating...")
+
+            # Create evaluator using SDK
+            result = client.evaluators.create(
+                evaluator_id=evaluator_id,
+                evaluator=evaluator_params,
+            )
+            logger.info(f"Successfully created evaluator '{evaluator_id}'")
 
             # Upload code as tar.gz to GCS
             evaluator_name = result.name  # e.g., "accounts/pyroworks/evaluators/test-123"
@@ -226,25 +229,6 @@ class Evaluator:
                     f"Cannot proceed with code upload. Response: {result}"
                 )
 
-            evaluator_version_param: EvaluatorVersionParam = {}
-            if "commit_hash" in evaluator_params:
-                evaluator_version_param["commit_hash"] = evaluator_params["commit_hash"]
-            if "entry_point" in evaluator_params:
-                evaluator_version_param["entry_point"] = evaluator_params["entry_point"]
-            if "requirements" in evaluator_params:
-                evaluator_version_param["requirements"] = evaluator_params["requirements"]
-
-            evaluator_version = client.evaluator_versions.create(
-                evaluator_id=evaluator_id,
-                evaluator_version=evaluator_version_param,
-            )
-            evaluator_version_id = evaluator_version.name.split("/")[-1] if evaluator_version.name else None
-            if not evaluator_version_id:
-                raise ValueError(
-                    "Create evaluator version response missing 'name' field. "
-                    f"Cannot proceed with code upload. Response: {evaluator_version}"
-                )
-
             try:
                 # Create tar.gz of current directory
                 cwd = os.getcwd()
@@ -256,8 +240,7 @@ class Evaluator:
 
                 # Call GetEvaluatorUploadEndpoint using SDK
                 logger.info(f"Requesting upload endpoint for {tar_filename}")
-                upload_response = client.evaluator_versions.get_upload_endpoint(
-                    version_id=evaluator_version_id,
+                upload_response = client.evaluators.get_upload_endpoint(
                     evaluator_id=evaluator_id,
                     filename_to_size={tar_filename: str(tar_size)},
                 )
@@ -338,9 +321,9 @@ class Evaluator:
                             raise
 
                 # Step 3: Validate upload using SDK
-                client.evaluator_versions.validate_upload(
-                    version_id=evaluator_version_id,
+                client.evaluators.validate_upload(
                     evaluator_id=evaluator_id,
+                    body={},
                 )
                 logger.info("Upload validated successfully")
 
@@ -351,10 +334,8 @@ class Evaluator:
             except Exception as upload_error:
                 logger.warning(f"Code upload failed (evaluator created but code not uploaded): {upload_error}")
                 # Don't fail - evaluator is created, just code upload failed
-                # Return None for version_id since upload failed
-                return result, None
 
-            return result, evaluator_version_id  # Return evaluator result and version ID
+            return result  # Return after attempting upload
         except fireworks.APIStatusError as e:
             logger.error(f"Error creating evaluator: {str(e)}")
             logger.error(f"Status code: {e.status_code}, Response: {e.response.text}")
@@ -380,6 +361,7 @@ def create_evaluation(
     evaluator_id: str,
     display_name: Optional[str] = None,
     description: Optional[str] = None,
+    force: bool = False,
     account_id: Optional[str] = None,
     api_key: Optional[str] = None,
     entry_point: Optional[str] = None,
@@ -391,13 +373,10 @@ def create_evaluation(
         evaluator_id: Unique identifier for the evaluator
         display_name: Display name for the evaluator
         description: Description for the evaluator
+        force: If True, delete and recreate if evaluator exists
         account_id: Optional Fireworks account ID
         api_key: Optional Fireworks API key
         entry_point: Optional entry point (module::function or path::function)
-
-    Returns:
-        A tuple of (evaluator_result, version_id) where version_id is the ID of the
-        created evaluator version, or None if upload failed.
     """
     evaluator = Evaluator(
         account_id=account_id,
@@ -405,4 +384,4 @@ def create_evaluation(
         entry_point=entry_point,
     )
 
-    return evaluator.create(evaluator_id, display_name, description)
+    return evaluator.create(evaluator_id, display_name, description, force)