eval-protocol 0.2.78__tar.gz → 0.2.80__tar.gz

{eval_protocol-0.2.78/eval_protocol.egg-info → eval_protocol-0.2.80}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.78
+Version: 0.2.80
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.78 → eval_protocol-0.2.80}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-11-03T11:40:49-0800",
+ "date": "2025-11-04T15:41:02-0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "fd090d994b7ee9738c2310e1e61427d027bad38d",
- "version": "0.2.78"
+ "full-revisionid": "9303a224e55ef4f6e47aaa9e9e596ebd1c83cc56",
+ "version": "0.2.80"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.2.78 → eval_protocol-0.2.80}/eval_protocol/adapters/fireworks_tracing.py

@@ -273,7 +273,12 @@ class FireworksTracingAdapter(BaseAdapter):
         if not tags:
             raise ValueError("At least one tag is required to fetch logs")
 
-        headers = {"Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}"}
+        from ..common_utils import get_user_agent
+
+        headers = {
+            "Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}",
+            "User-Agent": get_user_agent(),
+        }
         params: Dict[str, Any] = {"tags": tags, "limit": limit, "hours_back": hours_back, "program": "eval_protocol"}
 
         # Try /logs first, fall back to /v1/logs if not found
@@ -398,7 +403,12 @@ class FireworksTracingAdapter(BaseAdapter):
         else:
             url = f"{self.base_url}/v1/traces/pointwise"
 
-        headers = {"Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}"}
+        from ..common_utils import get_user_agent
+
+        headers = {
+            "Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}",
+            "User-Agent": get_user_agent(),
+        }
 
         result = None
         try:

{eval_protocol-0.2.78 → eval_protocol-0.2.80}/eval_protocol/auth.py

@@ -242,9 +242,16 @@ def verify_api_key_and_get_account_id(
         if not resolved_key:
             return None
         resolved_base = api_base or get_fireworks_api_base()
+
+        from .common_utils import get_user_agent
+
         url = f"{resolved_base.rstrip('/')}/verifyApiKey"
-        headers = {"Authorization": f"Bearer {resolved_key}"}
+        headers = {
+            "Authorization": f"Bearer {resolved_key}",
+            "User-Agent": get_user_agent(),
+        }
         resp = requests.get(url, headers=headers, timeout=10)
+
         if resp.status_code != 200:
             logger.debug("verifyApiKey returned status %s", resp.status_code)
             return None

{eval_protocol-0.2.78 → eval_protocol-0.2.80}/eval_protocol/cli.py

@@ -355,6 +355,10 @@ def parse_args(args=None):
         action="store_true",
         help="Non-interactive: upload all discovered evaluation tests",
     )
+    upload_parser.add_argument(
+        "--env-file",
+        help="Path to .env file containing secrets to upload (default: .env in current directory)",
+    )
 
     # Create command group
     create_parser = subparsers.add_parser(

{eval_protocol-0.2.78 → eval_protocol-0.2.80}/eval_protocol/cli_commands/create_rft.py

@@ -5,12 +5,15 @@ import time
 import argparse
 from typing import Any, Dict, Optional
 
+import requests
+
 from ..auth import (
     get_fireworks_account_id,
     get_fireworks_api_base,
     get_fireworks_api_key,
     verify_api_key_and_get_account_id,
 )
+from ..common_utils import get_user_agent
 from ..fireworks_rft import (
     _map_api_host_to_app_host,
     build_default_output_model,
@@ -263,6 +266,67 @@ def _auto_select_evaluator_id(cwd: str) -> Optional[str]:
     return None
 
 
+def _poll_evaluator_status(
+    evaluator_resource_name: str, api_key: str, api_base: str, timeout_minutes: int = 5
+) -> bool:
+    """
+    Poll evaluator status until it becomes ACTIVE or times out.
+
+    Args:
+        evaluator_resource_name: Full evaluator resource name (e.g., accounts/xxx/evaluators/yyy)
+        api_key: Fireworks API key
+        api_base: Fireworks API base URL
+        timeout_minutes: Maximum time to wait in minutes
+
+    Returns:
+        True if evaluator becomes ACTIVE, False if timeout or BUILD_FAILED
+    """
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+        "User-Agent": get_user_agent(),
+    }
+
+    check_url = f"{api_base}/v1/{evaluator_resource_name}"
+    timeout_seconds = timeout_minutes * 60
+    poll_interval = 10  # seconds
+    start_time = time.time()
+
+    print(f"Polling evaluator status (timeout: {timeout_minutes}m, interval: {poll_interval}s)...")
+
+    while time.time() - start_time < timeout_seconds:
+        try:
+            response = requests.get(check_url, headers=headers, timeout=30)
+            response.raise_for_status()
+
+            evaluator_data = response.json()
+            state = evaluator_data.get("state", "STATE_UNSPECIFIED")
+            status = evaluator_data.get("status", "")
+
+            if state == "ACTIVE":
+                print("✅ Evaluator is ACTIVE and ready!")
+                return True
+            elif state == "BUILD_FAILED":
+                print(f"❌ Evaluator build failed. Status: {status}")
+                return False
+            elif state == "BUILDING":
+                elapsed_minutes = (time.time() - start_time) / 60
+                print(f"⏳ Evaluator is still building... ({elapsed_minutes:.1f}m elapsed)")
+            else:
+                print(f"⏳ Evaluator state: {state}, status: {status}")
+
+        except requests.exceptions.RequestException as e:
+            print(f"Warning: Failed to check evaluator status: {e}")
+
+        # Wait before next poll
+        time.sleep(poll_interval)
+
+    # Timeout reached
+    elapsed_minutes = (time.time() - start_time) / 60
+    print(f"⏰ Timeout after {elapsed_minutes:.1f}m - evaluator is not yet ACTIVE")
+    return False
+
+
 def create_rft_command(args) -> int:
     evaluator_id: Optional[str] = getattr(args, "evaluator_id", None)
     non_interactive: bool = bool(getattr(args, "yes", False))
@@ -328,10 +392,28 @@ def create_rft_command(args) -> int:
             description=None,
             force=False,
             yes=True,
+            env_file=None,  # Add the new env_file parameter
         )
         rc = upload_command(upload_args)
         if rc == 0:
             print(f"✓ Uploaded/ensured evaluator: {evaluator_id}")
+
+            # Poll for evaluator status
+            print(f"Waiting for evaluator '{evaluator_id}' to become ACTIVE...")
+            is_active = _poll_evaluator_status(
+                evaluator_resource_name=evaluator_resource_name, api_key=api_key, api_base=api_base, timeout_minutes=5
+            )
+
+            if not is_active:
+                # Print helpful message with dashboard link
+                app_base = _map_api_host_to_app_host(api_base)
+                evaluator_slug = _extract_terminal_segment(evaluator_id)
+                dashboard_url = f"{app_base}/dashboard/evaluators/{evaluator_slug}"
+
+                print("\n❌ Evaluator is not ready within the timeout period.")
+                print(f"📊 Please check the evaluator status at: {dashboard_url}")
+                print("   Wait for it to become ACTIVE, then run 'eval-protocol create rft' again.")
+                return 1
         else:
             print("Warning: Evaluator upload did not complete successfully; proceeding to RFT creation.")
     except Exception as e:

{eval_protocol-0.2.78 → eval_protocol-0.2.80}/eval_protocol/cli_commands/upload.py

@@ -9,7 +9,7 @@ import runpy
 import sys
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Callable, Iterable, Optional
+from typing import Any, Dict, Iterable
 
 import pytest
 from eval_protocol.auth import (
@@ -551,6 +551,35 @@ def _prompt_select(tests: list[DiscoveredTest], non_interactive: bool) -> list[D
     return _prompt_select_interactive(tests)
 
 
+def _load_secrets_from_env_file(env_file_path: str) -> Dict[str, str]:
+    """
+    Load secrets from a .env file that should be uploaded to Fireworks.
+
+    Returns a dictionary of secret key-value pairs that contain 'API_KEY' in the name.
+    """
+    if not os.path.exists(env_file_path):
+        return {}
+
+    # Load the .env file into a temporary environment
+    env_vars = {}
+    with open(env_file_path, "r") as f:
+        for line in f:
+            line = line.strip()
+            if line and not line.startswith("#") and "=" in line:
+                key, value = line.split("=", 1)
+                key = key.strip()
+                value = value.strip().strip('"').strip("'")  # Remove quotes
+                env_vars[key] = value
+
+    # Filter for secrets that look like API keys
+    secrets = {}
+    for key, value in env_vars.items():
+        if "API_KEY" in key.upper() and value:
+            secrets[key] = value
+
+    return secrets
+
+
 def upload_command(args: argparse.Namespace) -> int:
     root = os.path.abspath(getattr(args, "path", "."))
     entries_arg = getattr(args, "entry", None)
@@ -585,11 +614,27 @@ def upload_command(args: argparse.Namespace) -> int:
     display_name = getattr(args, "display_name", None)
     description = getattr(args, "description", None)
     force = bool(getattr(args, "force", False))
+    env_file = getattr(args, "env_file", None)
 
-    # Ensure FIREWORKS_API_KEY is available to the remote by storing it as a Fireworks secret
+    # Load secrets from .env file and ensure they're available on Fireworks
     try:
         fw_account_id = get_fireworks_account_id()
+
+        # Determine .env file path
+        if env_file:
+            env_file_path = env_file
+        else:
+            env_file_path = os.path.join(root, ".env")
+
+        # Load secrets from .env file
+        secrets_from_file = _load_secrets_from_env_file(env_file_path)
+        secrets_from_env_file = secrets_from_file.copy()  # Track what came from .env file
+
+        # Also ensure FIREWORKS_API_KEY from environment is included
         fw_api_key_value = get_fireworks_api_key()
+        if fw_api_key_value:
+            secrets_from_file["FIREWORKS_API_KEY"] = fw_api_key_value
+
         if not fw_account_id and fw_api_key_value:
             # Attempt to verify and resolve account id from server headers
             resolved = verify_api_key_and_get_account_id(api_key=fw_api_key_value, api_base=get_fireworks_api_base())
@@ -598,21 +643,27 @@ def upload_command(args: argparse.Namespace) -> int:
                 # Propagate to environment so downstream calls use it if needed
                 os.environ["FIREWORKS_ACCOUNT_ID"] = fw_account_id
                 print(f"Resolved FIREWORKS_ACCOUNT_ID via API verification: {fw_account_id}")
-        if fw_account_id and fw_api_key_value:
-            print("Ensuring FIREWORKS_API_KEY is registered as a secret on Fireworks for rollout...")
-            if create_or_update_fireworks_secret(
-                account_id=fw_account_id,
-                key_name="FIREWORKS_API_KEY",
-                secret_value=fw_api_key_value,
-            ):
-                print("✓ FIREWORKS_API_KEY secret created/updated on Fireworks.")
-            else:
-                print("Warning: Failed to create/update FIREWORKS_API_KEY secret on Fireworks.")
+
+        if fw_account_id and secrets_from_file:
+            print(f"Found {len(secrets_from_file)} API keys to upload as Fireworks secrets...")
+            if secrets_from_env_file and os.path.exists(env_file_path):
+                print(f"Loading secrets from: {env_file_path}")
+
+            for secret_name, secret_value in secrets_from_file.items():
+                print(f"Ensuring {secret_name} is registered as a secret on Fireworks for rollout...")
+                if create_or_update_fireworks_secret(
+                    account_id=fw_account_id,
+                    key_name=secret_name,
+                    secret_value=secret_value,
+                ):
+                    print(f"✓ {secret_name} secret created/updated on Fireworks.")
+                else:
+                    print(f"Warning: Failed to create/update {secret_name} secret on Fireworks.")
         else:
             if not fw_account_id:
-                print("Warning: FIREWORKS_ACCOUNT_ID not found; cannot register FIREWORKS_API_KEY secret.")
-            if not fw_api_key_value:
-                print("Warning: FIREWORKS_API_KEY not found locally; cannot register secret.")
+                print("Warning: FIREWORKS_ACCOUNT_ID not found; cannot register secrets.")
+            if not secrets_from_file:
+                print("Warning: No API keys found in environment or .env file; no secrets to register.")
     except Exception as e:
         print(f"Warning: Skipped Fireworks secret registration due to error: {e}")
 

{eval_protocol-0.2.78 → eval_protocol-0.2.80}/eval_protocol/common_utils.py

@@ -5,6 +5,23 @@ from typing import Any, Dict, List
 import requests
 
 
+def get_user_agent() -> str:
+    """
+    Returns the user-agent string for eval-protocol CLI requests.
+
+    Format: eval-protocol-cli/{version}
+
+    Returns:
+        User-agent string identifying the eval-protocol CLI and version.
+    """
+    try:
+        from . import __version__
+
+        return f"eval-protocol/{__version__}"
+    except Exception:
+        return "eval-protocol/unknown"
+
+
 def load_jsonl(file_path: str) -> List[Dict[str, Any]]:
     """
     Reads a JSONL file where each line is a valid JSON object and returns a list of these objects.

{eval_protocol-0.2.78 → eval_protocol-0.2.80}/eval_protocol/evaluation.py

@@ -20,6 +20,7 @@ from eval_protocol.auth import (
     get_fireworks_api_key,
     verify_api_key_and_get_account_id,
 )
+from eval_protocol.common_utils import get_user_agent
 from eval_protocol.typed_interface import EvaluationMode
 
 from eval_protocol.get_pep440_version import get_pep440_version
@@ -405,6 +406,7 @@ class Evaluator:
         headers = {
             "Authorization": f"Bearer {auth_token}",
             "Content-Type": "application/json",
+            "User-Agent": get_user_agent(),
         }
         logger.info(f"Previewing evaluator using API endpoint: {url} with account: {account_id}")
         logger.debug(f"Preview API Request URL: {url}")
@@ -748,6 +750,7 @@ class Evaluator:
         headers = {
             "Authorization": f"Bearer {auth_token}",
             "Content-Type": "application/json",
+            "User-Agent": get_user_agent(),
         }
 
         self._ensure_requirements_present(os.getcwd())

{eval_protocol-0.2.78 → eval_protocol-0.2.80}/eval_protocol/fireworks_rft.py

@@ -11,6 +11,7 @@ from typing import Any, Callable, Dict, Iterable, Optional, Tuple
 import requests
 
 from .auth import get_fireworks_account_id, get_fireworks_api_base, get_fireworks_api_key
+from .common_utils import get_user_agent
 
 
 def _map_api_host_to_app_host(api_base: str) -> str:
@@ -157,12 +158,17 @@ def create_dataset_from_jsonl(
     display_name: Optional[str],
     jsonl_path: str,
 ) -> Tuple[str, Dict[str, Any]]:
-    headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+        "User-Agent": get_user_agent(),
+    }
     # Count examples quickly
     example_count = 0
     with open(jsonl_path, "r", encoding="utf-8") as f:
         for _ in f:
             example_count += 1
+
     dataset_url = f"{api_base.rstrip('/')}/v1/accounts/{account_id}/datasets"
     payload = {
         "dataset": {
@@ -181,7 +187,10 @@ def create_dataset_from_jsonl(
     upload_url = f"{api_base.rstrip('/')}/v1/accounts/{account_id}/datasets/{dataset_id}:upload"
     with open(jsonl_path, "rb") as f:
         files = {"file": f}
-        up_headers = {"Authorization": f"Bearer {api_key}"}
+        up_headers = {
+            "Authorization": f"Bearer {api_key}",
+            "User-Agent": get_user_agent(),
+        }
         up_resp = requests.post(upload_url, files=files, headers=up_headers, timeout=600)
     if up_resp.status_code not in (200, 201):
         raise RuntimeError(f"Dataset upload failed: {up_resp.status_code} {up_resp.text}")
@@ -195,7 +204,12 @@ def create_reinforcement_fine_tuning_job(
     body: Dict[str, Any],
 ) -> Dict[str, Any]:
     url = f"{api_base.rstrip('/')}/v1/accounts/{account_id}/reinforcementFineTuningJobs"
-    headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json", "Accept": "application/json"}
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+        "Accept": "application/json",
+        "User-Agent": get_user_agent(),
+    }
     resp = requests.post(url, json=body, headers=headers, timeout=60)
     if resp.status_code not in (200, 201):
         raise RuntimeError(f"RFT job creation failed: {resp.status_code} {resp.text}")

{eval_protocol-0.2.78 → eval_protocol-0.2.80}/eval_protocol/generation/clients.py

@@ -13,6 +13,8 @@ import aiohttp
 from omegaconf import DictConfig
 from pydantic import BaseModel  # Added for new models
 
+from ..common_utils import get_user_agent
+
 logger = logging.getLogger(__name__)
 
 
@@ -101,6 +103,7 @@ class FireworksModelClient(ModelClient):
             "Authorization": f"Bearer {self.api_key}",
             "Content-Type": "application/json",
             "Accept": "application/json",
+            "User-Agent": get_user_agent(),
         }
 
         debug_payload_log = json.loads(json.dumps(payload))

{eval_protocol-0.2.78 → eval_protocol-0.2.80}/eval_protocol/platform_api.py

@@ -11,6 +11,7 @@ from eval_protocol.auth import (
     get_fireworks_api_base,
     get_fireworks_api_key,
 )
+from eval_protocol.common_utils import get_user_agent
 
 logger = logging.getLogger(__name__)
 
@@ -95,6 +96,7 @@ def create_or_update_fireworks_secret(
     headers = {
         "Authorization": f"Bearer {resolved_api_key}",
         "Content-Type": "application/json",
+        "User-Agent": get_user_agent(),
     }
 
     # The secret_id for GET/PATCH/DELETE operations is the key_name.
@@ -107,10 +109,10 @@ def create_or_update_fireworks_secret(
 
     # Check if secret exists using GET (path uses normalized resource id)
     resource_id = _normalize_secret_resource_id(key_name)
-    get_url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{resource_id}"
     secret_exists = False
     try:
-        response = requests.get(get_url, headers=headers, timeout=10)
+        url = f"{resolved_api_base}/v1/accounts/{resolved_account_id}/secrets/{resource_id}"
+        response = requests.get(url, headers=headers, timeout=10)
         if response.status_code == 200:
             secret_exists = True
             logger.info(f"Secret '{key_name}' already exists. Will attempt to update.")
@@ -131,7 +133,6 @@ def create_or_update_fireworks_secret(
 
     if secret_exists:
         # Update existing secret (PATCH)
-        patch_url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{resource_id}"
         # Body for PATCH requires 'keyName' and 'value'.
         # Transform key_name for payload: uppercase and underscores
         payload_key_name = key_name.upper().replace("-", "_")
@@ -146,7 +147,8 @@ def create_or_update_fireworks_secret(
         payload = {"keyName": payload_key_name, "value": secret_value}
         try:
             logger.debug(f"PATCH payload for '{key_name}': {payload}")
-            response = requests.patch(patch_url, headers=headers, json=payload, timeout=30)
+            url = f"{resolved_api_base}/v1/accounts/{resolved_account_id}/secrets/{resource_id}"
+            response = requests.patch(url, json=payload, headers=headers, timeout=30)
             response.raise_for_status()
             logger.info(f"Successfully updated secret '{key_name}' on Fireworks platform.")
             return True
@@ -158,7 +160,6 @@ def create_or_update_fireworks_secret(
             return False
     else:
         # Create new secret (POST)
-        post_url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets"
         # Body for POST is gatewaySecret. 'name' field in payload is the resource path.
         # Let's assume for POST, the 'name' in payload can be omitted or is the key_name.
         # The API should ideally use 'keyName' from URL or a specific 'secretId' in payload for creation if 'name' is server-assigned.
@@ -183,7 +184,8 @@ def create_or_update_fireworks_secret(
         }
         try:
             logger.debug(f"POST payload for '{key_name}': {payload}")
-            response = requests.post(post_url, headers=headers, json=payload, timeout=30)
+            url = f"{resolved_api_base}/v1/accounts/{resolved_account_id}/secrets"
+            response = requests.post(url, json=payload, headers=headers, timeout=30)
             response.raise_for_status()
             logger.info(
                 f"Successfully created secret '{key_name}' on Fireworks platform. Full name: {response.json().get('name')}"
@@ -217,11 +219,14 @@ def get_fireworks_secret(
         logger.error("Missing Fireworks API key, base URL, or account ID for getting secret.")
         return None
 
-    headers = {"Authorization": f"Bearer {resolved_api_key}"}
+    headers = {
+        "Authorization": f"Bearer {resolved_api_key}",
+        "User-Agent": get_user_agent(),
+    }
     resource_id = _normalize_secret_resource_id(key_name)
-    url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{resource_id}"
 
     try:
+        url = f"{resolved_api_base}/v1/accounts/{resolved_account_id}/secrets/{resource_id}"
         response = requests.get(url, headers=headers, timeout=10)
         if response.status_code == 200:
             logger.info(f"Successfully retrieved secret '{key_name}'.")
@@ -254,11 +259,14 @@ def delete_fireworks_secret(
         logger.error("Missing Fireworks API key, base URL, or account ID for deleting secret.")
         return False
 
-    headers = {"Authorization": f"Bearer {resolved_api_key}"}
+    headers = {
+        "Authorization": f"Bearer {resolved_api_key}",
+        "User-Agent": get_user_agent(),
+    }
     resource_id = _normalize_secret_resource_id(key_name)
-    url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{resource_id}"
 
     try:
+        url = f"{resolved_api_base}/v1/accounts/{resolved_account_id}/secrets/{resource_id}"
         response = requests.delete(url, headers=headers, timeout=30)
         if response.status_code == 200 or response.status_code == 204:  # 204 No Content is also success for DELETE
             logger.info(f"Successfully deleted secret '{key_name}'.")

{eval_protocol-0.2.78 → eval_protocol-0.2.80}/eval_protocol/pytest/handle_persist_flow.py

@@ -7,9 +7,11 @@ import pathlib
 import re
 from typing import Any
 
+from eval_protocol.common_utils import get_user_agent
 from eval_protocol.directory_utils import find_eval_protocol_dir
 from eval_protocol.models import EvaluationRow
 from eval_protocol.pytest.store_experiment_link import store_experiment_link
+
 import requests
 
 
@@ -127,10 +129,14 @@ def handle_persist_flow(all_results: list[list[EvaluationRow]], test_func_name:
                         )
                         continue
 
-                    headers = {"Authorization": f"Bearer {fireworks_api_key}", "Content-Type": "application/json"}
+                    api_base = "https://api.fireworks.ai"
+                    headers = {
+                        "Authorization": f"Bearer {fireworks_api_key}",
+                        "Content-Type": "application/json",
+                        "User-Agent": get_user_agent(),
+                    }
 
                     # Make dataset first
-                    dataset_url = f"https://api.fireworks.ai/v1/accounts/{fireworks_account_id}/datasets"
 
                     dataset_payload = {  # pyright: ignore[reportUnknownVariableType]
                         "dataset": {
@@ -142,6 +148,7 @@ def handle_persist_flow(all_results: list[list[EvaluationRow]], test_func_name:
                         "datasetId": dataset_name,
                     }
 
+                    dataset_url = f"{api_base}/v1/accounts/{fireworks_account_id}/datasets"
                     dataset_response = requests.post(dataset_url, json=dataset_payload, headers=headers)  # pyright: ignore[reportUnknownArgumentType]
 
                     # Skip if dataset creation failed
@@ -157,13 +164,13 @@ def handle_persist_flow(all_results: list[list[EvaluationRow]], test_func_name:
                     dataset_id = dataset_data.get("datasetId", dataset_name)  # pyright: ignore[reportAny]
 
                     # Upload the JSONL file content
-                    upload_url = (
-                        f"https://api.fireworks.ai/v1/accounts/{fireworks_account_id}/datasets/{dataset_id}:upload"
-                    )
-                    upload_headers = {"Authorization": f"Bearer {fireworks_api_key}"}
-
+                    upload_url = f"{api_base}/v1/accounts/{fireworks_account_id}/datasets/{dataset_id}:upload"
                     with open(exp_file, "rb") as f:
                         files = {"file": f}
+                        upload_headers = {
+                            "Authorization": f"Bearer {fireworks_api_key}",
+                            "User-Agent": get_user_agent(),
+                        }
                         upload_response = requests.post(upload_url, files=files, headers=upload_headers)
 
                     # Skip if upload failed
@@ -176,7 +183,6 @@ def handle_persist_flow(all_results: list[list[EvaluationRow]], test_func_name:
                         continue
 
                     # Create evaluation job (optional - don't skip experiment if this fails)
-                    eval_job_url = f"https://api.fireworks.ai/v1/accounts/{fireworks_account_id}/evaluationJobs"
                     # Truncate job ID to fit 63 character limit
                     job_id_base = f"{dataset_name}-job"
                     if len(job_id_base) > 63:
@@ -194,6 +200,7 @@ def handle_persist_flow(all_results: list[list[EvaluationRow]], test_func_name:
                         },
                     }
 
+                    eval_job_url = f"{api_base}/v1/accounts/{fireworks_account_id}/evaluationJobs"
                     eval_response = requests.post(eval_job_url, json=eval_job_payload, headers=headers)
 
                     if eval_response.status_code in [200, 201]:

{eval_protocol-0.2.78 → eval_protocol-0.2.80/eval_protocol.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.78
+Version: 0.2.80
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT