eval-protocol 0.2.64.dev1__tar.gz → 0.2.84.dev1__tar.gz

{eval_protocol-0.2.64.dev1/eval_protocol.egg-info → eval_protocol-0.2.84.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.64.dev1
+Version: 0.2.84.dev1
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.64.dev1 → eval_protocol-0.2.84.dev1}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-27T01:07:36-0700",
+ "date": "2025-11-10T16:26:12-0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "2827c52b7a4d7d9e652934e8308e424645bcc52e",
- "version": "0.2.64-dev1"
+ "full-revisionid": "66542cf6410b379a35d2aec3de041fb37e18b0e2",
+ "version": "0.2.84.dev.1"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.2.64.dev1 → eval_protocol-0.2.84.dev1}/eval_protocol/adapters/fireworks_tracing.py

@@ -273,7 +273,12 @@ class FireworksTracingAdapter(BaseAdapter):
         if not tags:
             raise ValueError("At least one tag is required to fetch logs")
 
-        headers = {"Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}"}
+        from ..common_utils import get_user_agent
+
+        headers = {
+            "Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}",
+            "User-Agent": get_user_agent(),
+        }
         params: Dict[str, Any] = {"tags": tags, "limit": limit, "hours_back": hours_back, "program": "eval_protocol"}
 
         # Try /logs first, fall back to /v1/logs if not found
@@ -398,7 +403,12 @@ class FireworksTracingAdapter(BaseAdapter):
         else:
             url = f"{self.base_url}/v1/traces/pointwise"
 
-        headers = {"Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}"}
+        from ..common_utils import get_user_agent
+
+        headers = {
+            "Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}",
+            "User-Agent": get_user_agent(),
+        }
 
         result = None
         try:

{eval_protocol-0.2.64.dev1 → eval_protocol-0.2.84.dev1}/eval_protocol/auth.py

@@ -136,6 +136,56 @@ def _get_credential_from_config_file(key_name: str) -> Optional[str]:
     return None
 
 
+def _get_credentials_from_config_file() -> Dict[str, Optional[str]]:
+    """
+    Retrieve both api_key and account_id from auth.ini with a single read/parse.
+    Tries simple parsing first for both keys, then falls back to configparser for any missing ones.
+    Returns a dict with up to two keys: 'api_key' and 'account_id'.
+    """
+    results: Dict[str, Optional[str]] = {}
+    auth_ini_path = _get_auth_ini_file()
+    if not auth_ini_path.exists():
+        return results
+
+    # 1) Simple key=value parsing
+    try:
+        simple_creds = _parse_simple_auth_file(auth_ini_path)
+        if "api_key" in simple_creds and simple_creds["api_key"]:
+            results["api_key"] = simple_creds["api_key"]
+        if "account_id" in simple_creds and simple_creds["account_id"]:
+            results["account_id"] = simple_creds["account_id"]
+        if "api_key" in results and "account_id" in results:
+            return results
+    except Exception as e:
+        logger.warning("Error during simple parsing of %s: %s", str(auth_ini_path), e)
+
+    # 2) ConfigParser for any missing keys
+    try:
+        config = configparser.ConfigParser()
+        config.read(auth_ini_path)
+        for key_name in ("api_key", "account_id"):
+            if key_name in results and results[key_name]:
+                continue
+            if "fireworks" in config and config.has_option("fireworks", key_name):
+                value_from_file = config.get("fireworks", key_name)
+                if value_from_file:
+                    results[key_name] = value_from_file
+                    continue
+            if config.has_option(config.default_section, key_name):
+                value_from_default = config.get(config.default_section, key_name)
+                if value_from_default:
+                    results[key_name] = value_from_default
+    except configparser.MissingSectionHeaderError:
+        # Purely key=value file without section headers; simple parsing should have handled it already.
+        logger.debug("%s has no section headers; falling back to simple parsing results.", str(auth_ini_path))
+    except configparser.Error as e_config:
+        logger.warning("Configparser error reading %s: %s", str(auth_ini_path), e_config)
+    except Exception as e_general:
+        logger.warning("Unexpected error reading %s: %s", str(auth_ini_path), e_general)
+
+    return results
+
+
 def get_fireworks_api_key() -> Optional[str]:
     """
     Retrieves the Fireworks API key.
@@ -177,13 +227,15 @@ def get_fireworks_account_id() -> Optional[str]:
     The Account ID is sourced in the following order:
     1. FIREWORKS_ACCOUNT_ID environment variable.
     2. 'account_id' from the [fireworks] section of ~/.fireworks/auth.ini.
+    3. If an API key is available (env or auth.ini), resolve via verifyApiKey.
 
     Returns:
         The Account ID if found, otherwise None.
     """
     # If a profile is active, prefer profile file first, then env
     if _is_profile_active():
-        account_id_from_file = _get_credential_from_config_file("account_id")
+        creds = _get_credentials_from_config_file()
+        account_id_from_file = creds.get("account_id")
         if account_id_from_file:
             return account_id_from_file
         account_id = os.environ.get("FIREWORKS_ACCOUNT_ID")
@@ -196,11 +248,24 @@ def get_fireworks_account_id() -> Optional[str]:
         if account_id:
             logger.debug("Using FIREWORKS_ACCOUNT_ID from environment variable.")
             return account_id
-        account_id_from_file = _get_credential_from_config_file("account_id")
+        creds = _get_credentials_from_config_file()
+        account_id_from_file = creds.get("account_id")
         if account_id_from_file:
             return account_id_from_file
 
-    logger.debug("Fireworks Account ID not found in environment variables or auth.ini.")
+    # 3) Fallback: if API key is present, attempt to resolve via verifyApiKey (env or auth.ini)
+    try:
+        # Intentionally use get_fireworks_api_key to centralize precedence (env vs file)
+        api_key_for_verify = get_fireworks_api_key()
+        if api_key_for_verify:
+            resolved = verify_api_key_and_get_account_id(api_key=api_key_for_verify, api_base=get_fireworks_api_base())
+            if resolved:
+                logger.debug("Using FIREWORKS_ACCOUNT_ID resolved via verifyApiKey: %s", resolved)
+                return resolved
+    except Exception as e:
+        logger.debug("Failed to resolve FIREWORKS_ACCOUNT_ID via verifyApiKey: %s", e)
+
+    logger.debug("Fireworks Account ID not found in environment variables, auth.ini, or via verifyApiKey.")
     return None
 
 
@@ -242,9 +307,16 @@ def verify_api_key_and_get_account_id(
         if not resolved_key:
             return None
         resolved_base = api_base or get_fireworks_api_base()
+
+        from .common_utils import get_user_agent
+
         url = f"{resolved_base.rstrip('/')}/verifyApiKey"
-        headers = {"Authorization": f"Bearer {resolved_key}"}
+        headers = {
+            "Authorization": f"Bearer {resolved_key}",
+            "User-Agent": get_user_agent(),
+        }
         resp = requests.get(url, headers=headers, timeout=10)
+
         if resp.status_code != 200:
             logger.debug("verifyApiKey returned status %s", resp.status_code)
             return None

{eval_protocol-0.2.64.dev1 → eval_protocol-0.2.84.dev1}/eval_protocol/benchmarks/test_frozen_lake.py

@@ -46,7 +46,7 @@ def frozen_lake_to_evaluation_row(data: List[Dict[str, Any]]) -> List[Evaluation
     num_runs=1,
     max_concurrent_rollouts=3,
     mode="pointwise",
-    server_script_path="examples/frozen_lake_mcp/server.py",
+    server_script_path="eval_protocol/mcp_servers/frozen_lake/server.py",
 )
 def test_frozen_lake_evaluation(row: EvaluationRow) -> EvaluationRow:
     """

{eval_protocol-0.2.64.dev1 → eval_protocol-0.2.84.dev1}/eval_protocol/cli.py

@@ -355,6 +355,77 @@ def parse_args(args=None):
         action="store_true",
         help="Non-interactive: upload all discovered evaluation tests",
     )
+    upload_parser.add_argument(
+        "--env-file",
+        help="Path to .env file containing secrets to upload (default: .env in current directory)",
+    )
+
+    # Create command group
+    create_parser = subparsers.add_parser(
+        "create",
+        help="Resource creation commands",
+    )
+    create_subparsers = create_parser.add_subparsers(dest="create_command")
+    rft_parser = create_subparsers.add_parser(
+        "rft",
+        help="Create a Reinforcement Fine-tuning Job on Fireworks",
+    )
+    rft_parser.add_argument(
+        "--evaluator-id",
+        help="Evaluator ID used during upload; if omitted, derive from local traces or a single discovered test",
+    )
+    # Dataset options
+    rft_parser.add_argument(
+        "--dataset-id",
+        help="Use existing Fireworks dataset id (skip local materialization)",
+    )
+    rft_parser.add_argument(
+        "--dataset-jsonl",
+        help="Path to JSONL to upload as a new Fireworks dataset",
+    )
+    rft_parser.add_argument(
+        "--dataset-builder",
+        help="Explicit dataset builder spec (module::function or path::function)",
+    )
+    rft_parser.add_argument(
+        "--dataset-display-name",
+        help="Display name for dataset on Fireworks (defaults to dataset id)",
+    )
+    # Training config and evaluator/job settings
+    rft_parser.add_argument("--base-model", help="Base model resource id")
+    rft_parser.add_argument("--warm-start-from", help="Addon model to warm start from")
+    rft_parser.add_argument("--output-model", help="Output model id (defaults from evaluator)")
+    rft_parser.add_argument("--epochs", type=int, default=1)
+    rft_parser.add_argument("--batch-size", type=int, default=128000)
+    rft_parser.add_argument("--learning-rate", type=float, default=3e-5)
+    rft_parser.add_argument("--max-context-length", type=int, default=65536)
+    rft_parser.add_argument("--lora-rank", type=int, default=16)
+    rft_parser.add_argument("--accelerator-count", type=int, default=1)
+    rft_parser.add_argument("--region", help="Fireworks region enum value")
+    rft_parser.add_argument("--display-name", help="RFT job display name")
+    rft_parser.add_argument("--evaluation-dataset", help="Optional separate eval dataset id")
+    rft_parser.add_argument("--eval-auto-carveout", dest="eval_auto_carveout", action="store_true", default=True)
+    rft_parser.add_argument("--no-eval-auto-carveout", dest="eval_auto_carveout", action="store_false")
+    # Rollout chunking
+    rft_parser.add_argument("--chunk-size", type=int, default=10, help="Data chunk size for rollout batching")
+    # Inference params
+    rft_parser.add_argument("--temperature", type=float)
+    rft_parser.add_argument("--top-p", type=float)
+    rft_parser.add_argument("--top-k", type=int)
+    rft_parser.add_argument("--max-tokens", type=int, default=32768)
+    rft_parser.add_argument("--n", type=int, default=8)
+    rft_parser.add_argument("--inference-extra-body", help="JSON string for extra inference params")
+    # Wandb
+    rft_parser.add_argument("--wandb-enabled", action="store_true")
+    rft_parser.add_argument("--wandb-project")
+    rft_parser.add_argument("--wandb-entity")
+    rft_parser.add_argument("--wandb-run-id")
+    rft_parser.add_argument("--wandb-api-key")
+    # Misc
+    rft_parser.add_argument("--rft-job-id", help="Specify an explicit RFT job id")
+    rft_parser.add_argument("--yes", "-y", action="store_true", help="Non-interactive mode")
+    rft_parser.add_argument("--dry-run", action="store_true", help="Print planned REST calls without sending")
+    rft_parser.add_argument("--force", action="store_true", help="Overwrite existing evaluator with the same ID")
 
     # Run command (for Hydra-based evaluations)
     # This subparser intentionally defines no arguments itself.
@@ -481,6 +552,13 @@ def main():
         from .cli_commands.upload import upload_command
 
         return upload_command(args)
+    elif args.command == "create":
+        if args.create_command == "rft":
+            from .cli_commands.create_rft import create_rft_command
+
+            return create_rft_command(args)
+        print("Error: missing subcommand for 'create'. Try: eval-protocol create rft")
+        return 1
     elif args.command == "run":
         # For the 'run' command, Hydra takes over argument parsing.