eval-protocol 0.2.70.dev1__tar.gz → 0.2.83.dev1__tar.gz

{eval_protocol-0.2.70.dev1/eval_protocol.egg-info → eval_protocol-0.2.83.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.70.dev1
+Version: 0.2.83.dev1
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.70.dev1 → eval_protocol-0.2.83.dev1}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-29T17:18:36-0700",
+ "date": "2025-11-10T00:07:09-0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "0ebd0177dafc55bfa302a49b2d674c0487516eff",
- "version": "0.2.70-dev1"
+ "full-revisionid": "622929889e1714f7bfa6bca820470f2d27a12fb2",
+ "version": "0.2.83.dev1"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.2.70.dev1 → eval_protocol-0.2.83.dev1}/eval_protocol/adapters/fireworks_tracing.py

@@ -273,7 +273,12 @@ class FireworksTracingAdapter(BaseAdapter):
         if not tags:
             raise ValueError("At least one tag is required to fetch logs")
 
-        headers = {"Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}"}
+        from ..common_utils import get_user_agent
+
+        headers = {
+            "Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}",
+            "User-Agent": get_user_agent(),
+        }
         params: Dict[str, Any] = {"tags": tags, "limit": limit, "hours_back": hours_back, "program": "eval_protocol"}
 
         # Try /logs first, fall back to /v1/logs if not found
@@ -398,7 +403,12 @@ class FireworksTracingAdapter(BaseAdapter):
         else:
             url = f"{self.base_url}/v1/traces/pointwise"
 
-        headers = {"Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}"}
+        from ..common_utils import get_user_agent
+
+        headers = {
+            "Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}",
+            "User-Agent": get_user_agent(),
+        }
 
         result = None
         try:

{eval_protocol-0.2.70.dev1 → eval_protocol-0.2.83.dev1}/eval_protocol/auth.py

@@ -242,9 +242,16 @@ def verify_api_key_and_get_account_id(
         if not resolved_key:
             return None
         resolved_base = api_base or get_fireworks_api_base()
+
+        from .common_utils import get_user_agent
+
         url = f"{resolved_base.rstrip('/')}/verifyApiKey"
-        headers = {"Authorization": f"Bearer {resolved_key}"}
+        headers = {
+            "Authorization": f"Bearer {resolved_key}",
+            "User-Agent": get_user_agent(),
+        }
         resp = requests.get(url, headers=headers, timeout=10)
+
         if resp.status_code != 200:
             logger.debug("verifyApiKey returned status %s", resp.status_code)
             return None

{eval_protocol-0.2.70.dev1 → eval_protocol-0.2.83.dev1}/eval_protocol/benchmarks/test_frozen_lake.py

@@ -46,7 +46,7 @@ def frozen_lake_to_evaluation_row(data: List[Dict[str, Any]]) -> List[Evaluation
     num_runs=1,
     max_concurrent_rollouts=3,
     mode="pointwise",
-    server_script_path="examples/frozen_lake_mcp/server.py",
+    server_script_path="eval_protocol/mcp_servers/frozen_lake/server.py",
 )
 def test_frozen_lake_evaluation(row: EvaluationRow) -> EvaluationRow:
     """

{eval_protocol-0.2.70.dev1 → eval_protocol-0.2.83.dev1}/eval_protocol/cli.py

@@ -355,6 +355,10 @@ def parse_args(args=None):
         action="store_true",
         help="Non-interactive: upload all discovered evaluation tests",
     )
+    upload_parser.add_argument(
+        "--env-file",
+        help="Path to .env file containing secrets to upload (default: .env in current directory)",
+    )
 
     # Create command group
     create_parser = subparsers.add_parser(
@@ -391,23 +395,25 @@ def parse_args(args=None):
     rft_parser.add_argument("--base-model", help="Base model resource id")
     rft_parser.add_argument("--warm-start-from", help="Addon model to warm start from")
     rft_parser.add_argument("--output-model", help="Output model id (defaults from evaluator)")
-    rft_parser.add_argument("--epochs", type=int)
-    rft_parser.add_argument("--batch-size", type=int)
-    rft_parser.add_argument("--learning-rate", type=float)
-    rft_parser.add_argument("--max-context-length", type=int)
-    rft_parser.add_argument("--lora-rank", type=int)
-    rft_parser.add_argument("--accelerator-count", type=int)
+    rft_parser.add_argument("--epochs", type=int, default=8)
+    rft_parser.add_argument("--batch-size", type=int, default=128000)
+    rft_parser.add_argument("--learning-rate", type=float, default=3e-5)
+    rft_parser.add_argument("--max-context-length", type=int, default=65536)
+    rft_parser.add_argument("--lora-rank", type=int, default=16)
+    rft_parser.add_argument("--accelerator-count", type=int, default=1)
     rft_parser.add_argument("--region", help="Fireworks region enum value")
     rft_parser.add_argument("--display-name", help="RFT job display name")
     rft_parser.add_argument("--evaluation-dataset", help="Optional separate eval dataset id")
     rft_parser.add_argument("--eval-auto-carveout", dest="eval_auto_carveout", action="store_true", default=True)
     rft_parser.add_argument("--no-eval-auto-carveout", dest="eval_auto_carveout", action="store_false")
+    # Rollout chunking
+    rft_parser.add_argument("--chunk-size", type=int, default=10, help="Data chunk size for rollout batching")
     # Inference params
     rft_parser.add_argument("--temperature", type=float)
     rft_parser.add_argument("--top-p", type=float)
     rft_parser.add_argument("--top-k", type=int)
-    rft_parser.add_argument("--max-tokens", type=int)
-    rft_parser.add_argument("--n", type=int)
+    rft_parser.add_argument("--max-tokens", type=int, default=32768)
+    rft_parser.add_argument("--n", type=int, default=8)
     rft_parser.add_argument("--inference-extra-body", help="JSON string for extra inference params")
     # Wandb
     rft_parser.add_argument("--wandb-enabled", action="store_true")
@@ -419,6 +425,7 @@ def parse_args(args=None):
     rft_parser.add_argument("--rft-job-id", help="Specify an explicit RFT job id")
     rft_parser.add_argument("--yes", "-y", action="store_true", help="Non-interactive mode")
     rft_parser.add_argument("--dry-run", action="store_true", help="Print planned REST calls without sending")
+    rft_parser.add_argument("--force", action="store_true", help="Overwrite existing evaluator with the same ID")
 
     # Run command (for Hydra-based evaluations)
     # This subparser intentionally defines no arguments itself.