eval-protocol 0.2.41__tar.gz → 0.2.43__tar.gz

{eval_protocol-0.2.41/eval_protocol.egg-info → eval_protocol-0.2.43}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.41
+Version: 0.2.43
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.41 → eval_protocol-0.2.43}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-07T15:43:37-0700",
+ "date": "2025-10-08T08:52:41-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "289abc56fc56935b45b11da011712fe48d956af1",
- "version": "0.2.41"
+ "full-revisionid": "535169e7193e6500d8d323e7dbc31c14dca98b96",
+ "version": "0.2.43"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.2.41 → eval_protocol-0.2.43}/eval_protocol/adapters/fireworks_tracing.py

@@ -371,9 +371,7 @@ class FireworksTracingAdapter(BaseAdapter):
                         error_msg = error_detail or e.response.text
 
                         # Retry on 404 if it's due to incomplete/missing traces (backend still indexing)
-                        if e.response.status_code == 404 and (
-                            "Incomplete traces" in error_detail or "No traces found" in error_detail
-                        ):
+                        if e.response.status_code == 404:
                             should_retry = True
                     except Exception:
                         error_msg = e.response.text

{eval_protocol-0.2.41 → eval_protocol-0.2.43}/eval_protocol/auth.py

@@ -6,10 +6,48 @@ from typing import Dict, Optional  # Added Dict
 
 logger = logging.getLogger(__name__)
 
+# Default locations (used for tests and as fallback). Actual resolution is dynamic via _get_auth_ini_file().
 FIREWORKS_CONFIG_DIR = Path.home() / ".fireworks"
 AUTH_INI_FILE = FIREWORKS_CONFIG_DIR / "auth.ini"
 
 
+def _get_profile_base_dir() -> Path:
+    """
+    Resolve the Fireworks configuration base directory following firectl behavior:
+    - Default: ~/.fireworks
+    - If FIREWORKS_PROFILE is set and non-empty: ~/.fireworks/profiles/<profile>
+    """
+    profile_name = os.environ.get("FIREWORKS_PROFILE", "").strip()
+    base_dir = Path.home() / ".fireworks"
+    if profile_name:
+        base_dir = base_dir / "profiles" / profile_name
+    return base_dir
+
+
+def _get_auth_ini_file() -> Path:
+    """
+    Determine the auth.ini file path.
+    Priority:
+      1) FIREWORKS_AUTH_FILE env var when set
+      2) ~/.fireworks[/profiles/<profile>]/auth.ini (profile driven)
+    """
+    auth_file_env = os.environ.get("FIREWORKS_AUTH_FILE")
+    if auth_file_env:
+        return Path(auth_file_env)
+    return _get_profile_base_dir() / "auth.ini"
+
+
+def _is_profile_active() -> bool:
+    """
+    Returns True if a specific profile or explicit auth file is active.
+    In this case, profile-based credentials should take precedence over env vars.
+    """
+    if os.environ.get("FIREWORKS_AUTH_FILE"):
+        return True
+    prof = os.environ.get("FIREWORKS_PROFILE", "").strip()
+    return bool(prof)
+
+
 def _parse_simple_auth_file(file_path: Path) -> Dict[str, str]:
     """
     Parses an auth file with simple key=value lines.
@@ -20,7 +58,7 @@ def _parse_simple_auth_file(file_path: Path) -> Dict[str, str]:
     if not file_path.exists():
         return creds
     try:
-        with open(file_path, "r") as f:
+        with open(file_path, "r", encoding="utf-8") as f:
             for line in f:
                 line = line.strip()
                 if not line or line.startswith("#") or line.startswith(";"):
@@ -39,7 +77,7 @@ def _parse_simple_auth_file(file_path: Path) -> Dict[str, str]:
                     if key in ["api_key", "account_id"] and value:
                         creds[key] = value
     except Exception as e:
-        logger.warning(f"Error during simple parsing of {file_path}: {e}")
+        logger.warning("Error during simple parsing of %s: %s", str(file_path), e)
     return creds
 
 
@@ -48,13 +86,14 @@ def _get_credential_from_config_file(key_name: str) -> Optional[str]:
     Helper to get a specific credential (api_key or account_id) from auth.ini.
     Tries simple parsing first, then configparser.
     """
-    if not AUTH_INI_FILE.exists():
+    auth_ini_path = _get_auth_ini_file()
+    if not auth_ini_path.exists():
         return None
 
     # 1. Try simple key-value parsing first
-    simple_creds = _parse_simple_auth_file(AUTH_INI_FILE)
+    simple_creds = _parse_simple_auth_file(auth_ini_path)
     if key_name in simple_creds:
-        logger.debug(f"Using {key_name} from simple key-value parsing of {AUTH_INI_FILE}.")
+        logger.debug("Using %s from simple key-value parsing of %s.", key_name, str(auth_ini_path))
         return simple_creds[key_name]
 
     # 2. Fallback to configparser if not found via simple parsing or if simple parsing failed
@@ -62,30 +101,35 @@ def _get_credential_from_config_file(key_name: str) -> Optional[str]:
     #    but only if simple parsing didn't yield the key.
     try:
         config = configparser.ConfigParser()
-        config.read(AUTH_INI_FILE)
+        config.read(auth_ini_path)
 
         # Try [fireworks] section
         if "fireworks" in config and config.has_option("fireworks", key_name):
             value_from_file = config.get("fireworks", key_name)
             if value_from_file:
-                logger.debug(f"Using {key_name} from [fireworks] section in {AUTH_INI_FILE}.")
+                logger.debug("Using %s from [fireworks] section in %s.", key_name, str(auth_ini_path))
                 return value_from_file
 
         # Try default section (configparser might place items without section header here)
         if config.has_option(config.default_section, key_name):
             value_from_default = config.get(config.default_section, key_name)
             if value_from_default:
-                logger.debug(f"Using {key_name} from default section [{config.default_section}] in {AUTH_INI_FILE}.")
+                logger.debug(
+                    "Using %s from default section [%s] in %s.",
+                    key_name,
+                    config.default_section,
+                    str(auth_ini_path),
+                )
                 return value_from_default
 
     except configparser.MissingSectionHeaderError:
         # This error implies the file is purely key-value, which simple parsing should have handled.
         # If simple parsing failed to get the key, then it's likely not there or malformed.
-        logger.debug(f"{AUTH_INI_FILE} has no section headers, and simple parsing did not find {key_name}.")
+        logger.debug("%s has no section headers, and simple parsing did not find %s.", str(auth_ini_path), key_name)
     except configparser.Error as e_config:
-        logger.warning(f"Configparser error reading {AUTH_INI_FILE} for {key_name}: {e_config}")
+        logger.warning("Configparser error reading %s for %s: %s", str(auth_ini_path), key_name, e_config)
     except Exception as e_general:
-        logger.warning(f"Unexpected error reading {AUTH_INI_FILE} for {key_name}: {e_general}")
+        logger.warning("Unexpected error reading %s for %s: %s", str(auth_ini_path), key_name, e_general)
 
     return None
 
@@ -101,14 +145,24 @@ def get_fireworks_api_key() -> Optional[str]:
     Returns:
         The API key if found, otherwise None.
     """
-    api_key = os.environ.get("FIREWORKS_API_KEY")
-    if api_key:
-        logger.debug("Using FIREWORKS_API_KEY from environment variable.")
-        return api_key
-
-    api_key_from_file = _get_credential_from_config_file("api_key")
-    if api_key_from_file:
-        return api_key_from_file
+    # If a profile is active, prefer profile file first, then env
+    if _is_profile_active():
+        api_key_from_file = _get_credential_from_config_file("api_key")
+        if api_key_from_file:
+            return api_key_from_file
+        api_key = os.environ.get("FIREWORKS_API_KEY")
+        if api_key:
+            logger.debug("Using FIREWORKS_API_KEY from environment variable (profile active but file missing).")
+            return api_key
+    else:
+        # Default behavior: env overrides file
+        api_key = os.environ.get("FIREWORKS_API_KEY")
+        if api_key:
+            logger.debug("Using FIREWORKS_API_KEY from environment variable.")
+            return api_key
+        api_key_from_file = _get_credential_from_config_file("api_key")
+        if api_key_from_file:
+            return api_key_from_file
 
     logger.debug("Fireworks API key not found in environment variables or auth.ini.")
     return None
@@ -125,14 +179,24 @@ def get_fireworks_account_id() -> Optional[str]:
     Returns:
         The Account ID if found, otherwise None.
     """
-    account_id = os.environ.get("FIREWORKS_ACCOUNT_ID")
-    if account_id:
-        logger.debug("Using FIREWORKS_ACCOUNT_ID from environment variable.")
-        return account_id
-
-    account_id_from_file = _get_credential_from_config_file("account_id")
-    if account_id_from_file:
-        return account_id_from_file
+    # If a profile is active, prefer profile file first, then env
+    if _is_profile_active():
+        account_id_from_file = _get_credential_from_config_file("account_id")
+        if account_id_from_file:
+            return account_id_from_file
+        account_id = os.environ.get("FIREWORKS_ACCOUNT_ID")
+        if account_id:
+            logger.debug("Using FIREWORKS_ACCOUNT_ID from environment variable (profile active but file missing).")
+            return account_id
+    else:
+        # Default behavior: env overrides file
+        account_id = os.environ.get("FIREWORKS_ACCOUNT_ID")
+        if account_id:
+            logger.debug("Using FIREWORKS_ACCOUNT_ID from environment variable.")
+            return account_id
+        account_id_from_file = _get_credential_from_config_file("account_id")
+        if account_id_from_file:
+            return account_id_from_file
 
     logger.debug("Fireworks Account ID not found in environment variables or auth.ini.")
     return None
@@ -152,5 +216,5 @@ def get_fireworks_api_base() -> str:
     if os.environ.get("FIREWORKS_API_BASE"):
         logger.debug("Using FIREWORKS_API_BASE from environment variable.")
     else:
-        logger.debug(f"FIREWORKS_API_BASE not set in environment, defaulting to {api_base}.")
+        logger.debug("FIREWORKS_API_BASE not set in environment, defaulting to %s.", api_base)
     return api_base

{eval_protocol-0.2.41 → eval_protocol-0.2.43}/eval_protocol/cli.py

@@ -3,32 +3,47 @@ Command-line interface for Eval Protocol.
 """
 
 import argparse
-import asyncio
-import json
 import logging
 import os
 import sys
-import traceback
-import uuid
 from pathlib import Path
+from typing import Any, cast
 
 logger = logging.getLogger(__name__)
 
 
-from .cli_commands.agent_eval_cmd import agent_eval_command
 from .cli_commands.common import setup_logging
-from .cli_commands.deploy import deploy_command
-from .cli_commands.deploy_mcp import deploy_mcp_command
-from .cli_commands.logs import logs_command
-from .cli_commands.preview import preview_command
-from .cli_commands.run_eval_cmd import hydra_cli_entry_point
-from .cli_commands.upload import upload_command
+
+# Re-export deploy_command for backward compatibility with tests importing from eval_protocol.cli
+try:  # pragma: no cover - import-time alias for tests
+    from .cli_commands import deploy as _deploy_mod
+
+    deploy_command = _deploy_mod.deploy_command  # type: ignore[attr-defined]
+except Exception:  # pragma: no cover
+    # If import fails in constrained environments, tests that import it will surface the issue
+    deploy_command = None  # type: ignore[assignment]
+
+# Re-export preview_command for backward compatibility with tests importing from eval_protocol.cli
+try:  # pragma: no cover - import-time alias for tests
+    from .cli_commands import preview as _preview_mod
+
+    preview_command = _preview_mod.preview_command  # type: ignore[attr-defined]
+except Exception:  # pragma: no cover
+    preview_command = None  # type: ignore[assignment]
 
 
 def parse_args(args=None):
     """Parse command line arguments"""
     parser = argparse.ArgumentParser(description="eval-protocol: Tools for evaluation and reward modeling")
     parser.add_argument("--verbose", "-v", action="store_true", help="Enable verbose logging")
+    parser.add_argument(
+        "--profile",
+        help="Fireworks profile to use (reads ~/.fireworks/profiles/<name>/auth.ini and settings.ini)",
+    )
+    parser.add_argument(
+        "--server",
+        help="Fireworks API server hostname or URL (e.g., dev.api.fireworks.ai or https://dev.api.fireworks.ai)",
+    )
 
     subparsers = parser.add_subparsers(dest="command", help="Command to run")
 
@@ -356,12 +371,68 @@ def main():
             os.environ["PYTHONPATH"] = f"{current_dir}{os.pathsep}{current_pythonpath}"
         else:
             os.environ["PYTHONPATH"] = current_dir
-        logger.debug(f"Added current directory to PYTHONPATH: {current_dir}")
+        logger.debug("Added current directory to PYTHONPATH: %s", current_dir)
 
         # Also add to sys.path so it takes effect immediately for the current process
         if current_dir not in sys.path:
             sys.path.insert(0, current_dir)
 
+    # Pre-scan raw argv for global flags anywhere (before parsing or imports)
+    raw_argv = sys.argv[1:]
+
+    def _extract_flag_value(argv_list, flag_name):
+        # Supports --flag value and --flag=value
+        for i, tok in enumerate(argv_list):
+            if tok == flag_name:
+                if i + 1 < len(argv_list):
+                    return argv_list[i + 1]
+            elif tok.startswith(flag_name + "="):
+                return tok.split("=", 1)[1]
+        return None
+
+    pre_profile = _extract_flag_value(raw_argv, "--profile")
+    pre_server = _extract_flag_value(raw_argv, "--server")
+
+    # Handle Fireworks profile selection early so downstream modules see the env
+    profile = pre_profile
+    if profile:
+        try:
+            os.environ["FIREWORKS_PROFILE"] = profile
+            # Mirror firectl behavior: ~/.fireworks[/profiles/<profile>]
+            base_dir = Path.home() / ".fireworks"
+            if profile:
+                base_dir = base_dir / "profiles" / profile
+            os.makedirs(str(base_dir), mode=0o700, exist_ok=True)
+
+            # Provide helpful env hints for consumers (optional)
+            os.environ["FIREWORKS_AUTH_FILE"] = str(base_dir / "auth.ini")
+            os.environ["FIREWORKS_SETTINGS_FILE"] = str(base_dir / "settings.ini")
+            logger.debug("Using Fireworks profile '%s' at %s", profile, base_dir)
+        except OSError as e:
+            logger.warning("Failed to initialize Fireworks profile '%s': %s", profile, e)
+
+        # Proactively resolve and export account_id from the active profile to avoid stale .env overrides
+        try:
+            from eval_protocol.auth import get_fireworks_account_id as _resolve_account_id
+
+            resolved_account = _resolve_account_id()
+            if resolved_account:
+                os.environ["FIREWORKS_ACCOUNT_ID"] = resolved_account
+                logger.debug("Resolved account_id from profile '%s': %s", profile, resolved_account)
+        except Exception as e:  # noqa: B902
+            logger.debug("Unable to resolve account_id from profile '%s': %s", profile, e)
+
+    # Handle Fireworks server selection early
+    server = pre_server
+    if server:
+        # Normalize to full URL if just a hostname is supplied
+        normalized = server.strip()
+        if not normalized.startswith("http://") and not normalized.startswith("https://"):
+            normalized = f"https://{normalized}"
+        os.environ["FIREWORKS_API_BASE"] = normalized
+        logger.debug("Using Fireworks API base: %s", normalized)
+
+    # Now parse args normally (so help/commands work), after globals applied
     # Store original sys.argv[0] because Hydra might manipulate it
     # and we need it if we're not calling a Hydra app.
     original_script_name = sys.argv[0]
@@ -370,16 +441,28 @@ def main():
     setup_logging(args.verbose, getattr(args, "debug", False))
 
     if args.command == "preview":
+        if preview_command is None:
+            raise ImportError("preview_command is unavailable")
         return preview_command(args)
     elif args.command == "deploy":
+        if deploy_command is None:
+            raise ImportError("deploy_command is unavailable")
         return deploy_command(args)
     elif args.command == "deploy-mcp":
+        from .cli_commands.deploy_mcp import deploy_mcp_command
+
         return deploy_mcp_command(args)
     elif args.command == "agent-eval":
+        from .cli_commands.agent_eval_cmd import agent_eval_command
+
         return agent_eval_command(args)
     elif args.command == "logs":
+        from .cli_commands.logs import logs_command
+
         return logs_command(args)
     elif args.command == "upload":
+        from .cli_commands.upload import upload_command
+
         return upload_command(args)
     elif args.command == "run":
         # For the 'run' command, Hydra takes over argument parsing.
@@ -393,7 +476,7 @@ def main():
         local_conf_dir = os.path.join(current_dir, "conf")
 
         if not has_config_path and os.path.isdir(local_conf_dir):
-            logger.info(f"Auto-detected local conf directory: {local_conf_dir}")
+            logger.info("Auto-detected local conf directory: %s", local_conf_dir)
             hydra_specific_args = [
                 "--config-path",
                 local_conf_dir,
@@ -410,18 +493,21 @@ def main():
                     path_val = hydra_specific_args[i]
                     abs_path = os.path.abspath(path_val)
                     logger.debug(
-                        f"Converting relative --config-path '{path_val}' (space separated) to absolute '{abs_path}'"
+                        "Converting relative --config-path '%s' (space separated) to absolute '%s'",
+                        path_val,
+                        abs_path,
                     )
                     processed_hydra_args.append(abs_path)
                 else:
                     logger.error("--config-path specified without a value.")
-                    pass
             elif arg.startswith("--config-path="):
                 flag_part, path_val = arg.split("=", 1)
                 processed_hydra_args.append(flag_part)
                 abs_path = os.path.abspath(path_val)
                 logger.debug(
-                    f"Converting relative --config-path '{path_val}' (equals separated) to absolute '{abs_path}'"
+                    "Converting relative --config-path '%s' (equals separated) to absolute '%s'",
+                    path_val,
+                    abs_path,
                 )
                 processed_hydra_args.append(abs_path)
             else:
@@ -429,14 +515,17 @@ def main():
             i += 1
 
         sys.argv = [sys.argv[0]] + processed_hydra_args
-        logger.info(f"SYSCALL_ARGV_FOR_HYDRA (after potential abspath conversion): {sys.argv}")
+        logger.info("SYSCALL_ARGV_FOR_HYDRA (after potential abspath conversion): %s", sys.argv)
 
         try:
-            hydra_cli_entry_point()
+            from .cli_commands.run_eval_cmd import hydra_cli_entry_point
+
+            hydra_entry = cast(Any, hydra_cli_entry_point)
+            hydra_entry()  # type: ignore  # pylint: disable=no-value-for-parameter
             return 0
-        except Exception as e:
+        except Exception as e:  # pylint: disable=broad-except
             error_msg = str(e)
-            logger.error(f"Evaluation failed: {e}")
+            logger.error("Evaluation failed: %s", e)
 
             # Provide helpful suggestions for common Hydra/config errors
             if "Cannot find primary config" in error_msg: