eval-protocol 0.0.3__py3-none-any.whl

eval_protocol/agent/tool_registry.py

@@ -0,0 +1,111 @@
+"""
+Tool Registry for the Agent Evaluation Framework.
+Provides a mechanism to register and manage tools.
+"""
+
+import functools
+import inspect
+from typing import Any, Callable, Dict, List, Optional
+
+
+class ToolRegistry:
+    """
+    Registry for tools that can be used by agents.
+
+    Attributes:
+        name: Unique identifier for this tool registry
+        tools: Dictionary mapping tool names to tool functions
+        descriptions: Dictionary mapping tool names to their descriptions
+        parameters: Dictionary mapping tool names to their parameter specifications
+    """
+
+    def __init__(self, name: str):
+        """
+        Initialize a new tool registry.
+
+        Args:
+            name: Unique identifier for this registry
+        """
+        self.name = name
+        self.tools: Dict[str, Callable] = {}
+        self.descriptions: Dict[str, str] = {}
+        self.parameters: Dict[str, Dict[str, Any]] = {}
+
+    def tool(self, description: str, parameters: Dict[str, Any]) -> Callable:
+        """
+        Decorator to register a function as a tool.
+
+        Args:
+            description: Human-readable description of the tool
+            parameters: Parameter specifications for the tool
+
+        Returns:
+            Decorator function that registers the decorated function
+        """
+
+        def decorator(func: Callable) -> Callable:
+            tool_name = func.__name__
+            self.tools[tool_name] = func
+            self.descriptions[tool_name] = description
+            self.parameters[tool_name] = parameters
+
+            @functools.wraps(func)
+            def wrapper(*args, **kwargs):
+                return func(*args, **kwargs)
+
+            return wrapper
+
+        return decorator
+
+    def get_tool(self, tool_name: str) -> Optional[Callable]:
+        """
+        Get a tool function by name.
+
+        Args:
+            tool_name: Name of the tool to retrieve
+
+        Returns:
+            The tool function if found, None otherwise
+        """
+        return self.tools.get(tool_name)
+
+    def get_tools(self) -> Dict[str, Callable]:
+        """
+        Get all tools in this registry.
+
+        Returns:
+            Dictionary mapping tool names to tool functions
+        """
+        return self.tools
+
+    def get_openai_tools(self) -> List[Dict[str, Any]]:
+        """
+        Get tool specifications in OpenAI function calling format.
+
+        Returns:
+            List of tool specifications compatible with OpenAI API
+        """
+        tools = []
+        for tool_name in self.tools:
+            tools.append(
+                {
+                    "name": tool_name,
+                    "description": self.descriptions.get(tool_name, ""),
+                    "parameters": {
+                        "type": "object",
+                        "properties": self.parameters.get(tool_name, {}),
+                        "required": list(self.parameters.get(tool_name, {}).keys()),
+                    },
+                }
+            )
+        return tools
+
+    def create_fastapi_app(self):
+        """
+        Create a FastAPI app with endpoints for each tool.
+
+        Returns:
+            A FastAPI app instance with tool endpoints
+        """
+        # This is a stub implementation
+        return {"app_type": "FastAPI", "tools": list(self.tools.keys())}

eval_protocol/auth.py

@@ -0,0 +1,156 @@
+import configparser
+import logging
+import os
+from pathlib import Path
+from typing import Dict, Optional  # Added Dict
+
+logger = logging.getLogger(__name__)
+
+FIREWORKS_CONFIG_DIR = Path.home() / ".fireworks"
+AUTH_INI_FILE = FIREWORKS_CONFIG_DIR / "auth.ini"
+
+
+def _parse_simple_auth_file(file_path: Path) -> Dict[str, str]:
+    """
+    Parses an auth file with simple key=value lines.
+    Handles comments starting with # or ;.
+    Strips whitespace and basic quotes from values.
+    """
+    creds = {}
+    if not file_path.exists():
+        return creds
+    try:
+        with open(file_path, "r") as f:
+            for line in f:
+                line = line.strip()
+                if not line or line.startswith("#") or line.startswith(";"):
+                    continue
+                if "=" in line:
+                    key, value = line.split("=", 1)
+                    key = key.strip()
+                    value = value.strip()
+                    # Remove surrounding quotes if present
+                    if value and (
+                        (value.startswith('"') and value.endswith('"'))
+                        or (value.startswith("'") and value.endswith("'"))
+                    ):
+                        value = value[1:-1]
+
+                    if key in ["api_key", "account_id"] and value:
+                        creds[key] = value
+    except Exception as e:
+        logger.warning(f"Error during simple parsing of {file_path}: {e}")
+    return creds
+
+
+def _get_credential_from_config_file(key_name: str) -> Optional[str]:
+    """
+    Helper to get a specific credential (api_key or account_id) from auth.ini.
+    Tries simple parsing first, then configparser.
+    """
+    if not AUTH_INI_FILE.exists():
+        return None
+
+    # 1. Try simple key-value parsing first
+    simple_creds = _parse_simple_auth_file(AUTH_INI_FILE)
+    if key_name in simple_creds:
+        logger.debug(f"Using {key_name} from simple key-value parsing of {AUTH_INI_FILE}.")
+        return simple_creds[key_name]
+
+    # 2. Fallback to configparser if not found via simple parsing or if simple parsing failed
+    #    This path will also generate the "no section headers" warning if applicable,
+    #    but only if simple parsing didn't yield the key.
+    try:
+        config = configparser.ConfigParser()
+        config.read(AUTH_INI_FILE)
+
+        # Try [fireworks] section
+        if "fireworks" in config and config.has_option("fireworks", key_name):
+            value_from_file = config.get("fireworks", key_name)
+            if value_from_file:
+                logger.debug(f"Using {key_name} from [fireworks] section in {AUTH_INI_FILE}.")
+                return value_from_file
+
+        # Try default section (configparser might place items without section header here)
+        if config.has_option(config.default_section, key_name):
+            value_from_default = config.get(config.default_section, key_name)
+            if value_from_default:
+                logger.debug(f"Using {key_name} from default section [{config.default_section}] in {AUTH_INI_FILE}.")
+                return value_from_default
+
+    except configparser.MissingSectionHeaderError:
+        # This error implies the file is purely key-value, which simple parsing should have handled.
+        # If simple parsing failed to get the key, then it's likely not there or malformed.
+        logger.debug(f"{AUTH_INI_FILE} has no section headers, and simple parsing did not find {key_name}.")
+    except configparser.Error as e_config:
+        logger.warning(f"Configparser error reading {AUTH_INI_FILE} for {key_name}: {e_config}")
+    except Exception as e_general:
+        logger.warning(f"Unexpected error reading {AUTH_INI_FILE} for {key_name}: {e_general}")
+
+    return None
+
+
+def get_fireworks_api_key() -> Optional[str]:
+    """
+    Retrieves the Fireworks API key.
+
+    The key is sourced in the following order:
+    1. FIREWORKS_API_KEY environment variable.
+    2. 'api_key' from the [fireworks] section of ~/.fireworks/auth.ini.
+
+    Returns:
+        The API key if found, otherwise None.
+    """
+    api_key = os.environ.get("FIREWORKS_API_KEY")
+    if api_key:
+        logger.debug("Using FIREWORKS_API_KEY from environment variable.")
+        return api_key
+
+    api_key_from_file = _get_credential_from_config_file("api_key")
+    if api_key_from_file:
+        return api_key_from_file
+
+    logger.debug("Fireworks API key not found in environment variables or auth.ini.")
+    return None
+
+
+def get_fireworks_account_id() -> Optional[str]:
+    """
+    Retrieves the Fireworks Account ID.
+
+    The Account ID is sourced in the following order:
+    1. FIREWORKS_ACCOUNT_ID environment variable.
+    2. 'account_id' from the [fireworks] section of ~/.fireworks/auth.ini.
+
+    Returns:
+        The Account ID if found, otherwise None.
+    """
+    account_id = os.environ.get("FIREWORKS_ACCOUNT_ID")
+    if account_id:
+        logger.debug("Using FIREWORKS_ACCOUNT_ID from environment variable.")
+        return account_id
+
+    account_id_from_file = _get_credential_from_config_file("account_id")
+    if account_id_from_file:
+        return account_id_from_file
+
+    logger.debug("Fireworks Account ID not found in environment variables or auth.ini.")
+    return None
+
+
+def get_fireworks_api_base() -> str:
+    """
+    Retrieves the Fireworks API base URL.
+
+    The base URL is sourced from the FIREWORKS_API_BASE environment variable.
+    If not set, it defaults to "https://api.fireworks.ai".
+
+    Returns:
+        The API base URL.
+    """
+    api_base = os.environ.get("FIREWORKS_API_BASE", "https://api.fireworks.ai")
+    if os.environ.get("FIREWORKS_API_BASE"):
+        logger.debug("Using FIREWORKS_API_BASE from environment variable.")
+    else:
+        logger.debug(f"FIREWORKS_API_BASE not set in environment, defaulting to {api_base}.")
+    return api_base

eval_protocol/cli.py

@@ -0,0 +1,425 @@
+"""
+Command-line interface for reward-kit.
+"""
+
+import argparse
+import asyncio
+import json
+import logging
+import os
+import sys
+import traceback
+import uuid
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+from eval_protocol.evaluation import create_evaluation, preview_evaluation
+
+from .cli_commands.agent_eval_cmd import agent_eval_command
+from .cli_commands.common import (
+    check_agent_environment,
+    check_environment,
+    setup_logging,
+)
+from .cli_commands.deploy import deploy_command
+from .cli_commands.deploy_mcp import deploy_mcp_command
+from .cli_commands.preview import preview_command
+from .cli_commands.run_eval_cmd import hydra_cli_entry_point
+
+
+def parse_args(args=None):
+    """Parse command line arguments"""
+    parser = argparse.ArgumentParser(description="eval-protocol: Tools for evaluation and reward modeling")
+    parser.add_argument("--verbose", "-v", action="store_true", help="Enable verbose logging")
+
+    subparsers = parser.add_subparsers(dest="command", help="Command to run")
+
+    # Preview command
+    preview_parser = subparsers.add_parser("preview", help="Preview an evaluator with sample data")
+    preview_parser.add_argument(
+        "--metrics-folders",
+        "-m",
+        nargs="+",
+        help="Metric folders in format 'name=path', e.g., 'clarity=./metrics/clarity'",
+    )
+
+    # Make samples optional to allow HF dataset option
+    preview_parser.add_argument(
+        "--samples",
+        "-s",
+        required=False,
+        help="Path to JSONL file containing sample data",
+    )
+    preview_parser.add_argument(
+        "--max-samples",
+        type=int,
+        default=5,
+        help="Maximum number of samples to process (default: 5)",
+    )
+
+    # Add HuggingFace dataset options
+    hf_group = preview_parser.add_argument_group("HuggingFace Dataset Options")
+    hf_group.add_argument(
+        "--huggingface-dataset",
+        "--hf",
+        help="HuggingFace dataset name (e.g., 'deepseek-ai/DeepSeek-ProverBench')",
+    )
+    hf_group.add_argument(
+        "--huggingface-split",
+        default="train",
+        help="Dataset split to use (default: 'train')",
+    )
+    hf_group.add_argument(
+        "--huggingface-prompt-key",
+        default="prompt",
+        help="Key in the dataset containing the prompt text (default: 'prompt')",
+    )
+    hf_group.add_argument(
+        "--huggingface-response-key",
+        default="response",
+        help="Key in the dataset containing the response text (default: 'response')",
+    )
+    hf_group.add_argument(
+        "--huggingface-key-map",
+        help="JSON mapping of dataset keys to reward-kit message keys",
+    )
+    preview_parser.add_argument(
+        "--remote-url",
+        help="URL of a remote reward function endpoint to preview against. If provided, metrics-folders might be ignored.",
+    )
+
+    # Deploy command
+    deploy_parser = subparsers.add_parser("deploy", help="Create and deploy an evaluator, or register a remote one")
+    deploy_parser.add_argument("--id", required=True, help="ID for the evaluator")
+    deploy_parser.add_argument(
+        "--metrics-folders",
+        "-m",
+        nargs="+",
+        required=False,  # No longer strictly required if --remote-url is used
+        help="Metric folders in format 'name=path', e.g., 'clarity=./metrics/clarity'. Required if not using --remote-url.",
+    )
+    deploy_parser.add_argument(
+        "--display-name",
+        help="Display name for the evaluator (defaults to ID if not provided)",
+    )
+    deploy_parser.add_argument("--description", help="Description for the evaluator")
+    deploy_parser.add_argument(
+        "--force",
+        "-f",
+        action="store_true",
+        help="Force update if evaluator already exists",
+    )
+
+    # Add HuggingFace dataset options to deploy command
+    hf_deploy_group = deploy_parser.add_argument_group("HuggingFace Dataset Options")
+    hf_deploy_group.add_argument(
+        "--huggingface-dataset",
+        "--hf",
+        help="HuggingFace dataset name (e.g., 'deepseek-ai/DeepSeek-ProverBench')",
+    )
+    hf_deploy_group.add_argument(
+        "--huggingface-split",
+        default="train",
+        help="Dataset split to use (default: 'train')",
+    )
+    hf_deploy_group.add_argument(
+        "--huggingface-prompt-key",
+        default="prompt",
+        help="Key in the dataset containing the prompt text (default: 'prompt')",
+    )
+    hf_deploy_group.add_argument(
+        "--huggingface-response-key",
+        default="response",
+        help="Key in the dataset containing the response text (default: 'response')",
+    )
+    hf_deploy_group.add_argument(
+        "--huggingface-key-map",
+        help="JSON mapping of dataset keys to reward-kit message keys",
+    )
+    deploy_parser.add_argument(
+        "--remote-url",
+        help="URL of a pre-deployed remote reward function. If provided, deploys by registering this URL with Fireworks AI.",
+    )
+
+    # Deployment target options
+    target_group = deploy_parser.add_argument_group("Deployment Target Options")
+    target_group.add_argument(
+        "--target",
+        choices=["fireworks", "gcp-cloud-run", "local-serve"],
+        default="fireworks",
+        help="Deployment target. 'fireworks' for standard Fireworks platform deployment, 'gcp-cloud-run' for Google Cloud Run, 'local-serve' for local serving with Serveo tunneling.",
+    )
+    target_group.add_argument(
+        "--function-ref",
+        help="Reference to the reward function to deploy (e.g., 'my_module.reward_func'). Required for 'gcp-cloud-run' and 'local-serve' targets.",
+    )
+
+    # Local serving options (relevant if --target is local-serve)
+    local_serve_group = deploy_parser.add_argument_group("Local Serving Options (used if --target is local-serve)")
+    local_serve_group.add_argument(
+        "--local-port",
+        type=int,
+        default=8001,
+        help="Port for the local reward function server to listen on (default: 8001). Used with --target local-serve.",
+    )
+
+    # GCP deployment options
+    gcp_group = deploy_parser.add_argument_group(
+        "GCP Cloud Run Deployment Options (used if --target is gcp-cloud-run)"
+    )
+    # --function-ref is now in target_group
+    gcp_group.add_argument(
+        "--gcp-project",
+        required=False,
+        help="Google Cloud Project ID. Must be provided via CLI or rewardkit.yaml.",
+    )
+    gcp_group.add_argument(
+        "--gcp-region",
+        required=False,
+        help="Google Cloud Region for deployment (e.g., 'us-central1'). Must be provided via CLI or rewardkit.yaml.",
+    )
+    gcp_group.add_argument(
+        "--gcp-ar-repo",
+        required=False,
+        help="Google Artifact Registry repository name. Optional, defaults to value in rewardkit.yaml or 'reward-kit-evaluators' if not specified.",
+    )
+    gcp_group.add_argument(
+        "--service-account",
+        help="Email of the GCP service account to run the Cloud Run service. Optional.",
+    )
+    gcp_group.add_argument(
+        "--entry-point",
+        default="reward_function",
+        help="The name of the entry point function within your --function-ref module (default: reward_function). Only for gcp-cloud-run.",
+    )
+    gcp_group.add_argument(
+        "--runtime",
+        default="python311",  # Or a sensible default
+        help="The Cloud Functions/Run runtime (e.g., python311). Only for gcp-cloud-run.",
+    )
+    gcp_group.add_argument(
+        "--gcp-auth-mode",
+        choices=["open", "api-key"],  # Add 'iam' later
+        default=None,  # Default will be resolved in deploy_command
+        help="Authentication mode for the deployed GCP Cloud Run service. "
+        "'open': Publicly accessible. "
+        "'api-key': Service is publicly accessible but requires an API key in requests (handled by the application). "
+        "If not specified, defaults to value in rewardkit.yaml or 'api-key'. Optional.",
+    )
+
+    # Deploy MCP command
+    deploy_mcp_parser = subparsers.add_parser("deploy-mcp", help="Deploy an MCP server to Google Cloud Run")
+    deploy_mcp_parser.add_argument("--id", required=True, help="Unique ID for the MCP server deployment")
+    deploy_mcp_parser.add_argument(
+        "--mcp-server-module",
+        help="Python module containing the MCP server (e.g., 'examples.frozen_lake_mcp.frozen_lake_mcp_server'). Required if --dockerfile is not provided.",
+    )
+    deploy_mcp_parser.add_argument(
+        "--dockerfile",
+        help="Path to Dockerfile to use for deployment (recommended for tested local Dockerfiles). When provided, --mcp-server-module is not required.",
+    )
+    deploy_mcp_parser.add_argument(
+        "--gcp-project",
+        help="Google Cloud Project ID. Can also be set in rewardkit.yaml",
+    )
+    deploy_mcp_parser.add_argument(
+        "--gcp-region",
+        help="Google Cloud Region (e.g., 'us-central1'). Can also be set in rewardkit.yaml",
+    )
+    deploy_mcp_parser.add_argument(
+        "--gcp-ar-repo",
+        help="Google Artifact Registry repository name. Defaults to 'reward-kit-mcp-servers'",
+    )
+    deploy_mcp_parser.add_argument(
+        "--port",
+        type=int,
+        default=8000,
+        help="Port for the MCP server to listen on (default: 8000)",
+    )
+    deploy_mcp_parser.add_argument(
+        "--python-version",
+        default="3.11",
+        help="Python version for the container (default: 3.11)",
+    )
+    deploy_mcp_parser.add_argument("--requirements", help="Additional pip requirements (newline separated)")
+    deploy_mcp_parser.add_argument("--env-vars", nargs="*", help="Environment variables in KEY=VALUE format")
+
+    # Agent-eval command
+    agent_eval_parser = subparsers.add_parser(
+        "agent-eval", help="Run agent evaluation using the ForkableResource framework."
+    )
+    agent_eval_parser.add_argument(
+        "--task-def",
+        required=True,
+        help="Path to task definition file or directory containing task definitions.",
+    )
+    agent_eval_parser.add_argument(
+        "--parallel",
+        action="store_true",
+        help="Execute tasks in parallel when multiple tasks are specified.",
+    )
+    agent_eval_parser.add_argument(
+        "--max-concurrency",
+        type=int,
+        default=3,
+        help="Maximum number of tasks to execute in parallel (default: 3).",
+    )
+    agent_eval_parser.add_argument(
+        "--filter",
+        nargs="+",
+        help="Run only tasks matching the specified task IDs.",
+    )
+    agent_eval_parser.add_argument(
+        "--output-dir",
+        default="./agent_runs",
+        help="Directory to store agent evaluation run results (default: ./agent_runs).",
+    )
+    agent_eval_parser.add_argument(
+        "--model",
+        help="Override MODEL_AGENT environment variable (format: provider/model_name).",
+    )
+    agent_eval_parser.add_argument(
+        "--num-rollouts",
+        type=int,
+        help="Override the number of parallel rollouts to execute for each task.",
+    )
+
+    # Run command (for Hydra-based evaluations)
+    # This subparser intentionally defines no arguments itself.
+    # All arguments after 'run' will be passed to Hydra by parse_known_args.
+    subparsers.add_parser(
+        "run",
+        help="Run an evaluation using a Hydra configuration. All arguments after 'run' are passed to Hydra.",
+    )
+
+    # Use parse_known_args to allow Hydra to handle its own arguments
+    return parser.parse_known_args(args)
+
+
+def main():
+    """Main entry point for the CLI"""
+    try:
+        from dotenv import load_dotenv
+
+        # .env.dev for development-specific overrides, .env for general
+        load_dotenv(dotenv_path=Path(".") / ".env.dev", override=True)
+        load_dotenv(override=True)
+    except ImportError:
+        pass
+
+    # Automatic PYTHONPATH enhancement - add current directory to Python path
+    # This needs to happen early, before any module loading occurs
+    current_dir = os.getcwd()
+    current_pythonpath = os.environ.get("PYTHONPATH", "")
+    if current_dir not in current_pythonpath.split(os.pathsep):
+        if current_pythonpath:
+            os.environ["PYTHONPATH"] = f"{current_dir}{os.pathsep}{current_pythonpath}"
+        else:
+            os.environ["PYTHONPATH"] = current_dir
+        logger.debug(f"Added current directory to PYTHONPATH: {current_dir}")
+
+        # Also add to sys.path so it takes effect immediately for the current process
+        if current_dir not in sys.path:
+            sys.path.insert(0, current_dir)
+
+    # Store original sys.argv[0] because Hydra might manipulate it
+    # and we need it if we're not calling a Hydra app.
+    original_script_name = sys.argv[0]
+    args, remaining_argv = parse_args()  # Use parse_known_args
+
+    setup_logging(args.verbose, getattr(args, "debug", False))
+
+    if args.command == "preview":
+        return preview_command(args)
+    elif args.command == "deploy":
+        return deploy_command(args)
+    elif args.command == "deploy-mcp":
+        return deploy_mcp_command(args)
+    elif args.command == "agent-eval":
+        return agent_eval_command(args)
+    elif args.command == "run":
+        # For the 'run' command, Hydra takes over argument parsing.
+
+        # Filter out the initial '--' if present in remaining_argv, which parse_known_args might add
+        hydra_specific_args = [arg for arg in remaining_argv if arg != "--"]
+
+        # Auto-detect local conf directory and add it to config path if not explicitly provided
+        has_config_path = any(arg.startswith("--config-path") for arg in hydra_specific_args)
+        current_dir = os.getcwd()
+        local_conf_dir = os.path.join(current_dir, "conf")
+
+        if not has_config_path and os.path.isdir(local_conf_dir):
+            logger.info(f"Auto-detected local conf directory: {local_conf_dir}")
+            hydra_specific_args = [
+                "--config-path",
+                local_conf_dir,
+            ] + hydra_specific_args
+
+        processed_hydra_args = []
+        i = 0
+        while i < len(hydra_specific_args):
+            arg = hydra_specific_args[i]
+            if arg == "--config-path":
+                processed_hydra_args.append(arg)
+                i += 1
+                if i < len(hydra_specific_args):
+                    path_val = hydra_specific_args[i]
+                    abs_path = os.path.abspath(path_val)
+                    logger.debug(
+                        f"Converting relative --config-path '{path_val}' (space separated) to absolute '{abs_path}'"
+                    )
+                    processed_hydra_args.append(abs_path)
+                else:
+                    logger.error("--config-path specified without a value.")
+                    pass
+            elif arg.startswith("--config-path="):
+                flag_part, path_val = arg.split("=", 1)
+                processed_hydra_args.append(flag_part)
+                abs_path = os.path.abspath(path_val)
+                logger.debug(
+                    f"Converting relative --config-path '{path_val}' (equals separated) to absolute '{abs_path}'"
+                )
+                processed_hydra_args.append(abs_path)
+            else:
+                processed_hydra_args.append(arg)
+            i += 1
+
+        sys.argv = [sys.argv[0]] + processed_hydra_args
+        logger.info(f"SYSCALL_ARGV_FOR_HYDRA (after potential abspath conversion): {sys.argv}")
+
+        try:
+            hydra_cli_entry_point()
+            return 0
+        except Exception as e:
+            error_msg = str(e)
+            logger.error(f"Evaluation failed: {e}")
+
+            # Provide helpful suggestions for common Hydra/config errors
+            if "Cannot find primary config" in error_msg:
+                logger.error("HINT: Configuration file not found.")
+                logger.error("SOLUTION: Ensure you have a config file in ./conf/ directory")
+                logger.error("Try: reward-kit run --config-name simple_uipath_eval")
+            elif "missing from config" in error_msg or "MissingMandatoryValue" in error_msg:
+                logger.error("HINT: Required configuration values are missing.")
+                logger.error("SOLUTION: Check your config file for missing required fields")
+            elif "Config search path" in error_msg:
+                logger.error("HINT: Hydra cannot find the configuration directory.")
+                logger.error("SOLUTION: Create a ./conf directory with your config files")
+            elif "ValidationError" in error_msg:
+                logger.error("HINT: Configuration validation failed.")
+                logger.error("SOLUTION: Run 'reward-kit validate-data --file your_data.jsonl' to check data")
+
+            logger.error("\nQuick fix suggestions:")
+            logger.error("1. Use the simplified setup: reward-kit run --config-name simple_uipath_eval")
+            logger.error("2. Validate your data first: reward-kit validate-data --file data.jsonl --schema agent")
+            logger.error("3. Ensure you have: ./conf/simple_uipath_eval.yaml and ./uipath_reward.py")
+            return 1
+    else:
+        temp_parser = argparse.ArgumentParser(prog=os.path.basename(original_script_name))
+        temp_parser.print_help()
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

eval_protocol/cli_commands/__init__.py

@@ -0,0 +1 @@
+# This package will contain modules for different CLI commands and common utilities.