eval-protocol 0.0.3__py3-none-any.whl

eval_protocol/cli_commands/deploy_mcp.py

@@ -0,0 +1,287 @@
+"""
+CLI command for deploying MCP servers to Google Cloud Run.
+"""
+
+import importlib
+import os
+import sys
+import tempfile
+from pathlib import Path
+from typing import Dict, Optional
+
+from eval_protocol.config import GCPCloudRunConfig, RewardKitConfig
+from eval_protocol.config import _config_file_path as global_loaded_config_path
+from eval_protocol.config import get_config
+from eval_protocol.gcp_tools import (
+    build_and_push_docker_image,
+    deploy_to_cloud_run,
+    ensure_artifact_registry_repo_exists,
+)
+
+from .common import check_environment
+
+
+def _generate_mcp_dockerfile_content(
+    mcp_server_module: str,
+    python_version: str = "3.11",
+    service_port: int = 8000,
+    additional_requirements: Optional[str] = None,
+) -> str:
+    """
+    Generate Dockerfile content for MCP server deployment.
+
+    Args:
+        mcp_server_module: The Python module containing the MCP server (e.g., 'frozen_lake_mcp_server')
+        python_version: Python version to use in the container
+        service_port: Port the MCP server will listen on
+        additional_requirements: Additional pip requirements
+
+    Returns:
+        Dockerfile content as string
+    """
+
+    # Base requirements for MCP servers - matching setup.py dependencies
+    base_requirements = [
+        "fastmcp>=0.1.0",
+        # Core reward-kit dependencies from setup.py
+        "requests>=2.25.0",
+        "pydantic>=2.0.0",
+        "dataclasses-json>=0.5.7",
+        "fastapi>=0.68.0",
+        "uvicorn>=0.15.0",
+        "python-dotenv>=0.19.0",
+        "openai==1.78.1",
+        "aiosqlite",
+        "aiohttp",
+        "mcp>=1.9.2",
+        "PyYAML>=5.0",
+        "datasets==3.6.0",
+        "fsspec==2025.3.0",
+        "hydra-core>=1.3.2",
+        "omegaconf>=2.3.0",
+        "gymnasium>=0.29.0",
+        "httpx>=0.24.0",
+        "fireworks-ai>=0.17.19",
+    ]
+
+    if additional_requirements:
+        requirements = base_requirements + [req.strip() for req in additional_requirements.split("\n") if req.strip()]
+    else:
+        requirements = base_requirements
+
+    # Generate pip install lines with proper escaping
+    pip_install_lines = []
+    for req in requirements[:-1]:
+        pip_install_lines.append(f"    {req} \\")
+    pip_install_lines.append(f"    {requirements[-1]}")
+    pip_install_section = "\n".join(pip_install_lines)
+
+    dockerfile_content = f"""# Multi-stage build for MCP server deployment
+FROM python:{python_version}-slim as builder
+
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \\
+    build-essential \\
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements and install Python dependencies
+RUN pip install --no-cache-dir --upgrade pip
+
+# Install MCP server requirements
+RUN pip install --no-cache-dir \\
+{pip_install_section}
+
+# Production stage
+FROM python:{python_version}-slim
+
+WORKDIR /app
+
+# Install runtime dependencies
+RUN apt-get update && apt-get install -y \\
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy Python packages from builder
+COPY --from=builder /usr/local/lib/python{python_version}/site-packages /usr/local/lib/python{python_version}/site-packages
+COPY --from=builder /usr/local/bin /usr/local/bin
+
+# Copy the MCP server code
+COPY . .
+
+# Set environment variables for Cloud Run
+# FastMCP uses HOST and PORT environment variables for streamable-http transport
+ENV HOST=0.0.0.0
+ENV PORT={service_port}
+ENV PYTHONPATH=/app
+ENV PYTHONUNBUFFERED=1
+
+# Expose the port
+EXPOSE {service_port}
+
+# Run the MCP server with proper host and port for Cloud Run
+CMD ["python", "-m", "{mcp_server_module}", "--host", "0.0.0.0", "--port", "{service_port}"]
+"""
+
+    return dockerfile_content
+
+
+def _deploy_mcp_to_gcp_cloud_run(args, current_config, gcp_config_from_yaml):
+    """Deploy MCP server to GCP Cloud Run."""
+    print(f"Starting MCP server deployment to GCP Cloud Run for '{args.id}'...")
+
+    # Validate required arguments - either dockerfile or mcp-server-module must be provided
+    if not args.dockerfile and not args.mcp_server_module:
+        print("Error: Either --dockerfile or --mcp-server-module is required for MCP server deployment.")
+        return None
+
+    # Resolve GCP configuration
+    gcp_project_id = args.gcp_project
+    if not gcp_project_id and gcp_config_from_yaml:
+        gcp_project_id = gcp_config_from_yaml.project_id
+    if not gcp_project_id:
+        print("Error: GCP Project ID must be provided via --gcp-project or rewardkit.yaml.")
+        return None
+
+    gcp_region = args.gcp_region
+    if not gcp_region and gcp_config_from_yaml:
+        gcp_region = gcp_config_from_yaml.region
+    if not gcp_region:
+        print("Error: GCP Region must be provided via --gcp-region or rewardkit.yaml.")
+        return None
+
+    gcp_ar_repo_name = args.gcp_ar_repo
+    if not gcp_ar_repo_name and gcp_config_from_yaml:
+        gcp_ar_repo_name = gcp_config_from_yaml.artifact_registry_repository
+    if not gcp_ar_repo_name:
+        gcp_ar_repo_name = "reward-kit-mcp-servers"
+
+    print(f"Using GCP Project: {gcp_project_id}, Region: {gcp_region}, AR Repo: {gcp_ar_repo_name}")
+
+    # Ensure Artifact Registry repository exists
+    if not ensure_artifact_registry_repo_exists(
+        project_id=gcp_project_id, region=gcp_region, repo_name=gcp_ar_repo_name
+    ):
+        print(f"Failed to ensure Artifact Registry repository '{gcp_ar_repo_name}' exists. Aborting.")
+        return None
+
+    # Determine Dockerfile content - use provided file or generate
+    dockerfile_content = None
+    if hasattr(args, "dockerfile") and args.dockerfile:
+        # Use provided Dockerfile
+        dockerfile_path = Path(args.dockerfile)
+        if not dockerfile_path.exists():
+            print(f"Error: Dockerfile not found at {dockerfile_path}")
+            return None
+        print(f"Using provided Dockerfile: {dockerfile_path}")
+        try:
+            with open(dockerfile_path, "r") as f:
+                dockerfile_content = f.read()
+        except Exception as e:
+            print(f"Error reading Dockerfile at {dockerfile_path}: {e}")
+            return None
+    else:
+        # Generate Dockerfile content (legacy approach)
+        print("Generating Dockerfile content from mcp-server-module...")
+        dockerfile_content = _generate_mcp_dockerfile_content(
+            mcp_server_module=args.mcp_server_module,
+            python_version=getattr(args, "python_version", "3.11"),
+            service_port=getattr(args, "port", 8000),
+            additional_requirements=getattr(args, "requirements", None),
+        )
+
+    if not dockerfile_content:
+        print("Failed to obtain Dockerfile content. Aborting.")
+        return None
+
+    # Build and push Docker image
+    image_tag = "latest"
+    image_name_tag = f"{gcp_region}-docker.pkg.dev/{gcp_project_id}/{gcp_ar_repo_name}/{args.id}:{image_tag}"
+    build_context_dir = os.getcwd()
+
+    if not build_and_push_docker_image(
+        image_name_tag=image_name_tag,
+        dockerfile_content=dockerfile_content,
+        build_context_dir=build_context_dir,
+        gcp_project_id=gcp_project_id,
+    ):
+        print(f"Failed to build and push Docker image {image_name_tag}. Aborting.")
+        return None
+
+    print(f"Successfully built and pushed Docker image: {image_name_tag}")
+
+    # Deploy to Cloud Run
+    service_port = getattr(args, "port", 8000)
+    env_vars = {}
+
+    # Add any custom environment variables
+    if hasattr(args, "env_vars") and args.env_vars:
+        for env_pair in args.env_vars:
+            if "=" in env_pair:
+                key, value = env_pair.split("=", 1)
+                env_vars[key] = value
+
+    cloud_run_service_url = deploy_to_cloud_run(
+        service_name=args.id,
+        image_name_tag=image_name_tag,
+        gcp_project_id=gcp_project_id,
+        gcp_region=gcp_region,
+        allow_unauthenticated=True,  # MCP servers typically need to be publicly accessible
+        env_vars=env_vars if env_vars else None,
+        service_port=service_port,
+    )
+
+    if not cloud_run_service_url:
+        print("Failed to deploy to Cloud Run or retrieve service URL. Aborting.")
+        return None
+
+    print(f"🚀 Successfully deployed MCP server to Cloud Run!")
+    print(f"📍 Service URL: {cloud_run_service_url}")
+    print(f"🔗 MCP Connection URL: {cloud_run_service_url}")
+    print(f"📋 Service Name: {args.id}")
+    deployment_method = (
+        "local Dockerfile" if (hasattr(args, "dockerfile") and args.dockerfile) else "auto-generated Dockerfile"
+    )
+    print(f"🐳 Deployment Method: {deployment_method}")
+    print()
+    print("🎯 Next steps:")
+    print(f"   1. Test your MCP server: curl {cloud_run_service_url}/health")
+    print(f"   2. Connect MCP clients to: {cloud_run_service_url}")
+    print(
+        f"   3. Monitor logs: gcloud logging read 'resource.type=cloud_run_revision AND resource.labels.service_name={args.id}' --project {gcp_project_id}"
+    )
+
+    return cloud_run_service_url
+
+
+def deploy_mcp_command(args):
+    """Main entry point for MCP server deployment command."""
+
+    # Check environment (similar to existing deploy command)
+    if not check_environment():
+        print("Environment check failed. Please resolve the issues above before deploying.")
+        return False
+
+    try:
+        # Load configuration
+        current_config = get_config()
+        gcp_config_from_yaml: Optional[GCPCloudRunConfig] = None
+        if current_config and current_config.gcp_cloud_run:
+            gcp_config_from_yaml = current_config.gcp_cloud_run
+
+        # Deploy to GCP Cloud Run
+        service_url = _deploy_mcp_to_gcp_cloud_run(args, current_config, gcp_config_from_yaml)
+
+        if service_url:
+            print(f"✅ MCP server '{args.id}' successfully deployed!")
+            return True
+        else:
+            print(f"❌ Failed to deploy MCP server '{args.id}'")
+            return False
+
+    except Exception as e:
+        print(f"Error during MCP server deployment: {e}")
+        import traceback
+
+        traceback.print_exc()
+        return False

eval_protocol/cli_commands/preview.py

@@ -0,0 +1,186 @@
+"""
+CLI command for previewing an evaluator.
+"""
+
+import json
+import sys  # For sys.exit
+from pathlib import Path
+from typing import Any, Dict, Iterator, List, Optional, Union
+
+import requests  # For making HTTP requests
+
+from eval_protocol.evaluation import preview_evaluation
+
+# Assuming EvaluationRequest is defined in generic_server.
+# For loose coupling, it might be better in models.py or a shared types module.
+from eval_protocol.generic_server import EvaluationRequest
+from eval_protocol.models import EvaluateResult, Message
+
+# Assuming these helper functions exist or will be created in .common
+# If not, their logic for loading samples would need to be integrated here or called differently.
+from .common import (
+    check_environment,
+    load_samples_from_file,
+    load_samples_from_huggingface,
+)
+
+
+def preview_command(args):
+    """Preview an evaluator with sample data"""
+
+    # Check environment variables
+    if not check_environment():
+        return 1
+
+    # Validate --remote-url and --metrics-folders usage
+    if args.remote_url and args.metrics_folders:
+        print("Info: --metrics-folders are ignored when --remote-url is specified.")
+
+    if not args.remote_url and not args.metrics_folders:
+        print("Error: Either --remote-url or --metrics-folders must be specified.")
+        return 1
+
+    # Ensure either samples or huggingface_dataset is provided (still needed for remote_url)
+    if not args.samples and not args.huggingface_dataset:
+        print("Error: Either sample file (--samples) or HuggingFace dataset (--huggingface-dataset) is required.")
+        return 1
+
+    # If using samples file, verify it exists
+    if args.samples and not Path(args.samples).exists():
+        print(f"Error: Sample file '{args.samples}' not found")
+        return 1
+
+    # Process HuggingFace key mapping if provided
+    huggingface_message_key_map = None
+    if args.huggingface_key_map:
+        try:
+            huggingface_message_key_map = json.loads(args.huggingface_key_map)
+        except json.JSONDecodeError:
+            print("Error: Invalid JSON format for --huggingface-key-map")
+            return 1
+
+    if args.remote_url:
+        # Handle previewing against a remote URL
+        print(f"Previewing against remote URL: {args.remote_url}")
+
+        # Ensure the remote URL is a valid base URL
+        if not (args.remote_url.startswith("http://") or args.remote_url.startswith("https://")):
+            print(f"Error: Invalid --remote-url '{args.remote_url}'. Must start with http:// or https://")
+            return 1
+
+        evaluate_endpoint = f"{args.remote_url.rstrip('/')}/evaluate"
+
+        samples_iterator: Union[List[Any], Iterator[Dict[str, Any]]] = []
+        try:
+            if args.samples:
+                # Assuming load_samples_from_file yields dicts with "messages" and optional "ground_truth"
+                samples_iterator = load_samples_from_file(args.samples, args.max_samples)
+            elif args.huggingface_dataset:
+                # Assuming load_samples_from_huggingface yields dicts with "messages" and optional "ground_truth"
+                samples_iterator = load_samples_from_huggingface(
+                    dataset_name=args.huggingface_dataset,
+                    split=args.huggingface_split,
+                    prompt_key=args.huggingface_prompt_key,
+                    response_key=args.huggingface_response_key,
+                    key_map=huggingface_message_key_map,
+                    max_samples=args.max_samples,
+                )
+        except Exception as e:
+            print(f"Error loading samples: {e}")
+            return 1
+
+        results_count = 0
+        for i, sample_data in enumerate(samples_iterator):
+            # The load_samples_from_* helpers should ideally cap at max_samples,
+            # but we double-check here.
+            if i >= args.max_samples:
+                break
+            results_count += 1
+
+            messages_payload = sample_data.get("messages", [])
+            ground_truth_payload = sample_data.get("ground_truth")
+            # Allow passing other sample fields as kwargs to the reward function
+            sample_kwargs = {k: v for k, v in sample_data.items() if k not in ["messages", "ground_truth"]}
+
+            processed_messages = []
+            for msg_item in messages_payload:
+                if isinstance(msg_item, Message):  # If helpers return Message objects
+                    processed_messages.append(msg_item.model_dump(exclude_none=True))
+                elif isinstance(msg_item, dict):  # If helpers return dicts
+                    processed_messages.append(msg_item)
+                else:
+                    print(
+                        f"Warning: Sample {i+1} has unexpected message item type: {type(msg_item)}. Skipping this message item."
+                    )
+
+            try:
+                request_obj = EvaluationRequest(
+                    messages=processed_messages,
+                    ground_truth=ground_truth_payload,
+                    kwargs=sample_kwargs,
+                )
+            except Exception as e:  # Pydantic validation for EvaluationRequest
+                print(f"\n--- Sample {i+1} ---")
+                print(f"  Error creating request payload for sample: {e}")
+                print(f"  Sample data: {sample_data}")
+                print("--- End Sample ---")
+                continue  # Skip to next sample
+
+            print(f"\n--- Sample {i+1} ---")
+
+            try:
+                response = requests.post(
+                    evaluate_endpoint,
+                    json=request_obj.model_dump(),  # Use model_dump() for Pydantic v2, or .dict() for v1
+                    timeout=30,
+                )
+                response.raise_for_status()
+
+                result_json = response.json()
+                evaluate_result = EvaluateResult(**result_json)
+
+                print(f"  Score: {evaluate_result.score}")
+                print(f"  Reason: {evaluate_result.reason if evaluate_result.reason else 'N/A'}")
+                print(f"  Is Valid: {evaluate_result.is_score_valid}")
+                if evaluate_result.metrics:
+                    for k, v_metric in evaluate_result.metrics.items():
+                        print(
+                            f"  Metric '{k}': Score={v_metric.score}, Valid={v_metric.is_score_valid}, Reason={v_metric.reason}"
+                        )
+
+            except requests.exceptions.RequestException as e:
+                print(f"  Error calling remote URL '{evaluate_endpoint}': {e}")
+            except json.JSONDecodeError:
+                print(
+                    f"  Error: Invalid JSON response from remote URL. Status: {response.status_code}. Response text: {response.text[:200]}..."
+                )
+            except Exception as e:
+                print(f"  Error processing response from remote URL: {e}")
+            print("--- End Sample ---")
+
+        if results_count == 0:
+            print("No samples were processed. Check sample source or loading functions.")
+        return 0
+
+    else:
+        # Original behavior: preview using local metrics_folders
+        # This path is taken if args.remote_url is None (or empty string)
+        # We already checked above that if not remote_url, then metrics_folders must be present.
+
+        try:
+            preview_result = preview_evaluation(
+                metric_folders=args.metrics_folders,
+                sample_file=args.samples if args.samples else None,
+                max_samples=args.max_samples,
+                huggingface_dataset=args.huggingface_dataset,
+                huggingface_split=args.huggingface_split,
+                huggingface_prompt_key=args.huggingface_prompt_key,
+                huggingface_response_key=args.huggingface_response_key,
+                huggingface_message_key_map=huggingface_message_key_map,
+            )
+
+            preview_result.display()
+            return 0
+        except Exception as e:
+            print(f"Error previewing evaluator (local mode): {str(e)}")
+            return 1

eval_protocol/cli_commands/run_eval_cmd.py

@@ -0,0 +1,202 @@
+"""
+CLI command for running the full evaluation pipeline (generation + evaluation).
+This script is intended to be a Hydra application.
+"""
+
+import asyncio
+import logging
+import sys
+
+import hydra
+
+# Ensure hydra.core.hydra_config is available if used for output_dir
+from hydra.core.hydra_config import HydraConfig
+from omegaconf import (  # Ensure MISSING is imported if used in configs
+    MISSING,
+    DictConfig,
+    OmegaConf,
+)
+
+from eval_protocol.execution.pipeline import EvaluationPipeline
+
+logger = logging.getLogger(__name__)
+
+
+def run_evaluation_command_logic(cfg: DictConfig) -> None:
+    """
+    Main logic for the 'run-evaluation' command.
+    """
+    logger.info("Starting 'run-evaluation' command with resolved Hydra config.")
+
+    # Make Hydra's runtime output directory available to the pipeline if needed
+    # This assumes 'hydra_output_dir' is a valid field in the pipeline's config if it uses it.
+    # A cleaner way is for the pipeline to be Hydra-aware or for this function to pass it explicitly.
+    # For now, let's add it to the cfg object that pipeline receives.
+    # Ensure the config is not frozen if we add keys, then restore its original struct state.
+    was_struct = OmegaConf.is_struct(cfg)
+    if was_struct:
+        OmegaConf.set_struct(cfg, False)
+    cfg.hydra_output_dir = HydraConfig.get().runtime.output_dir
+    if was_struct:
+        OmegaConf.set_struct(cfg, True)
+
+    logger.debug(f"Full configuration for pipeline:\n{OmegaConf.to_yaml(cfg)}")
+
+    try:
+        pipeline = EvaluationPipeline(pipeline_cfg=cfg)
+        all_results = asyncio.run(pipeline.run())  # Store the results
+        logger.info("'run-evaluation' command finished successfully.")
+
+        # --- Add Summary Report ---
+        if all_results:
+            total_samples = len(all_results)
+            errors = [r for r in all_results if "error" in r and r["error"]]
+            # Consider a result successful for summary if it has a score and no critical error string
+            successful_evals = [
+                r for r in all_results if r.get("evaluation_score") is not None and not ("error" in r and r["error"])
+            ]
+
+            num_errors = len(errors)
+            num_successful = len(successful_evals)
+
+            summary_lines = [
+                "\n--- Evaluation Summary ---",
+                f"Total samples processed: {total_samples}",
+                f"Successful evaluations: {num_successful}",
+                f"Evaluation errors: {num_errors}",
+            ]
+
+            if num_successful > 0:
+                scores = [
+                    r["evaluation_score"]
+                    for r in successful_evals
+                    if isinstance(r.get("evaluation_score"), (int, float))
+                ]
+                if scores:
+                    avg_score = sum(scores) / len(scores)
+                    min_score = min(scores)
+                    max_score = max(scores)
+                    summary_lines.append(f"Average score: {avg_score:.2f}")
+                    summary_lines.append(f"Min score: {min_score:.2f}")
+                    summary_lines.append(f"Max score: {max_score:.2f}")
+
+                    # Score distribution (example: 5 bins)
+                    bins = [
+                        0.0,
+                        0.2,
+                        0.4,
+                        0.6,
+                        0.8,
+                        1.01,
+                    ]  # 1.01 to include 1.0 in last bin
+                    score_counts = [0] * (len(bins) - 1)
+                    for score in scores:
+                        for i in range(len(bins) - 1):
+                            if bins[i] <= score < bins[i + 1]:
+                                score_counts[i] += 1
+                                break
+                    summary_lines.append("Score distribution:")
+                    for i in range(len(bins) - 1):
+                        # Ensure bin upper bound is displayed correctly as 1.0 for the last bin
+                        upper_bin_display = 1.0 if bins[i + 1] > 1.0 else bins[i + 1]
+                        summary_lines.append(f"  [{bins[i]:.1f} - {upper_bin_display:.1f}): {score_counts[i]}")
+
+            if num_errors > 0:
+                summary_lines.append("\nError details (first 5):")
+                for i, err_item in enumerate(errors[:5]):
+                    err_id = err_item.get("id", "N/A")
+                    err_msg = err_item.get("error", "Unknown error")
+                    # Truncate long error messages for summary
+                    if len(err_msg) > 100:
+                        err_msg = err_msg[:100] + "..."
+                    summary_lines.append(f"  Sample ID {err_id}: {err_msg}")
+
+            summary_lines.append("--- End of Summary ---")
+
+            # Use logger.info for summary to respect overall logging settings
+            for line in summary_lines:
+                logger.info(line)
+        else:
+            logger.info("No results to summarize.")
+
+    except ValueError as ve:
+        error_msg = str(ve)
+        logger.error(f"Configuration or Value error in pipeline: {ve}")
+
+        # Provide helpful suggestions based on common errors
+        if "final_columns" in error_msg:
+            logger.error(
+                "HINT: This error suggests your dataset config has 'final_columns' which conflicts with the datasets library."
+            )
+            logger.error("SOLUTION: Remove 'final_columns' from your dataset config or use the simplified config.")
+        elif "user_query" in error_msg and "missing" in error_msg.lower():
+            logger.error("HINT: Your data is missing the 'user_query' column.")
+            logger.error("SOLUTION: Run 'reward-kit validate-data --file your_data.jsonl' to check data schema.")
+        elif "import" in error_msg.lower() or "module" in error_msg.lower():
+            logger.error("HINT: Cannot import your reward function module.")
+            logger.error("SOLUTION: Ensure your reward function file is in the current directory.")
+        elif "config" in error_msg.lower() and "not found" in error_msg.lower():
+            logger.error("HINT: Configuration file not found.")
+            logger.error("SOLUTION: Ensure your config file is in ./conf/ directory or use --config-path.")
+
+        sys.exit(1)  # Exit with error code for critical failures
+    except Exception as e:
+        error_msg = str(e)
+        logger.error(f"An unexpected error occurred during the evaluation pipeline: {e}")
+
+        # Provide helpful suggestions for common issues
+        if "unexpected keyword argument" in error_msg:
+            logger.error("HINT: This suggests a configuration parameter is being passed incorrectly.")
+            logger.error("SOLUTION: Check your dataset config for extra parameters like 'final_columns'.")
+        elif "No module named" in error_msg:
+            logger.error("HINT: Cannot find Python module for reward function.")
+            logger.error("SOLUTION: Ensure your reward function file is in the current directory.")
+        elif "not enough values to unpack" in error_msg:
+            logger.error("HINT: Data format mismatch.")
+            logger.error("SOLUTION: Run 'reward-kit validate-data --file your_data.jsonl' to check data format.")
+
+        logger.error("For more help, try:")
+        logger.error("1. Run 'reward-kit validate-data --file your_data.jsonl' to check data")
+        logger.error("2. Use the simplified config: --config-name simple_uipath_eval")
+        logger.error("3. Check that all files are in the correct locations")
+
+        sys.exit(1)  # Exit with error code
+
+
+# This is the Hydra entry point for this command.
+# It needs a config_path relative to where this script is, or an absolute one.
+# If reward-kit is installed, conf might not be easily found via relative paths.
+# Using `pkg://` provider is more robust for installed packages.
+# For now, assume a `conf` dir at project root, and this script is in `eval_protocol/cli_commands`.
+import os  # Ensure os is imported for path manipulation
+
+# So, `config_path` would be `../../conf`.
+# The `config_name` will be the primary config for this `run` command.
+# Let's point directly to the example's config for now to simplify debugging Hydra pathing.
+# Path from eval_protocol/cli_commands/ to examples/math_example/conf/
+# Construct an absolute path or a file:// URL to make it more robust.
+_RUN_EVAL_CMD_DIR = os.path.dirname(os.path.abspath(__file__))
+# Default config_path for @hydra.main, relative to this file.
+# Points to the project's top-level 'conf' directory.
+_DEFAULT_HYDRA_CONFIG_PATH = os.path.abspath(os.path.join(_RUN_EVAL_CMD_DIR, "..", "..", "conf"))
+
+
+@hydra.main(config_path=_DEFAULT_HYDRA_CONFIG_PATH, config_name=None, version_base="1.3")
+def hydra_cli_entry_point(cfg: DictConfig) -> None:
+    # config_path and config_name from CLI will override the defaults in the decorator.
+    # If --config-name is not provided via CLI, Hydra would look for a default config
+    # (e.g., config.yaml) in the _DEFAULT_HYDRA_CONFIG_PATH.
+    # However, our reward-kit run command will always pass --config-path and --config-name.
+    # passed to `reward-kit run` (e.g., --config-path, --config-name)
+    # or by Hydra's default search behavior if not provided via CLI.
+    run_evaluation_command_logic(cfg)
+
+
+# This allows running `python -m eval_protocol.cli_commands.run_eval_cmd` (if __main__.py in folder)
+# or if this file itself is made executable.
+if __name__ == "__main__":
+    # This will parse sys.argv for Hydra overrides.
+    # Example: python eval_protocol/cli_commands/run_eval_cmd.py dataset=gsm8k_local_prompts generation.enabled=false
+    import sys  # Required for sys.exit
+
+    hydra_cli_entry_point()