eval-protocol 0.0.3__py3-none-any.whl

eval_protocol/cli_commands/agent_eval_cmd.py

@@ -0,0 +1,264 @@
+"""
+CLI command for running agent evaluations using the ForkableResource framework.
+"""
+
+import asyncio
+
+try:
+    import yaml
+except ImportError:
+    import sys
+    import types
+
+    # Create a stub module if yaml is not installed
+    yaml = types.ModuleType("yaml")
+
+    def dummy_safe_load(x):
+        return None
+
+    def dummy_dump(x, **kwargs):
+        return None
+
+    yaml.safe_load = dummy_safe_load  # type: ignore[assignment]
+    yaml.dump = dummy_dump  # type: ignore[assignment]
+
+import json  # Fallback or for explicit JSON files
+import logging  # For logger instance
+import os  # For environment variables
+from pathlib import Path
+
+from pydantic import ValidationError
+
+from eval_protocol.agent import Orchestrator
+from eval_protocol.agent.task_manager import TaskManager
+from eval_protocol.models import TaskDefinitionModel  # Import the new Pydantic model
+
+# setup_logging is already called in cli.py's main, but good for standalone use if any
+# from .common import setup_logging
+
+
+def agent_eval_command(args):
+    """
+    Run agent evaluation using the Orchestrator and ForkableResource framework.
+    """
+    logger = logging.getLogger("agent_eval")
+    logger.info("Starting agent-eval command.")
+
+    task_manager = TaskManager()
+
+    if not args.task_def:
+        logger.error("Error: --task-def (path to task definition YAML file or directory) is required.")
+        return 1
+
+    task_def_path = Path(args.task_def)
+
+    registered_task_ids = []
+    if task_def_path.is_file():
+        task_def = task_manager._load_task_from_file(str(task_def_path))
+        if task_def:
+            task_id = task_manager.register_task(task_def)
+            registered_task_ids.append(task_id)
+        else:
+            logger.error(f"Failed to load task definition from {task_def_path}")
+            return 1
+    elif task_def_path.is_dir():
+        registered_task_ids = task_manager.register_tasks_from_directory(str(task_def_path))
+        if not registered_task_ids:
+            logger.error(f"No valid task definitions found in directory: {task_def_path}")
+            return 1
+    else:
+        logger.error(f"Task definition path not found or invalid: {task_def_path}")
+        return 1
+
+    logger.info(f"Registered {len(registered_task_ids)} tasks: {registered_task_ids}")
+
+    try:
+
+        async def main_flow():
+            if getattr(args, "model", None):
+                original_model = os.environ.get("MODEL_AGENT")
+                os.environ["MODEL_AGENT"] = args.model
+                logger.info(f"Model overridden to: {args.model}")
+
+            parallel = getattr(args, "parallel", False)
+            max_concurrency = getattr(args, "max_concurrency", 3)
+            filter_tasks = getattr(args, "filter", None)
+
+            tasks_to_run = registered_task_ids
+            if filter_tasks:
+                tasks_to_run = [tid for tid in registered_task_ids if tid in filter_tasks]
+                if not tasks_to_run:
+                    logger.warning(f"No tasks match the specified filter: {filter_tasks}")
+                    return
+
+            try:
+                num_rollouts_override = getattr(args, "num_rollouts", None)
+                results = await task_manager.execute_tasks(
+                    task_ids=tasks_to_run,
+                    parallel=parallel,
+                    max_concurrency=max_concurrency,
+                    num_rollouts_override=num_rollouts_override,
+                )
+
+                logger.info(f"Execution completed for {len(results)} tasks")
+                for task_id, result in results.items():
+                    if isinstance(result, dict) and "error" in result:
+                        logger.error(f"Task '{task_id}' failed: {result['error']}")
+                    elif isinstance(result, dict) and result.get("aggregated", False):
+                        # Handle aggregated results from multiple rollouts
+                        logger.info(f"Task '{task_id}' batch results:")
+                        logger.info(
+                            f"  - Rollouts: {result['successful_rollouts']}/{result['num_rollouts']} successful ({result.get('failed_rollouts', 0)} failed)"
+                        )
+                        logger.info(f"  - Success rate: {result['success_rate']:.2%}")
+                        logger.info(f"  - Average score: {result['avg_score']:.4f}")
+                        logger.info(f"  - Standard deviation: {result.get('std_dev', 0.0):.4f}")
+                        logger.info(f"  - Score range: {result['min_score']:.4f} - {result['max_score']:.4f}")
+                        if "aggregated_metrics" in result:
+                            logger.info(f"  - Aggregated metrics:")
+                            for metric_name, metric_data in result["aggregated_metrics"].items():
+                                logger.info(
+                                    f"    * {metric_name}: avg={metric_data['avg_score']:.4f}, range={metric_data['min_score']:.4f}-{metric_data['max_score']:.4f}"
+                                )
+
+                        # Log path to detailed results file
+                        if result.get("timestamp"):
+                            timestamp = (
+                                result["timestamp"].replace(":", "").replace("-", "").replace("T", "_").split(".")[0]
+                            )
+                            # Use the trajectory filename format that matches TaskManager
+                            trajectory_file = f"trajectory_{task_id}_{timestamp}.jsonl"
+                            logger.info(f"  - Trajectory data saved to: {trajectory_file}")
+                    elif isinstance(result, dict) and "score" in result:
+                        logger.info(f"Task '{task_id}' score: {result['score']}")
+                    else:
+                        logger.info(f"Task '{task_id}' completed")
+            finally:
+                await task_manager.cleanup()
+
+        asyncio.run(main_flow())
+        logger.info("agent-eval command finished successfully.")
+        return 0
+    except Exception as e:
+        logger.error(f"Error during agent-eval execution: {e}")
+        import traceback
+
+        logger.debug(traceback.format_exc())
+        return 1
+
+
+def bfcl_eval_command(args):
+    """
+    Run BFCL agent evaluations using the refactored framework.
+    This command specifically manages BFCL task evaluation.
+    """
+    logger = logging.getLogger("bfcl_eval")
+    logger.info("Starting BFCL evaluation command.")
+
+    task_manager = TaskManager()
+
+    task_dir = Path(args.task_dir)
+    if not task_dir.is_dir():
+        logger.error(f"Task directory not found: {task_dir}")
+        return 1
+
+    logger.info(f"Registering BFCL tasks from {task_dir}")
+
+    try:
+        registered_task_ids = []
+
+        if args.task_id:
+            task_path = task_dir / f"{args.task_id}.yaml"
+            if not task_path.exists():
+                logger.error(f"Task file not found: {task_path}")
+                return 1
+
+            task_def = task_manager._load_task_from_file(str(task_path))
+            if task_def:
+                task_id = task_manager.register_task(task_def)
+                registered_task_ids.append(task_id)
+                logger.info(f"Registered task: {task_id}")
+            else:
+                logger.error(f"Failed to load task from {task_path}")
+                return 1
+        else:
+            registered_task_ids = task_manager.register_tasks_from_directory(str(task_dir))
+            if not registered_task_ids:
+                logger.error(f"No valid BFCL tasks found in directory: {task_dir}")
+                return 1
+            logger.info(f"Registered {len(registered_task_ids)} BFCL tasks")
+
+        async def main_flow():
+            if args.model:
+                original_model = os.environ.get("MODEL_AGENT")
+                os.environ["MODEL_AGENT"] = args.model
+                logger.info(f"Model overridden to: {args.model}")
+
+            if args.output_dir:
+                output_path = Path(args.output_dir)
+                output_path.mkdir(parents=True, exist_ok=True)
+                logger.info(f"Results will be saved to {output_path}")
+
+            try:
+                results = await task_manager.execute_tasks(
+                    task_ids=registered_task_ids,
+                    parallel=args.parallel,
+                    max_concurrency=args.max_concurrency,
+                )
+
+                logger.info(f"BFCL evaluation completed for {len(results)} tasks")
+                for task_id, result in results.items():
+                    if isinstance(result, dict) and "error" in result:
+                        logger.error(f"Task '{task_id}' failed: {result['error']}")
+                    elif isinstance(result, dict) and "score" in result:
+                        logger.info(f"Task '{task_id}' score: {result['score']}")
+
+                        # More detailed results for BFCL
+                        if "format_score" in result:
+                            logger.info(f"Task '{task_id}' format score: {result['format_score']}")
+                        if "state_match" in result:
+                            logger.info(f"Task '{task_id}' state match: {result['state_match']}")
+                    else:
+                        logger.info(f"Task '{task_id}' completed with result: {result}")
+
+                if args.output_dir:
+                    results_file = Path(args.output_dir) / "bfcl_results.json"
+
+                    # Convert results to JSON-serializable format
+                    serializable_results = {}
+                    for task_id, result in results.items():
+                        if hasattr(result, "dict"):
+                            # Handle Pydantic models
+                            serializable_results[task_id] = result.dict()
+                        elif isinstance(result, dict):
+                            # Handle dictionaries with potentially non-serializable values
+                            serializable_dict = {}
+                            for k, v in result.items():
+                                if hasattr(v, "dict"):
+                                    serializable_dict[k] = v.dict()
+                                elif hasattr(v, "__dict__"):
+                                    serializable_dict[k] = str(v)
+                                else:
+                                    serializable_dict[k] = v
+                            serializable_results[task_id] = serializable_dict
+                        else:
+                            # Handle other objects by converting to string
+                            serializable_results[task_id] = str(result)
+
+                    with open(results_file, "w") as f:
+                        json.dump(serializable_results, f, indent=2)
+                    logger.info(f"Results saved to {results_file}")
+
+            finally:
+                await task_manager.cleanup()
+
+        asyncio.run(main_flow())
+        logger.info("BFCL evaluation completed successfully.")
+        return 0
+
+    except Exception as e:
+        logger.error(f"Error during BFCL evaluation: {e}")
+        import traceback
+
+        logger.debug(traceback.format_exc())
+        return 1

eval_protocol/cli_commands/common.py

@@ -0,0 +1,242 @@
+"""
+Common utility functions for the Reward Kit CLI.
+"""
+
+import json
+import logging
+import os
+from typing import Any, Dict, Iterator, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+def setup_logging(verbose=False, debug=False):
+    """Setup logging configuration"""
+    if debug:
+        log_level = logging.DEBUG
+        # More detailed format for debug
+        format_str = "[%(asctime)s][%(name)s][%(levelname)s] - %(pathname)s:%(lineno)d - %(message)s"
+    elif verbose:  # --verbose flag
+        log_level = logging.INFO
+        # Consistent format, similar to user's logs but with name
+        format_str = "[%(asctime)s][%(name)s][%(levelname)s] - %(message)s"
+    else:  # Default (neither --verbose nor --debug)
+        log_level = logging.INFO  # Changed from WARNING to INFO
+        # Use the same format as verbose for default INFO level
+        format_str = "[%(asctime)s][%(name)s][%(levelname)s] - %(message)s"
+
+    logging.basicConfig(level=log_level, format=format_str, datefmt="%Y-%m-%d %H:%M:%S")
+
+    # Set higher levels for noisy libraries unless in full debug mode
+    if not debug:
+        noisy_loggers = ["httpx", "mcp", "urllib3", "asyncio", "hpack", "httpcore"]
+        for logger_name in noisy_loggers:
+            logging.getLogger(logger_name).setLevel(logging.WARNING)
+
+    # Ensure eval_protocol's own loggers respect the overall log_level,
+    # overriding any specific DEBUG settings in submodules unless --debug is used.
+    # If log_level is WARNING (default), eval_protocol INFO and DEBUG logs will be suppressed.
+    # If log_level is INFO (--verbose), eval_protocol DEBUG logs will be suppressed.
+    # If log_level is DEBUG (--debug), all eval_protocol logs show.
+    logging.getLogger("eval_protocol").setLevel(log_level)
+
+
+def check_environment():
+    """Check if required environment variables are set for general commands."""
+    if not os.environ.get("FIREWORKS_API_KEY"):
+        logger.warning("FIREWORKS_API_KEY environment variable is not set.")
+        logger.warning("This is required for API calls. Set this variable before running the command.")
+        logger.warning("Example: FIREWORKS_API_KEY=$DEV_FIREWORKS_API_KEY reward-kit [command]")
+        return False
+    return True
+
+
+def check_agent_environment(test_mode=False):
+    """Check if required environment variables are set for agent evaluation commands."""
+    missing_vars = []
+    if not os.environ.get("MODEL_AGENT"):
+        missing_vars.append("MODEL_AGENT")
+
+    if test_mode:
+        if missing_vars:
+            logger.info(f"Note: The following environment variables are not set: {', '.join(missing_vars)}")
+            logger.info("Since you're running in test mode, these are not strictly required for all operations.")
+        return True
+
+    if missing_vars:
+        logger.warning(f"The following environment variables are not set: {', '.join(missing_vars)}")
+        logger.warning(
+            "These are typically required for full agent evaluation. Set these variables for full functionality."
+        )
+        logger.warning("Example: MODEL_AGENT=openai/gpt-4o-mini reward-kit agent-eval [args]")
+        logger.warning("Alternatively, use --test-mode for certain validation tasks without requiring all API keys.")
+        return False
+    return True
+
+
+# --- Sample Loading Helper Functions ---
+
+
+def _validate_sample_messages(messages: Any, sample_index: int, line_number: int) -> bool:
+    """Helper to validate the 'messages' field of a sample."""
+    if not isinstance(messages, list):
+        logger.warning(f"Sample {sample_index} (line {line_number}): 'messages' field is not a list. Skipping sample.")
+        return False
+    if not messages:
+        logger.warning(f"Sample {sample_index} (line {line_number}): 'messages' list is empty. Skipping sample.")
+        return False
+    for i, msg in enumerate(messages):
+        if not isinstance(msg, dict):
+            logger.warning(
+                f"Sample {sample_index} (line {line_number}): message item {i} is not a dictionary. Skipping sample."
+            )
+            return False
+        role = msg.get("role")
+        content = msg.get("content")
+        if not isinstance(role, str) or not isinstance(content, str):
+            logger.warning(
+                f"Sample {sample_index} (line {line_number}): message item {i} missing 'role' or 'content' string fields. Skipping sample."
+            )
+            return False
+    return True
+
+
+def load_samples_from_file(filepath: str, max_samples: int) -> Iterator[Dict[str, Any]]:
+    """
+    Loads samples from a JSONL file.
+    Each line should be a JSON object.
+    Each sample must contain a 'messages' key with a list of message dicts (each having 'role' and 'content').
+    Yields valid sample dictionaries up to max_samples.
+    """
+    count = 0
+    line_number = 0
+    try:
+        with open(filepath, "r", encoding="utf-8") as f:
+            for line in f:
+                line_number += 1
+                if count >= max_samples:
+                    logger.info(f"Reached max_samples ({max_samples}). Stopping sample loading from {filepath}.")
+                    break
+                line_content = line.strip()
+                if not line_content:
+                    continue
+                try:
+                    sample = json.loads(line_content)
+                except json.JSONDecodeError:
+                    logger.warning(f"Line {line_number}: Invalid JSON. Skipping line: {line_content[:100]}...")
+                    continue
+                if not isinstance(sample, dict):
+                    logger.warning(f"Line {line_number}: Content is not a JSON object. Skipping line.")
+                    continue
+                messages = sample.get("messages")
+                if messages is None:
+                    logger.warning(f"Sample (line {line_number}): 'messages' field is missing. Skipping sample.")
+                    continue
+                if not _validate_sample_messages(messages, count + 1, line_number):
+                    continue
+                yield sample
+                count += 1
+    except FileNotFoundError:
+        logger.error(f"Sample file not found: {filepath}")
+    except Exception as e:
+        logger.error(f"Error reading or processing sample file {filepath}: {e}")
+    if count == 0:
+        logger.info(f"No valid samples loaded from {filepath} after processing {line_number} lines.")
+
+
+def load_samples_from_huggingface(
+    dataset_name: str,
+    split: str,
+    prompt_key: str,
+    response_key: str,
+    key_map: Optional[Dict[str, str]],
+    max_samples: int,
+) -> Iterator[Dict[str, Any]]:
+    """
+    Loads samples from a HuggingFace dataset using the 'datasets' library.
+    Constructs 'messages' from prompt_key and response_key.
+    Uses key_map to map other dataset fields to custom keys in the output sample.
+    Yields valid sample dictionaries up to max_samples.
+    """
+    try:
+        from datasets import (
+            Dataset,
+            DatasetDict,
+            IterableDataset,
+            IterableDatasetDict,
+            load_dataset,
+        )
+
+        # Also consider specific exceptions from datasets like DatasetNotFoundError
+    except ImportError:
+        logger.error(
+            "The 'datasets' library is required to load samples from HuggingFace. "
+            "Please install it with 'pip install datasets'."
+        )
+        return
+
+    count = 0
+    processed_records = 0
+    try:
+        hf_dataset = load_dataset(dataset_name, split=split, streaming=True)  # Use streaming
+    except Exception as e:  # Broad exception for now, can be more specific
+        logger.error(f"Error loading HuggingFace dataset '{dataset_name}' (split: {split}): {e}")
+        return
+
+    if not isinstance(
+        hf_dataset, (DatasetDict, Dataset, IterableDatasetDict, IterableDataset)
+    ):  # Should be IterableDataset due to streaming=True
+        logger.error(f"Loaded HuggingFace dataset '{dataset_name}' is not a recognized Dataset type.")
+        return
+
+    logger.info(f"Streaming samples from HuggingFace dataset '{dataset_name}' (split: {split}).")
+    for record in hf_dataset:
+        processed_records += 1
+        if count >= max_samples:
+            logger.info(f"Reached max_samples ({max_samples}). Stopping HuggingFace sample loading.")
+            break
+
+        if not isinstance(record, dict):
+            logger.warning(f"HuggingFace dataset record {processed_records} is not a dictionary. Skipping.")
+            continue
+
+        prompt = record.get(prompt_key)
+        response_content = record.get(response_key)
+
+        if not isinstance(prompt, str):
+            logger.warning(
+                f"HuggingFace record {processed_records}: Prompt key '{prompt_key}' (value: {str(prompt)[:50]}...) did not yield a string. Skipping sample."
+            )
+            continue
+        if not isinstance(response_content, str):
+            logger.warning(
+                f"HuggingFace record {processed_records}: Response key '{response_key}' (value: {str(response_content)[:50]}...) did not yield a string. Skipping sample."
+            )
+            continue
+
+        messages = [
+            {"role": "user", "content": prompt},
+            {"role": "assistant", "content": response_content},
+        ]
+
+        if not _validate_sample_messages(messages, count + 1, processed_records):
+            continue
+
+        sample_output: Dict[str, Any] = {"messages": messages}
+
+        if key_map:
+            for source_key_in_record, target_key_in_sample in key_map.items():
+                if source_key_in_record in record:
+                    sample_output[target_key_in_sample] = record[source_key_in_record]
+                else:
+                    logger.warning(
+                        f"HuggingFace record {processed_records}: Key '{source_key_in_record}' from key_map not found. It will be omitted."
+                    )
+
+        yield sample_output
+        count += 1
+
+    if count == 0:
+        logger.info(
+            f"No valid samples loaded from HuggingFace dataset '{dataset_name}' (split: {split}) after processing {processed_records} records."
+        )