eval-protocol 0.0.3__py3-none-any.whl

eval_protocol/utils/packaging_utils.py

@@ -0,0 +1,108 @@
+import logging
+import os
+import subprocess
+import sys
+import tempfile
+from typing import List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+def get_pip_executable(venv_pip_path: Optional[str] = None) -> List[str]:
+    """Determines the pip executable command parts."""
+    if venv_pip_path and os.path.exists(venv_pip_path) and os.access(venv_pip_path, os.X_OK):
+        logger.info(f"Using specified pip executable: {venv_pip_path}")
+        return [venv_pip_path]
+
+    # Try to find pip in the current virtual environment's scripts/bin directory
+    # sys.executable should be /path/to/.venv/bin/python
+    # So, pip should be /path/to/.venv/bin/pip
+    # On Windows, it might be /path/to/.venv/Scripts/pip.exe
+    potential_pip_path = os.path.join(os.path.dirname(sys.executable), "pip")
+    if os.name == "nt":  # Windows check
+        potential_pip_path += ".exe"
+
+    if os.path.exists(potential_pip_path) and os.access(potential_pip_path, os.X_OK):
+        logger.info(f"Using auto-detected pip executable: {potential_pip_path}")
+        return [potential_pip_path]
+
+    # Fallback to sys.executable -m pip (should generally work if python is from the venv)
+    logger.info(f"Using pip via: {sys.executable} -m pip")
+    return [sys.executable, "-m", "pip"]
+
+
+def install_requirements(
+    requirements_list: List[str],
+    venv_pip_path: Optional[str] = None,  # User can specify e.g. ".venv/bin/pip"
+    log_output: bool = True,
+) -> None:
+    """
+    Installs a list of Python package requirements using pip.
+
+    Args:
+        requirements_list: A list of requirement strings (e.g., ["package_a==1.0", "package_b>=2.0"]).
+        venv_pip_path: Optional path to the specific pip executable to use.
+        log_output: If True, logs the stdout and stderr of the pip command.
+    """
+    if not requirements_list:
+        logger.debug("No requirements provided to install.")
+        return
+
+    unique_requirements = sorted(list(set(req.strip() for req in requirements_list if req.strip())))
+    if not unique_requirements:
+        logger.debug("No unique, non-empty requirements to install after stripping.")
+        return
+
+    pip_command_parts = get_pip_executable(venv_pip_path)
+
+    # Create a temporary requirements file
+    # delete=False is used because on Windows, a file opened for writing cannot be opened by another process.
+    # We will manually delete it in the finally block.
+    tmp_req_fd, tmp_req_file_path = tempfile.mkstemp(suffix=".txt", prefix="rk_reqs_")
+
+    try:
+        with os.fdopen(tmp_req_fd, "w") as tmp_req_file:
+            for req in unique_requirements:
+                tmp_req_file.write(req + "\n")
+
+        logger.info(
+            f"Attempting to install requirements: {unique_requirements} using pip command: {' '.join(pip_command_parts)} -r {tmp_req_file_path}"
+        )
+
+        command = pip_command_parts + ["install", "-r", tmp_req_file_path]
+
+        process = subprocess.run(
+            command,
+            check=True,  # Raise CalledProcessError on non-zero exit
+            capture_output=True,
+            text=True,  # Decodes stdout/stderr as text
+            encoding="utf-8",  # Explicit encoding
+            errors="replace",  # Handle potential encoding errors in pip output
+        )
+        if log_output and process.stdout:
+            logger.info(f"Pip install stdout:\n{process.stdout.strip()}")
+        # pip often uses stderr for progress/warnings even on success
+        if log_output and process.stderr:
+            logger.info(f"Pip install stderr:\n{process.stderr.strip()}")
+        logger.info(f"Successfully installed requirements: {unique_requirements}")
+
+    except subprocess.CalledProcessError as e:
+        error_message = f"Error installing requirements from {tmp_req_file_path}.\n"
+        error_message += f"Command: {' '.join(e.cmd)}\n"
+        if e.stdout:
+            error_message += f"Pip stdout:\n{e.stdout.strip()}\n"
+        if e.stderr:
+            error_message += f"Pip stderr:\n{e.stderr.strip()}\n"
+        logger.error(error_message)
+        raise RuntimeError(
+            f"Failed to install requirements: {unique_requirements}. Details:\n{e.stderr or e.stdout or str(e)}"
+        )
+    except FileNotFoundError:
+        logger.error(
+            f"Pip executable not found: {' '.join(pip_command_parts)}. Please ensure pip is installed and in PATH, or venv_pip_path is correct."
+        )
+        raise
+    finally:
+        if os.path.exists(tmp_req_file_path):
+            os.remove(tmp_req_file_path)
+            logger.debug(f"Removed temporary requirements file: {tmp_req_file_path}")

eval_protocol/utils/static_policy.py

@@ -0,0 +1,305 @@
+"""
+General Static Policy for MCP Environment Testing
+
+This policy provides a deterministic, non-LLM action sequence for fast iteration
+across different MCP environments. It can be configured with custom tool names
+and action sequences.
+
+This is useful for:
+- Fast testing of multi-session functionality
+- Debugging environment behavior
+- Performance testing without LLM overhead
+"""
+
+import asyncio
+import json
+import logging
+import os
+import random
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+# Import the base policy and types for proper recording functionality
+from eval_protocol.mcp.types import LLMUsageStats, MCPToolCall
+from eval_protocol.playback_policy import PlaybackPolicyBase
+
+logger = logging.getLogger(__name__)
+
+
+class StaticPolicy(PlaybackPolicyBase):
+    """
+    Static policy that follows a predetermined action sequence.
+
+    Can be configured for different environments with custom tool names and actions.
+    """
+
+    def __init__(
+        self,
+        tool_name: str,
+        action_sequence: Optional[List[str]] = None,
+        available_actions: Optional[List[str]] = None,
+        **kwargs,
+    ):
+        """
+        Initialize static policy with recording/playback support.
+
+        Args:
+            tool_name: Name of the tool to call for actions (e.g., "lake_move", "lander_action")
+            action_sequence: List of actions to execute. If None, uses a default sequence.
+            available_actions: List of all available actions for this environment.
+            **kwargs: Additional arguments passed to PlaybackPolicyBase
+        """
+        # Initialize parent class for recording/playback functionality
+        super().__init__(**kwargs)
+
+        self.tool_name = tool_name
+        self.available_actions = available_actions or []
+
+        # Set default action sequence if not provided
+        if action_sequence is None:
+            if self.available_actions:
+                # Use first few actions as default sequence
+                self.action_sequence = self.available_actions[: min(6, len(self.available_actions))]
+            else:
+                self.action_sequence = ["DEFAULT_ACTION"]
+        else:
+            self.action_sequence = action_sequence
+
+        self.step_counts = {}  # Track step count per environment
+
+    async def _generate_live_tool_calls(
+        self,
+        tool_schemas: List[Dict],
+        env_index: int,
+        conversation_history: List[Dict[str, Any]],
+    ) -> Tuple[List[MCPToolCall], LLMUsageStats]:
+        """
+        Generate tool calls in live mode using the static action sequence.
+
+        This implements the abstract method from PlaybackPolicyBase.
+
+        Args:
+            tool_schemas: Available tools for this environment
+            env_index: Environment index
+            conversation_history: Current conversation history for this environment
+
+        Returns:
+            List of MCPToolCall objects
+        """
+        # Get current step count for this environment
+        step_count = self.step_counts.get(env_index, 0)
+
+        # Determine action based on step count
+        if step_count < len(self.action_sequence):
+            action = self.action_sequence[step_count]
+        else:
+            # After sequence completes, repeat the last action
+            action = self.action_sequence[-1]
+
+        # Create tool call in MCPToolCall format
+        tool_call = MCPToolCall(tool_name=self.tool_name, arguments={"action": action})
+
+        # Update step count
+        self.step_counts[env_index] = step_count + 1
+
+        logger.debug(f"🎮 Env {env_index} step {step_count}: {action}")
+
+        return [tool_call], None
+
+    def add_tool_response(
+        self,
+        env_index: int,
+        tool_call: MCPToolCall,
+        tool_response: Union[str, List[Dict[str, Any]]],
+        conversation_history: List[Dict[str, Any]],
+        reward: float = 0.0,
+        terminated: bool = False,
+        info: Dict[str, Any] = None,
+    ):
+        """Add tool call and response to conversation history for recording."""
+
+        # Find the most recent assistant message with tool calls to get the correct call_id
+        call_id = None
+        for i in range(len(conversation_history) - 1, -1, -1):
+            if conversation_history[i]["role"] == "assistant" and "tool_calls" in conversation_history[i]:
+                # Find the tool call that matches our tool_name
+                for tc in conversation_history[i]["tool_calls"]:
+                    if tc["function"]["name"] == tool_call.tool_name:
+                        call_id = tc["id"]
+                        break
+                if call_id:
+                    break
+
+        # Fallback if no matching tool call found
+        if not call_id:
+            call_id = f"call_{env_index}_{len(conversation_history)}"
+
+        # Add tool response with control plane metadata
+        tool_message = {
+            "role": "tool",
+            "tool_call_id": call_id,
+            "content": tool_response,
+        }
+
+        # Add control plane metadata if provided
+        if reward != 0.0 or terminated or info:
+            tool_message["metadata"] = {
+                "reward": reward,
+                "terminated": terminated,
+                "info": info or {},
+            }
+
+        conversation_history.append(tool_message)
+
+    def log_conversation_state_for_playback(
+        self, env_index: int, step: int, conversation_history: List[Dict[str, Any]]
+    ):
+        """
+        Log the current conversation state in the format required for playback.
+
+        Expected format: {"env_index": 0, "step": 0, "messages": [{..}, {..}]}
+
+        Args:
+            env_index: Environment index
+            step: Current step number
+            conversation_history: List of conversation messages
+        """
+        # Use EP_PLAYBACK_FILE environment variable for recording
+        playback_file = os.environ.get("EP_PLAYBACK_FILE")
+        if not playback_file:
+            return  # No recording file specified
+
+        playback_entry = {
+            "env_index": env_index,
+            "step": step,
+            "messages": conversation_history.copy(),
+        }
+
+        with open(playback_file, "a") as f:
+            f.write(json.dumps(playback_entry) + "\n")
+
+    @property
+    def model_id(self) -> str:
+        """Model identifier for static policy."""
+        return f"static-policy-{self.tool_name}-v1"
+
+
+class RandomPolicy(PlaybackPolicyBase):
+    """
+    Random policy that selects random actions.
+    Useful for testing environment robustness.
+    """
+
+    def __init__(
+        self,
+        tool_name: str,
+        available_actions: List[str],
+        seed: Optional[int] = None,
+        **kwargs,
+    ):
+        """
+        Initialize random policy with recording/playback support.
+
+        Args:
+            tool_name: Name of the tool to call for actions
+            available_actions: List of all available actions for this environment
+            seed: Random seed for reproducibility
+            **kwargs: Additional arguments passed to PlaybackPolicyBase
+        """
+        # Initialize parent class for recording/playback functionality
+        super().__init__(**kwargs)
+
+        self.tool_name = tool_name
+        self.available_actions = available_actions
+        self.random = random.Random(seed)
+
+    async def _generate_live_tool_calls(
+        self,
+        tool_schemas: List[Dict],
+        env_index: int,
+        conversation_history: List[Dict[str, Any]],
+    ) -> Tuple[List[MCPToolCall], LLMUsageStats]:
+        """
+        Generate random tool calls in live mode.
+
+        Args:
+            tool_schemas: Available tools for this environment
+            env_index: Environment index
+            conversation_history: Current conversation history for this environment
+
+        Returns:
+            List of MCPToolCall objects
+        """
+        # Select random action
+        action = self.random.choice(self.available_actions)
+
+        # Create tool call
+        tool_call = MCPToolCall(tool_name=self.tool_name, arguments={"action": action})
+
+        logger.debug(f"🎲 Env {env_index}: {action}")
+
+        return [tool_call], None
+
+    def add_tool_response(
+        self,
+        env_index: int,
+        tool_call: MCPToolCall,
+        tool_response: Union[str, List[Dict[str, Any]]],
+        conversation_history: List[Dict[str, Any]],
+        reward: float = 0.0,
+        terminated: bool = False,
+        info: Dict[str, Any] = None,
+    ):
+        """Add tool call and response to conversation history for recording."""
+
+        # Find the most recent assistant message with tool calls
+        call_id = None
+        for i in range(len(conversation_history) - 1, -1, -1):
+            if conversation_history[i]["role"] == "assistant" and "tool_calls" in conversation_history[i]:
+                for tc in conversation_history[i]["tool_calls"]:
+                    if tc["function"]["name"] == tool_call.tool_name:
+                        call_id = tc["id"]
+                        break
+                if call_id:
+                    break
+
+        if not call_id:
+            call_id = f"call_{env_index}_{len(conversation_history)}"
+
+        # Add tool response with control plane metadata
+        tool_message = {
+            "role": "tool",
+            "tool_call_id": call_id,
+            "content": tool_response,
+        }
+
+        # Add control plane metadata if provided
+        if reward != 0.0 or terminated or info:
+            tool_message["metadata"] = {
+                "reward": reward,
+                "terminated": terminated,
+                "info": info or {},
+            }
+
+        conversation_history.append(tool_message)
+
+    def log_conversation_state_for_playback(
+        self, env_index: int, step: int, conversation_history: List[Dict[str, Any]]
+    ):
+        """Log the current conversation state for playback recording."""
+        playback_file = os.environ.get("EP_PLAYBACK_FILE")
+        if not playback_file:
+            return
+
+        playback_entry = {
+            "env_index": env_index,
+            "step": step,
+            "messages": conversation_history.copy(),
+        }
+
+        with open(playback_file, "a") as f:
+            f.write(json.dumps(playback_entry) + "\n")
+
+    @property
+    def model_id(self) -> str:
+        """Model identifier for random policy."""
+        return f"random-policy-{self.tool_name}-v1"