eval-protocol 0.0.3__py3-none-any.whl

eval_protocol/agent/resource_abc.py

@@ -0,0 +1,89 @@
+"""
+Abstract Base Class for Forkable Resources in the Agent Evaluation Framework V2.
+"""
+
+from abc import ABC, abstractmethod
+from typing import (  # Callable removed as not directly used in ABC signatures
+    Any,
+    Dict,
+    List,
+    Optional,
+)
+
+
+class ForkableResource(ABC):
+    """
+    Abstract base class defining the interface for a forkable, checkpointable,
+    and interactive environment resource for agent evaluation.
+    """
+
+    @abstractmethod
+    async def setup(self, config: Dict[str, Any]) -> None:
+        """
+        Initializes the resource with a given configuration.
+        This method should prepare the resource for its first use or fork.
+        For example, setting up a database schema, creating a base file system,
+        or starting a base Docker container.
+        """
+        pass
+
+    @abstractmethod
+    async def fork(self) -> "ForkableResource":
+        """
+        Creates and returns a new, independent instance of this resource
+        with an identical copy of the current state of the resource it was forked from.
+        This new instance is typically an EpisodeResource, used for a single agent rollout.
+        """
+        pass
+
+    @abstractmethod
+    async def checkpoint(self) -> Any:
+        """
+        Returns a serializable representation of the resource's current state.
+        The format of this state (e.g., bytes, dict, path to a file) is specific
+        to the resource implementation but must be restorable by `restore()`.
+        """
+        pass
+
+    @abstractmethod
+    async def restore(self, state_data: Any) -> None:
+        """
+        Restores the resource's state from previously checkpointed `state_data`.
+        The resource should be in the same state as when `checkpoint()` was called.
+        """
+        pass
+
+    @abstractmethod
+    async def step(self, action_name: str, action_params: Dict[str, Any]) -> Any:
+        """
+        Executes a named action with given parameters on the resource.
+        This typically modifies the resource's state.
+        Returns an observation or result of the action, specific to the resource and action.
+        """
+        pass
+
+    @abstractmethod
+    async def get_observation(self) -> Any:
+        """
+        Returns the current observable state of the resource for the agent.
+        The format of the observation is resource-specific.
+        """
+        pass
+
+    @abstractmethod
+    async def get_tools_spec(self) -> List[Dict[str, Any]]:
+        """
+        Returns a list of tool specifications (e.g., OpenAI function calling format)
+        that are currently available or applicable to this resource's state.
+        This can be dynamic, changing based on the resource's current state.
+        """
+        pass
+
+    @abstractmethod
+    async def close(self) -> None:
+        """
+        Performs any necessary cleanup for the resource.
+        This includes releasing acquired resources like database connections,
+        stopping containers, deleting temporary files or directories, etc.
+        """
+        pass

eval_protocol/agent/resource_pool.py

@@ -0,0 +1,184 @@
+"""
+Resource Pool for the Agent Evaluation Framework V2.
+Manages and allocates resources to specific tasks.
+"""
+
+import asyncio
+import logging
+from typing import Any, Dict, List, Optional, Set, Type
+
+from .resource_abc import ForkableResource
+
+
+class ResourcePool:
+    """
+    Manages a pool of ForkableResources that can be shared and reused across tasks.
+    Provides tracking and lifecycle management for resources.
+    """
+
+    def __init__(self):
+        """Initialize an empty resource pool."""
+        self.resources: Dict[str, ForkableResource] = {}  # resource_id -> resource instance
+        self.resource_tasks: Dict[str, Set[str]] = {}  # resource_id -> set of task_ids using it
+        self.task_resources: Dict[str, Set[str]] = {}  # task_id -> set of resource_ids used by it
+        self.logger = logging.getLogger("ResourcePool")
+
+    async def create_resource(
+        self,
+        resource_type: Type[ForkableResource],
+        resource_id: str,
+        config: Dict[str, Any],
+        task_id: Optional[str] = None,
+    ) -> Optional[ForkableResource]:
+        """
+        Create a new resource of the specified type and add it to the pool.
+
+        Args:
+            resource_type: The ForkableResource class to instantiate
+            resource_id: Unique identifier for the resource
+            config: Configuration dictionary for the resource setup
+            task_id: Optional task ID to associate with this resource
+
+        Returns:
+            The created resource or None if creation fails
+        """
+        if resource_id in self.resources:
+            self.logger.warning(f"Resource '{resource_id}' already exists in the pool. Returning existing instance.")
+            if task_id:
+                self._associate_task_with_resource(task_id, resource_id)
+            return self.resources[resource_id]
+
+        try:
+            resource = resource_type()
+            await resource.setup(config)
+
+            self.resources[resource_id] = resource
+            self.resource_tasks[resource_id] = set()
+
+            if task_id:
+                self._associate_task_with_resource(task_id, resource_id)
+
+            self.logger.info(f"Created resource '{resource_id}' of type {resource_type.__name__}")
+            return resource
+        except Exception as e:
+            self.logger.error(f"Failed to create resource '{resource_id}': {e}")
+            return None
+
+    def get_resource(self, resource_id: str) -> Optional[ForkableResource]:
+        """
+        Get a resource from the pool by its ID.
+
+        Args:
+            resource_id: The identifier of the resource to retrieve
+
+        Returns:
+            The resource instance or None if not found
+        """
+        return self.resources.get(resource_id)
+
+    def _associate_task_with_resource(self, task_id: str, resource_id: str) -> None:
+        """
+        Associate a task with a resource for tracking purposes.
+
+        Args:
+            task_id: The task identifier
+            resource_id: The resource identifier
+        """
+        if resource_id not in self.resources:
+            self.logger.warning(f"Cannot associate task '{task_id}' with non-existent resource '{resource_id}'.")
+            return
+
+        # Add task to resource's task set
+        if resource_id not in self.resource_tasks:
+            self.resource_tasks[resource_id] = set()
+        self.resource_tasks[resource_id].add(task_id)
+
+        # Add resource to task's resource set
+        if task_id not in self.task_resources:
+            self.task_resources[task_id] = set()
+        self.task_resources[task_id].add(resource_id)
+
+        self.logger.debug(f"Associated task '{task_id}' with resource '{resource_id}'.")
+
+    async def fork_resource_for_task(self, resource_id: str, task_id: str) -> Optional[ForkableResource]:
+        """
+        Fork a resource for a specific task.
+
+        Args:
+            resource_id: The identifier of the resource to fork
+            task_id: The task that will use the forked resource
+
+        Returns:
+            The forked resource instance or None if forking fails
+        """
+        base_resource = self.get_resource(resource_id)
+        if not base_resource:
+            self.logger.error(f"Cannot fork non-existent resource '{resource_id}'.")
+            return None
+
+        try:
+            forked_resource = await base_resource.fork()
+            # We don't track forked resources in the pool, as they are typically
+            # short-lived and managed by the Orchestrator
+            self.logger.debug(f"Forked resource '{resource_id}' for task '{task_id}'.")
+            return forked_resource
+        except Exception as e:
+            self.logger.error(f"Failed to fork resource '{resource_id}' for task '{task_id}': {e}")
+            return None
+
+    async def cleanup_task_resources(self, task_id: str) -> None:
+        """
+        Clean up all resources associated with a task.
+
+        Args:
+            task_id: The task identifier
+        """
+        if task_id not in self.task_resources:
+            self.logger.debug(f"No resources to clean up for task '{task_id}'.")
+            return
+
+        resource_ids = list(self.task_resources[task_id])
+        for resource_id in resource_ids:
+            # Remove task from resource's task set
+            if resource_id in self.resource_tasks:
+                self.resource_tasks[resource_id].discard(task_id)
+
+                # If resource has no more tasks, close and remove it
+                if not self.resource_tasks[resource_id]:
+                    await self.close_resource(resource_id)
+
+        # Clear task's resource tracking
+        self.task_resources.pop(task_id, None)
+        self.logger.info(f"Cleaned up resources for task '{task_id}'.")
+
+    async def close_resource(self, resource_id: str) -> None:
+        """
+        Close a resource and remove it from the pool.
+
+        Args:
+            resource_id: The identifier of the resource to close
+        """
+        if resource_id not in self.resources:
+            self.logger.debug(f"Cannot close non-existent resource '{resource_id}'.")
+            return
+
+        resource = self.resources[resource_id]
+        try:
+            await resource.close()
+            self.resources.pop(resource_id)
+            self.resource_tasks.pop(resource_id, None)
+            self.logger.info(f"Closed and removed resource '{resource_id}' from pool.")
+        except Exception as e:
+            self.logger.error(f"Error closing resource '{resource_id}': {e}")
+
+    async def close_all_resources(self) -> None:
+        """Close all resources in the pool and clear it."""
+        resource_ids = list(self.resources.keys())
+        for resource_id in resource_ids:
+            await self.close_resource(resource_id)
+
+        # Clear all tracking dictionaries
+        self.resources.clear()
+        self.resource_tasks.clear()
+        self.task_resources.clear()
+        self.logger.info("Closed all resources in the pool.")

eval_protocol/agent/resources/__init__.py

@@ -0,0 +1,44 @@
+"""
+Resources for the Reward Kit Agent V2 Framework.
+
+This package contains concrete implementations of the ForkableResource ABC.
+"""
+
+from .bfcl_sim_api_resource import BFCLSimAPIResource
+from .docker_resource import DockerResource
+from .filesystem_resource import FileSystemResource
+
+# HTTP Rollout Protocol types for server implementations
+from .http_rollout_protocol import (
+    EndEpisodeRequest,
+    EndEpisodeResponse,
+    GameObservation,
+    HealthResponse,
+    HttpRolloutConfig,
+    StartEpisodeRequest,
+    StartEpisodeResponse,
+    StepRequest,
+    StepResponse,
+)
+from .http_rollout_resource import HttpRolloutResource
+from .python_state_resource import PythonStateResource
+from .sql_resource import SQLResource
+
+__all__ = [
+    "PythonStateResource",
+    "SQLResource",
+    "FileSystemResource",
+    "DockerResource",
+    "BFCLSimAPIResource",
+    "HttpRolloutResource",
+    # HTTP Rollout Protocol
+    "HttpRolloutConfig",
+    "StartEpisodeRequest",
+    "StartEpisodeResponse",
+    "StepRequest",
+    "StepResponse",
+    "EndEpisodeRequest",
+    "EndEpisodeResponse",
+    "HealthResponse",
+    "GameObservation",
+]

eval_protocol/agent/resources/bfcl_envs/__init__.py

@@ -0,0 +1 @@
+# BFCL environment implementations

eval_protocol/agent/resources/bfcl_envs/gorilla_file_system.py

@@ -0,0 +1,342 @@
+"""Implementation of GorillaFileSystem."""
+
+from typing import Dict, Optional, Union
+
+
+class File:
+    """A file in the Gorilla File System."""
+
+    def __init__(
+        self, name: str = "", content: str = "", parent: Optional["Directory"] = None
+    ):  # 'Directory' as string literal
+        self.name: str = name
+        self.content: str = content
+        self.parent: Optional["Directory"] = parent
+
+    def __repr__(self):
+        return f"<File: {self.name}, Content: '{self.content[:20]}{'...' if len(self.content) > 20 else ''}'>"
+
+    def __eq__(self, other):
+        if not isinstance(other, File):
+            return False
+        return self.name == other.name and self.content == other.content
+
+
+class Directory:
+    """A directory in the Gorilla File System."""
+
+    def __init__(
+        self,
+        name: str = "",
+        parent: Optional["Directory"] = None,  # Changed to string literal
+        contents: Optional[Dict[str, Union[File, "Directory"]]] = None,
+    ):
+        self.name: str = name
+        self.parent: Optional["Directory"] = parent  # Changed to string literal
+        self.contents: Dict[str, Union[File, Directory]] = contents or {}
+
+    def __repr__(self):
+        parent_name = self.parent.name if self.parent else None
+        return f"<Directory: {self.name}, Parent: {parent_name}, Keys: {list(self.contents.keys())}>"
+
+    def __eq__(self, other):
+        if not isinstance(other, Directory):
+            return False
+        return self.name == other.name and self.contents == other.contents
+
+
+class GorillaFileSystem:
+    """A file system for BFCL evaluation."""
+
+    def __init__(self):
+        self.root: Directory = Directory(name="workspace", parent=None)
+        self.current_dir: Directory = self.root
+        self.long_context: bool = False
+
+    def _load_scenario(self, config: Dict):
+        """Load the file system from configuration."""
+        # self.root and self.current_dir are already initialized.
+        # We will only overwrite them if loading is successful.
+        if "root" in config:
+            try:
+                loaded_dir: Optional[Directory] = None
+                root_config = config["root"]
+                if isinstance(root_config, dict) and "type" in root_config:
+                    loaded_dir = self._load_directory_from_config("workspace", None, root_config)
+                elif isinstance(root_config, dict):  # Assuming if not 'type', it's the other YAML format
+                    loaded_dir = self._load_directory_from_yaml_config("workspace", None, root_config)
+
+                if loaded_dir:  # Check if loading returned a Directory
+                    self.root = loaded_dir
+                    self.current_dir = self.root
+                # If loaded_dir is None, self.root and self.current_dir retain their initial default values.
+            except Exception as e:
+                print(f"Error loading GorillaFileSystem scenario: {e}")
+                # If an exception occurred during loading, reset to a fresh default.
+                self.root = Directory(name="workspace", parent=None)
+                self.current_dir = self.root
+
+        if "long_context" in config:
+            self.long_context = config.get("long_context", False)
+
+    def _load_directory_from_config(self, name: str, parent: Optional[Directory], config: Dict) -> Optional[Directory]:
+        """Create a directory structure from configuration."""
+        if config.get("type") == "directory":
+            directory = Directory(name=name, parent=parent)
+            contents: Dict[str, Union[File, Directory]] = {}
+            for item_name, item_config in config.get("contents", {}).items():
+                item_type = item_config.get("type")
+                if item_type == "directory":
+                    loaded_item = self._load_directory_from_config(item_name, directory, item_config)
+                    if loaded_item:
+                        contents[item_name] = loaded_item
+                elif item_type == "file":
+                    contents[item_name] = File(
+                        name=item_name,
+                        content=item_config.get("content", ""),
+                        parent=directory,
+                    )
+            directory.contents = contents
+            return directory
+        return None
+
+    def _load_directory_from_yaml_config(self, name: str, parent: Optional[Directory], config: Dict) -> Directory:
+        """Create a directory structure from YAML configuration format."""
+        directory = Directory(name=name, parent=parent)
+        contents: Dict[str, Union[File, Directory]] = {}
+
+        # Ensure config.get("contents") is treated as a dictionary
+        config_contents = config.get("contents", {})
+        if not isinstance(config_contents, dict):
+            config_contents = {}  # Default to empty dict if not a dict
+
+        for item_name, item_config in config_contents.items():
+            if isinstance(item_config, dict):
+                if "contents" in item_config:  # Heuristic for directory
+                    loaded_subdir = self._load_directory_from_yaml_config(item_name, directory, item_config)
+                    if loaded_subdir:  # Ensure it's not None, though current impl always returns Directory
+                        contents[item_name] = loaded_subdir
+                elif "content" in item_config:  # Heuristic for file
+                    contents[item_name] = File(
+                        name=item_name,
+                        content=item_config.get("content", ""),
+                        parent=directory,
+                    )
+                elif item_config.get("type") == "directory":
+                    loaded_subdir = self._load_directory_from_yaml_config(item_name, directory, item_config)
+                    if loaded_subdir:
+                        contents[item_name] = loaded_subdir
+                elif item_config.get("type") == "file":
+                    contents[item_name] = File(
+                        name=item_name,
+                        content=item_config.get("content", ""),
+                        parent=directory,
+                    )
+        directory.contents = contents
+        return directory
+
+    def ls(self, path: Optional[str] = None) -> Dict:
+        """List directory contents."""
+        target_dir: Directory = self.current_dir
+        if path:
+            found_node = self._find_path(path)
+            if not isinstance(found_node, Directory):
+                return {"error": f"Path not found or not a directory: {path}"}
+            target_dir = found_node
+
+        items: Dict[str, Dict[str, str]] = {}
+        # target_dir is now guaranteed to be a Directory.
+        for name, item in target_dir.contents.items():
+            if isinstance(item, Directory):
+                items[name] = {"type": "directory"}
+            elif isinstance(item, File):
+                items[name] = {"type": "file"}
+
+        return {"current_directory": target_dir.name, "contents": items}
+
+    def cd(self, folder: str) -> Dict:
+        """Change current directory."""
+        if folder == "..":
+            parent_dir = self.current_dir.parent
+            if parent_dir is not None:
+                self.current_dir = parent_dir
+                return {
+                    "status": "success",
+                    "message": f"Changed to {self.current_dir.name}",
+                }
+            else:  # Parent is None, so we are at root
+                return {"status": "error", "message": "Already at root directory"}
+
+        # self.current_dir is always a Directory. Accessing .contents is safe.
+        target_item = self.current_dir.contents.get(folder)
+        if isinstance(target_item, Directory):
+            self.current_dir = target_item
+            return {"status": "success", "message": f"Changed to {folder}"}
+
+        return {"status": "error", "message": f"Directory {folder} not found"}
+
+    def mkdir(self, dir_name: str) -> Dict:
+        """Create a new directory."""
+        # self.current_dir is always a Directory. Accessing .contents is safe.
+        if dir_name in self.current_dir.contents:
+            return {
+                "status": "error",
+                "message": f"Directory {dir_name} already exists",
+            }
+
+        self.current_dir.contents[dir_name] = Directory(name=dir_name, parent=self.current_dir)
+        return {"status": "success", "message": f"Created directory {dir_name}"}
+
+    def cat(self, file_name: str) -> Dict:
+        """Display file contents."""
+        # self.current_dir is always a Directory. Accessing .contents is safe.
+        item = self.current_dir.contents.get(file_name)
+        if not isinstance(item, File):
+            return {"status": "error", "message": f"File {file_name} not found"}
+
+        return {
+            "status": "success",
+            "content": item.content,  # item is File, .content is safe
+        }
+
+    def mv(self, source: str, destination: str) -> Dict:
+        """Move a file or directory."""
+        source_item = self.current_dir.contents.get(source)
+        if source_item is None:
+            return {"status": "error", "message": f"Source {source} not found"}
+
+        parts = destination.split("/")
+        dest_name = parts[-1]
+        target_dir_path = "/".join(parts[:-1])
+
+        final_target_dir: Directory = self.current_dir
+        if target_dir_path:  # If destination includes a path
+            found_dir = self._find_path(target_dir_path)
+            if not isinstance(found_dir, Directory):
+                return {
+                    "status": "error",
+                    "message": f"Target directory path {target_dir_path} not found or not a directory",
+                }
+            final_target_dir = found_dir
+
+        if dest_name in final_target_dir.contents:
+            return {
+                "status": "error",
+                "message": f"Destination {destination} already exists",
+            }
+
+        # Move item
+        del self.current_dir.contents[source]  # Remove from old location
+        source_item.name = dest_name
+        source_item.parent = final_target_dir
+        final_target_dir.contents[dest_name] = source_item
+
+        return {"status": "success", "message": f"Moved {source} to {destination}"}
+
+    def grep(self, file_name: str, pattern: str) -> Dict:
+        """Search for a pattern in a file."""
+        item = self.current_dir.contents.get(file_name)
+        if not isinstance(item, File):
+            return {"status": "error", "message": f"File {file_name} not found"}
+
+        content = item.content  # item is File, .content is safe
+        lines = content.split("\n")
+        matches = [line for line in lines if pattern in line]
+
+        return {"status": "success", "matches": matches, "count": len(matches)}
+
+    def sort(self, file_name: str) -> Dict:
+        """Sort the lines in a file."""
+        item = self.current_dir.contents.get(file_name)
+        if not isinstance(item, File):
+            return {"status": "error", "message": f"File {file_name} not found"}
+
+        content = item.content  # item is File, .content is safe
+        lines = content.split("\n")
+        sorted_lines = sorted(lines)
+
+        item.content = "\n".join(sorted_lines)  # item is File, assigning .content is safe
+
+        return {"status": "success", "message": f"Sorted {file_name}"}
+
+    def diff(self, file_name1: str, file_name2: str) -> Dict:
+        """Compare two files."""
+        item1 = self.current_dir.contents.get(file_name1)
+        item2 = self.current_dir.contents.get(file_name2)
+
+        if not isinstance(item1, File):
+            return {"status": "error", "message": f"File {file_name1} not found"}
+        if not isinstance(item2, File):
+            return {"status": "error", "message": f"File {file_name2} not found"}
+
+        content1 = item1.content  # item1 is File
+        content2 = item2.content  # item2 is File
+
+        if content1 == content2:
+            return {
+                "status": "success",
+                "message": "Files are identical",
+                "differences": [],
+            }
+        else:
+            lines1 = content1.split("\n")
+            lines2 = content2.split("\n")
+            differences = []
+            for i in range(max(len(lines1), len(lines2))):
+                line1_val = lines1[i] if i < len(lines1) else None
+                line2_val = lines2[i] if i < len(lines2) else None
+                if line1_val != line2_val:
+                    differences.append({"line": i + 1, "file1": line1_val, "file2": line2_val})
+            return {
+                "status": "success",
+                "message": f"Found {len(differences)} differences",
+                "differences": differences,
+            }
+
+    def _find_path(self, path: str) -> Optional[Union[File, Directory]]:
+        """Helper to find a File or Directory by path. Returns None if not found."""
+        current_node: Optional[Directory]
+        parts: list[str]
+
+        if path.startswith("/"):
+            current_node = self.root
+            path_str = path.strip("/")
+            parts = path_str.split("/") if path_str else []
+        else:
+            current_node = self.current_dir
+            parts = path.split("/")
+
+        if (
+            not path or path == "." or (path == "/" and not parts)
+        ):  # Handle current dir or root for empty/special paths
+            return self.current_dir if (not path.startswith("/")) and (path == "." or not path) else self.root
+
+        for i, part_name in enumerate(parts):
+            if current_node is None:  # Should not happen if logic is correct and current_node starts as Directory
+                return None
+
+            if not part_name:  # Skip empty parts resulting from multiple slashes e.g. /dir1//file
+                if i == 0 and path.startswith("/"):  # special case for absolute path like "//file"
+                    continue
+                elif i > 0:
+                    continue
+
+            if part_name == "..":
+                current_node = current_node.parent  # Parent can be None
+                if current_node is None:  # Moved up from root
+                    return None
+                continue  # Successfully moved to parent
+
+            # current_node is a Directory here.
+            found_item = current_node.contents.get(part_name)
+
+            if i == len(parts) - 1:  # This is the last part of the path
+                return found_item  # Return File, Directory, or None if not found
+
+            if isinstance(found_item, Directory):
+                current_node = found_item  # Navigate into subdirectory
+            else:  # Path part is not a directory or not found, and it's not the last part
+                return None
+
+        # This return is for cases like path="dir" and it's a directory, or path="/"
+        return current_node