palace-eval 1.0.7__py3-none-any.whl

palace/__init__.py

@@ -0,0 +1,30 @@
+# Copyright (C) 2025 European Union
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the European Union Public Licence (EUPL) v. 1.2
+# as published by the European Union.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# European Union Public Licence for more details.
+#
+# You should have received a copy of the European Union Public Licence
+# along with this program. If not, see <https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12>.
+
+from .evaluation.judge_config import JudgeConfig
+
+__all__ = ["JudgeConfig"]
+
+
+def __getattr__(name):
+    # Lazy imports for heavy modules
+    if name == "Evaluation":
+        from .evaluation.orchestrator import Evaluation
+
+        return Evaluation
+    if name == "evaluate":
+        from .evaluation.orchestrator import evaluate
+
+        return evaluate
+    raise AttributeError(f"module 'palace' has no attribute {name!r}")

palace/agents/__init__.py

@@ -0,0 +1,20 @@
+# Copyright (C) 2025 European Union
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the European Union Public Licence (EUPL) v. 1.2
+# as published by the European Union.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# European Union Public Licence for more details.
+#
+# You should have received a copy of the European Union Public Licence
+# along with this program. If not, see <https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12>.
+
+# ruff: noqa: I001
+from palace.agents.base_agent import Agent as Agent
+from palace.agents.mcp_agent import MCPAgent as MCPAgent
+from palace.agents.api_agent import APIAgent as APIAgent
+from palace.agents.vivarium_agent import VivariumAgent as VivariumAgent
+from palace.utils.exceptions import ModelNotFoundError as ModelNotFoundError

palace/agents/api_agent.py

@@ -0,0 +1,148 @@
+# Copyright (C) 2025 European Union
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the European Union Public Licence (EUPL) v. 1.2
+# as published by the European Union.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# European Union Public Licence for more details.
+#
+# You should have received a copy of the European Union Public Licence
+# along with this program. If not, see <https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12>.
+
+import logging
+from typing import TYPE_CHECKING
+
+from palace.agents import Agent
+from palace.evaluation.types import AgentResult
+from palace.models.api_model import create_api_model
+from palace.utils.exceptions import ModelNotFoundError
+from palace.utils.multimodal import build_multimodal_content
+from palace.utils.printing import print
+
+if TYPE_CHECKING:
+    from palace.evaluation.types import Attachment
+
+_logger = logging.getLogger("palace.api_agent")
+
+# Patterns that indicate a deterministic model capability limitation (not transient)
+_UNSUPPORTED_PATTERNS = [
+    "context_length_exceeded",
+    "too many tokens",
+    "prompt is too long",
+    "maximum context length",
+    "input too long",
+    "exceeds the model's maximum",
+    "content_too_large",
+    "request too large",
+    "input tokens exceed",
+]
+
+# Patterns indicating the model doesn't exist — should abort evaluation
+_MODEL_NOT_FOUND_PATTERNS = [
+    "model not found",
+    "not available",
+    "does not exist",
+    "unknown model",
+    "invalid model",
+    "no such model",
+]
+
+
+def _is_unsupported_error(e: Exception) -> bool:
+    """Detect deterministic capability limitations from API errors.
+
+    Returns True for errors that indicate the model cannot process the input
+    (e.g., context too long, unsupported modality). These are permanent for
+    the given input and should not be retried.
+    """
+    # OpenAI / vLLM: BadRequestError has a .code attribute
+    if hasattr(e, "code") and e.code == "context_length_exceeded":
+        return True
+    # Message-based detection (Anthropic, vLLM variants, other providers)
+    msg = str(e).lower()
+    return any(p in msg for p in _UNSUPPORTED_PATTERNS)
+
+
+def _is_model_not_found(e: Exception) -> bool:
+    """Detect model-not-found errors that should abort the evaluation.
+
+    Returns True for errors that indicate the model doesn't exist on the endpoint.
+    These are configuration errors, not per-task issues.
+    """
+    msg = str(e).lower()
+    return any(p in msg for p in _MODEL_NOT_FOUND_PATTERNS)
+
+
+class APIAgent(Agent):
+    """Agent that calls an API endpoint (OpenAI-compatible, Azure OpenAI, or Anthropic).
+
+    Can be used for both agentic and non-agentic (black-box LLM) evaluation.
+    """
+
+    def __init__(
+        self,
+        /,
+        name: str,
+        url: str,
+        token: str | None = None,
+        api_type: str | None = None,
+        extra_params: dict | None = None,
+    ):
+        """Initialize an APIAgent.
+
+        Args:
+            name: The name of the agent, corresponding to a model ID on the API server.
+            url: The URL of the API server.
+            token: The API token for authentication. Defaults to None.
+            api_type: The API type to use ("openai", "anthropic", or "azure").
+                Auto-detected from model name if not specified.
+            extra_params: Extra kwargs to merge into every API call for this model.
+        """
+        if api_type is not None and api_type not in ["openai", "anthropic", "azure"]:
+            raise ValueError("api_type must be 'openai', 'anthropic', or 'azure'")
+        self._name = name
+        self.url = url
+        self.token = token
+        self.api_type = api_type
+        self._model = create_api_model(
+            model_id=name, url=url, token=token, api_type=api_type, extra_params=extra_params
+        )
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    async def run(
+        self, prompt: str, attachments: "list[Attachment] | None" = None, *, task_id: str | None = None
+    ) -> AgentResult:
+        self._model.quiet = not self.verbose
+        # Inline text attachments into prompt (non-agentic presentation)
+        if attachments:
+            prompt = self._inline_text_attachments(prompt, attachments)
+        content = build_multimodal_content(prompt, attachments)
+        try:
+            output = await self._model.generate([{"role": "user", "content": content}])
+        except Exception as e:
+            _logger.warning(f"Agent error: {e}")
+            if self.verbose:
+                print(f"[bold red]OpenAIAPI agent error: {e}[/]")
+            # Model not found is a fatal configuration error — abort evaluation
+            if _is_model_not_found(e):
+                raise ModelNotFoundError(f"Model '{self._name}' not found on {self.url}: {e}") from e
+            if _is_unsupported_error(e):
+                return AgentResult(outcome="unsupported", reason=f"unsupported: {e}")
+            return AgentResult(outcome="error", reason=f"agent_error: {e}")
+
+        return AgentResult(answer=output, metrics={})
+
+    @staticmethod
+    def _inline_text_attachments(prompt: str, attachments: "list[Attachment]") -> str:
+        """Inline text file content into the prompt for non-agentic evaluation."""
+        for att in attachments:
+            text = att.read_text()
+            if text:
+                prompt = f"Start of text attachment >>>\n{text}\n<<< End of text attachment\n\n{prompt}"
+        return prompt

palace/agents/base_agent.py

@@ -0,0 +1,66 @@
+# Copyright (C) 2025 European Union
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the European Union Public Licence (EUPL) v. 1.2
+# as published by the European Union.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# European Union Public Licence for more details.
+#
+# You should have received a copy of the European Union Public Licence
+# along with this program. If not, see <https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12>.
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from palace.evaluation.types import AgentResult, Attachment
+    from palace.task_types.base import ExecutionEnvironment, Task
+
+
+class Agent(ABC):
+    """Base class that defines the agent interface."""
+
+    verbose: bool = True
+    agentic: bool = False
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        pass
+
+    @abstractmethod
+    async def run(
+        self, prompt: str, attachments: "list[Attachment] | None" = None, *, task_id: str | None = None
+    ) -> "AgentResult":
+        """Run the agent on the given prompt and return a result.
+
+        Args:
+            prompt: The text prompt for the agent
+            attachments: Optional list of Attachment objects for multimodal tasks
+            task_id: Optional task identifier for agents that need to correlate
+                with per-task state (e.g. VivariumAgent environments)
+
+        Returns:
+            AgentResult with answer, metrics, and skip status.
+        """
+        pass
+
+    async def on_tasklist_start(self, tasklist_path: Path, info: dict) -> None:
+        """Called before evaluating a tasklist. Override for setup."""
+        pass
+
+    async def on_tasklist_end(self) -> None:
+        """Called after evaluating a tasklist. Override for cleanup."""
+        pass
+
+    async def on_task_start(self, task: "Task") -> "ExecutionEnvironment | None":
+        """Called before each task. Return execution environment for agentic verification."""
+        return None
+
+    async def on_task_end(self, task: "Task") -> None:
+        """Called after each task (including verify). Override for per-task cleanup."""
+        pass

palace/agents/mcp_agent.py

@@ -0,0 +1,119 @@
+# Copyright (C) 2025 European Union
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the European Union Public Licence (EUPL) v. 1.2
+# as published by the European Union.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# European Union Public Licence for more details.
+#
+# You should have received a copy of the European Union Public Licence
+# along with this program. If not, see <https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12>.
+
+import json
+from typing import Any, Callable
+
+from mcp.types import CallToolResult, TextContent
+from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential
+
+from palace.agents import Agent
+from palace.evaluation.types import AgentResult
+from palace.mcp_utils.mcp_client import _call_tool, list_tools
+
+
+class MCPAgent(Agent):
+    """Agent that connects to a remote MCP server."""
+
+    def __init__(
+        self,
+        url: str,
+        token: str | None = None,
+        name: str | None = None,
+        params: dict[str, Any] | None = None,
+        output_processor: Callable[[CallToolResult], str] | None = None,
+    ):
+        """Initialize the MCPAgent.
+
+        Args:
+            url: The URL of the MCP server.
+            token: Authentication token for the MCP server.
+            name: Name of the agent/tool to connect to.
+            params: Custom parameters to pass to the agent/tool.
+            output_processor: Function to process the agent/tool output.
+        """
+        self.url = url
+        self.token = token
+        self.params = params
+        self.output_processor = output_processor
+
+        available_agents = list_tools(url, token).tools
+
+        if len(available_agents) == 0:
+            raise ValueError(f"There is no agent or tool at {url}.")
+
+        if name is not None and name in [a.name for a in available_agents]:
+            self._name = name
+        elif name is not None and name not in [a.name for a in available_agents]:
+            raise ValueError(
+                f"There is no agent with the provided name {name} at {url}, only found: {available_agents}."
+            )
+        elif name is None and len(available_agents) > 1:
+            raise ValueError(
+                f"There is more than one agent at {url} but provided name is {name}. Specify one of {available_agents}."
+            )
+        else:
+            self._name = available_agents[0].name
+
+        if self.params is not None and "main" in self.params:
+            self._input_parameter = self.params["main"]
+        else:
+            try:
+                self._input_parameter = list(
+                    [a for a in available_agents if a.name == self._name][0].inputSchema["properties"].keys()
+                )[0]
+            except Exception as e:
+                raise ValueError(f"Can't find the input parameter for the agent {self._name} at {url}.") from e
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @retry(
+        stop=stop_after_attempt(5),
+        wait=wait_exponential(multiplier=5, max=60),
+        retry=retry_if_exception_type((ConnectionError, TimeoutError, OSError)),
+    )
+    async def _run_with_retry(self, task: str) -> AgentResult:
+        params = {self._input_parameter: task}
+        if self.params is not None and "custom" in self.params:
+            params |= self.params["custom"]
+
+        output: CallToolResult = await _call_tool(self.url, self.name, params, self.token)
+
+        if self.output_processor is not None:
+            answer = self.output_processor(output)
+        else:
+            if not isinstance(output.content[0], TextContent):
+                raise ValueError(f"MCPAgent expected TextContent, got: {type(output.content[0])}")
+            answer = output.content[0].text
+            if not isinstance(answer, str) or answer.strip() == "":
+                raise ValueError(f"MCPAgent answer not found in output content. Got: {output.content}")
+
+        try:
+            assert isinstance(output.content[1], TextContent)
+            metrics = json.loads(output.content[1].text)
+            assert isinstance(metrics, dict)
+        except Exception:
+            metrics = None
+
+        return AgentResult(answer=answer, metrics=metrics)
+
+    async def run(
+        self, prompt: str, attachments: "list[Any] | None" = None, *, task_id: str | None = None
+    ) -> AgentResult:
+        if attachments:
+            # TODO: pass image attachments as ImageContent when MCP SDK supports it in tool calls
+            return AgentResult(outcome="error", reason="unsupported_attachment")
+        return await self._run_with_retry(prompt)