palace-eval 1.0.0__py3-none-any.whl

___dashboard/node_modules/flatted/python/flatted.py

@@ -0,0 +1,163 @@
+# Copyright (C) 2025 European Union
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the European Union Public Licence (EUPL) v. 1.2
+# as published by the European Union.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# European Union Public Licence for more details.
+#
+# You should have received a copy of the European Union Public Licence
+# along with this program. If not, see <https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12>.
+
+# ISC License
+#
+# Copyright (c) 2018-2025, Andrea Giammarchi, @WebReflection
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+# PERFORMANCE OF THIS SOFTWARE.
+
+import json as _json
+
+class _Known:
+    def __init__(self):
+        self.key = []
+        self.value = []
+
+class _String:
+    def __init__(self, value):
+        self.value = value
+
+
+def _array_keys(value):
+    keys = []
+    i = 0
+    for _ in value:
+        keys.append(i)
+        i += 1
+    return keys
+
+def _object_keys(value):
+    keys = []
+    for key in value:
+        keys.append(key)
+    return keys
+
+def _is_array(value):
+    return isinstance(value, (list, tuple))
+
+def _is_object(value):
+    return isinstance(value, dict)
+
+def _is_string(value):
+    return isinstance(value, str)
+
+def _index(known, input, value):
+    input.append(value)
+    index = str(len(input) - 1)
+    known.key.append(value)
+    known.value.append(index)
+    return index
+
+def _loop(keys, input, known, output):
+    for key in keys:
+        value = output[key]
+        if isinstance(value, _String):
+            _ref(key, input[int(value.value)], input, known, output)
+
+    return output
+
+def _ref(key, value, input, known, output):
+    if _is_array(value) and value not in known:
+        known.append(value)
+        value = _loop(_array_keys(value), input, known, value)
+    elif _is_object(value) and value not in known:
+        known.append(value)
+        value = _loop(_object_keys(value), input, known, value)
+
+    output[key] = value
+
+def _relate(known, input, value):
+    if _is_string(value) or _is_array(value) or _is_object(value):
+        try:
+            return known.value[known.key.index(value)]
+        except:
+            return _index(known, input, value)
+
+    return value
+
+def _transform(known, input, value):
+    if _is_array(value):
+        output = []
+        for val in value:
+            output.append(_relate(known, input, val))
+        return output
+
+    if _is_object(value):
+        obj = {}
+        for key in value:
+            obj[key] = _relate(known, input, value[key])
+        return obj
+
+    return value
+
+def _wrap(value):
+    if _is_string(value):
+        return _String(value)
+
+    if _is_array(value):
+        i = 0
+        for val in value:
+            value[i] = _wrap(val)
+            i += 1
+
+    elif _is_object(value):
+        for key in value:
+            value[key] = _wrap(value[key])
+
+    return value
+
+def parse(value, *args, **kwargs):
+    json = _json.loads(value, *args, **kwargs)
+    wrapped = []
+    for value in json:
+        wrapped.append(_wrap(value))
+
+    input = []
+    for value in wrapped:
+        if isinstance(value, _String):
+            input.append(value.value)
+        else:
+            input.append(value)
+
+    value = input[0]
+
+    if _is_array(value):
+        return _loop(_array_keys(value), input, [value], value)
+
+    if _is_object(value):
+        return _loop(_object_keys(value), input, [value], value)
+
+    return value
+
+
+def stringify(value, *args, **kwargs):
+    known = _Known()
+    input = []
+    output = []
+    i = int(_index(known, input, value))
+    while i < len(input):
+        output.append(_transform(known, input, input[i]))
+        i += 1
+    return _json.dumps(output, *args, **kwargs)

palace/__init__.py

@@ -0,0 +1,25 @@
+# Copyright (C) 2025 European Union
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the European Union Public Licence (EUPL) v. 1.2
+# as published by the European Union.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# European Union Public Licence for more details.
+#
+# You should have received a copy of the European Union Public Licence
+# along with this program. If not, see <https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12>.
+
+
+def __getattr__(name):
+    if name == "Evaluation":
+        from .evaluation.orchestrator import Evaluation
+
+        return Evaluation
+    if name == "evaluate":
+        from .evaluation.orchestrator import evaluate
+
+        return evaluate
+    raise AttributeError(f"module 'palace' has no attribute {name!r}")

palace/agents/__init__.py

@@ -0,0 +1,19 @@
+# Copyright (C) 2025 European Union
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the European Union Public Licence (EUPL) v. 1.2
+# as published by the European Union.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# European Union Public Licence for more details.
+#
+# You should have received a copy of the European Union Public Licence
+# along with this program. If not, see <https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12>.
+
+# ruff: noqa: I001
+from palace.agents.base_agent import Agent as Agent
+from palace.agents.mcp_agent import MCPAgent as MCPAgent
+from palace.agents.api_agent import APIAgent as APIAgent
+from palace.agents.vivarium_agent import VivariumAgent as VivariumAgent

palace/agents/api_agent.py

@@ -0,0 +1,124 @@
+# Copyright (C) 2025 European Union
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the European Union Public Licence (EUPL) v. 1.2
+# as published by the European Union.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# European Union Public Licence for more details.
+#
+# You should have received a copy of the European Union Public Licence
+# along with this program. If not, see <https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12>.
+
+import logging
+from typing import TYPE_CHECKING
+
+from palace.agents import Agent
+from palace.evaluation.types import AgentResult
+from palace.models.api_model import create_api_model
+from palace.utils.multimodal import build_multimodal_content
+from palace.utils.printing import print
+
+if TYPE_CHECKING:
+    from palace.evaluation.types import Attachment
+
+_logger = logging.getLogger("palace.api_agent")
+
+# Patterns that indicate a deterministic model capability limitation (not transient)
+_UNSUPPORTED_PATTERNS = [
+    "context_length_exceeded",
+    "too many tokens",
+    "prompt is too long",
+    "maximum context length",
+    "input too long",
+    "exceeds the model's maximum",
+    "content_too_large",
+    "request too large",
+    "input tokens exceed",
+]
+
+
+def _is_unsupported_error(e: Exception) -> bool:
+    """Detect deterministic capability limitations from API errors.
+
+    Returns True for errors that indicate the model cannot process the input
+    (e.g., context too long, unsupported modality). These are permanent for
+    the given input and should not be retried.
+    """
+    # OpenAI / vLLM: BadRequestError has a .code attribute
+    if hasattr(e, "code") and e.code == "context_length_exceeded":
+        return True
+    # Message-based detection (Anthropic, vLLM variants, other providers)
+    msg = str(e).lower()
+    return any(p in msg for p in _UNSUPPORTED_PATTERNS)
+
+
+class APIAgent(Agent):
+    """Agent that calls an API endpoint (OpenAI-compatible, Azure OpenAI, or Anthropic).
+
+    Can be used for both agentic and non-agentic (black-box LLM) evaluation.
+    """
+
+    def __init__(
+        self,
+        /,
+        name: str,
+        url: str,
+        token: str | None = None,
+        api_type: str | None = None,
+        extra_params: dict | None = None,
+    ):
+        """Initialize an APIAgent.
+
+        Args:
+            name: The name of the agent, corresponding to a model ID on the API server.
+            url: The URL of the API server.
+            token: The API token for authentication. Defaults to None.
+            api_type: The API type to use ("openai", "anthropic", or "azure").
+                Auto-detected from model name if not specified.
+            extra_params: Extra kwargs to merge into every API call for this model.
+        """
+        if api_type is not None and api_type not in ["openai", "anthropic", "azure"]:
+            raise ValueError("api_type must be 'openai', 'anthropic', or 'azure'")
+        self._name = name
+        self.url = url
+        self.token = token
+        self.api_type = api_type
+        self._model = create_api_model(
+            model_id=name, url=url, token=token, api_type=api_type, extra_params=extra_params
+        )
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    async def run(
+        self, prompt: str, attachments: "list[Attachment] | None" = None, *, task_id: str | None = None
+    ) -> AgentResult:
+        self._model.quiet = not self.verbose
+        # Inline text attachments into prompt (non-agentic presentation)
+        if attachments:
+            prompt = self._inline_text_attachments(prompt, attachments)
+        content = build_multimodal_content(prompt, attachments)
+        try:
+            output = await self._model.generate([{"role": "user", "content": content}])
+        except Exception as e:
+            _logger.warning(f"Agent error: {e}")
+            if self.verbose:
+                print(f"[bold red]OpenAIAPI agent error: {e}[/]")
+            if _is_unsupported_error(e):
+                return AgentResult(outcome="unsupported", reason=f"unsupported: {e}")
+            return AgentResult(outcome="error", reason=f"agent_error: {e}")
+
+        return AgentResult(answer=output, metrics={})
+
+    @staticmethod
+    def _inline_text_attachments(prompt: str, attachments: "list[Attachment]") -> str:
+        """Inline text file content into the prompt for non-agentic evaluation."""
+        for att in attachments:
+            text = att.read_text()
+            if text:
+                prompt = f"Start of text attachment >>>\n{text}\n<<< End of text attachment\n\n{prompt}"
+        return prompt

palace/agents/base_agent.py

@@ -0,0 +1,66 @@
+# Copyright (C) 2025 European Union
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the European Union Public Licence (EUPL) v. 1.2
+# as published by the European Union.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# European Union Public Licence for more details.
+#
+# You should have received a copy of the European Union Public Licence
+# along with this program. If not, see <https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12>.
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from palace.evaluation.types import AgentResult, Attachment
+    from palace.task_types.base import ExecutionEnvironment, Task
+
+
+class Agent(ABC):
+    """Base class that defines the agent interface."""
+
+    verbose: bool = True
+    agentic: bool = False
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        pass
+
+    @abstractmethod
+    async def run(
+        self, prompt: str, attachments: "list[Attachment] | None" = None, *, task_id: str | None = None
+    ) -> "AgentResult":
+        """Run the agent on the given prompt and return a result.
+
+        Args:
+            prompt: The text prompt for the agent
+            attachments: Optional list of Attachment objects for multimodal tasks
+            task_id: Optional task identifier for agents that need to correlate
+                with per-task state (e.g. VivariumAgent environments)
+
+        Returns:
+            AgentResult with answer, metrics, and skip status.
+        """
+        pass
+
+    async def on_tasklist_start(self, tasklist_path: Path, info: dict) -> None:
+        """Called before evaluating a tasklist. Override for setup."""
+        pass
+
+    async def on_tasklist_end(self) -> None:
+        """Called after evaluating a tasklist. Override for cleanup."""
+        pass
+
+    async def on_task_start(self, task: "Task") -> "ExecutionEnvironment | None":
+        """Called before each task. Return execution environment for agentic verification."""
+        return None
+
+    async def on_task_end(self, task: "Task") -> None:
+        """Called after each task (including verify). Override for per-task cleanup."""
+        pass

palace/agents/mcp_agent.py

@@ -0,0 +1,119 @@
+# Copyright (C) 2025 European Union
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the European Union Public Licence (EUPL) v. 1.2
+# as published by the European Union.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# European Union Public Licence for more details.
+#
+# You should have received a copy of the European Union Public Licence
+# along with this program. If not, see <https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12>.
+
+import json
+from typing import Any, Callable
+
+from mcp.types import CallToolResult, TextContent
+from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential
+
+from palace.agents import Agent
+from palace.evaluation.types import AgentResult
+from palace.mcp_utils.mcp_client import _call_tool, list_tools
+
+
+class MCPAgent(Agent):
+    """Agent that connects to a remote MCP server."""
+
+    def __init__(
+        self,
+        url: str,
+        token: str | None = None,
+        name: str | None = None,
+        params: dict[str, Any] | None = None,
+        output_processor: Callable[[CallToolResult], str] | None = None,
+    ):
+        """Initialize the MCPAgent.
+
+        Args:
+            url: The URL of the MCP server.
+            token: Authentication token for the MCP server.
+            name: Name of the agent/tool to connect to.
+            params: Custom parameters to pass to the agent/tool.
+            output_processor: Function to process the agent/tool output.
+        """
+        self.url = url
+        self.token = token
+        self.params = params
+        self.output_processor = output_processor
+
+        available_agents = list_tools(url, token).tools
+
+        if len(available_agents) == 0:
+            raise ValueError(f"There is no agent or tool at {url}.")
+
+        if name is not None and name in [a.name for a in available_agents]:
+            self._name = name
+        elif name is not None and name not in [a.name for a in available_agents]:
+            raise ValueError(
+                f"There is no agent with the provided name {name} at {url}, only found: {available_agents}."
+            )
+        elif name is None and len(available_agents) > 1:
+            raise ValueError(
+                f"There is more than one agent at {url} but provided name is {name}. Specify one of {available_agents}."
+            )
+        else:
+            self._name = available_agents[0].name
+
+        if self.params is not None and "main" in self.params:
+            self._input_parameter = self.params["main"]
+        else:
+            try:
+                self._input_parameter = list(
+                    [a for a in available_agents if a.name == self._name][0].inputSchema["properties"].keys()
+                )[0]
+            except Exception as e:
+                raise ValueError(f"Can't find the input parameter for the agent {self._name} at {url}.") from e
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @retry(
+        stop=stop_after_attempt(5),
+        wait=wait_exponential(multiplier=5, max=60),
+        retry=retry_if_exception_type((ConnectionError, TimeoutError, OSError)),
+    )
+    async def _run_with_retry(self, task: str) -> AgentResult:
+        params = {self._input_parameter: task}
+        if self.params is not None and "custom" in self.params:
+            params |= self.params["custom"]
+
+        output: CallToolResult = await _call_tool(self.url, self.name, params, self.token)
+
+        if self.output_processor is not None:
+            answer = self.output_processor(output)
+        else:
+            if not isinstance(output.content[0], TextContent):
+                raise ValueError(f"MCPAgent expected TextContent, got: {type(output.content[0])}")
+            answer = output.content[0].text
+            if not isinstance(answer, str) or answer.strip() == "":
+                raise ValueError(f"MCPAgent answer not found in output content. Got: {output.content}")
+
+        try:
+            assert isinstance(output.content[1], TextContent)
+            metrics = json.loads(output.content[1].text)
+            assert isinstance(metrics, dict)
+        except Exception:
+            metrics = None
+
+        return AgentResult(answer=answer, metrics=metrics)
+
+    async def run(
+        self, prompt: str, attachments: "list[Any] | None" = None, *, task_id: str | None = None
+    ) -> AgentResult:
+        if attachments:
+            # TODO: pass image attachments as ImageContent when MCP SDK supports it in tool calls
+            return AgentResult(outcome="error", reason="unsupported_attachment")
+        return await self._run_with_retry(prompt)