magma-bench 2.0.0b1__tar.gz

magma_bench-2.0.0b1/LICENSE

@@ -0,0 +1,24 @@
+BSD 2-Clause License
+
+Copyright (c) 2026, MAGMA-s
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

magma_bench-2.0.0b1/PKG-INFO

@@ -0,0 +1,108 @@
+Metadata-Version: 2.4
+Name: magma_bench
+Version: 2.0.0b1
+Summary: Evaluate agents on interactive robotic tasks with MAGMA
+Project-URL: Documentation, https://magma-rob.github.io/docs/intro
+Project-URL: Repository, https://github.com/MAGMA-rob/magma-bench
+Project-URL: Issues, https://github.com/MAGMA-rob/magma-bench/issues
+Classifier: Development Status :: 4 - Beta
+Requires-Python: <3.13,>=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: magma_core[simulation]<3.0.0,>=2.0.0b1
+Requires-Dist: magma_scenarios<3.0.0,>=2.0.0
+Requires-Dist: pydantic<3,>=2.4
+Requires-Dist: requests<3,>=2.34.2
+Requires-Dist: torch==2.13.0
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Provides-Extra: video
+Requires-Dist: imageio-ffmpeg>=0.5; extra == "video"
+Requires-Dist: numpy>=1.23; extra == "video"
+Requires-Dist: Pillow>=10; extra == "video"
+Dynamic: license-file
+
+# MAGMA-BENCH
+
+Evaluate agents on interactive robotic tasks with MAGMA. MAGMA-BENCH loads
+compiled benchmark episodes, runs them in simulation, communicates with an agent
+through the MAGMA HTTP protocol, and saves detailed results and metrics.
+
+**Version 2.0.0b1 is the first beta of v2.** APIs, command-line options, and
+result formats may change before the stable release. End-to-end validation is
+still in progress.
+
+[Official documentation](https://magma-rob.github.io/docs/intro) ·
+[Report an issue](https://github.com/MAGMA-rob/magma-bench/issues)
+
+## Installation
+
+Python 3.12 is required. Simulation installation has been checked on Linux
+x86_64. GPU simulation and rendering require compatible system drivers.
+
+```bash
+python -m pip install "magma_bench==2.0.0b1"
+```
+
+This installs `magma_core[simulation]>=2.0.0b1,<3.0.0` and
+`magma_scenarios>=2.0.0,<3.0.0` with their Python dependencies. The benchmark
+files and the agent server are separate inputs and are not bundled with this
+package.
+
+The version is pinned explicitly because this is a prerelease. To request the
+latest version, including prereleases, use:
+
+```bash
+python -m pip install --upgrade --pre magma_bench
+```
+
+## Usage
+
+Start a compatible MAGMA agent server, then run a compiled benchmark directory:
+
+```bash
+magma-bench run \
+  --benchmark-root /path/to/compiled-benchmark \
+  --agent-address http://127.0.0.1:8888 \
+  --run-name experiment-1
+```
+
+The agent must expose `/health`, `/v1/info`, and `/v1/responses` using protocol
+version 2.0. Configuration can be supplied with `--config-path`; command-line
+options override its benchmark settings.
+
+Useful options include:
+
+```bash
+magma-bench run --help
+magma-bench run --benchmark-root /path/to/benchmark --scenarios coffee_comp
+magma-bench run --benchmark-root /path/to/benchmark --skip-judge
+magma-bench run --benchmark-root /path/to/benchmark --model-logs --videos
+```
+
+Video output uses the optional dependencies:
+
+```bash
+python -m pip install "magma_bench[video]==2.0.0b1"
+```
+
+Version 2 uses result schema `2.0`. Resume a compatible run with
+`--results-path`; older result schemas and agent adapters are unsupported.
+
+See the [official documentation](https://magma-rob.github.io/docs/intro) for
+benchmark preparation, configuration, agent setup, metrics, and result formats.
+
+## Install from source
+
+Install the tagged release from GitHub:
+
+```bash
+python -m pip install "magma_bench @ git+https://github.com/MAGMA-rob/magma-bench.git@v2.0.0b1"
+```
+
+Dependencies are resolved from PyPI. For local development, clone the repository
+and run `python -m pip install -e ".[dev,video]"`.
+
+## License
+
+[BSD 2-Clause](https://github.com/MAGMA-rob/magma-bench/blob/main/LICENSE).

magma_bench-2.0.0b1/README.md

@@ -0,0 +1,84 @@
+# MAGMA-BENCH
+
+Evaluate agents on interactive robotic tasks with MAGMA. MAGMA-BENCH loads
+compiled benchmark episodes, runs them in simulation, communicates with an agent
+through the MAGMA HTTP protocol, and saves detailed results and metrics.
+
+**Version 2.0.0b1 is the first beta of v2.** APIs, command-line options, and
+result formats may change before the stable release. End-to-end validation is
+still in progress.
+
+[Official documentation](https://magma-rob.github.io/docs/intro) ·
+[Report an issue](https://github.com/MAGMA-rob/magma-bench/issues)
+
+## Installation
+
+Python 3.12 is required. Simulation installation has been checked on Linux
+x86_64. GPU simulation and rendering require compatible system drivers.
+
+```bash
+python -m pip install "magma_bench==2.0.0b1"
+```
+
+This installs `magma_core[simulation]>=2.0.0b1,<3.0.0` and
+`magma_scenarios>=2.0.0,<3.0.0` with their Python dependencies. The benchmark
+files and the agent server are separate inputs and are not bundled with this
+package.
+
+The version is pinned explicitly because this is a prerelease. To request the
+latest version, including prereleases, use:
+
+```bash
+python -m pip install --upgrade --pre magma_bench
+```
+
+## Usage
+
+Start a compatible MAGMA agent server, then run a compiled benchmark directory:
+
+```bash
+magma-bench run \
+  --benchmark-root /path/to/compiled-benchmark \
+  --agent-address http://127.0.0.1:8888 \
+  --run-name experiment-1
+```
+
+The agent must expose `/health`, `/v1/info`, and `/v1/responses` using protocol
+version 2.0. Configuration can be supplied with `--config-path`; command-line
+options override its benchmark settings.
+
+Useful options include:
+
+```bash
+magma-bench run --help
+magma-bench run --benchmark-root /path/to/benchmark --scenarios coffee_comp
+magma-bench run --benchmark-root /path/to/benchmark --skip-judge
+magma-bench run --benchmark-root /path/to/benchmark --model-logs --videos
+```
+
+Video output uses the optional dependencies:
+
+```bash
+python -m pip install "magma_bench[video]==2.0.0b1"
+```
+
+Version 2 uses result schema `2.0`. Resume a compatible run with
+`--results-path`; older result schemas and agent adapters are unsupported.
+
+See the [official documentation](https://magma-rob.github.io/docs/intro) for
+benchmark preparation, configuration, agent setup, metrics, and result formats.
+
+## Install from source
+
+Install the tagged release from GitHub:
+
+```bash
+python -m pip install "magma_bench @ git+https://github.com/MAGMA-rob/magma-bench.git@v2.0.0b1"
+```
+
+Dependencies are resolved from PyPI. For local development, clone the repository
+and run `python -m pip install -e ".[dev,video]"`.
+
+## License
+
+[BSD 2-Clause](https://github.com/MAGMA-rob/magma-bench/blob/main/LICENSE).

magma_bench-2.0.0b1/pyproject.toml

@@ -0,0 +1,44 @@
+# SPDX-License-Identifier: BSD-2-Clause
+# Copyright (c) 2026, Loan Bernat
+
+[project]
+name = "magma_bench"
+version = "2.0.0b1"
+description = "Evaluate agents on interactive robotic tasks with MAGMA"
+readme = "README.md"
+requires-python = ">=3.12,<3.13"
+classifiers = ["Development Status :: 4 - Beta"]
+dependencies = [
+    "magma_core[simulation]>=2.0.0b1,<3.0.0",
+    "magma_scenarios>=2.0.0,<3.0.0",
+    "pydantic>=2.4,<3",
+    "requests>=2.34.2,<3",
+    "torch==2.13.0",
+]
+
+[project.urls]
+Documentation = "https://magma-rob.github.io/docs/intro"
+Repository = "https://github.com/MAGMA-rob/magma-bench"
+Issues = "https://github.com/MAGMA-rob/magma-bench/issues"
+
+[project.optional-dependencies]
+dev = ["pytest"]
+video = [
+    "imageio-ffmpeg>=0.5",
+    "numpy>=1.23",
+    "Pillow>=10",
+]
+
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[project.scripts]
+magma-bench = "magma_bench.command_line:main"
+
+[tool.pytest.ini_options]
+addopts = "-p no:launch_testing -p no:launch_ros"
+testpaths = ["tests"]

magma_bench-2.0.0b1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

magma_bench-2.0.0b1/src/magma_bench/__main__.py

@@ -0,0 +1,4 @@
+from .command_line import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

magma_bench-2.0.0b1/src/magma_bench/agents/__init__.py

@@ -0,0 +1,3 @@
+from .base import BenchmarkAgent
+
+__all__ = ["BenchmarkAgent"]

magma_bench-2.0.0b1/src/magma_bench/agents/base/__init__.py

@@ -0,0 +1,5 @@
+from .benchmark_agent import BenchmarkAgent
+
+__all__ = [
+    "BenchmarkAgent",
+]

magma_bench-2.0.0b1/src/magma_bench/agents/base/benchmark_agent.py

@@ -0,0 +1,202 @@
+from __future__ import annotations
+
+from copy import deepcopy
+import json
+import threading
+import time
+from typing import Any, Dict, List, Optional
+from uuid import uuid4
+
+import requests
+
+from magma_core.domain.agent_call import Call
+from magma_core.protocol.agent import (
+    PROTOCOL_VERSION, AgentHealth, AgentInfo, AgentInput, AgentInstruction,
+    AgentOutput, AgentRequest, AgentResponse, JsonObject,
+)
+from magma_core.simulation.agents import AgentAnswer, BadAgentAnswer, ValidAgentAnswer
+from magma_core.simulation.data_structures import EmptyInstruction
+from magma_bench.data_structures import BenchmarkAgentResult, EpisodeSituation
+
+
+class BenchmarkAgent:
+    """Batched HTTP client for a single protocol-v2 agent runtime."""
+
+    def __init__(
+        self,
+        agent_url: str,
+        agent_name: str | None = None,
+        extra_keys: JsonObject | None = None,
+        timeout: float = 360,
+        collect_model_logs: bool = False,
+    ) -> None:
+        self.agent_url = agent_url.rstrip("/")
+        self.timeout = timeout
+        self.extra_keys = deepcopy(extra_keys) if extra_keys is not None else {}
+        self.collect_model_logs = collect_model_logs
+        health = requests.get(f"{self.agent_url}/health", timeout=timeout)
+        health.raise_for_status()
+        AgentHealth.model_validate(health.json())
+        info = requests.get(f"{self.agent_url}/v1/info", timeout=timeout)
+        info.raise_for_status()
+        self.info = AgentInfo.model_validate(info.json())
+        if self.info.protocol_version != PROTOCOL_VERSION:
+            raise ValueError(f"Unsupported agent protocol {self.info.protocol_version!r}")
+        if not self.info.capabilities.get("inference", False):
+            raise ValueError("Agent runtime does not support inference")
+        self.agent_name = agent_name or self.info.agent_id
+        self.lock = threading.Lock()
+        self.completed_results: List[BenchmarkAgentResult] = []
+        self.waiting_inputs: Dict[int, EpisodeSituation] = {}
+        self.thread_exception: Optional[Exception] = None
+        self.thread_running = True
+        self.thread = threading.Thread(
+            target=self._periodic_computing_of_answers, args=(4,), daemon=True,
+        )
+        self.thread.start()
+
+    def compute_agent_results(
+        self, batch_inputs: Dict[int, EpisodeSituation],
+    ) -> List[BenchmarkAgentResult]:
+        if not batch_inputs:
+            return []
+        inputs: list[AgentInput] = []
+        for env_idx, situation in batch_inputs.items():
+            instruction = situation.current_instruction
+            if isinstance(instruction, EmptyInstruction):
+                raise ValueError("EmptyInstruction requires the suspended-skill resume path")
+            role = instruction.get_role()
+            if role not in {"USER", "SYSTEM"}:
+                raise ValueError(f"Unsupported input role {role!r}")
+            inputs.append(AgentInput(
+                id=env_idx,
+                instruction=AgentInstruction(
+                    type="user" if role == "USER" else "env",
+                    content=instruction.get_content(),
+                ),
+                tools=deepcopy(situation.tools),
+                attributes=deepcopy(situation.attributes),
+                memory=deepcopy(situation.memory),
+                num_outputs=1,
+                extra_keys=deepcopy(self.extra_keys),
+            ))
+        request = AgentRequest(request_id=uuid4().hex, inputs=inputs)
+        response = self.send_to_agent(request)
+        results: list[BenchmarkAgentResult] = []
+        for output in response.root:
+            updated = batch_inputs[output.source_id].snapshot()
+            updated.memory = deepcopy(output.memory)
+            diagnostics: list[dict[str, Any]] = []
+            if self.collect_model_logs:
+                diagnostics.append({
+                    "internal_steps": deepcopy(output.internal_steps),
+                    "error": (
+                        output.error.model_dump(mode="json")
+                        if output.error is not None
+                        else None
+                    ),
+                })
+            results.append(BenchmarkAgentResult(
+                answer=self.normalize_model_response(output),
+                situation=updated,
+                model_diagnostics=diagnostics,
+            ))
+        return results
+
+    def send_to_agent(self, request: AgentRequest) -> AgentResponse:
+        response = requests.post(
+            f"{self.agent_url}/v1/responses",
+            json=request.model_dump(mode="json"), timeout=self.timeout,
+        )
+        response.raise_for_status()
+        parsed = AgentResponse.model_validate(response.json())
+        parsed.validate_request(request)
+        return parsed
+
+    def normalize_model_response(
+        self, response: AgentOutput, agent_step_id: int = 0,
+    ) -> AgentAnswer:
+        if response.status == "error":
+            assert response.error is not None
+            return BadAgentAnswer(
+                source_node_id=response.source_id, agent_step_id=agent_step_id, say="",
+                raw_action=json.dumps(response.model_dump(mode="json"), ensure_ascii=False),
+                reason=response.error.message,
+            )
+        assert response.output is not None
+        return ValidAgentAnswer(
+            source_node_id=response.source_id, agent_step_id=agent_step_id,
+            say=response.output.say,
+            calls=[
+                Call(
+                    name=call.name,
+                    arguments=deepcopy(call.arguments),
+                    target_robot_name=call.target_robot_name,
+                )
+                for call in response.output.tool_calls
+            ],
+        )
+
+    def get_agent_card(self) -> Dict[str, Any]:
+        return {
+            "agent": self.agent_name,
+            "agent_id": self.info.agent_id,
+            "agent_version": self.info.agent_version,
+            "protocol_version": self.info.protocol_version,
+            "extra_keys": deepcopy(self.extra_keys),
+        }
+
+    def get_pending_results(self) -> List[BenchmarkAgentResult]:
+        with self.lock:
+            if self.thread_exception is not None:
+                raise RuntimeError(
+                    "The benchmark-agent background thread failed"
+                ) from self.thread_exception
+            out = self.completed_results.copy()
+            self.completed_results.clear()
+        return out
+
+    def add_inputs(self, inputs: Dict[int, EpisodeSituation]) -> None:
+        with self.lock:
+            if self.thread_exception is not None:
+                raise RuntimeError(
+                    "Cannot add inputs after the benchmark-agent thread failed"
+                ) from self.thread_exception
+            if not self.thread_running:
+                raise RuntimeError("Cannot add inputs to a stopped benchmark agent")
+            for idx, situation in inputs.items():
+                if idx in self.waiting_inputs:
+                    raise RuntimeError(
+                        f"Environment {idx} already waits for an agent answer"
+                    )
+                self.waiting_inputs[idx] = situation.snapshot()
+
+    def stop(self) -> None:
+        with self.lock:
+            self.thread_running = False
+        self.thread.join(timeout=5)
+
+    def _periodic_computing_of_answers(self, interval: float = 4.0) -> None:
+        try:
+            while self.thread_running:
+                start_time = time.time()
+                to_do: Dict[int, EpisodeSituation] = {}
+                with self.lock:
+                    if self.waiting_inputs:
+                        to_do = self.waiting_inputs.copy()
+                        self.waiting_inputs.clear()
+                
+                if to_do:
+                    results = self.compute_agent_results(to_do)
+                    with self.lock:
+                        self.completed_results.extend(results)
+
+                elapsed = time.time() - start_time
+                sleep_time = max(0, interval - elapsed)
+                time.sleep(sleep_time)
+
+        except Exception as error:
+            with self.lock:
+                self.thread_exception = error
+        with self.lock:
+            self.thread_running = False

magma_bench-2.0.0b1/src/magma_bench/artifacts/__init__.py

@@ -0,0 +1,79 @@
+from .models import (
+    CONDITIONS,
+    Condition,
+    ActiveErrorSpec,
+    BenchmarkManifest,
+    CompiledEpisodeSpec,
+    DeclarativeStageSpec,
+    EpisodeIndexEntry,
+    EpisodeMetadata,
+    EpisodeSpec,
+    InstructionSpec,
+    InterventionSpec,
+    LogRuleSpec,
+    ObjectSpec,
+    ScenarioManifest,
+    SemanticManifest,
+    SerializedStageSpec,
+    SkeletonManifest,
+    Track,
+    VariantCondition,
+    StageInputSpec,
+    StagePresentationSpec,
+    StageSpec,
+    load_json_model,
+)
+from .runtime import (
+    DeclarativeStage,
+    apply_stage_presentation,
+    deserialize_error,
+    deserialize_goal,
+    deserialize_runtime_randomizer,
+    deserialize_serialized_stage,
+    deserialize_stage,
+    deserialize_task_stages,
+    serialize_error,
+    serialize_goal,
+    serialize_stage,
+    serialize_task_stages,
+    stage_presentation,
+)
+
+__all__ = [
+    "CONDITIONS",
+    "Condition",
+    "ActiveErrorSpec",
+    "BenchmarkManifest",
+    "CompiledEpisodeSpec",
+    "DeclarativeStage",
+    "DeclarativeStageSpec",
+    "EpisodeIndexEntry",
+    "EpisodeMetadata",
+    "EpisodeSpec",
+    "InstructionSpec",
+    "InterventionSpec",
+    "LogRuleSpec",
+    "ObjectSpec",
+    "ScenarioManifest",
+    "SemanticManifest",
+    "SerializedStageSpec",
+    "SkeletonManifest",
+    "Track",
+    "VariantCondition",
+    "StageInputSpec",
+    "StagePresentationSpec",
+    "StageSpec",
+    "apply_stage_presentation",
+    "deserialize_error",
+    "deserialize_goal",
+    "deserialize_runtime_randomizer",
+    "deserialize_serialized_stage",
+    "deserialize_stage",
+    "deserialize_task_stages",
+    "load_json_model",
+    "serialize_error",
+    "serialize_goal",
+    "serialize_stage",
+    "serialize_task_stages",
+    "stage_presentation",
+]