cogames 0.3.64__py3-none-any.whl → 0.3.68__py3-none-any.whl

cogames/cogs_vs_clips/variants.py

@@ -1,95 +1,189 @@
-from typing import override
+from __future__ import annotations
 
+from typing import TYPE_CHECKING, override
+
+from pydantic import Field
+
+from cogames.cogs_vs_clips.config import CvCConfig
 from cogames.cogs_vs_clips.evals.difficulty_variants import DIFFICULTY_VARIANTS
-from cogames.cogs_vs_clips.mission import Mission, MissionVariant
-from cogames.cogs_vs_clips.procedural import BaseHubVariant, MachinaArenaVariant
-from mettagrid.config.action_config import VibeTransfer
-from mettagrid.config.game_value import stat
-from mettagrid.config.reward_config import reward
+from cogames.cogs_vs_clips.terrain import BaseHubVariant, MachinaArenaVariant
+from cogames.core import CoGameMissionVariant
+from mettagrid.config.mettagrid_config import MettaGridConfig
 from mettagrid.map_builder.map_builder import MapBuilderConfig
 from mettagrid.mapgen.mapgen import MapGen
 from mettagrid.mapgen.scenes.base_hub import DEFAULT_EXTRACTORS as HUB_EXTRACTORS
 from mettagrid.mapgen.scenes.building_distributions import DistributionConfig, DistributionType
 
+if TYPE_CHECKING:
+    from cogames.cogs_vs_clips.mission import CvCMission
+
+
+def _apply_clips_settings(
+    mission: CvCMission,
+    *,
+    initial_clips_start: int | None = None,
+    initial_clips_spots: int | None = None,
+    scramble_start: int | None = None,
+    scramble_interval: int | None = None,
+    scramble_radius: int | None = None,
+    align_start: int | None = None,
+    align_interval: int | None = None,
+    align_radius: int | None = None,
+) -> None:
+    clips = mission.clips
+    if initial_clips_start is not None:
+        clips.initial_clips_start = initial_clips_start
+    if initial_clips_spots is not None:
+        clips.initial_clips_spots = initial_clips_spots
+    if scramble_start is not None:
+        clips.scramble_start = scramble_start
+    if scramble_interval is not None:
+        clips.scramble_interval = scramble_interval
+    if scramble_radius is not None:
+        clips.scramble_radius = scramble_radius
+    if align_start is not None:
+        clips.align_start = align_start
+    if align_interval is not None:
+        clips.align_interval = align_interval
+    if align_radius is not None:
+        clips.align_radius = align_radius
+
+
+class NumCogsVariant(CoGameMissionVariant):
+    name: str = "num_cogs"
+    description: str = "Set the number of cogs for the mission."
+    num_cogs: int
 
-class DarkSideVariant(MissionVariant):
-    name: str = "dark_side"
-    description: str = "You're on the dark side of the asteroid. You recharge slower."
+    @override
+    def modify_mission(self, mission: CvCMission) -> None:
+        if self.num_cogs < mission.site.min_cogs or self.num_cogs > mission.site.max_cogs:
+            raise ValueError(
+                f"Invalid number of cogs for {mission.site.name}: {self.num_cogs}. "
+                + f"Must be between {mission.site.min_cogs} and {mission.site.max_cogs}"
+            )
+
+        mission.num_cogs = self.num_cogs
+
+
+class ClipsEasyVariant(CoGameMissionVariant):
+    name: str = "clips_easy"
+    description: str = "Slow clips expansion with late pressure."
 
     @override
-    def modify_mission(self, mission):
-        assert isinstance(mission, Mission)
-        mission.cog.energy_regen = 0
+    def modify_mission(self, mission: CvCMission) -> None:
+        _apply_clips_settings(
+            mission,
+            initial_clips_start=50,
+            initial_clips_spots=1,
+            scramble_start=250,
+            scramble_interval=250,
+            scramble_radius=15,
+            align_start=300,
+            align_interval=250,
+            align_radius=15,
+        )
 
 
-class SuperChargedVariant(MissionVariant):
-    name: str = "super_charged"
-    description: str = "The sun is shining on you. You recharge faster."
+class ClipsMediumVariant(CoGameMissionVariant):
+    name: str = "clips_medium"
+    description: str = "Baseline clips pressure (Machina1 default)."
 
     @override
-    def modify_mission(self, mission):
-        assert isinstance(mission, Mission)
-        mission.cog.energy_regen += 2
+    def modify_mission(self, mission: CvCMission) -> None:
+        _apply_clips_settings(
+            mission,
+            initial_clips_start=10,
+            initial_clips_spots=1,
+            scramble_start=50,
+            scramble_interval=100,
+            scramble_radius=25,
+            align_start=100,
+            align_interval=100,
+            align_radius=25,
+        )
 
 
-class RoughTerrainVariant(MissionVariant):
-    name: str = "rough_terrain"
-    description: str = "The terrain is rough. Moving is more energy intensive."
+class ClipsHardVariant(CoGameMissionVariant):
+    name: str = "clips_hard"
+    description: str = "Fast clips pressure with wider influence."
 
     @override
-    def modify_mission(self, mission):
-        assert isinstance(mission, Mission)
-        mission.cog.move_energy_cost += 2
+    def modify_mission(self, mission: CvCMission) -> None:
+        _apply_clips_settings(
+            mission,
+            initial_clips_start=5,
+            initial_clips_spots=2,
+            scramble_start=25,
+            scramble_interval=50,
+            scramble_radius=35,
+            align_start=50,
+            align_interval=50,
+            align_radius=35,
+        )
 
 
-class PackRatVariant(MissionVariant):
-    name: str = "pack_rat"
-    description: str = "Raise heart, cargo, energy, and gear caps to 255."
+class ClipsWaveOnlyVariant(CoGameMissionVariant):
+    name: str = "clips_wave_only"
+    description: str = "Initial clips wave only, no further spread."
 
     @override
-    def modify_mission(self, mission):
-        assert isinstance(mission, Mission)
-        mission.cog.heart_limit = max(mission.cog.heart_limit, 255)
-        mission.cog.energy_limit = max(mission.cog.energy_limit, 255)
-        mission.cog.cargo_limit = max(mission.cog.cargo_limit, 255)
-        mission.cog.gear_limit = max(mission.cog.gear_limit, 255)
+    def modify_mission(self, mission: CvCMission) -> None:
+        disable_start = mission.max_steps + 1
+        _apply_clips_settings(
+            mission,
+            initial_clips_start=10,
+            initial_clips_spots=3,
+            scramble_start=disable_start,
+            scramble_interval=disable_start,
+            align_start=disable_start,
+            align_interval=disable_start,
+            scramble_radius=25,
+            align_radius=25,
+        )
 
 
-class EnergizedVariant(MissionVariant):
-    name: str = "energized"
-    description: str = "Max energy and full regen so agents never run dry."
+class DarkSideVariant(CoGameMissionVariant):
+    name: str = "dark_side"
+    description: str = "You're on the dark side of the asteroid. You recharge slower."
 
     @override
-    def modify_mission(self, mission):
-        assert isinstance(mission, Mission)
-        mission.cog.energy_limit = max(mission.cog.energy_limit, 255)
-        mission.cog.energy_regen = mission.cog.energy_limit
+    def modify_mission(self, mission: CvCMission) -> None:
+        mission.weather.day_deltas = {"solar": 0}
+        mission.weather.night_deltas = {"solar": 0}
+
+
+class SuperChargedVariant(CoGameMissionVariant):
+    name: str = "super_charged"
+    description: str = "The sun is shining on you. You recharge faster."
+
+    @override
+    def modify_mission(self, mission: CvCMission) -> None:
+        mission.weather.day_deltas = {k: v + 2 for k, v in mission.weather.day_deltas.items()}
+        mission.weather.night_deltas = {k: v + 2 for k, v in mission.weather.night_deltas.items()}
 
 
-class CompassVariant(MissionVariant):
-    name: str = "compass"
-    description: str = "Enable compass observation."
+class EnergizedVariant(CoGameMissionVariant):
+    name: str = "energized"
+    description: str = "Max energy and full regen so agents never run dry."
 
     @override
-    def modify_env(self, mission, env):
-        env.game.obs.global_obs.compass = True
+    def modify_mission(self, mission: CvCMission) -> None:
+        mission.cog.energy_limit = max(mission.cog.energy_limit, 255)
+        mission.weather.day_deltas = {"solar": 255}
+        mission.weather.night_deltas = {"solar": 255}
 
 
-class Small50Variant(MissionVariant):
+class Small50Variant(CoGameMissionVariant):
     name: str = "small_50"
     description: str = "Set map size to 50x50 for quick runs."
 
     def modify_env(self, mission, env) -> None:
         map_builder = env.game.map_builder
-        # Only set width/height if instance is a SceneConfig, not a MapBuilderConfig
-        # When instance is a MapBuilderConfig, width and height must be None
         if isinstance(map_builder, MapGen.Config) and isinstance(map_builder.instance, MapBuilderConfig):
-            # Skip setting width/height for MapBuilderConfig instances
             return
         env.game.map_builder = map_builder.model_copy(update={"width": 50, "height": 50})
 
 
-# Biome variants (weather) for procedural maps
 class DesertVariant(MachinaArenaVariant):
     name: str = "desert"
     description: str = "The desert sands make navigation challenging."
@@ -117,7 +211,6 @@ class CityVariant(MachinaArenaVariant):
     def modify_node(self, node):
         node.biome_weights = {"city": 1.0, "caves": 0.0, "desert": 0.0, "forest": 0.0}
         node.base_biome = "city"
-        # Fill almost the entire map with the city layer
         node.density_scale = 1.0
         node.biome_count = 1
         node.max_biome_zone_fraction = 0.95
@@ -140,32 +233,29 @@ class DistantResourcesVariant(MachinaArenaVariant):
 
     @override
     def modify_node(self, node):
-        # Bias buildings toward the map edges using bimodal clusters centered at
         node.building_coverage = 0.01
 
         vertical_edges = DistributionConfig(
             type=DistributionType.BIMODAL,
-            center1_x=0.92,  # top right corner
+            center1_x=0.92,
             center1_y=0.08,
-            center2_x=0.08,  # bottom left corner
+            center2_x=0.08,
             center2_y=0.92,
             cluster_std=0.18,
         )
         horizontal_edges = DistributionConfig(
             type=DistributionType.BIMODAL,
-            center1_x=0.08,  # top left corner
+            center1_x=0.08,
             center1_y=0.08,
-            center2_x=0.92,  # bottom right corner
+            center2_x=0.92,
             center2_y=0.92,
             cluster_std=0.18,
         )
 
-        # Apply edge-biased distributions to extractors; other buildings follow the global distribution
         names = list(self.building_names)
         node.building_distributions = {
             name: (vertical_edges if i % 2 == 0 else horizontal_edges) for i, name in enumerate(names)
         }
-        # Fallback for any unspecified building types
         node.distribution = DistributionConfig(type=DistributionType.UNIFORM)
 
 
@@ -180,10 +270,10 @@ class QuadrantBuildingsVariant(MachinaArenaVariant):
 
         names = list(node.building_names or self.building_names)
         centers = [
-            (0.25, 0.25),  # top-left
-            (0.75, 0.25),  # top-right
-            (0.25, 0.75),  # bottom-left
-            (0.75, 0.75),  # bottom-right
+            (0.25, 0.25),
+            (0.75, 0.25),
+            (0.25, 0.75),
+            (0.75, 0.75),
         ]
         dists: dict[str, DistributionConfig] = {}
         for i, name in enumerate(names):
@@ -206,8 +296,6 @@ class SingleResourceUniformVariant(MachinaArenaVariant):
 
     @override
     def modify_node(self, node):
-        # Resolve resource to a concrete building name
-        # Restrict building set to only the chosen building and enforce uniform distribution
         node.building_names = [self.building_name]
         node.building_weights = {self.building_name: 1.0}
         node.building_distributions = None
@@ -217,12 +305,10 @@ class SingleResourceUniformVariant(MachinaArenaVariant):
 class EmptyBaseVariant(BaseHubVariant):
     name: str = "empty_base"
     description: str = "Base hub with extractors removed from the four corners."
-    # Extractor object names to remove, e.g., ["oxygen_extractor"]
     missing: list[str] = list(HUB_EXTRACTORS)
 
     @override
     def modify_node(self, node):
-        # Use the default extractor order and blank out any that are missing
         missing_set = set(self.missing or [])
         corner_objects = [name if name not in missing_set else "" for name in HUB_EXTRACTORS]
         node.corner_objects = corner_objects
@@ -230,8 +316,6 @@ class EmptyBaseVariant(BaseHubVariant):
 
 
 class BalancedCornersVariant(MachinaArenaVariant):
-    """Enable corner balancing to ensure fair spawn distances."""
-
     name: str = "balanced_corners"
     description: str = "Balance path distances from center to corners for fair spawns."
     balance_tolerance: float = 1.5
@@ -244,65 +328,75 @@ class BalancedCornersVariant(MachinaArenaVariant):
         node.max_balance_shortcuts = self.max_balance_shortcuts
 
 
-class TraderVariant(MissionVariant):
-    name: str = "trader"
-    description: str = "Agents can trade resources with each other."
+class MultiTeamVariant(CoGameMissionVariant):
+    """Split the map into multiple team instances, each with their own hub and resources."""
+
+    name: str = "multi_team"
+    description: str = "Split map into separate team instances with independent hubs."
+    num_teams: int = Field(default=2, ge=2, le=2, description="Number of teams (max 2 supported)")
 
     @override
-    def modify_env(self, mission, env):
-        # Define vibe transfers for trading resources (actor gives, target receives)
-        trade_transfers = [
-            VibeTransfer(vibe="carbon_a", target={"carbon": 1}, actor={"carbon": -1}),
-            VibeTransfer(vibe="carbon_b", target={"carbon": 10}, actor={"carbon": -10}),
-            VibeTransfer(vibe="oxygen_a", target={"oxygen": 1}, actor={"oxygen": -1}),
-            VibeTransfer(vibe="oxygen_b", target={"oxygen": 10}, actor={"oxygen": -10}),
-            VibeTransfer(vibe="germanium_a", target={"germanium": 1}, actor={"germanium": -1}),
-            VibeTransfer(vibe="germanium_b", target={"germanium": 4}, actor={"germanium": -4}),
-            VibeTransfer(vibe="silicon_a", target={"silicon": 10}, actor={"silicon": -10}),
-            VibeTransfer(vibe="silicon_b", target={"silicon": 50}, actor={"silicon": -50}),
-            VibeTransfer(vibe="heart_a", target={"heart": 1}, actor={"heart": -1}),
-            VibeTransfer(vibe="heart_b", target={"heart": 4}, actor={"heart": -4}),
-        ]
-        # Enable transfer action with these vibes
-        env.game.actions.transfer.enabled = True
-        env.game.actions.transfer.vibe_transfers.extend(trade_transfers)
+    def modify_mission(self, mission: CvCMission) -> None:
+        team = next(iter(mission.teams.values()))
+        # Each team gets the original agent count; clear num_cogs so total is derived from teams
+        original_agents = mission.num_agents
+        mission.teams = {
+            name: team.model_copy(update={"name": name, "short_name": name, "num_agents": original_agents})
+            for name in ["cogs_green", "cogs_blue"][: self.num_teams]
+        }
+        mission.num_cogs = None
+
+    def modify_env(self, mission: CvCMission, env: MettaGridConfig) -> None:
+        original_builder = env.game.map_builder
+        # Shrink inner instance borders so teams are close together
+        if isinstance(original_builder, MapGen.Config):
+            original_builder.border_width = 1
+        env.game.map_builder = MapGen.Config(
+            instance=original_builder,
+            instances=self.num_teams,
+            set_team_by_instance=True,
+            instance_names=[t.short_name for t in mission.teams.values()],
+            instance_object_remap={
+                "c:hub": "{instance_name}:hub",
+                "c:chest": "{instance_name}:chest",
+                **{f"c:{g}": f"{{instance_name}}:{g}" for g in CvCConfig.GEAR},
+            },
+            # Connect instances: no added borders, clear walls at boundary
+            border_width=0,  # No outer border (inner instances have their own)
+            instance_border_width=0,  # No border between instances
+            instance_border_clear_radius=3,  # Clear walls near instance boundary
+        )
 
 
-class SharedRewardsVariant(MissionVariant):
-    name: str = "shared_rewards"
-    description: str = "Rewards for deposited hearts are shared among all agents."
+class NoClipsVariant(CoGameMissionVariant):
+    name: str = "no_clips"
+    description: str = "Disable clips behavior entirely."
 
     @override
-    def modify_env(self, mission, env):
-        num_cogs = mission.num_cogs if mission.num_cogs is not None else mission.site.min_cogs
-        env.game.agent.rewards["chest_heart_deposited_by_agent"] = reward(
-            stat("chest.heart.deposited_by_agent"), weight=0
-        )
-        env.game.agent.rewards["chest_heart_amount"] = reward(stat("chest.heart.amount"), weight=1 / num_cogs)
+    def modify_mission(self, mission: CvCMission) -> None:
+        mission.clips.disabled = True
 
 
-# TODO - validate that all variant names are unique
-VARIANTS: list[MissionVariant] = [
+VARIANTS: list[CoGameMissionVariant] = [
     CavesVariant(),
     CityVariant(),
-    CompassVariant(),
     DarkSideVariant(),
+    NoClipsVariant(),
     DesertVariant(),
     EmptyBaseVariant(),
     EnergizedVariant(),
     ForestVariant(),
-    PackRatVariant(),
+    MultiTeamVariant(),
     QuadrantBuildingsVariant(),
-    RoughTerrainVariant(),
-    SharedRewardsVariant(),
     SingleResourceUniformVariant(),
     Small50Variant(),
     SuperChargedVariant(),
-    TraderVariant(),
     *DIFFICULTY_VARIANTS,
 ]
 
-# Hidden variants registry: Remains usable but will NOT appear in `cogames variants` listing
-HIDDEN_VARIANTS: list[MissionVariant] = [
-    # Example: ExperimentalVariant(),  # keep empty by default
+HIDDEN_VARIANTS: list[CoGameMissionVariant] = [
+    ClipsEasyVariant(),
+    ClipsMediumVariant(),
+    ClipsHardVariant(),
+    ClipsWaveOnlyVariant(),
 ]

cogames/cogs_vs_clips/weather.py

@@ -0,0 +1,52 @@
+"""Weather system events for CogsGuard missions.
+
+Day/night cycle that applies resource deltas to entities at regular intervals.
+"""
+
+from pydantic import Field
+
+from mettagrid.base_config import Config
+from mettagrid.config.event_config import EventConfig, periodic
+from mettagrid.config.mutation import updateTarget
+from mettagrid.config.tag import typeTag
+
+
+class WeatherConfig(Config):
+    """Configuration for day/night weather cycle."""
+
+    day_length: int = Field(default=200)
+    day_deltas: dict[str, int] = Field(default_factory=lambda: {"solar": 3})
+    night_deltas: dict[str, int] = Field(default_factory=lambda: {"solar": 1})
+    target_tag: str = Field(default="agent")
+
+    def events(self, max_steps: int) -> dict[str, EventConfig]:
+        """Create weather events for a mission.
+
+        Returns:
+            Dictionary of event name to EventConfig.
+        """
+        events: dict[str, EventConfig] = {}
+        tag = typeTag(self.target_tag)
+        half = self.day_length // 2
+
+        def _merge(apply: dict[str, int], reverse: dict[str, int]) -> dict[str, int]:
+            keys = set(apply) | set(reverse)
+            return {k: apply.get(k, 0) - reverse.get(k, 0) for k in keys}
+
+        # Dawn: reverse night deltas, apply day deltas
+        events["weather_day"] = EventConfig(
+            name="weather_day",
+            target_tag=tag,
+            timesteps=periodic(start=0, period=self.day_length, end=max_steps),
+            mutations=[updateTarget(_merge(self.day_deltas, self.night_deltas))],
+        )
+
+        # Dusk: reverse day deltas, apply night deltas
+        events["weather_night"] = EventConfig(
+            name="weather_night",
+            target_tag=tag,
+            timesteps=periodic(start=half, period=self.day_length, end=max_steps),
+            mutations=[updateTarget(_merge(self.night_deltas, self.day_deltas))],
+        )
+
+        return events

cogames/core.py

@@ -0,0 +1,87 @@
+"""Core base classes for CoGame missions and variants."""
+
+from __future__ import annotations
+
+from abc import ABC
+from typing import TypeVar
+
+from pydantic import Field
+from typing_extensions import Self
+
+from mettagrid.base_config import Config
+from mettagrid.config.mettagrid_config import MettaGridConfig
+from mettagrid.map_builder.map_builder import AnyMapBuilderConfig
+
+# Type variable for mission types
+TMission = TypeVar("TMission", bound="CoGameMission")
+
+MAP_MISSION_DELIMITER = "."
+
+
+class CoGameMissionVariant(Config, ABC):
+    # Note: we could derive the name from the class name automatically, but it would make it
+    # harder to find the variant source code based on CLI interactions.
+    name: str
+    description: str = Field(default="")
+
+    def modify_mission(self, mission: CoGameMission) -> None:
+        # Override this method to modify the mission.
+        # Variants are allowed to modify the mission in-place - it's guaranteed to be a one-time only instance.
+        pass
+
+    def modify_env(self, mission: CoGameMission, env: MettaGridConfig) -> None:
+        # Override this method to modify the produced environment.
+        # Variants are allowed to modify the environment in-place.
+        pass
+
+    def compat(self, mission: CoGameMission) -> bool:
+        """Check if this variant is compatible with the given mission.
+
+        Returns True if the variant can be safely applied to the mission.
+        Override this method to add compatibility checks.
+        """
+        return True
+
+    def apply(self, mission: TMission) -> TMission:
+        mission = mission.model_copy(deep=True)
+        mission.variants.append(self)
+        self.modify_mission(mission)
+        return mission
+
+
+class CoGameSite(Config):
+    name: str
+    description: str
+    map_builder: AnyMapBuilderConfig
+
+    min_cogs: int = Field(default=1, ge=1)
+    max_cogs: int = Field(default=1000, ge=1)
+
+
+class CoGameMission(Config, ABC):
+    """Base class for Mission configurations with common fields and methods."""
+
+    name: str
+    description: str
+    site: CoGameSite
+    num_cogs: int | None = None
+
+    # Variants are applied to the mission immediately, and to its env when make_env is called
+    variants: list[CoGameMissionVariant] = Field(default_factory=list)
+
+    max_steps: int = Field(default=10000)
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        # Can't call `variant.apply` here because it will create a new mission instance
+        for variant in self.variants:
+            variant.modify_mission(self)
+
+    def with_variants(self, variants: list[CoGameMissionVariant]) -> Self:
+        mission = self
+        for variant in variants:
+            mission = variant.apply(mission)
+        return mission
+
+    def full_name(self) -> str:
+        return f"{self.site.name}{MAP_MISSION_DELIMITER}{self.name}"

cogames/docs/SCRIPTED_AGENT.md

@@ -261,11 +261,11 @@ uv run cogames play --mission evals.diagnostic_assemble_seeded_search -p baselin
 
 ```bash
 # Run full evaluation suite
-uv run python packages/cogames/scripts/run_evaluation.py
+uv run cogames diagnose ladybug -S all
 
 # Evaluate specific agent
-uv run python packages/cogames/scripts/run_evaluation.py --policy baseline
-uv run python packages/cogames/scripts/run_evaluation.py --policy ladybug
+uv run cogames diagnose baseline
+uv run cogames diagnose ladybug
 ```
 
 ## Evaluation Results

cogames/evaluate.py

@@ -13,10 +13,10 @@ from pydantic import BaseModel, ConfigDict
 from rich.console import Console
 from rich.table import Table
 
-from metta_alo.rollout import run_multi_episode_rollout
 from metta_alo.scoring import allocate_counts, validate_proportions
 from mettagrid import MettaGridConfig
 from mettagrid.policy.policy import PolicySpec
+from mettagrid.runner.rollout import run_multi_episode_rollout
 from mettagrid.simulator.multi_episode.rollout import MultiEpisodeRolloutResult
 from mettagrid.simulator.multi_episode.summary import MultiEpisodeRolloutSummary, build_multi_episode_rollout_summaries
 
@@ -46,6 +46,7 @@ def evaluate(
     episodes: int,
     action_timeout_ms: int,
     seed: int = 42,
+    device: Optional[str] = None,
     output_format: Optional[Literal["yaml", "json"]] = None,
     save_replay: Optional[str] = None,
 ) -> MissionResultsSummary:
@@ -70,7 +71,7 @@ def evaluate(
     all_replay_paths: list[str] = []
     for mission_name, env_cfg in missions:
         counts = allocate_counts(env_cfg.game.num_agents, proportions)
-        assignments = np.repeat(np.arange(len(counts), dtype=int), counts)
+        assignments = [i for i, c in enumerate(counts) for _ in range(c)]
 
         progress_label = f"Simulating ({mission_name})"
         with typer.progressbar(length=episodes, label=progress_label) as progress:
@@ -83,6 +84,7 @@ def evaluate(
                 max_action_time_ms=action_timeout_ms,
                 replay_dir=save_replay,
                 create_replay_dir=save_replay is not None,
+                device=device,
                 on_progress=lambda _episode_idx, _result: progress.update(1),
             )