plancraft 0.1.3__py3-none-any.whl → 0.2.0__py3-none-any.whl

plancraft/environments/tags/wooden_pressure_plates.json

@@ -0,0 +1,13 @@
+{
+  "replace": false,
+  "values": [
+    "minecraft:oak_pressure_plate",
+    "minecraft:spruce_pressure_plate",
+    "minecraft:birch_pressure_plate",
+    "minecraft:jungle_pressure_plate",
+    "minecraft:acacia_pressure_plate",
+    "minecraft:dark_oak_pressure_plate",
+    "minecraft:crimson_pressure_plate",
+    "minecraft:warped_pressure_plate"
+  ]
+}

plancraft/environments/tags/wooden_slabs.json

@@ -0,0 +1,13 @@
+{
+  "replace": false,
+  "values": [
+    "minecraft:oak_slab",
+    "minecraft:spruce_slab",
+    "minecraft:birch_slab",
+    "minecraft:jungle_slab",
+    "minecraft:acacia_slab",
+    "minecraft:dark_oak_slab",
+    "minecraft:crimson_slab",
+    "minecraft:warped_slab"
+  ]
+}

plancraft/environments/tags/wooden_stairs.json

@@ -0,0 +1,13 @@
+{
+  "replace": false,
+  "values": [
+    "minecraft:oak_stairs",
+    "minecraft:spruce_stairs",
+    "minecraft:birch_stairs",
+    "minecraft:jungle_stairs",
+    "minecraft:acacia_stairs",
+    "minecraft:dark_oak_stairs",
+    "minecraft:crimson_stairs",
+    "minecraft:warped_stairs"
+  ]
+}

plancraft/environments/tags/wooden_trapdoors.json

@@ -0,0 +1,13 @@
+{
+  "replace": false,
+  "values": [
+    "minecraft:acacia_trapdoor",
+    "minecraft:birch_trapdoor",
+    "minecraft:dark_oak_trapdoor",
+    "minecraft:jungle_trapdoor",
+    "minecraft:oak_trapdoor",
+    "minecraft:spruce_trapdoor",
+    "minecraft:crimson_trapdoor",
+    "minecraft:warped_trapdoor"
+  ]
+}

plancraft/environments/tags/wool.json

@@ -0,0 +1,21 @@
+{
+  "replace": false,
+  "values": [
+    "minecraft:white_wool",
+    "minecraft:orange_wool",
+    "minecraft:magenta_wool",
+    "minecraft:light_blue_wool",
+    "minecraft:yellow_wool",
+    "minecraft:lime_wool",
+    "minecraft:pink_wool",
+    "minecraft:gray_wool",
+    "minecraft:light_gray_wool",
+    "minecraft:cyan_wool",
+    "minecraft:purple_wool",
+    "minecraft:blue_wool",
+    "minecraft:brown_wool",
+    "minecraft:green_wool",
+    "minecraft:red_wool",
+    "minecraft:black_wool"
+  ]
+}

plancraft/evaluator.py

@@ -6,25 +6,26 @@ import time
 
 import imageio
 import pandas as pd
-import torch
 import wandb
 from loguru import logger
 from tqdm import tqdm
 
 from plancraft.config import EvalConfig, PlancraftExample
 from plancraft.environments.actions import StopAction
-from plancraft.environments.env_real import RealPlancraft
-from plancraft.environments.env_symbolic import SymbolicPlancraft
+from plancraft.environments.env import PlancraftEnv
 from plancraft.models import get_model
 
-wandb.require("core")
-
 
 class Evaluator:
     """
     The evaluator class handles the environment loop and model interaction
 
     The environment is created based on the configuration and the examples are loaded from the dataset.
+
+    The Evaluator uses the dataset examples and initializes the environment with the example's inventory.
+
+    It is also responsible for early stopping and verifying the target object has been craft.
+    Finally, it also saves the results of the evaluation and the images generated during the evaluation.
     """
 
     def __init__(self, cfg: EvalConfig):
@@ -39,18 +40,20 @@ class Evaluator:
         self.environment = self.create_env(cfg)
         self.model = get_model(cfg)
 
-        self.record_frames = not (cfg.plancraft.environment.symbolic)
-
-        # no_op action
-        self.no_op = self.environment.action_space.no_op()
-
     def evaluator_name(self) -> str:
-        symb_str = "real"
-        if self.cfg.plancraft.environment.symbolic:
-            symb_str = "symb"
+        if self.cfg.plancraft.use_text_inventory and self.cfg.plancraft.use_images:
+            name_str = "both"
+        elif self.cfg.plancraft.use_images:
+            name_str = "images"
+        elif self.cfg.plancraft.use_text_inventory:
+            name_str = "text"
+        else:
+            raise ValueError(
+                "At least one of use_text_inventory or use_images should be True"
+            )
 
         if self.cfg.plancraft.use_maskrcnn:
-            symb_str += "_mrcnn"
+            name_str += "_mrcnn"
 
         model_name = self.cfg.plancraft.model.split("/")[-1]
         if self.cfg.plancraft.adapter != "":
@@ -58,10 +61,10 @@ class Evaluator:
 
         mode = self.cfg.plancraft.mode
         if mode in ["dummy", "oracle"]:
-            return f"{mode}_{symb_str}"
+            return f"{mode}_{name_str}"
 
         actions = "|".join(self.cfg.plancraft.valid_actions)
-        return f"{self.cfg.plancraft.mode}_{symb_str}_{model_name}_{actions}"
+        return f"{self.cfg.plancraft.mode}_{name_str}_{model_name}_{actions}"
 
     def save_results_dict(self, example: PlancraftExample, results_dict: dict):
         output_dir = f"{self.output_dir}/{self.generation_number}"
@@ -69,7 +72,9 @@ class Evaluator:
         json_path = f"{output_dir}/{example.id}.json"
         with open(json_path, "w") as f:
             json.dump(results_dict, f, indent=4)
-        wandb.save(json_path, policy="now")
+
+        if wandb.run is not None:
+            wandb.save(json_path, policy="now")
 
     def save_images(self, example: PlancraftExample, frames: list):
         if len(frames) == 0:
@@ -78,7 +83,8 @@ class Evaluator:
         os.makedirs(output_dir, exist_ok=True)
         imageio.mimsave(f"{output_dir}/{example.id}.gif", frames)
         # upload to wandb
-        wandb.save(f"{output_dir}/{example.id}.gif", policy="now")
+        if wandb.run is not None:
+            wandb.save(f"{output_dir}/{example.id}.gif", policy="now")
 
     def load_results_dict(self, example: PlancraftExample) -> dict:
         path = f"{self.output_dir}/{self.generation_number}/{example.id}.json"
@@ -87,20 +93,12 @@ class Evaluator:
         with open(path, "r") as f:
             return json.load(f)
 
-    def create_env(self, cfg: EvalConfig) -> RealPlancraft | SymbolicPlancraft:
-        if cfg.plancraft.environment.symbolic:
-            return SymbolicPlancraft(inventory=[])
-        return RealPlancraft(
+    def create_env(self, cfg: EvalConfig) -> PlancraftEnv:
+        return PlancraftEnv(
             inventory=[],
-            symbolic_action_space=cfg.plancraft.environment.symbolic_action_space,
-            symbolic_observation_space=cfg.plancraft.environment.symbolic_observation_space,
-            preferred_spawn_biome=cfg.plancraft.environment.preferred_spawn_biome,
             resolution=cfg.plancraft.environment.resolution,
         )
 
-    def close(self):
-        self.environment.close()
-
     def load_dataset(self, dataset_split: str) -> list[PlancraftExample]:
         with open(f"data/{dataset_split}.json", "r") as f:
             dataset = json.load(f)
@@ -111,9 +109,9 @@ class Evaluator:
         example: PlancraftExample,
     ):
         current_inventory = example.slotted_inventory
-        self.environment.fast_reset(new_inventory=current_inventory)
+        self.environment.reset(new_inventory=current_inventory)
         # do a no op to an initial observation
-        obs, _, _, _ = self.environment.step(self.no_op)
+        obs = self.environment.step()
         # assert that the inventory is correct
         if "inventory" in obs:
             for item in current_inventory:
@@ -144,11 +142,8 @@ class Evaluator:
                     return True
         return False
 
-    @torch.no_grad()
     def eval_all_examples(self, progress_bar=False) -> list:
         results = []
-        action = self.no_op.copy()
-
         pbar = tqdm(
             total=len(self.examples),
             disable=not progress_bar,
@@ -165,7 +160,7 @@ class Evaluator:
             success = False
 
             self.reset(example)
-            action = self.no_op.copy()
+            action = None
 
             while (
                 not self.model.history.check_stuck()
@@ -179,12 +174,14 @@ class Evaluator:
                     break
 
                 # step action
-                observation, _, _, _ = self.environment.step(action)
+                observation = self.environment.step(action)
 
                 # check if the episode is done
                 success = self.check_done(observation["inventory"], example.target)
                 # exit if success
                 if success:
+                    # add final observation to history
+                    self.model.history.add_observation_to_history(observation)
                     break
 
                 # predict next action

plancraft/models/__init__.py

@@ -1,10 +1,9 @@
-from plancraft.models.base import ABCModel
-
 from plancraft.config import EvalConfig
+from plancraft.models.act import ActModel
+from plancraft.models.base import ABCModel
 from plancraft.models.dummy import DummyModel
-from plancraft.models.react import ReactModel
 from plancraft.models.oracle import OracleModel
-from plancraft.models.act import ActModel
+from plancraft.models.react import ReactModel
 
 
 def get_model(cfg: EvalConfig) -> ABCModel:

plancraft/models/act.py

@@ -32,9 +32,6 @@ class ActModel(ABCModel):
     """
 
     def __init__(self, cfg: EvalConfig):
-        assert (
-            cfg.plancraft.environment.symbolic_action_space
-        ), "Real action space unsupported"
         self.cfg = cfg
         self.env_is_multimodal = not cfg.plancraft.environment.symbolic
         self.use_maskrcnn = cfg.plancraft.use_maskrcnn
@@ -107,7 +104,7 @@ class ActModel(ABCModel):
         if not self.use_system_prompt:
             self.system_prompt = None
 
-        self.history = History(
+        self._history = History(
             initial_dialogue=examples,
             use_multimodal_content_format=self.use_multimodal_content_format,
         )
@@ -115,6 +112,10 @@ class ActModel(ABCModel):
         self.max_messages_window = cfg.plancraft.max_message_window
         self.kv_cache = None
 
+    @property
+    def history(self):
+        return self._history
+
     def reset_history(
         self,
         objective: str,

plancraft/models/base.py

@@ -1,11 +1,11 @@
 import abc
-
-from copy import copy
 from collections import Counter
+from copy import copy
+
+from loguru import logger
 
 from plancraft.environments.actions import (
     SymbolicMoveAction,
-    RealActionInteraction,
     SymbolicSmeltAction,
 )
 
@@ -29,7 +29,7 @@ class History:
 
     def add_message_to_history(self, content: str | dict, role="user"):
         if role == "assistant":
-            print(content)
+            logger.info(content)
 
         if isinstance(content, dict):
             assert "content" in content, "content key not found in message"
@@ -48,9 +48,7 @@ class History:
             else:
                 self.dialogue_history.append({"role": role, "content": content})
 
-    def add_action_to_history(
-        self, action: SymbolicSmeltAction | RealActionInteraction | SymbolicMoveAction
-    ):
+    def add_action_to_history(self, action: SymbolicSmeltAction | SymbolicMoveAction):
         if action is None:
             return
         self.action_history.append(action.model_dump())
@@ -139,7 +137,7 @@ class ABCModel(abc.ABC):
     @abc.abstractmethod
     def step(
         self, observation: list[dict]
-    ) -> list[SymbolicMoveAction | RealActionInteraction | SymbolicSmeltAction]:
+    ) -> list[SymbolicMoveAction | SymbolicSmeltAction]:
         """
         Model should output a valid action based on the 3 types available
 
@@ -148,5 +146,16 @@ class ABCModel(abc.ABC):
         """
         raise NotImplementedError()
 
+    @property
+    @abc.abstractmethod
+    def history(self) -> History:
+        """
+        Return the history object.
+        """
+        raise NotImplementedError()
+
     def reset_history(self, objective: str = ""):
+        """
+        Reset the history with a new objective.
+        """
         self.history.reset(objective=objective)

plancraft/models/bbox_model.py

@@ -1,13 +1,14 @@
+from typing import Literal
+
 import einops
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
 import torchvision.transforms.v2 as v2
 from huggingface_hub import PyTorchModelHubMixin
-from plancraft.environments.items import ALL_ITEMS
 from torchvision.models.detection.faster_rcnn import (
-    fasterrcnn_resnet50_fpn_v2,
     ResNet50_Weights,
+    fasterrcnn_resnet50_fpn_v2,
 )
 from torchvision.models.detection.roi_heads import (
     fastrcnn_loss,
@@ -18,15 +19,22 @@ from torchvision.models.detection.roi_heads import (
 )
 from torchvision.ops import boxes as box_ops
 
+from plancraft.environments.items import ALL_ITEMS
+
+from functools import lru_cache
+
 
-def slot_to_bbox(slot: int):
+@lru_cache
+def slot_to_bbox(
+    slot: int, resolution: Literal["low", "medium", "high"] = "high"
+) -> list[int]:
     # crafting slot
     if slot == 0:
         # slot size: 25x25
         # top left corner: (x= 118, y=30)
-        box_size = 25
-        left_x = 117
-        top_y = 29
+        box_size = 26
+        left_x = 119
+        top_y = 30
     # crafting grid
     elif slot < 10:
         # slot size: 18x18
@@ -34,8 +42,8 @@ def slot_to_bbox(slot: int):
         box_size = 18
         row = (slot - 1) // 3
         col = (slot - 1) % 3
-        left_x = 27 + (box_size * col)
-        top_y = 15 + (box_size * row)
+        left_x = 29 + (box_size * col)
+        top_y = 16 + (box_size * row)
     # inventory
     elif slot < 37:
         # slot size: 18x18
@@ -43,29 +51,22 @@ def slot_to_bbox(slot: int):
         box_size = 18
         row = (slot - 10) // 9
         col = (slot - 10) % 9
-        left_x = 5 + (box_size * col)
-        top_y = 82 + (box_size * row)
+        left_x = 7 + (box_size * col)
+        top_y = 83 + (box_size * row)
     # hotbar
     else:
         # slot size: 18x18
         # top left corner: (x= 6 + 18 * col, y=141)
         box_size = 18
         col = (slot - 37) % 9
-        left_x = 5 + (box_size * col)
-        top_y = 140
-    return [left_x, top_y, left_x + box_size, top_y + box_size]
-
-
-def precompute_slot_bboxes():
-    """Precompute the bounding boxes for all slots."""
-    slot_bboxes = {}
-    for slot in range(46):  # Assuming slot indices range from 0 to 45
-        slot_bboxes[slot] = slot_to_bbox(slot)
-    return slot_bboxes
-
-
-# Precompute all slot bounding boxes
-IDX_TO_BBOX = precompute_slot_bboxes()
+        left_x = 7 + (box_size * col)
+        top_y = 141
+    bounding_box = [left_x, top_y, left_x + box_size, top_y + box_size]
+    if resolution == "medium":
+        bounding_box = [i * 2 for i in bounding_box]
+    elif resolution == "high":
+        bounding_box = [i * 4 for i in bounding_box]
+    return bounding_box
 
 
 def postprocess_detections_custom(
@@ -391,10 +392,15 @@ def calculate_iou(boxA, boxB):
     return iou
 
 
-def bbox_to_slot_index_iou(bbox: tuple[int, int, int, int]) -> int:
+def bbox_to_slot_index_iou(bbox: tuple[int, int, int, int], resolution="high") -> int:
     """Assign the given bounding box to the slot with the highest IoU."""
     best_slot = None
     best_iou = -1
+
+    IDX_TO_BBOX = {
+        slot: slot_to_bbox(slot, resolution=resolution) for slot in range(46)
+    }
+
     # Iterate through all precomputed slot bounding boxes
     for slot, slot_bbox in IDX_TO_BBOX.items():
         iou = calculate_iou(bbox, slot_bbox)
@@ -422,7 +428,7 @@ class IntegratedBoundingBoxModel(nn.Module, PyTorchModelHubMixin):
             image_mean=[0.63, 0.63, 0.63],
             image_std=[0.21, 0.21, 0.21],
             min_size=128,
-            max_size=256,
+            max_size=512,
             num_classes=len(ALL_ITEMS),
             box_score_thresh=0.05,
             rpn_batch_size_per_image=64,
@@ -449,7 +455,7 @@ class IntegratedBoundingBoxModel(nn.Module, PyTorchModelHubMixin):
             preds = self.model(x)
             return preds
 
-    def get_inventory(self, pil_image):
+    def get_inventory(self, pil_image, resolution="high"):
         """
         Predict boxes and quantities
         """
@@ -458,14 +464,16 @@ class IntegratedBoundingBoxModel(nn.Module, PyTorchModelHubMixin):
             img_tensor = img_tensor.cuda()
         with torch.no_grad():
             predictions = self.model(img_tensor.unsqueeze(0))
-        return self.prediction_to_inventory(predictions[0])
+        return self.prediction_to_inventory(predictions[0], resolution=resolution)
 
     @staticmethod
-    def prediction_to_inventory(prediction, threshold=0.9) -> list[dict]:
+    def prediction_to_inventory(
+        prediction, threshold=0.9, resolution="high"
+    ) -> list[dict]:
         inventory = []
         seen_slots = set()
         for i in range(len(prediction["boxes"])):
-            slot = bbox_to_slot_index_iou(prediction["boxes"][i])
+            slot = bbox_to_slot_index_iou(prediction["boxes"][i], resolution=resolution)
             score = prediction["scores"][i]
             label_idx = prediction["labels"][i].item()
             label = ALL_ITEMS[label_idx]

plancraft/models/dummy.py

@@ -2,7 +2,6 @@ import random
 
 from plancraft.config import EvalConfig
 from plancraft.environments.actions import (
-    RealActionInteraction,
     SymbolicMoveAction,
     SymbolicSmeltAction,
 )
@@ -15,8 +14,11 @@ class DummyModel(ABCModel):
     """
 
     def __init__(self, cfg: EvalConfig):
-        self.symbolic_move_action = cfg.plancraft.environment.symbolic_action_space
-        self.history = History(objective="")
+        self._history = History(objective="")
+
+    @property
+    def history(self):
+        return self._history
 
     def random_select(self, observation):
         if observation is None or "inventory" not in observation:
@@ -36,17 +38,12 @@ class DummyModel(ABCModel):
             slot_from=random_slot_from, slot_to=random_slot_to, quantity=1
         )
 
-    def step(
-        self, observation: dict
-    ) -> list[SymbolicMoveAction | RealActionInteraction | SymbolicSmeltAction]:
+    def step(self, observation: dict) -> list[SymbolicMoveAction | SymbolicSmeltAction]:
         # add observation to history
         self.history.add_observation_to_history(observation)
 
         # get action
-        if self.symbolic_move_action:
-            action = self.random_select(observation)
-        else:
-            action = RealActionInteraction()
+        action = self.random_select(observation)
 
         # add action to history
         self.history.add_action_to_history(action)

plancraft/models/oracle.py

@@ -3,7 +3,6 @@ from collections import Counter
 
 from plancraft.config import EvalConfig
 from plancraft.environments.actions import (
-    RealActionInteraction,
     StopAction,
     SymbolicMoveAction,
     SymbolicSmeltAction,
@@ -132,13 +131,14 @@ class OracleModel(ABCModel):
     """
 
     def __init__(self, cfg: EvalConfig):
-        assert (
-            cfg.plancraft.environment.symbolic_action_space
-        ), "Only symbolic actions are supported for oracle"
-        self.history = History(objective="")
+        self._history = History(objective="")
         self.plans = []
         self.subplans = []
 
+    @property
+    def history(self):
+        return self._history
+
     def reset_history(self, objective: str = ""):
         self.history.reset(objective=objective)
         self.plans = []
@@ -245,9 +245,7 @@ class OracleModel(ABCModel):
 
         return self.subplans.pop(0)
 
-    def step(
-        self, observation: dict
-    ) -> list[SymbolicMoveAction | RealActionInteraction | SymbolicSmeltAction]:
+    def step(self, observation: dict) -> list[SymbolicMoveAction | SymbolicSmeltAction]:
         # add observation to history
         self.history.add_observation_to_history(observation)
 

plancraft/train/dataset.py

@@ -3,6 +3,7 @@ import json
 import random
 
 import torch
+from loguru import logger
 from torch.utils.data import Dataset
 
 TEMPLATES = {
@@ -34,7 +35,7 @@ class PlancraftDialogueDataset(Dataset):
 
         assert trace_mode in ["oa", "ota"], f"Invalid trace mode {trace_mode}"
 
-        print("Loading dialogue dataset")
+        logger.info("Loading dialogue dataset")
         data = []
         for example_path in sorted(
             glob.glob(f"{dataset_dir}/{split}/{trace_mode}/*.json")

plancraft/utils.py

@@ -5,6 +5,7 @@ import cv2
 import numpy as np
 import torch
 import torch.nn.functional as F
+from loguru import logger
 from PIL import Image
 
 
@@ -28,7 +29,7 @@ def get_torch_device() -> torch.device:
         device = torch.device("cuda", 0)
     elif torch.backends.mps.is_available():
         if not torch.backends.mps.is_built():
-            print(
+            logger.info(
                 "MPS not available because the current PyTorch install was not built with MPS enabled."
             )
         else:
@@ -37,9 +38,9 @@ def get_torch_device() -> torch.device:
 
 
 def resize_image(img, target_resolution=(128, 128)):
-    if type(img) == np.ndarray:
+    if type(img) is np.ndarray:
         img = cv2.resize(img, target_resolution, interpolation=cv2.INTER_LINEAR)
-    elif type(img) == torch.Tensor:
+    elif type(img) is torch.Tensor:
         img = F.interpolate(img, size=target_resolution, mode="bilinear")
     else:
         raise ValueError("Unsupported type for img")