plancraft 0.1.4__py3-none-any.whl → 0.2.0__py3-none-any.whl

plancraft/models/base.py

@@ -1,11 +1,11 @@
 import abc
-
-from copy import copy
 from collections import Counter
+from copy import copy
+
+from loguru import logger
 
 from plancraft.environments.actions import (
     SymbolicMoveAction,
-    RealActionInteraction,
     SymbolicSmeltAction,
 )
 
@@ -29,7 +29,7 @@ class History:
 
     def add_message_to_history(self, content: str | dict, role="user"):
         if role == "assistant":
-            print(content)
+            logger.info(content)
 
         if isinstance(content, dict):
             assert "content" in content, "content key not found in message"
@@ -48,9 +48,7 @@ class History:
             else:
                 self.dialogue_history.append({"role": role, "content": content})
 
-    def add_action_to_history(
-        self, action: SymbolicSmeltAction | RealActionInteraction | SymbolicMoveAction
-    ):
+    def add_action_to_history(self, action: SymbolicSmeltAction | SymbolicMoveAction):
         if action is None:
             return
         self.action_history.append(action.model_dump())
@@ -139,7 +137,7 @@ class ABCModel(abc.ABC):
     @abc.abstractmethod
     def step(
         self, observation: list[dict]
-    ) -> list[SymbolicMoveAction | RealActionInteraction | SymbolicSmeltAction]:
+    ) -> list[SymbolicMoveAction | SymbolicSmeltAction]:
         """
         Model should output a valid action based on the 3 types available
 
@@ -148,5 +146,16 @@ class ABCModel(abc.ABC):
         """
         raise NotImplementedError()
 
+    @property
+    @abc.abstractmethod
+    def history(self) -> History:
+        """
+        Return the history object.
+        """
+        raise NotImplementedError()
+
     def reset_history(self, objective: str = ""):
+        """
+        Reset the history with a new objective.
+        """
         self.history.reset(objective=objective)

plancraft/models/bbox_model.py

@@ -1,13 +1,14 @@
+from typing import Literal
+
 import einops
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
 import torchvision.transforms.v2 as v2
 from huggingface_hub import PyTorchModelHubMixin
-from plancraft.environments.items import ALL_ITEMS
 from torchvision.models.detection.faster_rcnn import (
-    fasterrcnn_resnet50_fpn_v2,
     ResNet50_Weights,
+    fasterrcnn_resnet50_fpn_v2,
 )
 from torchvision.models.detection.roi_heads import (
     fastrcnn_loss,
@@ -18,15 +19,22 @@ from torchvision.models.detection.roi_heads import (
 )
 from torchvision.ops import boxes as box_ops
 
+from plancraft.environments.items import ALL_ITEMS
+
+from functools import lru_cache
+
 
-def slot_to_bbox(slot: int):
+@lru_cache
+def slot_to_bbox(
+    slot: int, resolution: Literal["low", "medium", "high"] = "high"
+) -> list[int]:
     # crafting slot
     if slot == 0:
         # slot size: 25x25
         # top left corner: (x= 118, y=30)
-        box_size = 25
-        left_x = 117
-        top_y = 29
+        box_size = 26
+        left_x = 119
+        top_y = 30
     # crafting grid
     elif slot < 10:
         # slot size: 18x18
@@ -34,8 +42,8 @@ def slot_to_bbox(slot: int):
         box_size = 18
         row = (slot - 1) // 3
         col = (slot - 1) % 3
-        left_x = 27 + (box_size * col)
-        top_y = 15 + (box_size * row)
+        left_x = 29 + (box_size * col)
+        top_y = 16 + (box_size * row)
     # inventory
     elif slot < 37:
         # slot size: 18x18
@@ -43,29 +51,22 @@ def slot_to_bbox(slot: int):
         box_size = 18
         row = (slot - 10) // 9
         col = (slot - 10) % 9
-        left_x = 5 + (box_size * col)
-        top_y = 82 + (box_size * row)
+        left_x = 7 + (box_size * col)
+        top_y = 83 + (box_size * row)
     # hotbar
     else:
         # slot size: 18x18
         # top left corner: (x= 6 + 18 * col, y=141)
         box_size = 18
         col = (slot - 37) % 9
-        left_x = 5 + (box_size * col)
-        top_y = 140
-    return [left_x, top_y, left_x + box_size, top_y + box_size]
-
-
-def precompute_slot_bboxes():
-    """Precompute the bounding boxes for all slots."""
-    slot_bboxes = {}
-    for slot in range(46):  # Assuming slot indices range from 0 to 45
-        slot_bboxes[slot] = slot_to_bbox(slot)
-    return slot_bboxes
-
-
-# Precompute all slot bounding boxes
-IDX_TO_BBOX = precompute_slot_bboxes()
+        left_x = 7 + (box_size * col)
+        top_y = 141
+    bounding_box = [left_x, top_y, left_x + box_size, top_y + box_size]
+    if resolution == "medium":
+        bounding_box = [i * 2 for i in bounding_box]
+    elif resolution == "high":
+        bounding_box = [i * 4 for i in bounding_box]
+    return bounding_box
 
 
 def postprocess_detections_custom(
@@ -391,10 +392,15 @@ def calculate_iou(boxA, boxB):
     return iou
 
 
-def bbox_to_slot_index_iou(bbox: tuple[int, int, int, int]) -> int:
+def bbox_to_slot_index_iou(bbox: tuple[int, int, int, int], resolution="high") -> int:
     """Assign the given bounding box to the slot with the highest IoU."""
     best_slot = None
     best_iou = -1
+
+    IDX_TO_BBOX = {
+        slot: slot_to_bbox(slot, resolution=resolution) for slot in range(46)
+    }
+
     # Iterate through all precomputed slot bounding boxes
     for slot, slot_bbox in IDX_TO_BBOX.items():
         iou = calculate_iou(bbox, slot_bbox)
@@ -422,7 +428,7 @@ class IntegratedBoundingBoxModel(nn.Module, PyTorchModelHubMixin):
             image_mean=[0.63, 0.63, 0.63],
             image_std=[0.21, 0.21, 0.21],
             min_size=128,
-            max_size=256,
+            max_size=512,
             num_classes=len(ALL_ITEMS),
             box_score_thresh=0.05,
             rpn_batch_size_per_image=64,
@@ -449,7 +455,7 @@ class IntegratedBoundingBoxModel(nn.Module, PyTorchModelHubMixin):
             preds = self.model(x)
             return preds
 
-    def get_inventory(self, pil_image):
+    def get_inventory(self, pil_image, resolution="high"):
         """
         Predict boxes and quantities
         """
@@ -458,14 +464,16 @@ class IntegratedBoundingBoxModel(nn.Module, PyTorchModelHubMixin):
             img_tensor = img_tensor.cuda()
         with torch.no_grad():
             predictions = self.model(img_tensor.unsqueeze(0))
-        return self.prediction_to_inventory(predictions[0])
+        return self.prediction_to_inventory(predictions[0], resolution=resolution)
 
     @staticmethod
-    def prediction_to_inventory(prediction, threshold=0.9) -> list[dict]:
+    def prediction_to_inventory(
+        prediction, threshold=0.9, resolution="high"
+    ) -> list[dict]:
         inventory = []
         seen_slots = set()
         for i in range(len(prediction["boxes"])):
-            slot = bbox_to_slot_index_iou(prediction["boxes"][i])
+            slot = bbox_to_slot_index_iou(prediction["boxes"][i], resolution=resolution)
             score = prediction["scores"][i]
             label_idx = prediction["labels"][i].item()
             label = ALL_ITEMS[label_idx]

plancraft/models/dummy.py

@@ -2,7 +2,6 @@ import random
 
 from plancraft.config import EvalConfig
 from plancraft.environments.actions import (
-    RealActionInteraction,
     SymbolicMoveAction,
     SymbolicSmeltAction,
 )
@@ -15,8 +14,11 @@ class DummyModel(ABCModel):
     """
 
     def __init__(self, cfg: EvalConfig):
-        self.symbolic_move_action = cfg.plancraft.environment.symbolic_action_space
-        self.history = History(objective="")
+        self._history = History(objective="")
+
+    @property
+    def history(self):
+        return self._history
 
     def random_select(self, observation):
         if observation is None or "inventory" not in observation:
@@ -36,17 +38,12 @@ class DummyModel(ABCModel):
             slot_from=random_slot_from, slot_to=random_slot_to, quantity=1
         )
 
-    def step(
-        self, observation: dict
-    ) -> list[SymbolicMoveAction | RealActionInteraction | SymbolicSmeltAction]:
+    def step(self, observation: dict) -> list[SymbolicMoveAction | SymbolicSmeltAction]:
         # add observation to history
         self.history.add_observation_to_history(observation)
 
         # get action
-        if self.symbolic_move_action:
-            action = self.random_select(observation)
-        else:
-            action = RealActionInteraction()
+        action = self.random_select(observation)
 
         # add action to history
         self.history.add_action_to_history(action)

plancraft/models/oracle.py

@@ -3,7 +3,6 @@ from collections import Counter
 
 from plancraft.config import EvalConfig
 from plancraft.environments.actions import (
-    RealActionInteraction,
     StopAction,
     SymbolicMoveAction,
     SymbolicSmeltAction,
@@ -132,13 +131,14 @@ class OracleModel(ABCModel):
     """
 
     def __init__(self, cfg: EvalConfig):
-        assert (
-            cfg.plancraft.environment.symbolic_action_space
-        ), "Only symbolic actions are supported for oracle"
-        self.history = History(objective="")
+        self._history = History(objective="")
         self.plans = []
         self.subplans = []
 
+    @property
+    def history(self):
+        return self._history
+
     def reset_history(self, objective: str = ""):
         self.history.reset(objective=objective)
         self.plans = []
@@ -245,9 +245,7 @@ class OracleModel(ABCModel):
 
         return self.subplans.pop(0)
 
-    def step(
-        self, observation: dict
-    ) -> list[SymbolicMoveAction | RealActionInteraction | SymbolicSmeltAction]:
+    def step(self, observation: dict) -> list[SymbolicMoveAction | SymbolicSmeltAction]:
         # add observation to history
         self.history.add_observation_to_history(observation)
 

plancraft/train/dataset.py

@@ -3,6 +3,7 @@ import json
 import random
 
 import torch
+from loguru import logger
 from torch.utils.data import Dataset
 
 TEMPLATES = {
@@ -34,7 +35,7 @@ class PlancraftDialogueDataset(Dataset):
 
         assert trace_mode in ["oa", "ota"], f"Invalid trace mode {trace_mode}"
 
-        print("Loading dialogue dataset")
+        logger.info("Loading dialogue dataset")
         data = []
         for example_path in sorted(
             glob.glob(f"{dataset_dir}/{split}/{trace_mode}/*.json")

plancraft/utils.py

@@ -5,6 +5,7 @@ import cv2
 import numpy as np
 import torch
 import torch.nn.functional as F
+from loguru import logger
 from PIL import Image
 
 
@@ -28,7 +29,7 @@ def get_torch_device() -> torch.device:
         device = torch.device("cuda", 0)
     elif torch.backends.mps.is_available():
         if not torch.backends.mps.is_built():
-            print(
+            logger.info(
                 "MPS not available because the current PyTorch install was not built with MPS enabled."
             )
         else:
@@ -37,9 +38,9 @@ def get_torch_device() -> torch.device:
 
 
 def resize_image(img, target_resolution=(128, 128)):
-    if type(img) == np.ndarray:
+    if type(img) is np.ndarray:
         img = cv2.resize(img, target_resolution, interpolation=cv2.INTER_LINEAR)
-    elif type(img) == torch.Tensor:
+    elif type(img) is torch.Tensor:
         img = F.interpolate(img, size=target_resolution, mode="bilinear")
     else:
         raise ValueError("Unsupported type for img")

plancraft-0.2.0.dist-info/METADATA

@@ -0,0 +1,74 @@
+Metadata-Version: 2.3
+Name: plancraft
+Version: 0.2.0
+Summary: Plancraft: an evaluation dataset for planning with LLM agents
+License: MIT License
+        
+        Copyright (c) 2024 Gautier Dagan
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+Requires-Python: >=3.9
+Requires-Dist: hydra-core>=1.3.2
+Requires-Dist: imageio>=2.36.0
+Requires-Dist: loguru
+Requires-Dist: networkx
+Requires-Dist: numpy
+Requires-Dist: openai
+Requires-Dist: pandas
+Requires-Dist: pillow>=8.0.0
+Requires-Dist: pydantic
+Requires-Dist: python-dotenv>=1.0.1
+Requires-Dist: setuptools
+Requires-Dist: tqdm
+Requires-Dist: wandb
+Provides-Extra: full
+Requires-Dist: accelerate; extra == 'full'
+Requires-Dist: einops; extra == 'full'
+Requires-Dist: hf-transfer; extra == 'full'
+Requires-Dist: huggingface-hub; extra == 'full'
+Requires-Dist: matplotlib; extra == 'full'
+Requires-Dist: peft; extra == 'full'
+Requires-Dist: seaborn; extra == 'full'
+Requires-Dist: torch>=2.5.0; extra == 'full'
+Requires-Dist: torchvision>=0.20.0; extra == 'full'
+Requires-Dist: transformers>=4.43.3; extra == 'full'
+Description-Content-Type: text/markdown
+
+# plancraft
+
+[![Test](https://github.com/gautierdag/plancraft/actions/workflows/test.yaml/badge.svg)](https://github.com/gautierdag/plancraft/actions/workflows/test.yaml)
+![Python Version](https://img.shields.io/badge/python-3.9+-blue)
+![Ruff](https://img.shields.io/badge/linter-ruff-blue)
+[![PyPI Version](https://img.shields.io/pypi/v/plancraft)](https://pypi.org/project/plancraft/)
+
+Plancraft is a minecraft environment and agent that innovates on planning LLM agents with a retriever
+
+You can install the package by running the following command:
+
+```bash
+pip install plancraft
+```
+
+Should you need the multimodal version of the package, you will also need a custom [fork](https://github.com/gautierdag/minerl.git) of the minerl package. You can install it by running the following command:
+
+```bash
+pip install git+hhttps://github.com/gautierdag/minerl.git
+```
+
+Note that you may need to follow the same installation instructions as in the [minerl documentation](https://minerl.readthedocs.io/en/latest/tutorials/index.html).