plancraft 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

plancraft/environments/tags/slabs.json

@@ -0,0 +1,38 @@
+{
+  "replace": false,
+  "values": [
+    "#minecraft:wooden_slabs",
+    "minecraft:stone_slab",
+    "minecraft:smooth_stone_slab",
+    "minecraft:stone_brick_slab",
+    "minecraft:sandstone_slab",
+    "minecraft:purpur_slab",
+    "minecraft:quartz_slab",
+    "minecraft:red_sandstone_slab",
+    "minecraft:brick_slab",
+    "minecraft:cobblestone_slab",
+    "minecraft:nether_brick_slab",
+    "minecraft:petrified_oak_slab",
+    "minecraft:prismarine_slab",
+    "minecraft:prismarine_brick_slab",
+    "minecraft:dark_prismarine_slab",
+    "minecraft:polished_granite_slab",
+    "minecraft:smooth_red_sandstone_slab",
+    "minecraft:mossy_stone_brick_slab",
+    "minecraft:polished_diorite_slab",
+    "minecraft:mossy_cobblestone_slab",
+    "minecraft:end_stone_brick_slab",
+    "minecraft:smooth_sandstone_slab",
+    "minecraft:smooth_quartz_slab",
+    "minecraft:granite_slab",
+    "minecraft:andesite_slab",
+    "minecraft:red_nether_brick_slab",
+    "minecraft:polished_andesite_slab",
+    "minecraft:diorite_slab",
+    "minecraft:cut_sandstone_slab",
+    "minecraft:cut_red_sandstone_slab",
+    "minecraft:blackstone_slab",
+    "minecraft:polished_blackstone_brick_slab",
+    "minecraft:polished_blackstone_slab"
+  ]
+}

plancraft/environments/tags/small_flowers.json

@@ -0,0 +1,18 @@
+{
+  "replace": false,
+  "values": [
+    "minecraft:dandelion",
+    "minecraft:poppy",
+    "minecraft:blue_orchid",
+    "minecraft:allium",
+    "minecraft:azure_bluet",
+    "minecraft:red_tulip",
+    "minecraft:orange_tulip",
+    "minecraft:white_tulip",
+    "minecraft:pink_tulip",
+    "minecraft:oxeye_daisy",
+    "minecraft:cornflower",
+    "minecraft:lily_of_the_valley",
+    "minecraft:wither_rose"
+  ]
+}

plancraft/environments/tags/soul_fire_base_blocks.json

@@ -0,0 +1,7 @@
+{
+  "replace": false,
+  "values": [
+    "minecraft:soul_sand",
+    "minecraft:soul_soil"
+  ]
+}

plancraft/environments/tags/spruce_logs.json

@@ -0,0 +1,9 @@
+{
+  "replace": false,
+  "values": [
+    "minecraft:spruce_log",
+    "minecraft:spruce_wood",
+    "minecraft:stripped_spruce_log",
+    "minecraft:stripped_spruce_wood"
+  ]
+}

plancraft/environments/tags/stairs.json

@@ -0,0 +1,34 @@
+{
+  "replace": false,
+  "values": [
+    "#minecraft:wooden_stairs",
+    "minecraft:cobblestone_stairs",
+    "minecraft:sandstone_stairs",
+    "minecraft:nether_brick_stairs",
+    "minecraft:stone_brick_stairs",
+    "minecraft:brick_stairs",
+    "minecraft:purpur_stairs",
+    "minecraft:quartz_stairs",
+    "minecraft:red_sandstone_stairs",
+    "minecraft:prismarine_brick_stairs",
+    "minecraft:prismarine_stairs",
+    "minecraft:dark_prismarine_stairs",
+    "minecraft:polished_granite_stairs",
+    "minecraft:smooth_red_sandstone_stairs",
+    "minecraft:mossy_stone_brick_stairs",
+    "minecraft:polished_diorite_stairs",
+    "minecraft:mossy_cobblestone_stairs",
+    "minecraft:end_stone_brick_stairs",
+    "minecraft:stone_stairs",
+    "minecraft:smooth_sandstone_stairs",
+    "minecraft:smooth_quartz_stairs",
+    "minecraft:granite_stairs",
+    "minecraft:andesite_stairs",
+    "minecraft:red_nether_brick_stairs",
+    "minecraft:polished_andesite_stairs",
+    "minecraft:diorite_stairs",
+    "minecraft:blackstone_stairs",
+    "minecraft:polished_blackstone_brick_stairs",
+    "minecraft:polished_blackstone_stairs"
+  ]
+}

plancraft/environments/tags/stone_bricks.json

@@ -0,0 +1,9 @@
+{
+  "replace": false,
+  "values": [
+    "minecraft:stone_bricks",
+    "minecraft:mossy_stone_bricks",
+    "minecraft:cracked_stone_bricks",
+    "minecraft:chiseled_stone_bricks"
+  ]
+}

plancraft/environments/tags/stone_crafting_materials.json

@@ -0,0 +1,7 @@
+{
+  "replace": false,
+  "values": [
+    "minecraft:cobblestone",
+    "minecraft:blackstone"
+  ]
+}

plancraft/environments/tags/stone_tool_materials.json

@@ -0,0 +1,7 @@
+{
+  "replace": false,
+  "values": [
+    "minecraft:cobblestone",
+    "minecraft:blackstone"
+  ]
+}

plancraft/environments/tags/tall_flowers.json

@@ -0,0 +1,9 @@
+{
+  "replace": false,
+  "values": [
+    "minecraft:sunflower",
+    "minecraft:lilac",
+    "minecraft:peony",
+    "minecraft:rose_bush"
+  ]
+}

plancraft/environments/tags/trapdoors.json

@@ -0,0 +1,7 @@
+{
+  "replace": false,
+  "values": [
+    "#minecraft:wooden_trapdoors",
+    "minecraft:iron_trapdoor"
+  ]
+}

plancraft/environments/tags/walls.json

@@ -0,0 +1,22 @@
+{
+  "replace": false,
+  "values": [
+    "minecraft:cobblestone_wall",
+    "minecraft:mossy_cobblestone_wall",
+    "minecraft:brick_wall",
+    "minecraft:prismarine_wall",
+    "minecraft:red_sandstone_wall",
+    "minecraft:mossy_stone_brick_wall",
+    "minecraft:granite_wall",
+    "minecraft:stone_brick_wall",
+    "minecraft:nether_brick_wall",
+    "minecraft:andesite_wall",
+    "minecraft:red_nether_brick_wall",
+    "minecraft:sandstone_wall",
+    "minecraft:end_stone_brick_wall",
+    "minecraft:diorite_wall",
+    "minecraft:blackstone_wall",
+    "minecraft:polished_blackstone_brick_wall",
+    "minecraft:polished_blackstone_wall"
+  ]
+}

plancraft/environments/tags/warped_stems.json

@@ -0,0 +1,9 @@
+{
+  "replace": false,
+  "values": [
+    "minecraft:warped_stem",
+    "minecraft:stripped_warped_stem",
+    "minecraft:warped_hyphae",
+    "minecraft:stripped_warped_hyphae"
+  ]
+}

plancraft/environments/tags/wooden_buttons.json

@@ -0,0 +1,13 @@
+{
+  "replace": false,
+  "values": [
+    "minecraft:oak_button",
+    "minecraft:spruce_button",
+    "minecraft:birch_button",
+    "minecraft:jungle_button",
+    "minecraft:acacia_button",
+    "minecraft:dark_oak_button",
+    "minecraft:crimson_button",
+    "minecraft:warped_button"
+  ]
+}

plancraft/environments/tags/wooden_doors.json

@@ -0,0 +1,13 @@
+{
+  "replace": false,
+  "values": [
+    "minecraft:oak_door",
+    "minecraft:spruce_door",
+    "minecraft:birch_door",
+    "minecraft:jungle_door",
+    "minecraft:acacia_door",
+    "minecraft:dark_oak_door",
+    "minecraft:crimson_door",
+    "minecraft:warped_door"
+  ]
+}

plancraft/environments/tags/wooden_fences.json

@@ -0,0 +1,13 @@
+{
+  "replace": false,
+  "values": [
+    "minecraft:oak_fence",
+    "minecraft:acacia_fence",
+    "minecraft:dark_oak_fence",
+    "minecraft:spruce_fence",
+    "minecraft:birch_fence",
+    "minecraft:jungle_fence",
+    "minecraft:crimson_fence",
+    "minecraft:warped_fence"
+  ]
+}

plancraft/environments/tags/wooden_pressure_plates.json

@@ -0,0 +1,13 @@
+{
+  "replace": false,
+  "values": [
+    "minecraft:oak_pressure_plate",
+    "minecraft:spruce_pressure_plate",
+    "minecraft:birch_pressure_plate",
+    "minecraft:jungle_pressure_plate",
+    "minecraft:acacia_pressure_plate",
+    "minecraft:dark_oak_pressure_plate",
+    "minecraft:crimson_pressure_plate",
+    "minecraft:warped_pressure_plate"
+  ]
+}

plancraft/environments/tags/wooden_slabs.json

@@ -0,0 +1,13 @@
+{
+  "replace": false,
+  "values": [
+    "minecraft:oak_slab",
+    "minecraft:spruce_slab",
+    "minecraft:birch_slab",
+    "minecraft:jungle_slab",
+    "minecraft:acacia_slab",
+    "minecraft:dark_oak_slab",
+    "minecraft:crimson_slab",
+    "minecraft:warped_slab"
+  ]
+}

plancraft/environments/tags/wooden_stairs.json

@@ -0,0 +1,13 @@
+{
+  "replace": false,
+  "values": [
+    "minecraft:oak_stairs",
+    "minecraft:spruce_stairs",
+    "minecraft:birch_stairs",
+    "minecraft:jungle_stairs",
+    "minecraft:acacia_stairs",
+    "minecraft:dark_oak_stairs",
+    "minecraft:crimson_stairs",
+    "minecraft:warped_stairs"
+  ]
+}

plancraft/environments/tags/wooden_trapdoors.json

@@ -0,0 +1,13 @@
+{
+  "replace": false,
+  "values": [
+    "minecraft:acacia_trapdoor",
+    "minecraft:birch_trapdoor",
+    "minecraft:dark_oak_trapdoor",
+    "minecraft:jungle_trapdoor",
+    "minecraft:oak_trapdoor",
+    "minecraft:spruce_trapdoor",
+    "minecraft:crimson_trapdoor",
+    "minecraft:warped_trapdoor"
+  ]
+}

plancraft/environments/tags/wool.json

@@ -0,0 +1,21 @@
+{
+  "replace": false,
+  "values": [
+    "minecraft:white_wool",
+    "minecraft:orange_wool",
+    "minecraft:magenta_wool",
+    "minecraft:light_blue_wool",
+    "minecraft:yellow_wool",
+    "minecraft:lime_wool",
+    "minecraft:pink_wool",
+    "minecraft:gray_wool",
+    "minecraft:light_gray_wool",
+    "minecraft:cyan_wool",
+    "minecraft:purple_wool",
+    "minecraft:blue_wool",
+    "minecraft:brown_wool",
+    "minecraft:green_wool",
+    "minecraft:red_wool",
+    "minecraft:black_wool"
+  ]
+}

plancraft/evaluator.py

@@ -0,0 +1,273 @@
+import json
+import os
+import random
+import string
+import time
+
+import imageio
+import pandas as pd
+import torch
+import wandb
+from loguru import logger
+from tqdm import tqdm
+
+from plancraft.config import EvalConfig, PlancraftExample
+from plancraft.environments.actions import StopAction
+from plancraft.environments.env_real import RealPlancraft
+from plancraft.environments.env_symbolic import SymbolicPlancraft
+from plancraft.models import get_model
+
+wandb.require("core")
+
+
+class Evaluator:
+    """
+    The evaluator class handles the environment loop and model interaction
+
+    The environment is created based on the configuration and the examples are loaded from the dataset.
+    """
+
+    def __init__(self, cfg: EvalConfig):
+        self.cfg = cfg
+        self.output_dir = (
+            f"{cfg.plancraft.output_dir}/{self.evaluator_name()}/{cfg.plancraft.split}"
+        )
+        self.generation_number = 0
+
+        self.examples: list[PlancraftExample] = self.load_dataset(cfg.plancraft.split)
+
+        self.environment = self.create_env(cfg)
+        self.model = get_model(cfg)
+
+        self.record_frames = not (cfg.plancraft.environment.symbolic)
+
+        # no_op action
+        self.no_op = self.environment.action_space.no_op()
+
+    def evaluator_name(self) -> str:
+        symb_str = "real"
+        if self.cfg.plancraft.environment.symbolic:
+            symb_str = "symb"
+
+        if self.cfg.plancraft.use_maskrcnn:
+            symb_str += "_mrcnn"
+
+        model_name = self.cfg.plancraft.model.split("/")[-1]
+        if self.cfg.plancraft.adapter != "":
+            model_name = self.cfg.plancraft.adapter.split("/")[-1]
+
+        mode = self.cfg.plancraft.mode
+        if mode in ["dummy", "oracle"]:
+            return f"{mode}_{symb_str}"
+
+        actions = "|".join(self.cfg.plancraft.valid_actions)
+        return f"{self.cfg.plancraft.mode}_{symb_str}_{model_name}_{actions}"
+
+    def save_results_dict(self, example: PlancraftExample, results_dict: dict):
+        output_dir = f"{self.output_dir}/{self.generation_number}"
+        os.makedirs(output_dir, exist_ok=True)
+        json_path = f"{output_dir}/{example.id}.json"
+        with open(json_path, "w") as f:
+            json.dump(results_dict, f, indent=4)
+        wandb.save(json_path, policy="now")
+
+    def save_images(self, example: PlancraftExample, frames: list):
+        if len(frames) == 0:
+            return
+        output_dir = f"{self.output_dir}/{self.generation_number}"
+        os.makedirs(output_dir, exist_ok=True)
+        imageio.mimsave(f"{output_dir}/{example.id}.gif", frames)
+        # upload to wandb
+        wandb.save(f"{output_dir}/{example.id}.gif", policy="now")
+
+    def load_results_dict(self, example: PlancraftExample) -> dict:
+        path = f"{self.output_dir}/{self.generation_number}/{example.id}.json"
+        if not os.path.exists(path) or not self.cfg.plancraft.resume:
+            return None
+        with open(path, "r") as f:
+            return json.load(f)
+
+    def create_env(self, cfg: EvalConfig) -> RealPlancraft | SymbolicPlancraft:
+        if cfg.plancraft.environment.symbolic:
+            return SymbolicPlancraft(inventory=[])
+        return RealPlancraft(
+            inventory=[],
+            symbolic_action_space=cfg.plancraft.environment.symbolic_action_space,
+            symbolic_observation_space=cfg.plancraft.environment.symbolic_observation_space,
+            preferred_spawn_biome=cfg.plancraft.environment.preferred_spawn_biome,
+            resolution=cfg.plancraft.environment.resolution,
+        )
+
+    def close(self):
+        self.environment.close()
+
+    def load_dataset(self, dataset_split: str) -> list[PlancraftExample]:
+        with open(f"data/{dataset_split}.json", "r") as f:
+            dataset = json.load(f)
+            return [PlancraftExample(**example) for example in dataset]
+
+    def reset(
+        self,
+        example: PlancraftExample,
+    ):
+        current_inventory = example.slotted_inventory
+        self.environment.fast_reset(new_inventory=current_inventory)
+        # do a no op to an initial observation
+        obs, _, _, _ = self.environment.step(self.no_op)
+        # assert that the inventory is correct
+        if "inventory" in obs:
+            for item in current_inventory:
+                slot = item["slot"]
+                if (
+                    obs["inventory"][slot]["type"] != item["type"]
+                    or obs["inventory"][slot]["quantity"] != item["quantity"]
+                ) and item["type"] != "air":
+                    logger.warning(f"Inventory does not match expected for slot {slot}")
+                    logger.warning(f"Expected {item}")
+                    logger.warning(f"Got {obs['inventory'][slot]}")
+                    # try again
+                    self.reset(example)
+
+        objective = f"Craft an item of type: {example.target}"
+        self.model.reset_history(objective=objective)
+
+    def check_done(self, inventory: list[dict[str, int]], target: str):
+        """
+        Check that target object is obtained
+        """
+        for item in inventory:
+            if target == item["type"]:
+                # ensure item is taken out of crafting slot
+                if "slot" in item and item["slot"] != 0:
+                    return True
+                if "index" in item and item["index"] != 0:
+                    return True
+        return False
+
+    @torch.no_grad()
+    def eval_all_examples(self, progress_bar=False) -> list:
+        results = []
+        action = self.no_op.copy()
+
+        pbar = tqdm(
+            total=len(self.examples),
+            disable=not progress_bar,
+        )
+        correct = 0
+        count = 0
+
+        for example in self.examples:
+            if resume_result := self.load_results_dict(example):
+                pbar.update(self.cfg.plancraft.max_steps)
+                results.append(resume_result)
+                continue
+
+            success = False
+
+            self.reset(example)
+            action = self.no_op.copy()
+
+            while (
+                not self.model.history.check_stuck()
+                and self.model.history.num_steps < self.cfg.plancraft.max_steps
+            ):
+                # if the action is stop then we end the episode
+                if isinstance(action, StopAction):
+                    # if the action is stop and task is impossible then success
+                    # otherwise we should not have stopped
+                    success = example.impossible
+                    break
+
+                # step action
+                observation, _, _, _ = self.environment.step(action)
+
+                # check if the episode is done
+                success = self.check_done(observation["inventory"], example.target)
+                # exit if success
+                if success:
+                    break
+
+                # predict next action
+                action = self.model.step(observation)
+
+            # save results and reset
+            result = {
+                "success": success,
+                "recipe_type": example.recipe_type,
+                "number_of_steps": self.model.history.num_steps,
+                "model_trace": self.model.history.trace(),
+                "example_id": example.id,
+                "impossible": example.impossible,
+            }
+            results.append(result)
+            self.save_results_dict(example, result)
+            self.save_images(example, self.model.history.images)
+
+            correct += int(result["success"])
+            count += 1
+
+            acc = correct / count
+            pbar.set_postfix(correct=correct, count=count, acc=acc)
+            pbar.update(1)
+
+        return results
+
+    def eval_all(self):
+        logger.info(
+            f"Running evaluation over {len(self.examples)} examples {self.cfg.plancraft.num_generations} times."
+        )
+        run_name = (
+            f"{self.evaluator_name()} {self.cfg.plancraft.split}".replace(" ", "_")
+            .replace(".", "_")
+            .strip()
+        )
+
+        for n in range(self.cfg.plancraft.num_generations):
+            logger.info(f"Generation {n+1}/{self.cfg.plancraft.num_generations}")
+            run_id = "".join(random.choices(string.ascii_lowercase, k=5))
+            generation_run_name = run_name + f"_{run_id}"
+
+            wandb.init(
+                name=generation_run_name,
+                project=self.cfg.wandb.project,
+                entity=self.cfg.wandb.entity,
+                mode=self.cfg.wandb.mode,
+                group=self.cfg.plancraft.model,
+                job_type=self.cfg.plancraft.mode,
+                config=self.cfg.model_dump(),
+            )
+            time_now = time.time()
+
+            results_list = self.eval_all_examples(progress_bar=True)
+
+            results_df = pd.DataFrame(results_list)
+
+            output = {
+                "avg_success_rate": results_df["success"].mean(),
+                "avg_number_of_steps": results_df["number_of_steps"].mean(),
+                "avg_num_tokens_used": results_df["model_trace"]
+                .apply(pd.Series)["tokens_used"]
+                .mean(),
+            }
+
+            # calculate success rate for each recipe type
+            recipe_types = results_df["recipe_type"].unique()
+            for recipe_type in recipe_types:
+                mask = results_df["recipe_type"] == recipe_type
+                success_rate = results_df[mask]["success"].mean()
+                output[f"{recipe_type}_success_rate"] = success_rate
+
+            time_elapsed = time.time() - time_now
+            logger.info(f"Time elapsed: {time_elapsed:.2f}s")
+
+            logger.info(output)
+            wandb.log(output)
+            table = wandb.Table(
+                dataframe=results_df[["success", "number_of_steps", "example_id"]]
+            )
+            wandb.log({"results": table})
+            wandb.finish()
+
+            self.generation_number += 1
+
+        logger.info("Done")

plancraft/models/__init__.py

@@ -0,0 +1,21 @@
+from plancraft.models.base import ABCModel
+
+from plancraft.config import EvalConfig
+from plancraft.models.dummy import DummyModel
+from plancraft.models.react import ReactModel
+from plancraft.models.oracle import OracleModel
+from plancraft.models.act import ActModel
+
+
+def get_model(cfg: EvalConfig) -> ABCModel:
+    """
+    Factory get model (default: ReactModel)
+    """
+    if cfg.plancraft.mode == "dummy":
+        return DummyModel(cfg)
+    elif cfg.plancraft.mode == "oracle":
+        return OracleModel(cfg)
+    elif cfg.plancraft.mode == "act":
+        return ActModel(cfg)
+    else:
+        return ReactModel(cfg)