plancraft 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- plancraft/config.py +155 -0
- plancraft/environments/__init__.py +0 -0
- plancraft/evaluator.py +273 -0
- plancraft/utils.py +84 -0
- plancraft-0.1.3.dist-info/METADATA +74 -0
- plancraft-0.1.3.dist-info/RECORD +30 -0
- {plancraft-0.1.1.dist-info → plancraft-0.1.3.dist-info}/WHEEL +1 -1
- plancraft-0.1.3.dist-info/top_level.txt +1 -0
- plancraft-0.1.1.dist-info/METADATA +0 -74
- plancraft-0.1.1.dist-info/RECORD +0 -26
- plancraft-0.1.1.dist-info/top_level.txt +0 -3
- {environments → plancraft}/__init__.py +0 -0
- {environments → plancraft/environments}/actions.py +0 -0
- {environments → plancraft/environments}/env_real.py +0 -0
- {environments → plancraft/environments}/env_symbolic.py +0 -0
- {environments → plancraft/environments}/items.py +0 -0
- {environments → plancraft/environments}/planner.py +0 -0
- {environments → plancraft/environments}/recipes.py +0 -0
- {environments → plancraft/environments}/sampler.py +0 -0
- {models → plancraft/models}/__init__.py +0 -0
- {models → plancraft/models}/act.py +0 -0
- {models → plancraft/models}/base.py +0 -0
- {models → plancraft/models}/bbox_model.py +0 -0
- {models → plancraft/models}/dummy.py +0 -0
- {models → plancraft/models}/few_shot_images/__init__.py +0 -0
- {models → plancraft/models}/generators.py +0 -0
- {models → plancraft/models}/oam.py +0 -0
- {models → plancraft/models}/oracle.py +0 -0
- {models → plancraft/models}/prompts.py +0 -0
- {models → plancraft/models}/react.py +0 -0
- {models → plancraft/models}/utils.py +0 -0
- {train → plancraft/train}/dataset.py +0 -0
- {plancraft-0.1.1.dist-info → plancraft-0.1.3.dist-info}/LICENSE +0 -0
plancraft/config.py
ADDED
@@ -0,0 +1,155 @@
|
|
1
|
+
from typing import Literal, Optional, Union
|
2
|
+
|
3
|
+
from pydantic import BaseModel, model_validator
|
4
|
+
|
5
|
+
try:
|
6
|
+
from plancraft.environments.recipes import RECIPES
|
7
|
+
except ImportError:
|
8
|
+
RECIPES = {}
|
9
|
+
|
10
|
+
DatasetSplit = Literal[
|
11
|
+
"train", "val", "val.small", "val.small.easy", "test", "test.small"
|
12
|
+
]
|
13
|
+
|
14
|
+
|
15
|
+
class EnvironmentConfig(BaseModel):
|
16
|
+
symbolic: bool
|
17
|
+
symbolic_observation_space: bool
|
18
|
+
symbolic_action_space: bool
|
19
|
+
preferred_spawn_biome: str = "plains"
|
20
|
+
resolution: list[int] = [512, 512]
|
21
|
+
|
22
|
+
|
23
|
+
class PlancraftConfig(BaseModel):
|
24
|
+
model: str
|
25
|
+
adapter: str = ""
|
26
|
+
tokenizer: str
|
27
|
+
num_generations: int
|
28
|
+
mode: Literal["react", "act", "oracle", "dummy"] = "react"
|
29
|
+
output_dir: str
|
30
|
+
max_steps: int = 30 # max number of steps (smelt/move) to take in the environment before stopping
|
31
|
+
quantize: Literal[False, "int4", "int8"]
|
32
|
+
environment: EnvironmentConfig
|
33
|
+
split: DatasetSplit = "val.small"
|
34
|
+
max_message_window: int = 30 # max number of messages to keep in dialogue history (30 is around 8k llama3 tokens)
|
35
|
+
hot_cache: bool = True # whether to cache the dialogue history between steps
|
36
|
+
resume: bool = True # resume inference
|
37
|
+
few_shot: bool = True # whether to use few-shot prompt
|
38
|
+
system_prompt: bool = True # whether to use system prompt
|
39
|
+
valid_actions: list[str] = ["move", "smelt", "think", "search", "impossible"]
|
40
|
+
use_maskrcnn: bool = False # whether to use maskrcnn for multimodal parsing
|
41
|
+
|
42
|
+
# observations
|
43
|
+
use_text_inventory: bool = True # whether to include inventory in text
|
44
|
+
use_images: bool = False # whether to include images in multimodal content
|
45
|
+
use_multimodal_content_format: bool = (
|
46
|
+
False # whether to use multimodal content format
|
47
|
+
)
|
48
|
+
|
49
|
+
@model_validator(mode="after")
|
50
|
+
def validate(self):
|
51
|
+
assert set(
|
52
|
+
self.valid_actions
|
53
|
+
).issubset(
|
54
|
+
{"move", "smelt", "think", "search", "impossible"}
|
55
|
+
), "valid_actions should be subset of {'move', 'smelt', 'think', 'search', 'impossible'}"
|
56
|
+
|
57
|
+
if self.use_images:
|
58
|
+
assert (
|
59
|
+
not self.environment.symbolic
|
60
|
+
), "Set environment.symbolic to False when using images"
|
61
|
+
|
62
|
+
return self
|
63
|
+
|
64
|
+
|
65
|
+
class WandbConfig(BaseModel):
|
66
|
+
project: str
|
67
|
+
entity: str
|
68
|
+
mode: str
|
69
|
+
|
70
|
+
|
71
|
+
class LaunchConfig(BaseModel):
|
72
|
+
command: str
|
73
|
+
job_name: str
|
74
|
+
gpu_limit: int
|
75
|
+
gpu_product: str
|
76
|
+
cpu_request: int
|
77
|
+
ram_request: str
|
78
|
+
interactive: bool = False
|
79
|
+
namespace: str = "informatics"
|
80
|
+
env_vars: dict[str, dict[str, str]]
|
81
|
+
|
82
|
+
|
83
|
+
class EvalConfig(BaseModel):
|
84
|
+
plancraft: PlancraftConfig
|
85
|
+
wandb: WandbConfig
|
86
|
+
launch: LaunchConfig
|
87
|
+
|
88
|
+
|
89
|
+
class TrainingArgs(BaseModel):
|
90
|
+
base_model: str = "llama3"
|
91
|
+
trace_mode: str = "oa"
|
92
|
+
push_to_hub: bool = False
|
93
|
+
|
94
|
+
# uses less space but not working with multi-gpu training..
|
95
|
+
qlora: bool = False
|
96
|
+
|
97
|
+
lora_alpha: int = 16
|
98
|
+
lora_dropout: float = 0.1
|
99
|
+
lora_r: int = 64
|
100
|
+
# training data args
|
101
|
+
seed: int = 42
|
102
|
+
# model args
|
103
|
+
batch_size: int = 1
|
104
|
+
max_seq_length: int = 8142
|
105
|
+
max_message_window: int = 100
|
106
|
+
only_assistant: bool = True
|
107
|
+
|
108
|
+
# training args
|
109
|
+
gradient_accumulation_steps: int = 4
|
110
|
+
learning_rate: float = 2e-4
|
111
|
+
max_grad_norm: float = 0.3
|
112
|
+
warmup_ratio: float = 0.03
|
113
|
+
num_train_epochs: int = 3
|
114
|
+
num_workers: int = 1
|
115
|
+
|
116
|
+
|
117
|
+
class TrainConfig(BaseModel):
|
118
|
+
training: TrainingArgs
|
119
|
+
wandb: WandbConfig
|
120
|
+
launch: LaunchConfig
|
121
|
+
|
122
|
+
|
123
|
+
class PlancraftExample(BaseModel):
|
124
|
+
target: str
|
125
|
+
inventory: dict[str, int]
|
126
|
+
slotted_inventory: list[dict[str, Union[str, int]]]
|
127
|
+
num_distractors: int
|
128
|
+
impossible: bool
|
129
|
+
optimal_path_length: Optional[int]
|
130
|
+
optimal_path: Optional[list[str]]
|
131
|
+
inventory_trace: Optional[list[dict[str, int]]]
|
132
|
+
items_used: Optional[int]
|
133
|
+
unique_items_used: Optional[int]
|
134
|
+
complexity: Optional[int]
|
135
|
+
complexity_bin: int
|
136
|
+
unseen_in_train: bool
|
137
|
+
unseen_in_val: bool
|
138
|
+
split: DatasetSplit
|
139
|
+
id: str
|
140
|
+
|
141
|
+
recipe_type: Optional[str] = ""
|
142
|
+
|
143
|
+
# post processing set recipe type
|
144
|
+
def model_post_init(self, __context):
|
145
|
+
recipe_types = set()
|
146
|
+
if self.optimal_path is None:
|
147
|
+
self.recipe_type = "impossible"
|
148
|
+
return
|
149
|
+
for step in self.optimal_path:
|
150
|
+
for r in RECIPES[step]:
|
151
|
+
recipe_types.add(r.recipe_type)
|
152
|
+
if len(recipe_types) == 1:
|
153
|
+
self.recipe_type = recipe_types.pop()
|
154
|
+
else:
|
155
|
+
self.recipe_type = "mixed"
|
File without changes
|
plancraft/evaluator.py
ADDED
@@ -0,0 +1,273 @@
|
|
1
|
+
import json
|
2
|
+
import os
|
3
|
+
import random
|
4
|
+
import string
|
5
|
+
import time
|
6
|
+
|
7
|
+
import imageio
|
8
|
+
import pandas as pd
|
9
|
+
import torch
|
10
|
+
import wandb
|
11
|
+
from loguru import logger
|
12
|
+
from tqdm import tqdm
|
13
|
+
|
14
|
+
from plancraft.config import EvalConfig, PlancraftExample
|
15
|
+
from plancraft.environments.actions import StopAction
|
16
|
+
from plancraft.environments.env_real import RealPlancraft
|
17
|
+
from plancraft.environments.env_symbolic import SymbolicPlancraft
|
18
|
+
from plancraft.models import get_model
|
19
|
+
|
20
|
+
wandb.require("core")
|
21
|
+
|
22
|
+
|
23
|
+
class Evaluator:
|
24
|
+
"""
|
25
|
+
The evaluator class handles the environment loop and model interaction
|
26
|
+
|
27
|
+
The environment is created based on the configuration and the examples are loaded from the dataset.
|
28
|
+
"""
|
29
|
+
|
30
|
+
def __init__(self, cfg: EvalConfig):
|
31
|
+
self.cfg = cfg
|
32
|
+
self.output_dir = (
|
33
|
+
f"{cfg.plancraft.output_dir}/{self.evaluator_name()}/{cfg.plancraft.split}"
|
34
|
+
)
|
35
|
+
self.generation_number = 0
|
36
|
+
|
37
|
+
self.examples: list[PlancraftExample] = self.load_dataset(cfg.plancraft.split)
|
38
|
+
|
39
|
+
self.environment = self.create_env(cfg)
|
40
|
+
self.model = get_model(cfg)
|
41
|
+
|
42
|
+
self.record_frames = not (cfg.plancraft.environment.symbolic)
|
43
|
+
|
44
|
+
# no_op action
|
45
|
+
self.no_op = self.environment.action_space.no_op()
|
46
|
+
|
47
|
+
def evaluator_name(self) -> str:
|
48
|
+
symb_str = "real"
|
49
|
+
if self.cfg.plancraft.environment.symbolic:
|
50
|
+
symb_str = "symb"
|
51
|
+
|
52
|
+
if self.cfg.plancraft.use_maskrcnn:
|
53
|
+
symb_str += "_mrcnn"
|
54
|
+
|
55
|
+
model_name = self.cfg.plancraft.model.split("/")[-1]
|
56
|
+
if self.cfg.plancraft.adapter != "":
|
57
|
+
model_name = self.cfg.plancraft.adapter.split("/")[-1]
|
58
|
+
|
59
|
+
mode = self.cfg.plancraft.mode
|
60
|
+
if mode in ["dummy", "oracle"]:
|
61
|
+
return f"{mode}_{symb_str}"
|
62
|
+
|
63
|
+
actions = "|".join(self.cfg.plancraft.valid_actions)
|
64
|
+
return f"{self.cfg.plancraft.mode}_{symb_str}_{model_name}_{actions}"
|
65
|
+
|
66
|
+
def save_results_dict(self, example: PlancraftExample, results_dict: dict):
|
67
|
+
output_dir = f"{self.output_dir}/{self.generation_number}"
|
68
|
+
os.makedirs(output_dir, exist_ok=True)
|
69
|
+
json_path = f"{output_dir}/{example.id}.json"
|
70
|
+
with open(json_path, "w") as f:
|
71
|
+
json.dump(results_dict, f, indent=4)
|
72
|
+
wandb.save(json_path, policy="now")
|
73
|
+
|
74
|
+
def save_images(self, example: PlancraftExample, frames: list):
|
75
|
+
if len(frames) == 0:
|
76
|
+
return
|
77
|
+
output_dir = f"{self.output_dir}/{self.generation_number}"
|
78
|
+
os.makedirs(output_dir, exist_ok=True)
|
79
|
+
imageio.mimsave(f"{output_dir}/{example.id}.gif", frames)
|
80
|
+
# upload to wandb
|
81
|
+
wandb.save(f"{output_dir}/{example.id}.gif", policy="now")
|
82
|
+
|
83
|
+
def load_results_dict(self, example: PlancraftExample) -> dict:
|
84
|
+
path = f"{self.output_dir}/{self.generation_number}/{example.id}.json"
|
85
|
+
if not os.path.exists(path) or not self.cfg.plancraft.resume:
|
86
|
+
return None
|
87
|
+
with open(path, "r") as f:
|
88
|
+
return json.load(f)
|
89
|
+
|
90
|
+
def create_env(self, cfg: EvalConfig) -> RealPlancraft | SymbolicPlancraft:
|
91
|
+
if cfg.plancraft.environment.symbolic:
|
92
|
+
return SymbolicPlancraft(inventory=[])
|
93
|
+
return RealPlancraft(
|
94
|
+
inventory=[],
|
95
|
+
symbolic_action_space=cfg.plancraft.environment.symbolic_action_space,
|
96
|
+
symbolic_observation_space=cfg.plancraft.environment.symbolic_observation_space,
|
97
|
+
preferred_spawn_biome=cfg.plancraft.environment.preferred_spawn_biome,
|
98
|
+
resolution=cfg.plancraft.environment.resolution,
|
99
|
+
)
|
100
|
+
|
101
|
+
def close(self):
|
102
|
+
self.environment.close()
|
103
|
+
|
104
|
+
def load_dataset(self, dataset_split: str) -> list[PlancraftExample]:
|
105
|
+
with open(f"data/{dataset_split}.json", "r") as f:
|
106
|
+
dataset = json.load(f)
|
107
|
+
return [PlancraftExample(**example) for example in dataset]
|
108
|
+
|
109
|
+
def reset(
|
110
|
+
self,
|
111
|
+
example: PlancraftExample,
|
112
|
+
):
|
113
|
+
current_inventory = example.slotted_inventory
|
114
|
+
self.environment.fast_reset(new_inventory=current_inventory)
|
115
|
+
# do a no op to an initial observation
|
116
|
+
obs, _, _, _ = self.environment.step(self.no_op)
|
117
|
+
# assert that the inventory is correct
|
118
|
+
if "inventory" in obs:
|
119
|
+
for item in current_inventory:
|
120
|
+
slot = item["slot"]
|
121
|
+
if (
|
122
|
+
obs["inventory"][slot]["type"] != item["type"]
|
123
|
+
or obs["inventory"][slot]["quantity"] != item["quantity"]
|
124
|
+
) and item["type"] != "air":
|
125
|
+
logger.warning(f"Inventory does not match expected for slot {slot}")
|
126
|
+
logger.warning(f"Expected {item}")
|
127
|
+
logger.warning(f"Got {obs['inventory'][slot]}")
|
128
|
+
# try again
|
129
|
+
self.reset(example)
|
130
|
+
|
131
|
+
objective = f"Craft an item of type: {example.target}"
|
132
|
+
self.model.reset_history(objective=objective)
|
133
|
+
|
134
|
+
def check_done(self, inventory: list[dict[str, int]], target: str):
|
135
|
+
"""
|
136
|
+
Check that target object is obtained
|
137
|
+
"""
|
138
|
+
for item in inventory:
|
139
|
+
if target == item["type"]:
|
140
|
+
# ensure item is taken out of crafting slot
|
141
|
+
if "slot" in item and item["slot"] != 0:
|
142
|
+
return True
|
143
|
+
if "index" in item and item["index"] != 0:
|
144
|
+
return True
|
145
|
+
return False
|
146
|
+
|
147
|
+
@torch.no_grad()
|
148
|
+
def eval_all_examples(self, progress_bar=False) -> list:
|
149
|
+
results = []
|
150
|
+
action = self.no_op.copy()
|
151
|
+
|
152
|
+
pbar = tqdm(
|
153
|
+
total=len(self.examples),
|
154
|
+
disable=not progress_bar,
|
155
|
+
)
|
156
|
+
correct = 0
|
157
|
+
count = 0
|
158
|
+
|
159
|
+
for example in self.examples:
|
160
|
+
if resume_result := self.load_results_dict(example):
|
161
|
+
pbar.update(self.cfg.plancraft.max_steps)
|
162
|
+
results.append(resume_result)
|
163
|
+
continue
|
164
|
+
|
165
|
+
success = False
|
166
|
+
|
167
|
+
self.reset(example)
|
168
|
+
action = self.no_op.copy()
|
169
|
+
|
170
|
+
while (
|
171
|
+
not self.model.history.check_stuck()
|
172
|
+
and self.model.history.num_steps < self.cfg.plancraft.max_steps
|
173
|
+
):
|
174
|
+
# if the action is stop then we end the episode
|
175
|
+
if isinstance(action, StopAction):
|
176
|
+
# if the action is stop and task is impossible then success
|
177
|
+
# otherwise we should not have stopped
|
178
|
+
success = example.impossible
|
179
|
+
break
|
180
|
+
|
181
|
+
# step action
|
182
|
+
observation, _, _, _ = self.environment.step(action)
|
183
|
+
|
184
|
+
# check if the episode is done
|
185
|
+
success = self.check_done(observation["inventory"], example.target)
|
186
|
+
# exit if success
|
187
|
+
if success:
|
188
|
+
break
|
189
|
+
|
190
|
+
# predict next action
|
191
|
+
action = self.model.step(observation)
|
192
|
+
|
193
|
+
# save results and reset
|
194
|
+
result = {
|
195
|
+
"success": success,
|
196
|
+
"recipe_type": example.recipe_type,
|
197
|
+
"number_of_steps": self.model.history.num_steps,
|
198
|
+
"model_trace": self.model.history.trace(),
|
199
|
+
"example_id": example.id,
|
200
|
+
"impossible": example.impossible,
|
201
|
+
}
|
202
|
+
results.append(result)
|
203
|
+
self.save_results_dict(example, result)
|
204
|
+
self.save_images(example, self.model.history.images)
|
205
|
+
|
206
|
+
correct += int(result["success"])
|
207
|
+
count += 1
|
208
|
+
|
209
|
+
acc = correct / count
|
210
|
+
pbar.set_postfix(correct=correct, count=count, acc=acc)
|
211
|
+
pbar.update(1)
|
212
|
+
|
213
|
+
return results
|
214
|
+
|
215
|
+
def eval_all(self):
|
216
|
+
logger.info(
|
217
|
+
f"Running evaluation over {len(self.examples)} examples {self.cfg.plancraft.num_generations} times."
|
218
|
+
)
|
219
|
+
run_name = (
|
220
|
+
f"{self.evaluator_name()} {self.cfg.plancraft.split}".replace(" ", "_")
|
221
|
+
.replace(".", "_")
|
222
|
+
.strip()
|
223
|
+
)
|
224
|
+
|
225
|
+
for n in range(self.cfg.plancraft.num_generations):
|
226
|
+
logger.info(f"Generation {n+1}/{self.cfg.plancraft.num_generations}")
|
227
|
+
run_id = "".join(random.choices(string.ascii_lowercase, k=5))
|
228
|
+
generation_run_name = run_name + f"_{run_id}"
|
229
|
+
|
230
|
+
wandb.init(
|
231
|
+
name=generation_run_name,
|
232
|
+
project=self.cfg.wandb.project,
|
233
|
+
entity=self.cfg.wandb.entity,
|
234
|
+
mode=self.cfg.wandb.mode,
|
235
|
+
group=self.cfg.plancraft.model,
|
236
|
+
job_type=self.cfg.plancraft.mode,
|
237
|
+
config=self.cfg.model_dump(),
|
238
|
+
)
|
239
|
+
time_now = time.time()
|
240
|
+
|
241
|
+
results_list = self.eval_all_examples(progress_bar=True)
|
242
|
+
|
243
|
+
results_df = pd.DataFrame(results_list)
|
244
|
+
|
245
|
+
output = {
|
246
|
+
"avg_success_rate": results_df["success"].mean(),
|
247
|
+
"avg_number_of_steps": results_df["number_of_steps"].mean(),
|
248
|
+
"avg_num_tokens_used": results_df["model_trace"]
|
249
|
+
.apply(pd.Series)["tokens_used"]
|
250
|
+
.mean(),
|
251
|
+
}
|
252
|
+
|
253
|
+
# calculate success rate for each recipe type
|
254
|
+
recipe_types = results_df["recipe_type"].unique()
|
255
|
+
for recipe_type in recipe_types:
|
256
|
+
mask = results_df["recipe_type"] == recipe_type
|
257
|
+
success_rate = results_df[mask]["success"].mean()
|
258
|
+
output[f"{recipe_type}_success_rate"] = success_rate
|
259
|
+
|
260
|
+
time_elapsed = time.time() - time_now
|
261
|
+
logger.info(f"Time elapsed: {time_elapsed:.2f}s")
|
262
|
+
|
263
|
+
logger.info(output)
|
264
|
+
wandb.log(output)
|
265
|
+
table = wandb.Table(
|
266
|
+
dataframe=results_df[["success", "number_of_steps", "example_id"]]
|
267
|
+
)
|
268
|
+
wandb.log({"results": table})
|
269
|
+
wandb.finish()
|
270
|
+
|
271
|
+
self.generation_number += 1
|
272
|
+
|
273
|
+
logger.info("Done")
|
plancraft/utils.py
ADDED
@@ -0,0 +1,84 @@
|
|
1
|
+
import glob
|
2
|
+
import pathlib
|
3
|
+
|
4
|
+
import cv2
|
5
|
+
import numpy as np
|
6
|
+
import torch
|
7
|
+
import torch.nn.functional as F
|
8
|
+
from PIL import Image
|
9
|
+
|
10
|
+
|
11
|
+
def get_downloaded_models() -> dict:
|
12
|
+
"""
|
13
|
+
Get the list of downloaded models on the NFS partition (EIDF).
|
14
|
+
"""
|
15
|
+
downloaded_models = {}
|
16
|
+
# known models on NFS partition
|
17
|
+
if pathlib.Path("/nfs").exists():
|
18
|
+
local_models = glob.glob("/nfs/public/hf/models/*/*")
|
19
|
+
downloaded_models = {
|
20
|
+
model.replace("/nfs/public/hf/models/", ""): model for model in local_models
|
21
|
+
}
|
22
|
+
return downloaded_models
|
23
|
+
|
24
|
+
|
25
|
+
def get_torch_device() -> torch.device:
|
26
|
+
device = torch.device("cpu")
|
27
|
+
if torch.cuda.is_available():
|
28
|
+
device = torch.device("cuda", 0)
|
29
|
+
elif torch.backends.mps.is_available():
|
30
|
+
if not torch.backends.mps.is_built():
|
31
|
+
print(
|
32
|
+
"MPS not available because the current PyTorch install was not built with MPS enabled."
|
33
|
+
)
|
34
|
+
else:
|
35
|
+
device = torch.device("mps")
|
36
|
+
return device
|
37
|
+
|
38
|
+
|
39
|
+
def resize_image(img, target_resolution=(128, 128)):
|
40
|
+
if type(img) == np.ndarray:
|
41
|
+
img = cv2.resize(img, target_resolution, interpolation=cv2.INTER_LINEAR)
|
42
|
+
elif type(img) == torch.Tensor:
|
43
|
+
img = F.interpolate(img, size=target_resolution, mode="bilinear")
|
44
|
+
else:
|
45
|
+
raise ValueError("Unsupported type for img")
|
46
|
+
return img
|
47
|
+
|
48
|
+
|
49
|
+
def save_frames_to_video(frames: list, out_path: str):
|
50
|
+
imgs = []
|
51
|
+
for id, (frame, goal) in enumerate(frames):
|
52
|
+
# if torch.is_tensor(frame):
|
53
|
+
# frame = frame.permute(0, 2, 3, 1).cpu().numpy()
|
54
|
+
|
55
|
+
frame = resize_image(frame, (320, 240)).astype("uint8")
|
56
|
+
cv2.putText(
|
57
|
+
frame,
|
58
|
+
f"FID: {id}",
|
59
|
+
(10, 25),
|
60
|
+
cv2.FONT_HERSHEY_SIMPLEX,
|
61
|
+
0.8,
|
62
|
+
(255, 255, 255),
|
63
|
+
2,
|
64
|
+
)
|
65
|
+
cv2.putText(
|
66
|
+
frame,
|
67
|
+
f"Goal: {goal}",
|
68
|
+
(10, 55),
|
69
|
+
cv2.FONT_HERSHEY_SIMPLEX,
|
70
|
+
0.8,
|
71
|
+
(255, 0, 0),
|
72
|
+
2,
|
73
|
+
)
|
74
|
+
imgs.append(Image.fromarray(frame))
|
75
|
+
imgs = imgs[::3]
|
76
|
+
imgs[0].save(
|
77
|
+
out_path,
|
78
|
+
save_all=True,
|
79
|
+
append_images=imgs[1:],
|
80
|
+
optimize=False,
|
81
|
+
quality=0,
|
82
|
+
duration=150,
|
83
|
+
loop=0,
|
84
|
+
)
|
@@ -0,0 +1,74 @@
|
|
1
|
+
Metadata-Version: 2.1
|
2
|
+
Name: plancraft
|
3
|
+
Version: 0.1.3
|
4
|
+
Summary: Plancraft: an evaluation dataset for planning with LLM agents
|
5
|
+
Requires-Python: >=3.9
|
6
|
+
Description-Content-Type: text/markdown
|
7
|
+
License-File: LICENSE
|
8
|
+
Requires-Dist: accelerate>=1.0.1
|
9
|
+
Requires-Dist: coloredlogs>=10.0
|
10
|
+
Requires-Dist: daemoniker>=0.2.3
|
11
|
+
Requires-Dist: datasets>=3.0.2
|
12
|
+
Requires-Dist: dill>=0.3.1.1
|
13
|
+
Requires-Dist: einops>=0.8.0
|
14
|
+
Requires-Dist: flaky>=3.8.1
|
15
|
+
Requires-Dist: hf-transfer>=0.1.8
|
16
|
+
Requires-Dist: huggingface-hub>=0.26.1
|
17
|
+
Requires-Dist: hydra-core>=1.3.2
|
18
|
+
Requires-Dist: imagehash>=4.0.0
|
19
|
+
Requires-Dist: imageio>=2.36.0
|
20
|
+
Requires-Dist: inflection>=0.3.1
|
21
|
+
Requires-Dist: ipython>=7.5.0
|
22
|
+
Requires-Dist: jinja2>=2.11.2
|
23
|
+
Requires-Dist: loguru>=0.7.2
|
24
|
+
Requires-Dist: lxml>=4.3.3
|
25
|
+
Requires-Dist: matplotlib>=3.9.2
|
26
|
+
Requires-Dist: networkx>=3.2.1
|
27
|
+
Requires-Dist: numpy<1.24,>=1.16.2
|
28
|
+
Requires-Dist: openai>=1.52.2
|
29
|
+
Requires-Dist: opencv-python>=4.1.0.25
|
30
|
+
Requires-Dist: pandas>=2.1.0
|
31
|
+
Requires-Dist: peft>=0.13.2
|
32
|
+
Requires-Dist: pillow>=8.0.0
|
33
|
+
Requires-Dist: psutil>=5.6.2
|
34
|
+
Requires-Dist: pydantic>=2.9.2
|
35
|
+
Requires-Dist: pyglet>=2.0.18
|
36
|
+
Requires-Dist: pyro4>=4.76
|
37
|
+
Requires-Dist: python-dotenv>=1.0.1
|
38
|
+
Requires-Dist: pyyaml>=6.0.2
|
39
|
+
Requires-Dist: requests>=2.20.0
|
40
|
+
Requires-Dist: seaborn>=0.13.2
|
41
|
+
Requires-Dist: setuptools>=49.2.0
|
42
|
+
Requires-Dist: tinydb>=4.8.2
|
43
|
+
Requires-Dist: torch>=2.5.0
|
44
|
+
Requires-Dist: torchvision>=0.20.0
|
45
|
+
Requires-Dist: tqdm>=4.32.2
|
46
|
+
Requires-Dist: transformers>=4.43.3
|
47
|
+
Requires-Dist: typing>=3.6.6
|
48
|
+
Requires-Dist: wandb>=0.18.5
|
49
|
+
Requires-Dist: xmltodict==0.12.0
|
50
|
+
Provides-Extra: full
|
51
|
+
Requires-Dist: gym<=0.23.1,>=0.19.0; extra == "full"
|
52
|
+
|
53
|
+
# plancraft
|
54
|
+
|
55
|
+
[](https://github.com/gautierdag/plancraft/actions/workflows/test.yaml)
|
56
|
+

|
57
|
+

|
58
|
+
[](https://pypi.org/project/plancraft/)
|
59
|
+
|
60
|
+
Plancraft is a minecraft environment and agent that innovates on planning LLM agents with a retriever
|
61
|
+
|
62
|
+
You can install the package by running the following command:
|
63
|
+
|
64
|
+
```bash
|
65
|
+
pip install plancraft
|
66
|
+
```
|
67
|
+
|
68
|
+
Should you need the multimodal version of the package, you will also need a custom [fork](https://github.com/gautierdag/minerl.git) of the minerl package. You can install it by running the following command:
|
69
|
+
|
70
|
+
```bash
|
71
|
+
pip install git+hhttps://github.com/gautierdag/minerl.git
|
72
|
+
```
|
73
|
+
|
74
|
+
Note that you may need to follow the same installation instructions as in the [minerl documentation](https://minerl.readthedocs.io/en/latest/tutorials/index.html).
|
@@ -0,0 +1,30 @@
|
|
1
|
+
plancraft/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
2
|
+
plancraft/config.py,sha256=k7Ac6Zh6g-Q-fuazPxVdXnSL_PMd3jsIqyiImNaJW5A,4472
|
3
|
+
plancraft/evaluator.py,sha256=zomShgwyUOHNVZbTERpimgEYiyD4JXzHzPi4tg45lsM,9926
|
4
|
+
plancraft/utils.py,sha256=J9K81zBXWCcyIBjwu_eHznnzWamXU-pQrz7BdNTVbLU,2289
|
5
|
+
plancraft/environments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
6
|
+
plancraft/environments/actions.py,sha256=SeeC9l1cJBs9pdba6BefQ_iQNfFf6FVTWm7HWkacbsY,6262
|
7
|
+
plancraft/environments/env_real.py,sha256=oETMvdq8-TspPoIRxioTDj9EEmuZRuuMxd9c3momvPk,10877
|
8
|
+
plancraft/environments/env_symbolic.py,sha256=ot4IStZ3oT7CYPqdqHsGl2BopLyZUwSF671SrHIiMLk,7777
|
9
|
+
plancraft/environments/items.py,sha256=1R56LyK6tqIssQJMqHst6A9DeEfOX5DN-OBAkumGncw,217
|
10
|
+
plancraft/environments/planner.py,sha256=2B-0aunllmTuiHE7Jn5jHzCg6mMgxZjisiTDdpSKupk,3954
|
11
|
+
plancraft/environments/recipes.py,sha256=nXvOLCRljiZ5IgeevXuosU9IgSs7oQWQJFiuyRNSVFs,19571
|
12
|
+
plancraft/environments/sampler.py,sha256=ZBYoENKdQQ7wbAyVk-c9UNRWKPE0omv9he8c8QZ6wXg,7625
|
13
|
+
plancraft/models/__init__.py,sha256=PasK3jpbhpD0kxF4iHukcccZqvZg6lL240zie3DfLDY,622
|
14
|
+
plancraft/models/act.py,sha256=jdZunT7FcbHvcaJZ_wUDLSuObHjU6JZybghR_B0QJ8Q,6548
|
15
|
+
plancraft/models/base.py,sha256=fFM2BV9PqvIFtUlTz8iz5HPemYRy3S0EituM1XdJJSQ,4927
|
16
|
+
plancraft/models/bbox_model.py,sha256=CoX-odH59S-djkPOH2ViEmbYWo1sefmHiOcBlFWiAkg,16814
|
17
|
+
plancraft/models/dummy.py,sha256=QjxTIiKsWSmhUMAuw7Yy-OKKCLi_x3rwll4hH7ZNXso,1732
|
18
|
+
plancraft/models/generators.py,sha256=c2avYUoXAAWOHRFq24ZYqHqNgoPs_L_XWJrR_TKFg9E,17358
|
19
|
+
plancraft/models/oam.py,sha256=_TpxsaCnGvQ7YJC6KWcpl2HzgBiQaHn7U1zRAXfdjOo,9943
|
20
|
+
plancraft/models/oracle.py,sha256=xRplR2cCW_39UGYtSjDLRDmp0UILhtIXYjuRJ-jokJ8,9598
|
21
|
+
plancraft/models/prompts.py,sha256=XwoRqd_5_VfCUXb10dCRFYXgw70mO2VoQocn3Z2zgs0,6165
|
22
|
+
plancraft/models/react.py,sha256=5yM4tv7tDfm_-5yUSAcw5C4wioWijF9fLEODbFqFDvg,3346
|
23
|
+
plancraft/models/utils.py,sha256=osKX0_uux9wzqYzq1ST0Cu5idrAnyfNvXrj0uO1eKo0,9424
|
24
|
+
plancraft/models/few_shot_images/__init__.py,sha256=nIkyB6w3ok-h4lfJKsrcMQzQF624Y9uxYV1FqCu3Lx0,351
|
25
|
+
plancraft/train/dataset.py,sha256=NrZjbIkosui1kaq7AIWSrYvvzrDxu_njH7FmGKY3xnI,5434
|
26
|
+
plancraft-0.1.3.dist-info/LICENSE,sha256=YGR8ehDB4t-T-lOQKMfKNR-2zsOU7E3E5NA8t25HKE0,1070
|
27
|
+
plancraft-0.1.3.dist-info/METADATA,sha256=XLudjSvyfZ-uYJZ1K1RAJUVOQhozFD3FLzwUQeikhK4,2631
|
28
|
+
plancraft-0.1.3.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
|
29
|
+
plancraft-0.1.3.dist-info/top_level.txt,sha256=yGGA9HtPKH2-pIJFrbyYhj2JF9Xj-4m0fxMPKT9FNzg,10
|
30
|
+
plancraft-0.1.3.dist-info/RECORD,,
|
@@ -0,0 +1 @@
|
|
1
|
+
plancraft
|
@@ -1,74 +0,0 @@
|
|
1
|
-
Metadata-Version: 2.1
|
2
|
-
Name: plancraft
|
3
|
-
Version: 0.1.1
|
4
|
-
Summary: Plancraft: an evaluation dataset for planning with LLM agents
|
5
|
-
Requires-Python: >=3.9
|
6
|
-
Description-Content-Type: text/markdown
|
7
|
-
License-File: LICENSE
|
8
|
-
Requires-Dist: accelerate >=1.0.1
|
9
|
-
Requires-Dist: coloredlogs >=10.0
|
10
|
-
Requires-Dist: daemoniker >=0.2.3
|
11
|
-
Requires-Dist: datasets >=3.0.2
|
12
|
-
Requires-Dist: dill >=0.3.1.1
|
13
|
-
Requires-Dist: einops >=0.8.0
|
14
|
-
Requires-Dist: flaky >=3.8.1
|
15
|
-
Requires-Dist: hf-transfer >=0.1.8
|
16
|
-
Requires-Dist: huggingface-hub >=0.26.1
|
17
|
-
Requires-Dist: hydra-core >=1.3.2
|
18
|
-
Requires-Dist: imagehash >=4.0.0
|
19
|
-
Requires-Dist: imageio >=2.36.0
|
20
|
-
Requires-Dist: inflection >=0.3.1
|
21
|
-
Requires-Dist: ipython >=7.5.0
|
22
|
-
Requires-Dist: jinja2 >=2.11.2
|
23
|
-
Requires-Dist: loguru >=0.7.2
|
24
|
-
Requires-Dist: lxml >=4.3.3
|
25
|
-
Requires-Dist: matplotlib >=3.9.2
|
26
|
-
Requires-Dist: networkx >=3.2.1
|
27
|
-
Requires-Dist: numpy <1.24,>=1.16.2
|
28
|
-
Requires-Dist: openai >=1.52.2
|
29
|
-
Requires-Dist: opencv-python >=4.1.0.25
|
30
|
-
Requires-Dist: pandas >=2.1.0
|
31
|
-
Requires-Dist: peft >=0.13.2
|
32
|
-
Requires-Dist: pillow >=8.0.0
|
33
|
-
Requires-Dist: psutil >=5.6.2
|
34
|
-
Requires-Dist: pydantic >=2.9.2
|
35
|
-
Requires-Dist: pyglet >=2.0.18
|
36
|
-
Requires-Dist: pyro4 >=4.76
|
37
|
-
Requires-Dist: python-dotenv >=1.0.1
|
38
|
-
Requires-Dist: pyyaml >=6.0.2
|
39
|
-
Requires-Dist: requests >=2.20.0
|
40
|
-
Requires-Dist: seaborn >=0.13.2
|
41
|
-
Requires-Dist: setuptools >=49.2.0
|
42
|
-
Requires-Dist: tinydb >=4.8.2
|
43
|
-
Requires-Dist: torch >=2.5.0
|
44
|
-
Requires-Dist: torchvision >=0.20.0
|
45
|
-
Requires-Dist: tqdm >=4.32.2
|
46
|
-
Requires-Dist: transformers >=4.43.3
|
47
|
-
Requires-Dist: typing >=3.6.6
|
48
|
-
Requires-Dist: wandb >=0.18.5
|
49
|
-
Requires-Dist: xmltodict ==0.12.0
|
50
|
-
Provides-Extra: full
|
51
|
-
Requires-Dist: gym <=0.23.1,>=0.19.0 ; extra == 'full'
|
52
|
-
|
53
|
-
# plancraft
|
54
|
-
|
55
|
-
[](https://github.com/gautierdag/plancraft/actions/workflows/test.yaml)
|
56
|
-

|
57
|
-

|
58
|
-
[](https://pypi.org/project/plancraft/)
|
59
|
-
|
60
|
-
Plancraft is a minecraft environment and agent that innovates on planning LLM agents with a retriever
|
61
|
-
|
62
|
-
You can install the package by running the following command:
|
63
|
-
|
64
|
-
```bash
|
65
|
-
pip install plancraft
|
66
|
-
```
|
67
|
-
|
68
|
-
Should you need the multimodal version of the package, you will also need a custom [fork](https://github.com/gautierdag/minerl.git) of the minerl package. You can install it by running the following command:
|
69
|
-
|
70
|
-
```bash
|
71
|
-
pip install git+hhttps://github.com/gautierdag/minerl.git
|
72
|
-
```
|
73
|
-
|
74
|
-
Note that you may need to follow the same installation instructions as in the [minerl documentation](https://minerl.readthedocs.io/en/latest/tutorials/index.html).
|
plancraft-0.1.1.dist-info/RECORD
DELETED
@@ -1,26 +0,0 @@
|
|
1
|
-
environments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
2
|
-
environments/actions.py,sha256=SeeC9l1cJBs9pdba6BefQ_iQNfFf6FVTWm7HWkacbsY,6262
|
3
|
-
environments/env_real.py,sha256=oETMvdq8-TspPoIRxioTDj9EEmuZRuuMxd9c3momvPk,10877
|
4
|
-
environments/env_symbolic.py,sha256=ot4IStZ3oT7CYPqdqHsGl2BopLyZUwSF671SrHIiMLk,7777
|
5
|
-
environments/items.py,sha256=1R56LyK6tqIssQJMqHst6A9DeEfOX5DN-OBAkumGncw,217
|
6
|
-
environments/planner.py,sha256=2B-0aunllmTuiHE7Jn5jHzCg6mMgxZjisiTDdpSKupk,3954
|
7
|
-
environments/recipes.py,sha256=nXvOLCRljiZ5IgeevXuosU9IgSs7oQWQJFiuyRNSVFs,19571
|
8
|
-
environments/sampler.py,sha256=ZBYoENKdQQ7wbAyVk-c9UNRWKPE0omv9he8c8QZ6wXg,7625
|
9
|
-
models/__init__.py,sha256=PasK3jpbhpD0kxF4iHukcccZqvZg6lL240zie3DfLDY,622
|
10
|
-
models/act.py,sha256=jdZunT7FcbHvcaJZ_wUDLSuObHjU6JZybghR_B0QJ8Q,6548
|
11
|
-
models/base.py,sha256=fFM2BV9PqvIFtUlTz8iz5HPemYRy3S0EituM1XdJJSQ,4927
|
12
|
-
models/bbox_model.py,sha256=CoX-odH59S-djkPOH2ViEmbYWo1sefmHiOcBlFWiAkg,16814
|
13
|
-
models/dummy.py,sha256=QjxTIiKsWSmhUMAuw7Yy-OKKCLi_x3rwll4hH7ZNXso,1732
|
14
|
-
models/generators.py,sha256=c2avYUoXAAWOHRFq24ZYqHqNgoPs_L_XWJrR_TKFg9E,17358
|
15
|
-
models/oam.py,sha256=_TpxsaCnGvQ7YJC6KWcpl2HzgBiQaHn7U1zRAXfdjOo,9943
|
16
|
-
models/oracle.py,sha256=xRplR2cCW_39UGYtSjDLRDmp0UILhtIXYjuRJ-jokJ8,9598
|
17
|
-
models/prompts.py,sha256=XwoRqd_5_VfCUXb10dCRFYXgw70mO2VoQocn3Z2zgs0,6165
|
18
|
-
models/react.py,sha256=5yM4tv7tDfm_-5yUSAcw5C4wioWijF9fLEODbFqFDvg,3346
|
19
|
-
models/utils.py,sha256=osKX0_uux9wzqYzq1ST0Cu5idrAnyfNvXrj0uO1eKo0,9424
|
20
|
-
models/few_shot_images/__init__.py,sha256=nIkyB6w3ok-h4lfJKsrcMQzQF624Y9uxYV1FqCu3Lx0,351
|
21
|
-
train/dataset.py,sha256=NrZjbIkosui1kaq7AIWSrYvvzrDxu_njH7FmGKY3xnI,5434
|
22
|
-
plancraft-0.1.1.dist-info/LICENSE,sha256=YGR8ehDB4t-T-lOQKMfKNR-2zsOU7E3E5NA8t25HKE0,1070
|
23
|
-
plancraft-0.1.1.dist-info/METADATA,sha256=Cj-_TAlOz4PGzed9FVVTrpI0Oc0wk2s94gOSd0XaNTE,2675
|
24
|
-
plancraft-0.1.1.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
|
25
|
-
plancraft-0.1.1.dist-info/top_level.txt,sha256=ZT60unZw3qNbZoGMCVc-V_0hI4YIYvTVGGScIgkCa88,26
|
26
|
-
plancraft-0.1.1.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|