plancraft 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- plancraft/config.py +155 -0
- plancraft/environments/__init__.py +0 -0
- plancraft/evaluator.py +273 -0
- plancraft/utils.py +84 -0
- plancraft-0.1.3.dist-info/METADATA +74 -0
- plancraft-0.1.3.dist-info/RECORD +30 -0
- {plancraft-0.1.1.dist-info → plancraft-0.1.3.dist-info}/WHEEL +1 -1
- plancraft-0.1.3.dist-info/top_level.txt +1 -0
- plancraft-0.1.1.dist-info/METADATA +0 -74
- plancraft-0.1.1.dist-info/RECORD +0 -26
- plancraft-0.1.1.dist-info/top_level.txt +0 -3
- {environments → plancraft}/__init__.py +0 -0
- {environments → plancraft/environments}/actions.py +0 -0
- {environments → plancraft/environments}/env_real.py +0 -0
- {environments → plancraft/environments}/env_symbolic.py +0 -0
- {environments → plancraft/environments}/items.py +0 -0
- {environments → plancraft/environments}/planner.py +0 -0
- {environments → plancraft/environments}/recipes.py +0 -0
- {environments → plancraft/environments}/sampler.py +0 -0
- {models → plancraft/models}/__init__.py +0 -0
- {models → plancraft/models}/act.py +0 -0
- {models → plancraft/models}/base.py +0 -0
- {models → plancraft/models}/bbox_model.py +0 -0
- {models → plancraft/models}/dummy.py +0 -0
- {models → plancraft/models}/few_shot_images/__init__.py +0 -0
- {models → plancraft/models}/generators.py +0 -0
- {models → plancraft/models}/oam.py +0 -0
- {models → plancraft/models}/oracle.py +0 -0
- {models → plancraft/models}/prompts.py +0 -0
- {models → plancraft/models}/react.py +0 -0
- {models → plancraft/models}/utils.py +0 -0
- {train → plancraft/train}/dataset.py +0 -0
- {plancraft-0.1.1.dist-info → plancraft-0.1.3.dist-info}/LICENSE +0 -0
plancraft/config.py
ADDED
@@ -0,0 +1,155 @@
|
|
1
|
+
from typing import Literal, Optional, Union
|
2
|
+
|
3
|
+
from pydantic import BaseModel, model_validator
|
4
|
+
|
5
|
+
try:
|
6
|
+
from plancraft.environments.recipes import RECIPES
|
7
|
+
except ImportError:
|
8
|
+
RECIPES = {}
|
9
|
+
|
10
|
+
DatasetSplit = Literal[
|
11
|
+
"train", "val", "val.small", "val.small.easy", "test", "test.small"
|
12
|
+
]
|
13
|
+
|
14
|
+
|
15
|
+
class EnvironmentConfig(BaseModel):
|
16
|
+
symbolic: bool
|
17
|
+
symbolic_observation_space: bool
|
18
|
+
symbolic_action_space: bool
|
19
|
+
preferred_spawn_biome: str = "plains"
|
20
|
+
resolution: list[int] = [512, 512]
|
21
|
+
|
22
|
+
|
23
|
+
class PlancraftConfig(BaseModel):
|
24
|
+
model: str
|
25
|
+
adapter: str = ""
|
26
|
+
tokenizer: str
|
27
|
+
num_generations: int
|
28
|
+
mode: Literal["react", "act", "oracle", "dummy"] = "react"
|
29
|
+
output_dir: str
|
30
|
+
max_steps: int = 30 # max number of steps (smelt/move) to take in the environment before stopping
|
31
|
+
quantize: Literal[False, "int4", "int8"]
|
32
|
+
environment: EnvironmentConfig
|
33
|
+
split: DatasetSplit = "val.small"
|
34
|
+
max_message_window: int = 30 # max number of messages to keep in dialogue history (30 is around 8k llama3 tokens)
|
35
|
+
hot_cache: bool = True # whether to cache the dialogue history between steps
|
36
|
+
resume: bool = True # resume inference
|
37
|
+
few_shot: bool = True # whether to use few-shot prompt
|
38
|
+
system_prompt: bool = True # whether to use system prompt
|
39
|
+
valid_actions: list[str] = ["move", "smelt", "think", "search", "impossible"]
|
40
|
+
use_maskrcnn: bool = False # whether to use maskrcnn for multimodal parsing
|
41
|
+
|
42
|
+
# observations
|
43
|
+
use_text_inventory: bool = True # whether to include inventory in text
|
44
|
+
use_images: bool = False # whether to include images in multimodal content
|
45
|
+
use_multimodal_content_format: bool = (
|
46
|
+
False # whether to use multimodal content format
|
47
|
+
)
|
48
|
+
|
49
|
+
@model_validator(mode="after")
|
50
|
+
def validate(self):
|
51
|
+
assert set(
|
52
|
+
self.valid_actions
|
53
|
+
).issubset(
|
54
|
+
{"move", "smelt", "think", "search", "impossible"}
|
55
|
+
), "valid_actions should be subset of {'move', 'smelt', 'think', 'search', 'impossible'}"
|
56
|
+
|
57
|
+
if self.use_images:
|
58
|
+
assert (
|
59
|
+
not self.environment.symbolic
|
60
|
+
), "Set environment.symbolic to False when using images"
|
61
|
+
|
62
|
+
return self
|
63
|
+
|
64
|
+
|
65
|
+
class WandbConfig(BaseModel):
|
66
|
+
project: str
|
67
|
+
entity: str
|
68
|
+
mode: str
|
69
|
+
|
70
|
+
|
71
|
+
class LaunchConfig(BaseModel):
|
72
|
+
command: str
|
73
|
+
job_name: str
|
74
|
+
gpu_limit: int
|
75
|
+
gpu_product: str
|
76
|
+
cpu_request: int
|
77
|
+
ram_request: str
|
78
|
+
interactive: bool = False
|
79
|
+
namespace: str = "informatics"
|
80
|
+
env_vars: dict[str, dict[str, str]]
|
81
|
+
|
82
|
+
|
83
|
+
class EvalConfig(BaseModel):
|
84
|
+
plancraft: PlancraftConfig
|
85
|
+
wandb: WandbConfig
|
86
|
+
launch: LaunchConfig
|
87
|
+
|
88
|
+
|
89
|
+
class TrainingArgs(BaseModel):
|
90
|
+
base_model: str = "llama3"
|
91
|
+
trace_mode: str = "oa"
|
92
|
+
push_to_hub: bool = False
|
93
|
+
|
94
|
+
# uses less space but not working with multi-gpu training..
|
95
|
+
qlora: bool = False
|
96
|
+
|
97
|
+
lora_alpha: int = 16
|
98
|
+
lora_dropout: float = 0.1
|
99
|
+
lora_r: int = 64
|
100
|
+
# training data args
|
101
|
+
seed: int = 42
|
102
|
+
# model args
|
103
|
+
batch_size: int = 1
|
104
|
+
max_seq_length: int = 8142
|
105
|
+
max_message_window: int = 100
|
106
|
+
only_assistant: bool = True
|
107
|
+
|
108
|
+
# training args
|
109
|
+
gradient_accumulation_steps: int = 4
|
110
|
+
learning_rate: float = 2e-4
|
111
|
+
max_grad_norm: float = 0.3
|
112
|
+
warmup_ratio: float = 0.03
|
113
|
+
num_train_epochs: int = 3
|
114
|
+
num_workers: int = 1
|
115
|
+
|
116
|
+
|
117
|
+
class TrainConfig(BaseModel):
|
118
|
+
training: TrainingArgs
|
119
|
+
wandb: WandbConfig
|
120
|
+
launch: LaunchConfig
|
121
|
+
|
122
|
+
|
123
|
+
class PlancraftExample(BaseModel):
|
124
|
+
target: str
|
125
|
+
inventory: dict[str, int]
|
126
|
+
slotted_inventory: list[dict[str, Union[str, int]]]
|
127
|
+
num_distractors: int
|
128
|
+
impossible: bool
|
129
|
+
optimal_path_length: Optional[int]
|
130
|
+
optimal_path: Optional[list[str]]
|
131
|
+
inventory_trace: Optional[list[dict[str, int]]]
|
132
|
+
items_used: Optional[int]
|
133
|
+
unique_items_used: Optional[int]
|
134
|
+
complexity: Optional[int]
|
135
|
+
complexity_bin: int
|
136
|
+
unseen_in_train: bool
|
137
|
+
unseen_in_val: bool
|
138
|
+
split: DatasetSplit
|
139
|
+
id: str
|
140
|
+
|
141
|
+
recipe_type: Optional[str] = ""
|
142
|
+
|
143
|
+
# post processing set recipe type
|
144
|
+
def model_post_init(self, __context):
|
145
|
+
recipe_types = set()
|
146
|
+
if self.optimal_path is None:
|
147
|
+
self.recipe_type = "impossible"
|
148
|
+
return
|
149
|
+
for step in self.optimal_path:
|
150
|
+
for r in RECIPES[step]:
|
151
|
+
recipe_types.add(r.recipe_type)
|
152
|
+
if len(recipe_types) == 1:
|
153
|
+
self.recipe_type = recipe_types.pop()
|
154
|
+
else:
|
155
|
+
self.recipe_type = "mixed"
|
File without changes
|
plancraft/evaluator.py
ADDED
@@ -0,0 +1,273 @@
|
|
1
|
+
import json
|
2
|
+
import os
|
3
|
+
import random
|
4
|
+
import string
|
5
|
+
import time
|
6
|
+
|
7
|
+
import imageio
|
8
|
+
import pandas as pd
|
9
|
+
import torch
|
10
|
+
import wandb
|
11
|
+
from loguru import logger
|
12
|
+
from tqdm import tqdm
|
13
|
+
|
14
|
+
from plancraft.config import EvalConfig, PlancraftExample
|
15
|
+
from plancraft.environments.actions import StopAction
|
16
|
+
from plancraft.environments.env_real import RealPlancraft
|
17
|
+
from plancraft.environments.env_symbolic import SymbolicPlancraft
|
18
|
+
from plancraft.models import get_model
|
19
|
+
|
20
|
+
wandb.require("core")
|
21
|
+
|
22
|
+
|
23
|
+
class Evaluator:
|
24
|
+
"""
|
25
|
+
The evaluator class handles the environment loop and model interaction
|
26
|
+
|
27
|
+
The environment is created based on the configuration and the examples are loaded from the dataset.
|
28
|
+
"""
|
29
|
+
|
30
|
+
def __init__(self, cfg: EvalConfig):
|
31
|
+
self.cfg = cfg
|
32
|
+
self.output_dir = (
|
33
|
+
f"{cfg.plancraft.output_dir}/{self.evaluator_name()}/{cfg.plancraft.split}"
|
34
|
+
)
|
35
|
+
self.generation_number = 0
|
36
|
+
|
37
|
+
self.examples: list[PlancraftExample] = self.load_dataset(cfg.plancraft.split)
|
38
|
+
|
39
|
+
self.environment = self.create_env(cfg)
|
40
|
+
self.model = get_model(cfg)
|
41
|
+
|
42
|
+
self.record_frames = not (cfg.plancraft.environment.symbolic)
|
43
|
+
|
44
|
+
# no_op action
|
45
|
+
self.no_op = self.environment.action_space.no_op()
|
46
|
+
|
47
|
+
def evaluator_name(self) -> str:
|
48
|
+
symb_str = "real"
|
49
|
+
if self.cfg.plancraft.environment.symbolic:
|
50
|
+
symb_str = "symb"
|
51
|
+
|
52
|
+
if self.cfg.plancraft.use_maskrcnn:
|
53
|
+
symb_str += "_mrcnn"
|
54
|
+
|
55
|
+
model_name = self.cfg.plancraft.model.split("/")[-1]
|
56
|
+
if self.cfg.plancraft.adapter != "":
|
57
|
+
model_name = self.cfg.plancraft.adapter.split("/")[-1]
|
58
|
+
|
59
|
+
mode = self.cfg.plancraft.mode
|
60
|
+
if mode in ["dummy", "oracle"]:
|
61
|
+
return f"{mode}_{symb_str}"
|
62
|
+
|
63
|
+
actions = "|".join(self.cfg.plancraft.valid_actions)
|
64
|
+
return f"{self.cfg.plancraft.mode}_{symb_str}_{model_name}_{actions}"
|
65
|
+
|
66
|
+
def save_results_dict(self, example: PlancraftExample, results_dict: dict):
|
67
|
+
output_dir = f"{self.output_dir}/{self.generation_number}"
|
68
|
+
os.makedirs(output_dir, exist_ok=True)
|
69
|
+
json_path = f"{output_dir}/{example.id}.json"
|
70
|
+
with open(json_path, "w") as f:
|
71
|
+
json.dump(results_dict, f, indent=4)
|
72
|
+
wandb.save(json_path, policy="now")
|
73
|
+
|
74
|
+
def save_images(self, example: PlancraftExample, frames: list):
|
75
|
+
if len(frames) == 0:
|
76
|
+
return
|
77
|
+
output_dir = f"{self.output_dir}/{self.generation_number}"
|
78
|
+
os.makedirs(output_dir, exist_ok=True)
|
79
|
+
imageio.mimsave(f"{output_dir}/{example.id}.gif", frames)
|
80
|
+
# upload to wandb
|
81
|
+
wandb.save(f"{output_dir}/{example.id}.gif", policy="now")
|
82
|
+
|
83
|
+
def load_results_dict(self, example: PlancraftExample) -> dict:
|
84
|
+
path = f"{self.output_dir}/{self.generation_number}/{example.id}.json"
|
85
|
+
if not os.path.exists(path) or not self.cfg.plancraft.resume:
|
86
|
+
return None
|
87
|
+
with open(path, "r") as f:
|
88
|
+
return json.load(f)
|
89
|
+
|
90
|
+
def create_env(self, cfg: EvalConfig) -> RealPlancraft | SymbolicPlancraft:
|
91
|
+
if cfg.plancraft.environment.symbolic:
|
92
|
+
return SymbolicPlancraft(inventory=[])
|
93
|
+
return RealPlancraft(
|
94
|
+
inventory=[],
|
95
|
+
symbolic_action_space=cfg.plancraft.environment.symbolic_action_space,
|
96
|
+
symbolic_observation_space=cfg.plancraft.environment.symbolic_observation_space,
|
97
|
+
preferred_spawn_biome=cfg.plancraft.environment.preferred_spawn_biome,
|
98
|
+
resolution=cfg.plancraft.environment.resolution,
|
99
|
+
)
|
100
|
+
|
101
|
+
def close(self):
|
102
|
+
self.environment.close()
|
103
|
+
|
104
|
+
def load_dataset(self, dataset_split: str) -> list[PlancraftExample]:
|
105
|
+
with open(f"data/{dataset_split}.json", "r") as f:
|
106
|
+
dataset = json.load(f)
|
107
|
+
return [PlancraftExample(**example) for example in dataset]
|
108
|
+
|
109
|
+
def reset(
|
110
|
+
self,
|
111
|
+
example: PlancraftExample,
|
112
|
+
):
|
113
|
+
current_inventory = example.slotted_inventory
|
114
|
+
self.environment.fast_reset(new_inventory=current_inventory)
|
115
|
+
# do a no op to an initial observation
|
116
|
+
obs, _, _, _ = self.environment.step(self.no_op)
|
117
|
+
# assert that the inventory is correct
|
118
|
+
if "inventory" in obs:
|
119
|
+
for item in current_inventory:
|
120
|
+
slot = item["slot"]
|
121
|
+
if (
|
122
|
+
obs["inventory"][slot]["type"] != item["type"]
|
123
|
+
or obs["inventory"][slot]["quantity"] != item["quantity"]
|
124
|
+
) and item["type"] != "air":
|
125
|
+
logger.warning(f"Inventory does not match expected for slot {slot}")
|
126
|
+
logger.warning(f"Expected {item}")
|
127
|
+
logger.warning(f"Got {obs['inventory'][slot]}")
|
128
|
+
# try again
|
129
|
+
self.reset(example)
|
130
|
+
|
131
|
+
objective = f"Craft an item of type: {example.target}"
|
132
|
+
self.model.reset_history(objective=objective)
|
133
|
+
|
134
|
+
def check_done(self, inventory: list[dict[str, int]], target: str):
|
135
|
+
"""
|
136
|
+
Check that target object is obtained
|
137
|
+
"""
|
138
|
+
for item in inventory:
|
139
|
+
if target == item["type"]:
|
140
|
+
# ensure item is taken out of crafting slot
|
141
|
+
if "slot" in item and item["slot"] != 0:
|
142
|
+
return True
|
143
|
+
if "index" in item and item["index"] != 0:
|
144
|
+
return True
|
145
|
+
return False
|
146
|
+
|
147
|
+
@torch.no_grad()
|
148
|
+
def eval_all_examples(self, progress_bar=False) -> list:
|
149
|
+
results = []
|
150
|
+
action = self.no_op.copy()
|
151
|
+
|
152
|
+
pbar = tqdm(
|
153
|
+
total=len(self.examples),
|
154
|
+
disable=not progress_bar,
|
155
|
+
)
|
156
|
+
correct = 0
|
157
|
+
count = 0
|
158
|
+
|
159
|
+
for example in self.examples:
|
160
|
+
if resume_result := self.load_results_dict(example):
|
161
|
+
pbar.update(self.cfg.plancraft.max_steps)
|
162
|
+
results.append(resume_result)
|
163
|
+
continue
|
164
|
+
|
165
|
+
success = False
|
166
|
+
|
167
|
+
self.reset(example)
|
168
|
+
action = self.no_op.copy()
|
169
|
+
|
170
|
+
while (
|
171
|
+
not self.model.history.check_stuck()
|
172
|
+
and self.model.history.num_steps < self.cfg.plancraft.max_steps
|
173
|
+
):
|
174
|
+
# if the action is stop then we end the episode
|
175
|
+
if isinstance(action, StopAction):
|
176
|
+
# if the action is stop and task is impossible then success
|
177
|
+
# otherwise we should not have stopped
|
178
|
+
success = example.impossible
|
179
|
+
break
|
180
|
+
|
181
|
+
# step action
|
182
|
+
observation, _, _, _ = self.environment.step(action)
|
183
|
+
|
184
|
+
# check if the episode is done
|
185
|
+
success = self.check_done(observation["inventory"], example.target)
|
186
|
+
# exit if success
|
187
|
+
if success:
|
188
|
+
break
|
189
|
+
|
190
|
+
# predict next action
|
191
|
+
action = self.model.step(observation)
|
192
|
+
|
193
|
+
# save results and reset
|
194
|
+
result = {
|
195
|
+
"success": success,
|
196
|
+
"recipe_type": example.recipe_type,
|
197
|
+
"number_of_steps": self.model.history.num_steps,
|
198
|
+
"model_trace": self.model.history.trace(),
|
199
|
+
"example_id": example.id,
|
200
|
+
"impossible": example.impossible,
|
201
|
+
}
|
202
|
+
results.append(result)
|
203
|
+
self.save_results_dict(example, result)
|
204
|
+
self.save_images(example, self.model.history.images)
|
205
|
+
|
206
|
+
correct += int(result["success"])
|
207
|
+
count += 1
|
208
|
+
|
209
|
+
acc = correct / count
|
210
|
+
pbar.set_postfix(correct=correct, count=count, acc=acc)
|
211
|
+
pbar.update(1)
|
212
|
+
|
213
|
+
return results
|
214
|
+
|
215
|
+
def eval_all(self):
|
216
|
+
logger.info(
|
217
|
+
f"Running evaluation over {len(self.examples)} examples {self.cfg.plancraft.num_generations} times."
|
218
|
+
)
|
219
|
+
run_name = (
|
220
|
+
f"{self.evaluator_name()} {self.cfg.plancraft.split}".replace(" ", "_")
|
221
|
+
.replace(".", "_")
|
222
|
+
.strip()
|
223
|
+
)
|
224
|
+
|
225
|
+
for n in range(self.cfg.plancraft.num_generations):
|
226
|
+
logger.info(f"Generation {n+1}/{self.cfg.plancraft.num_generations}")
|
227
|
+
run_id = "".join(random.choices(string.ascii_lowercase, k=5))
|
228
|
+
generation_run_name = run_name + f"_{run_id}"
|
229
|
+
|
230
|
+
wandb.init(
|
231
|
+
name=generation_run_name,
|
232
|
+
project=self.cfg.wandb.project,
|
233
|
+
entity=self.cfg.wandb.entity,
|
234
|
+
mode=self.cfg.wandb.mode,
|
235
|
+
group=self.cfg.plancraft.model,
|
236
|
+
job_type=self.cfg.plancraft.mode,
|
237
|
+
config=self.cfg.model_dump(),
|
238
|
+
)
|
239
|
+
time_now = time.time()
|
240
|
+
|
241
|
+
results_list = self.eval_all_examples(progress_bar=True)
|
242
|
+
|
243
|
+
results_df = pd.DataFrame(results_list)
|
244
|
+
|
245
|
+
output = {
|
246
|
+
"avg_success_rate": results_df["success"].mean(),
|
247
|
+
"avg_number_of_steps": results_df["number_of_steps"].mean(),
|
248
|
+
"avg_num_tokens_used": results_df["model_trace"]
|
249
|
+
.apply(pd.Series)["tokens_used"]
|
250
|
+
.mean(),
|
251
|
+
}
|
252
|
+
|
253
|
+
# calculate success rate for each recipe type
|
254
|
+
recipe_types = results_df["recipe_type"].unique()
|
255
|
+
for recipe_type in recipe_types:
|
256
|
+
mask = results_df["recipe_type"] == recipe_type
|
257
|
+
success_rate = results_df[mask]["success"].mean()
|
258
|
+
output[f"{recipe_type}_success_rate"] = success_rate
|
259
|
+
|
260
|
+
time_elapsed = time.time() - time_now
|
261
|
+
logger.info(f"Time elapsed: {time_elapsed:.2f}s")
|
262
|
+
|
263
|
+
logger.info(output)
|
264
|
+
wandb.log(output)
|
265
|
+
table = wandb.Table(
|
266
|
+
dataframe=results_df[["success", "number_of_steps", "example_id"]]
|
267
|
+
)
|
268
|
+
wandb.log({"results": table})
|
269
|
+
wandb.finish()
|
270
|
+
|
271
|
+
self.generation_number += 1
|
272
|
+
|
273
|
+
logger.info("Done")
|
plancraft/utils.py
ADDED
@@ -0,0 +1,84 @@
|
|
1
|
+
import glob
|
2
|
+
import pathlib
|
3
|
+
|
4
|
+
import cv2
|
5
|
+
import numpy as np
|
6
|
+
import torch
|
7
|
+
import torch.nn.functional as F
|
8
|
+
from PIL import Image
|
9
|
+
|
10
|
+
|
11
|
+
def get_downloaded_models() -> dict:
|
12
|
+
"""
|
13
|
+
Get the list of downloaded models on the NFS partition (EIDF).
|
14
|
+
"""
|
15
|
+
downloaded_models = {}
|
16
|
+
# known models on NFS partition
|
17
|
+
if pathlib.Path("/nfs").exists():
|
18
|
+
local_models = glob.glob("/nfs/public/hf/models/*/*")
|
19
|
+
downloaded_models = {
|
20
|
+
model.replace("/nfs/public/hf/models/", ""): model for model in local_models
|
21
|
+
}
|
22
|
+
return downloaded_models
|
23
|
+
|
24
|
+
|
25
|
+
def get_torch_device() -> torch.device:
|
26
|
+
device = torch.device("cpu")
|
27
|
+
if torch.cuda.is_available():
|
28
|
+
device = torch.device("cuda", 0)
|
29
|
+
elif torch.backends.mps.is_available():
|
30
|
+
if not torch.backends.mps.is_built():
|
31
|
+
print(
|
32
|
+
"MPS not available because the current PyTorch install was not built with MPS enabled."
|
33
|
+
)
|
34
|
+
else:
|
35
|
+
device = torch.device("mps")
|
36
|
+
return device
|
37
|
+
|
38
|
+
|
39
|
+
def resize_image(img, target_resolution=(128, 128)):
|
40
|
+
if type(img) == np.ndarray:
|
41
|
+
img = cv2.resize(img, target_resolution, interpolation=cv2.INTER_LINEAR)
|
42
|
+
elif type(img) == torch.Tensor:
|
43
|
+
img = F.interpolate(img, size=target_resolution, mode="bilinear")
|
44
|
+
else:
|
45
|
+
raise ValueError("Unsupported type for img")
|
46
|
+
return img
|
47
|
+
|
48
|
+
|
49
|
+
def save_frames_to_video(frames: list, out_path: str):
|
50
|
+
imgs = []
|
51
|
+
for id, (frame, goal) in enumerate(frames):
|
52
|
+
# if torch.is_tensor(frame):
|
53
|
+
# frame = frame.permute(0, 2, 3, 1).cpu().numpy()
|
54
|
+
|
55
|
+
frame = resize_image(frame, (320, 240)).astype("uint8")
|
56
|
+
cv2.putText(
|
57
|
+
frame,
|
58
|
+
f"FID: {id}",
|
59
|
+
(10, 25),
|
60
|
+
cv2.FONT_HERSHEY_SIMPLEX,
|
61
|
+
0.8,
|
62
|
+
(255, 255, 255),
|
63
|
+
2,
|
64
|
+
)
|
65
|
+
cv2.putText(
|
66
|
+
frame,
|
67
|
+
f"Goal: {goal}",
|
68
|
+
(10, 55),
|
69
|
+
cv2.FONT_HERSHEY_SIMPLEX,
|
70
|
+
0.8,
|
71
|
+
(255, 0, 0),
|
72
|
+
2,
|
73
|
+
)
|
74
|
+
imgs.append(Image.fromarray(frame))
|
75
|
+
imgs = imgs[::3]
|
76
|
+
imgs[0].save(
|
77
|
+
out_path,
|
78
|
+
save_all=True,
|
79
|
+
append_images=imgs[1:],
|
80
|
+
optimize=False,
|
81
|
+
quality=0,
|
82
|
+
duration=150,
|
83
|
+
loop=0,
|
84
|
+
)
|
@@ -0,0 +1,74 @@
|
|
1
|
+
Metadata-Version: 2.1
|
2
|
+
Name: plancraft
|
3
|
+
Version: 0.1.3
|
4
|
+
Summary: Plancraft: an evaluation dataset for planning with LLM agents
|
5
|
+
Requires-Python: >=3.9
|
6
|
+
Description-Content-Type: text/markdown
|
7
|
+
License-File: LICENSE
|
8
|
+
Requires-Dist: accelerate>=1.0.1
|
9
|
+
Requires-Dist: coloredlogs>=10.0
|
10
|
+
Requires-Dist: daemoniker>=0.2.3
|
11
|
+
Requires-Dist: datasets>=3.0.2
|
12
|
+
Requires-Dist: dill>=0.3.1.1
|
13
|
+
Requires-Dist: einops>=0.8.0
|
14
|
+
Requires-Dist: flaky>=3.8.1
|
15
|
+
Requires-Dist: hf-transfer>=0.1.8
|
16
|
+
Requires-Dist: huggingface-hub>=0.26.1
|
17
|
+
Requires-Dist: hydra-core>=1.3.2
|
18
|
+
Requires-Dist: imagehash>=4.0.0
|
19
|
+
Requires-Dist: imageio>=2.36.0
|
20
|
+
Requires-Dist: inflection>=0.3.1
|
21
|
+
Requires-Dist: ipython>=7.5.0
|
22
|
+
Requires-Dist: jinja2>=2.11.2
|
23
|
+
Requires-Dist: loguru>=0.7.2
|
24
|
+
Requires-Dist: lxml>=4.3.3
|
25
|
+
Requires-Dist: matplotlib>=3.9.2
|
26
|
+
Requires-Dist: networkx>=3.2.1
|
27
|
+
Requires-Dist: numpy<1.24,>=1.16.2
|
28
|
+
Requires-Dist: openai>=1.52.2
|
29
|
+
Requires-Dist: opencv-python>=4.1.0.25
|
30
|
+
Requires-Dist: pandas>=2.1.0
|
31
|
+
Requires-Dist: peft>=0.13.2
|
32
|
+
Requires-Dist: pillow>=8.0.0
|
33
|
+
Requires-Dist: psutil>=5.6.2
|
34
|
+
Requires-Dist: pydantic>=2.9.2
|
35
|
+
Requires-Dist: pyglet>=2.0.18
|
36
|
+
Requires-Dist: pyro4>=4.76
|
37
|
+
Requires-Dist: python-dotenv>=1.0.1
|
38
|
+
Requires-Dist: pyyaml>=6.0.2
|
39
|
+
Requires-Dist: requests>=2.20.0
|
40
|
+
Requires-Dist: seaborn>=0.13.2
|
41
|
+
Requires-Dist: setuptools>=49.2.0
|
42
|
+
Requires-Dist: tinydb>=4.8.2
|
43
|
+
Requires-Dist: torch>=2.5.0
|
44
|
+
Requires-Dist: torchvision>=0.20.0
|
45
|
+
Requires-Dist: tqdm>=4.32.2
|
46
|
+
Requires-Dist: transformers>=4.43.3
|
47
|
+
Requires-Dist: typing>=3.6.6
|
48
|
+
Requires-Dist: wandb>=0.18.5
|
49
|
+
Requires-Dist: xmltodict==0.12.0
|
50
|
+
Provides-Extra: full
|
51
|
+
Requires-Dist: gym<=0.23.1,>=0.19.0; extra == "full"
|
52
|
+
|
53
|
+
# plancraft
|
54
|
+
|
55
|
+
[![Test](https://github.com/gautierdag/plancraft/actions/workflows/test.yaml/badge.svg)](https://github.com/gautierdag/plancraft/actions/workflows/test.yaml)
|
56
|
+
![Python Version](https://img.shields.io/badge/python-3.9+-blue)
|
57
|
+
![Ruff](https://img.shields.io/badge/linter-ruff-blue)
|
58
|
+
[![PyPI Version](https://img.shields.io/pypi/v/plancraft)](https://pypi.org/project/plancraft/)
|
59
|
+
|
60
|
+
Plancraft is a minecraft environment and agent that innovates on planning LLM agents with a retriever
|
61
|
+
|
62
|
+
You can install the package by running the following command:
|
63
|
+
|
64
|
+
```bash
|
65
|
+
pip install plancraft
|
66
|
+
```
|
67
|
+
|
68
|
+
Should you need the multimodal version of the package, you will also need a custom [fork](https://github.com/gautierdag/minerl.git) of the minerl package. You can install it by running the following command:
|
69
|
+
|
70
|
+
```bash
|
71
|
+
pip install git+hhttps://github.com/gautierdag/minerl.git
|
72
|
+
```
|
73
|
+
|
74
|
+
Note that you may need to follow the same installation instructions as in the [minerl documentation](https://minerl.readthedocs.io/en/latest/tutorials/index.html).
|
@@ -0,0 +1,30 @@
|
|
1
|
+
plancraft/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
2
|
+
plancraft/config.py,sha256=k7Ac6Zh6g-Q-fuazPxVdXnSL_PMd3jsIqyiImNaJW5A,4472
|
3
|
+
plancraft/evaluator.py,sha256=zomShgwyUOHNVZbTERpimgEYiyD4JXzHzPi4tg45lsM,9926
|
4
|
+
plancraft/utils.py,sha256=J9K81zBXWCcyIBjwu_eHznnzWamXU-pQrz7BdNTVbLU,2289
|
5
|
+
plancraft/environments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
6
|
+
plancraft/environments/actions.py,sha256=SeeC9l1cJBs9pdba6BefQ_iQNfFf6FVTWm7HWkacbsY,6262
|
7
|
+
plancraft/environments/env_real.py,sha256=oETMvdq8-TspPoIRxioTDj9EEmuZRuuMxd9c3momvPk,10877
|
8
|
+
plancraft/environments/env_symbolic.py,sha256=ot4IStZ3oT7CYPqdqHsGl2BopLyZUwSF671SrHIiMLk,7777
|
9
|
+
plancraft/environments/items.py,sha256=1R56LyK6tqIssQJMqHst6A9DeEfOX5DN-OBAkumGncw,217
|
10
|
+
plancraft/environments/planner.py,sha256=2B-0aunllmTuiHE7Jn5jHzCg6mMgxZjisiTDdpSKupk,3954
|
11
|
+
plancraft/environments/recipes.py,sha256=nXvOLCRljiZ5IgeevXuosU9IgSs7oQWQJFiuyRNSVFs,19571
|
12
|
+
plancraft/environments/sampler.py,sha256=ZBYoENKdQQ7wbAyVk-c9UNRWKPE0omv9he8c8QZ6wXg,7625
|
13
|
+
plancraft/models/__init__.py,sha256=PasK3jpbhpD0kxF4iHukcccZqvZg6lL240zie3DfLDY,622
|
14
|
+
plancraft/models/act.py,sha256=jdZunT7FcbHvcaJZ_wUDLSuObHjU6JZybghR_B0QJ8Q,6548
|
15
|
+
plancraft/models/base.py,sha256=fFM2BV9PqvIFtUlTz8iz5HPemYRy3S0EituM1XdJJSQ,4927
|
16
|
+
plancraft/models/bbox_model.py,sha256=CoX-odH59S-djkPOH2ViEmbYWo1sefmHiOcBlFWiAkg,16814
|
17
|
+
plancraft/models/dummy.py,sha256=QjxTIiKsWSmhUMAuw7Yy-OKKCLi_x3rwll4hH7ZNXso,1732
|
18
|
+
plancraft/models/generators.py,sha256=c2avYUoXAAWOHRFq24ZYqHqNgoPs_L_XWJrR_TKFg9E,17358
|
19
|
+
plancraft/models/oam.py,sha256=_TpxsaCnGvQ7YJC6KWcpl2HzgBiQaHn7U1zRAXfdjOo,9943
|
20
|
+
plancraft/models/oracle.py,sha256=xRplR2cCW_39UGYtSjDLRDmp0UILhtIXYjuRJ-jokJ8,9598
|
21
|
+
plancraft/models/prompts.py,sha256=XwoRqd_5_VfCUXb10dCRFYXgw70mO2VoQocn3Z2zgs0,6165
|
22
|
+
plancraft/models/react.py,sha256=5yM4tv7tDfm_-5yUSAcw5C4wioWijF9fLEODbFqFDvg,3346
|
23
|
+
plancraft/models/utils.py,sha256=osKX0_uux9wzqYzq1ST0Cu5idrAnyfNvXrj0uO1eKo0,9424
|
24
|
+
plancraft/models/few_shot_images/__init__.py,sha256=nIkyB6w3ok-h4lfJKsrcMQzQF624Y9uxYV1FqCu3Lx0,351
|
25
|
+
plancraft/train/dataset.py,sha256=NrZjbIkosui1kaq7AIWSrYvvzrDxu_njH7FmGKY3xnI,5434
|
26
|
+
plancraft-0.1.3.dist-info/LICENSE,sha256=YGR8ehDB4t-T-lOQKMfKNR-2zsOU7E3E5NA8t25HKE0,1070
|
27
|
+
plancraft-0.1.3.dist-info/METADATA,sha256=XLudjSvyfZ-uYJZ1K1RAJUVOQhozFD3FLzwUQeikhK4,2631
|
28
|
+
plancraft-0.1.3.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
|
29
|
+
plancraft-0.1.3.dist-info/top_level.txt,sha256=yGGA9HtPKH2-pIJFrbyYhj2JF9Xj-4m0fxMPKT9FNzg,10
|
30
|
+
plancraft-0.1.3.dist-info/RECORD,,
|
@@ -0,0 +1 @@
|
|
1
|
+
plancraft
|
@@ -1,74 +0,0 @@
|
|
1
|
-
Metadata-Version: 2.1
|
2
|
-
Name: plancraft
|
3
|
-
Version: 0.1.1
|
4
|
-
Summary: Plancraft: an evaluation dataset for planning with LLM agents
|
5
|
-
Requires-Python: >=3.9
|
6
|
-
Description-Content-Type: text/markdown
|
7
|
-
License-File: LICENSE
|
8
|
-
Requires-Dist: accelerate >=1.0.1
|
9
|
-
Requires-Dist: coloredlogs >=10.0
|
10
|
-
Requires-Dist: daemoniker >=0.2.3
|
11
|
-
Requires-Dist: datasets >=3.0.2
|
12
|
-
Requires-Dist: dill >=0.3.1.1
|
13
|
-
Requires-Dist: einops >=0.8.0
|
14
|
-
Requires-Dist: flaky >=3.8.1
|
15
|
-
Requires-Dist: hf-transfer >=0.1.8
|
16
|
-
Requires-Dist: huggingface-hub >=0.26.1
|
17
|
-
Requires-Dist: hydra-core >=1.3.2
|
18
|
-
Requires-Dist: imagehash >=4.0.0
|
19
|
-
Requires-Dist: imageio >=2.36.0
|
20
|
-
Requires-Dist: inflection >=0.3.1
|
21
|
-
Requires-Dist: ipython >=7.5.0
|
22
|
-
Requires-Dist: jinja2 >=2.11.2
|
23
|
-
Requires-Dist: loguru >=0.7.2
|
24
|
-
Requires-Dist: lxml >=4.3.3
|
25
|
-
Requires-Dist: matplotlib >=3.9.2
|
26
|
-
Requires-Dist: networkx >=3.2.1
|
27
|
-
Requires-Dist: numpy <1.24,>=1.16.2
|
28
|
-
Requires-Dist: openai >=1.52.2
|
29
|
-
Requires-Dist: opencv-python >=4.1.0.25
|
30
|
-
Requires-Dist: pandas >=2.1.0
|
31
|
-
Requires-Dist: peft >=0.13.2
|
32
|
-
Requires-Dist: pillow >=8.0.0
|
33
|
-
Requires-Dist: psutil >=5.6.2
|
34
|
-
Requires-Dist: pydantic >=2.9.2
|
35
|
-
Requires-Dist: pyglet >=2.0.18
|
36
|
-
Requires-Dist: pyro4 >=4.76
|
37
|
-
Requires-Dist: python-dotenv >=1.0.1
|
38
|
-
Requires-Dist: pyyaml >=6.0.2
|
39
|
-
Requires-Dist: requests >=2.20.0
|
40
|
-
Requires-Dist: seaborn >=0.13.2
|
41
|
-
Requires-Dist: setuptools >=49.2.0
|
42
|
-
Requires-Dist: tinydb >=4.8.2
|
43
|
-
Requires-Dist: torch >=2.5.0
|
44
|
-
Requires-Dist: torchvision >=0.20.0
|
45
|
-
Requires-Dist: tqdm >=4.32.2
|
46
|
-
Requires-Dist: transformers >=4.43.3
|
47
|
-
Requires-Dist: typing >=3.6.6
|
48
|
-
Requires-Dist: wandb >=0.18.5
|
49
|
-
Requires-Dist: xmltodict ==0.12.0
|
50
|
-
Provides-Extra: full
|
51
|
-
Requires-Dist: gym <=0.23.1,>=0.19.0 ; extra == 'full'
|
52
|
-
|
53
|
-
# plancraft
|
54
|
-
|
55
|
-
[![Test](https://github.com/gautierdag/plancraft/actions/workflows/test.yaml/badge.svg)](https://github.com/gautierdag/plancraft/actions/workflows/test.yaml)
|
56
|
-
![Python Version](https://img.shields.io/badge/python-3.9+-blue)
|
57
|
-
![Ruff](https://img.shields.io/badge/linter-ruff-blue)
|
58
|
-
[![PyPI Version](https://img.shields.io/pypi/v/plancraft)](https://pypi.org/project/plancraft/)
|
59
|
-
|
60
|
-
Plancraft is a minecraft environment and agent that innovates on planning LLM agents with a retriever
|
61
|
-
|
62
|
-
You can install the package by running the following command:
|
63
|
-
|
64
|
-
```bash
|
65
|
-
pip install plancraft
|
66
|
-
```
|
67
|
-
|
68
|
-
Should you need the multimodal version of the package, you will also need a custom [fork](https://github.com/gautierdag/minerl.git) of the minerl package. You can install it by running the following command:
|
69
|
-
|
70
|
-
```bash
|
71
|
-
pip install git+hhttps://github.com/gautierdag/minerl.git
|
72
|
-
```
|
73
|
-
|
74
|
-
Note that you may need to follow the same installation instructions as in the [minerl documentation](https://minerl.readthedocs.io/en/latest/tutorials/index.html).
|
plancraft-0.1.1.dist-info/RECORD
DELETED
@@ -1,26 +0,0 @@
|
|
1
|
-
environments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
2
|
-
environments/actions.py,sha256=SeeC9l1cJBs9pdba6BefQ_iQNfFf6FVTWm7HWkacbsY,6262
|
3
|
-
environments/env_real.py,sha256=oETMvdq8-TspPoIRxioTDj9EEmuZRuuMxd9c3momvPk,10877
|
4
|
-
environments/env_symbolic.py,sha256=ot4IStZ3oT7CYPqdqHsGl2BopLyZUwSF671SrHIiMLk,7777
|
5
|
-
environments/items.py,sha256=1R56LyK6tqIssQJMqHst6A9DeEfOX5DN-OBAkumGncw,217
|
6
|
-
environments/planner.py,sha256=2B-0aunllmTuiHE7Jn5jHzCg6mMgxZjisiTDdpSKupk,3954
|
7
|
-
environments/recipes.py,sha256=nXvOLCRljiZ5IgeevXuosU9IgSs7oQWQJFiuyRNSVFs,19571
|
8
|
-
environments/sampler.py,sha256=ZBYoENKdQQ7wbAyVk-c9UNRWKPE0omv9he8c8QZ6wXg,7625
|
9
|
-
models/__init__.py,sha256=PasK3jpbhpD0kxF4iHukcccZqvZg6lL240zie3DfLDY,622
|
10
|
-
models/act.py,sha256=jdZunT7FcbHvcaJZ_wUDLSuObHjU6JZybghR_B0QJ8Q,6548
|
11
|
-
models/base.py,sha256=fFM2BV9PqvIFtUlTz8iz5HPemYRy3S0EituM1XdJJSQ,4927
|
12
|
-
models/bbox_model.py,sha256=CoX-odH59S-djkPOH2ViEmbYWo1sefmHiOcBlFWiAkg,16814
|
13
|
-
models/dummy.py,sha256=QjxTIiKsWSmhUMAuw7Yy-OKKCLi_x3rwll4hH7ZNXso,1732
|
14
|
-
models/generators.py,sha256=c2avYUoXAAWOHRFq24ZYqHqNgoPs_L_XWJrR_TKFg9E,17358
|
15
|
-
models/oam.py,sha256=_TpxsaCnGvQ7YJC6KWcpl2HzgBiQaHn7U1zRAXfdjOo,9943
|
16
|
-
models/oracle.py,sha256=xRplR2cCW_39UGYtSjDLRDmp0UILhtIXYjuRJ-jokJ8,9598
|
17
|
-
models/prompts.py,sha256=XwoRqd_5_VfCUXb10dCRFYXgw70mO2VoQocn3Z2zgs0,6165
|
18
|
-
models/react.py,sha256=5yM4tv7tDfm_-5yUSAcw5C4wioWijF9fLEODbFqFDvg,3346
|
19
|
-
models/utils.py,sha256=osKX0_uux9wzqYzq1ST0Cu5idrAnyfNvXrj0uO1eKo0,9424
|
20
|
-
models/few_shot_images/__init__.py,sha256=nIkyB6w3ok-h4lfJKsrcMQzQF624Y9uxYV1FqCu3Lx0,351
|
21
|
-
train/dataset.py,sha256=NrZjbIkosui1kaq7AIWSrYvvzrDxu_njH7FmGKY3xnI,5434
|
22
|
-
plancraft-0.1.1.dist-info/LICENSE,sha256=YGR8ehDB4t-T-lOQKMfKNR-2zsOU7E3E5NA8t25HKE0,1070
|
23
|
-
plancraft-0.1.1.dist-info/METADATA,sha256=Cj-_TAlOz4PGzed9FVVTrpI0Oc0wk2s94gOSd0XaNTE,2675
|
24
|
-
plancraft-0.1.1.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
|
25
|
-
plancraft-0.1.1.dist-info/top_level.txt,sha256=ZT60unZw3qNbZoGMCVc-V_0hI4YIYvTVGGScIgkCa88,26
|
26
|
-
plancraft-0.1.1.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|