plancraft 0.1.2__py3-none-any.whl → 0.1.3__py3-none-any.whl

plancraft/train/dataset.py

@@ -0,0 +1,187 @@
+import glob
+import json
+import random
+
+import torch
+from torch.utils.data import Dataset
+
+TEMPLATES = {
+    "idefics2": {
+        "assistant": "\nAssistant:",
+        "user": "\nUser:",
+    },
+    "llama3": {
+        "assistant": "<|start_header_id|>assistant<|end_header_id|>\n\n",
+        "user": "<|start_header_id|>user<|end_header_id|>\n\n",
+    },
+}
+
+
+class PlancraftDialogueDataset(Dataset):
+    def __init__(
+        self,
+        dataset_dir: str = "data/oracle",
+        use_images=False,
+        trace_mode="oa",
+        split="train",
+        max_message_window=30,
+    ):
+        super().__init__()
+        self.split = split
+        self.use_images = use_images
+        self.trace_mode = trace_mode
+        self.add_system_message = True
+
+        assert trace_mode in ["oa", "ota"], f"Invalid trace mode {trace_mode}"
+
+        print("Loading dialogue dataset")
+        data = []
+        for example_path in sorted(
+            glob.glob(f"{dataset_dir}/{split}/{trace_mode}/*.json")
+        ):
+            with open(example_path) as f:
+                example = json.load(f)
+            data.append(example)
+        self.dataset = data
+        self.max_message_window = max_message_window
+
+    def __len__(self) -> int:
+        return len(self.dataset)
+
+    def __getitem__(self, idx: int) -> tuple[dict, list]:
+        example = self.dataset[idx]
+        if len(example["messages"]) > self.max_message_window:
+            # add system message
+            if self.add_system_message:
+                messages = [example["messages"][0]]
+            else:
+                messages = []
+
+            # sample window
+            user_messages_idxs = list(
+                range(self.max_message_window, len(example["messages"]), 2)
+            )
+            end = random.choice(user_messages_idxs)
+            start = end - self.max_message_window + 1
+            assert start != 0
+            # add window
+            messages = messages + example["messages"][start : end + 1]
+
+        else:
+            messages = example["messages"]
+        return messages
+
+
+def track_assistant_response(
+    batch,
+    tokenizer,
+    template_name: str = "llama3",
+):
+    """
+    Mask that returns 1 for tokens in the assistant response and 0 otherwise.
+    """
+    assistant_template = TEMPLATES[template_name]["assistant"]
+    user_template = TEMPLATES[template_name]["user"]
+    start_seq = tokenizer.encode(
+        assistant_template,
+        add_special_tokens=False,
+        return_tensors="pt",
+    )[0]
+    end_seq = tokenizer.encode(
+        user_template,
+        add_special_tokens=False,
+        return_tensors="pt",
+    )[0]
+    encoded_label_ids = batch["labels"]
+    mask = torch.zeros_like(encoded_label_ids)
+    for seq_idx, seq in enumerate(encoded_label_ids):
+        in_masked_response = False
+        i = 0
+        while i < len(seq):
+            if i + len(start_seq) < len(seq) and torch.all(
+                seq[i : i + len(start_seq)].eq(start_seq)
+            ):
+                in_masked_response = True
+                i += len(start_seq)
+                continue
+            if i + len(end_seq) < len(seq) and torch.all(
+                seq[i : i + len(end_seq)].eq(end_seq)
+            ):
+                in_masked_response = False
+                i += len(end_seq)
+                continue
+            if in_masked_response:
+                mask[seq_idx, i] = 1
+            else:
+                mask[seq_idx, i] = 0
+            i += 1
+    return mask
+
+
+def get_collate_fn(
+    tokenizer,
+    max_length=8142,
+    only_assistant=False,
+    template_name: str = "llama3",
+):
+    assert template_name in TEMPLATES
+
+    def collate_fn(batch):
+        messages_batch = []
+        for messages in batch:
+            text = tokenizer.apply_chat_template(
+                messages, add_generation_prompt=False, tokenize=False
+            )
+            # remove BOS token since it will later be added again by the tokenizer
+            text = text.replace("<|begin_of_text|>", "")
+            messages_batch.append(text)
+
+        batch = tokenizer(
+            messages_batch,
+            truncation=True,
+            max_length=max_length,
+            return_tensors="pt",
+        )
+        labels = batch["input_ids"].clone()
+        batch["labels"] = labels
+
+        # add mask for assistant response
+        if only_assistant:
+            mask = track_assistant_response(
+                batch, tokenizer, template_name=template_name
+            )
+            labels[mask == 0] = -100
+
+        return batch
+
+    return collate_fn
+
+
+def get_dataset_and_collate(
+    tokenizer,
+    template_name: str = "llama3",
+    max_length: int = 8142,
+    max_message_window: int = 30,
+    trace_mode="oa",
+    only_assistant=False,
+):
+    if template_name == "llama3":
+        train_dataset = PlancraftDialogueDataset(
+            use_images=False,
+            max_message_window=max_message_window,
+            split="train",
+            trace_mode=trace_mode,
+        )
+        val_dataset = PlancraftDialogueDataset(
+            use_images=False,
+            max_message_window=max_message_window,
+            split="val",
+            trace_mode=trace_mode,
+        )
+        collate_fn = get_collate_fn(
+            tokenizer=tokenizer,
+            only_assistant=only_assistant,
+            template_name=template_name,
+            max_length=max_length,
+        )
+    return train_dataset, val_dataset, collate_fn

plancraft/utils.py

@@ -0,0 +1,84 @@
+import glob
+import pathlib
+
+import cv2
+import numpy as np
+import torch
+import torch.nn.functional as F
+from PIL import Image
+
+
+def get_downloaded_models() -> dict:
+    """
+    Get the list of downloaded models on the NFS partition (EIDF).
+    """
+    downloaded_models = {}
+    # known models on NFS partition
+    if pathlib.Path("/nfs").exists():
+        local_models = glob.glob("/nfs/public/hf/models/*/*")
+        downloaded_models = {
+            model.replace("/nfs/public/hf/models/", ""): model for model in local_models
+        }
+    return downloaded_models
+
+
+def get_torch_device() -> torch.device:
+    device = torch.device("cpu")
+    if torch.cuda.is_available():
+        device = torch.device("cuda", 0)
+    elif torch.backends.mps.is_available():
+        if not torch.backends.mps.is_built():
+            print(
+                "MPS not available because the current PyTorch install was not built with MPS enabled."
+            )
+        else:
+            device = torch.device("mps")
+    return device
+
+
+def resize_image(img, target_resolution=(128, 128)):
+    if type(img) == np.ndarray:
+        img = cv2.resize(img, target_resolution, interpolation=cv2.INTER_LINEAR)
+    elif type(img) == torch.Tensor:
+        img = F.interpolate(img, size=target_resolution, mode="bilinear")
+    else:
+        raise ValueError("Unsupported type for img")
+    return img
+
+
+def save_frames_to_video(frames: list, out_path: str):
+    imgs = []
+    for id, (frame, goal) in enumerate(frames):
+        # if torch.is_tensor(frame):
+        # frame = frame.permute(0, 2, 3, 1).cpu().numpy()
+
+        frame = resize_image(frame, (320, 240)).astype("uint8")
+        cv2.putText(
+            frame,
+            f"FID: {id}",
+            (10, 25),
+            cv2.FONT_HERSHEY_SIMPLEX,
+            0.8,
+            (255, 255, 255),
+            2,
+        )
+        cv2.putText(
+            frame,
+            f"Goal: {goal}",
+            (10, 55),
+            cv2.FONT_HERSHEY_SIMPLEX,
+            0.8,
+            (255, 0, 0),
+            2,
+        )
+        imgs.append(Image.fromarray(frame))
+    imgs = imgs[::3]
+    imgs[0].save(
+        out_path,
+        save_all=True,
+        append_images=imgs[1:],
+        optimize=False,
+        quality=0,
+        duration=150,
+        loop=0,
+    )

{plancraft-0.1.2.dist-info → plancraft-0.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: plancraft
-Version: 0.1.2
+Version: 0.1.3
 Summary: Plancraft: an evaluation dataset for planning with LLM agents
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown

plancraft-0.1.3.dist-info/RECORD

@@ -0,0 +1,30 @@
+plancraft/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+plancraft/config.py,sha256=k7Ac6Zh6g-Q-fuazPxVdXnSL_PMd3jsIqyiImNaJW5A,4472
+plancraft/evaluator.py,sha256=zomShgwyUOHNVZbTERpimgEYiyD4JXzHzPi4tg45lsM,9926
+plancraft/utils.py,sha256=J9K81zBXWCcyIBjwu_eHznnzWamXU-pQrz7BdNTVbLU,2289
+plancraft/environments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+plancraft/environments/actions.py,sha256=SeeC9l1cJBs9pdba6BefQ_iQNfFf6FVTWm7HWkacbsY,6262
+plancraft/environments/env_real.py,sha256=oETMvdq8-TspPoIRxioTDj9EEmuZRuuMxd9c3momvPk,10877
+plancraft/environments/env_symbolic.py,sha256=ot4IStZ3oT7CYPqdqHsGl2BopLyZUwSF671SrHIiMLk,7777
+plancraft/environments/items.py,sha256=1R56LyK6tqIssQJMqHst6A9DeEfOX5DN-OBAkumGncw,217
+plancraft/environments/planner.py,sha256=2B-0aunllmTuiHE7Jn5jHzCg6mMgxZjisiTDdpSKupk,3954
+plancraft/environments/recipes.py,sha256=nXvOLCRljiZ5IgeevXuosU9IgSs7oQWQJFiuyRNSVFs,19571
+plancraft/environments/sampler.py,sha256=ZBYoENKdQQ7wbAyVk-c9UNRWKPE0omv9he8c8QZ6wXg,7625
+plancraft/models/__init__.py,sha256=PasK3jpbhpD0kxF4iHukcccZqvZg6lL240zie3DfLDY,622
+plancraft/models/act.py,sha256=jdZunT7FcbHvcaJZ_wUDLSuObHjU6JZybghR_B0QJ8Q,6548
+plancraft/models/base.py,sha256=fFM2BV9PqvIFtUlTz8iz5HPemYRy3S0EituM1XdJJSQ,4927
+plancraft/models/bbox_model.py,sha256=CoX-odH59S-djkPOH2ViEmbYWo1sefmHiOcBlFWiAkg,16814
+plancraft/models/dummy.py,sha256=QjxTIiKsWSmhUMAuw7Yy-OKKCLi_x3rwll4hH7ZNXso,1732
+plancraft/models/generators.py,sha256=c2avYUoXAAWOHRFq24ZYqHqNgoPs_L_XWJrR_TKFg9E,17358
+plancraft/models/oam.py,sha256=_TpxsaCnGvQ7YJC6KWcpl2HzgBiQaHn7U1zRAXfdjOo,9943
+plancraft/models/oracle.py,sha256=xRplR2cCW_39UGYtSjDLRDmp0UILhtIXYjuRJ-jokJ8,9598
+plancraft/models/prompts.py,sha256=XwoRqd_5_VfCUXb10dCRFYXgw70mO2VoQocn3Z2zgs0,6165
+plancraft/models/react.py,sha256=5yM4tv7tDfm_-5yUSAcw5C4wioWijF9fLEODbFqFDvg,3346
+plancraft/models/utils.py,sha256=osKX0_uux9wzqYzq1ST0Cu5idrAnyfNvXrj0uO1eKo0,9424
+plancraft/models/few_shot_images/__init__.py,sha256=nIkyB6w3ok-h4lfJKsrcMQzQF624Y9uxYV1FqCu3Lx0,351
+plancraft/train/dataset.py,sha256=NrZjbIkosui1kaq7AIWSrYvvzrDxu_njH7FmGKY3xnI,5434
+plancraft-0.1.3.dist-info/LICENSE,sha256=YGR8ehDB4t-T-lOQKMfKNR-2zsOU7E3E5NA8t25HKE0,1070
+plancraft-0.1.3.dist-info/METADATA,sha256=XLudjSvyfZ-uYJZ1K1RAJUVOQhozFD3FLzwUQeikhK4,2631
+plancraft-0.1.3.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+plancraft-0.1.3.dist-info/top_level.txt,sha256=yGGA9HtPKH2-pIJFrbyYhj2JF9Xj-4m0fxMPKT9FNzg,10
+plancraft-0.1.3.dist-info/RECORD,,

plancraft-0.1.3.dist-info/top_level.txt

@@ -0,0 +1 @@
+plancraft

plancraft-0.1.2.dist-info/RECORD

@@ -1,5 +0,0 @@
-plancraft-0.1.2.dist-info/LICENSE,sha256=YGR8ehDB4t-T-lOQKMfKNR-2zsOU7E3E5NA8t25HKE0,1070
-plancraft-0.1.2.dist-info/METADATA,sha256=IDsYHFCvjx4Nhe4Jgyj0x7YsIVAR95ZyrSA-LIQBgeI,2631
-plancraft-0.1.2.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-plancraft-0.1.2.dist-info/top_level.txt,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-plancraft-0.1.2.dist-info/RECORD,,

plancraft-0.1.2.dist-info/top_level.txt

@@ -1 +0,0 @@
-