plancraft 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

{plancraft-0.2.0.dist-info → plancraft-0.3.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.26.3
+Generator: hatchling 1.27.0
 Root-Is-Purelib: true
 Tag: py3-none-any

plancraft/models/few_shot_images/__init__.py

@@ -1,16 +0,0 @@
-import os
-import glob
-
-import numpy as np
-import imageio
-
-
-def get_few_shot_images_path():
-    return os.path.dirname(__file__)
-
-
-def load_prompt_images() -> list[np.ndarray]:
-    current_dir = get_few_shot_images_path()
-    files = glob.glob(os.path.join(current_dir, "*.png"))
-    images = [imageio.imread(file) for file in files]
-    return images

plancraft/models/oam.py

@@ -1,283 +0,0 @@
-from typing import Optional
-
-from loguru import logger
-
-import torch
-import torch.nn as nn
-import torchvision.transforms.v2 as v2
-from transformers import (
-    AutoConfig,
-    AutoModelForCausalLM,
-    AutoTokenizer,
-    PretrainedConfig,
-    PreTrainedModel,
-)
-
-from plancraft.models.bbox_model import IntegratedBoundingBoxModel
-
-
-class PlancraftOAMConfig(PretrainedConfig):
-    model_type = "plancraft-aom"
-    is_composition = True
-
-    def __init__(
-        self,
-        from_llama=False,
-        **kwargs,
-    ):
-        self.from_llama = from_llama
-        super().__init__(**kwargs)
-
-
-class PlancraftOAM(PreTrainedModel):
-    config_class = PlancraftOAMConfig
-
-    def __init__(self, config: PlancraftOAMConfig):
-        super().__init__(config)
-
-        self.config = config
-        # load text model
-        if self.config.from_llama:
-            self.text_model = AutoModelForCausalLM.from_pretrained(
-                "meta-llama/Meta-Llama-3.1-8B-Instruct",
-            )
-        else:
-            text_model_config = AutoConfig.from_pretrained(
-                "meta-llama/Meta-Llama-3.1-8B-Instruct",
-            )
-            self.text_model = AutoModelForCausalLM.from_config(text_model_config)
-
-        # load vision model
-        self.vision_model = IntegratedBoundingBoxModel.from_pretrained(
-            "gautierdag/plancraft-maskrcnn"
-        )
-        self.vision_model.eval()
-
-        # convert vision features to text embedding
-        self.vision_to_text_embedding = nn.Linear(
-            1024, self.text_model.config.hidden_size
-        )
-        self.tokenizer = AutoTokenizer.from_pretrained(
-            "meta-llama/Meta-Llama-3.1-8B-Instruct",
-            trust_remote=True,
-        )
-        # add special tokens
-        self.tokenizer.add_special_tokens(
-            {
-                "additional_special_tokens": [
-                    "<|inventory|>",
-                ]
-            }
-        )
-        self.tokenizer.pad_token = self.tokenizer.eos_token
-        self.inventory_idx = self.tokenizer.convert_tokens_to_ids("<|inventory|>")
-
-        # resize token embeddings
-        self.text_model.resize_token_embeddings(len(self.tokenizer))
-        # image transforms
-        self.transforms = v2.Compose(
-            [v2.ToImage(), v2.ToDtype(torch.float32, scale=True)]
-        )
-
-    @torch.no_grad()
-    def extract_bboxes(self, images: list) -> list[dict]:
-        if len(images) == 0:
-            return []
-        img_tensors = torch.stack([self.transforms(img) for img in images])
-        img_tensors = img_tensors.cuda()
-        # disable gradients
-        self.vision_model.freeze()
-        # get bounding box predictions
-        bbox_preds = self.vision_model(img_tensors)
-        return bbox_preds
-
-    def prepare_messages(self, messages: list, bboxes: list[dict]) -> str:
-        # no bounding boxes
-        if len(bboxes) == 0:
-            text = self.tokenizer.apply_chat_template(
-                messages, add_generation_prompt=not self.training, tokenize=False
-            )
-            text = text.replace("<|begin_of_text|>", "")
-            return text
-
-        # expand <|inventory|> tokens into N tokens (N = number of bounding boxes)
-        new_messages = []
-        i_pred = 0
-        for m in messages:
-            new_message = m.copy()
-            if new_message["role"] == "user" and new_message["content"].endswith(
-                "<|inventory|>"
-            ):
-                # add inventory tokens for each bounding box
-                new_message["content"] = new_message["content"].replace(
-                    "<|inventory|>",
-                    "<|inventory|>" * (bboxes[i_pred]["features"].shape[0]),
-                )
-                i_pred += 1
-            new_messages.append(new_message)
-
-        assert i_pred == len(
-            bboxes
-        ), "Number of inventory tokens does not match number of images"
-        # add special tokens
-
-        text = self.tokenizer.apply_chat_template(
-            new_messages, add_generation_prompt=not self.training, tokenize=False
-        )
-        text = text.replace("<|begin_of_text|>", "")
-        return text
-
-    def inputs_merger(
-        self,
-        input_ids: torch.LongTensor,
-        inputs_embeds: Optional[torch.Tensor],
-        image_hidden_states: Optional[torch.Tensor],
-    ):
-        # along batch dimension
-        for i in range(len(image_hidden_states)):
-            if len(image_hidden_states[i]) == 0:
-                assert (
-                    input_ids[i] == self.inventory_idx
-                ).sum() == 0, "No images but inventory token is still present"
-                continue
-
-            # count the number of inventory tokens
-            n_inventory_tokens = (input_ids[i] == self.inventory_idx).sum()
-            if n_inventory_tokens != image_hidden_states[i].shape[0]:
-                logger.warning(
-                    f"Number of inventory tokens ({n_inventory_tokens}) does not match number of bounding boxes ({image_hidden_states[i].shape[0]}). Possible truncation."
-                )
-                # truncated from the start
-                image_hidden_states[i] = image_hidden_states[i][-n_inventory_tokens:]
-
-            # replace inventory tokens with bbox features
-            inputs_embeds[i, input_ids[i] == self.inventory_idx] = image_hidden_states[
-                i
-            ]
-        return inputs_embeds
-
-    def process_inputs(
-        self,
-        batch_messages: list[list[dict]] = [],  # list of list of messages (untokenized)
-        batch_images: list[list] = [],  # list of list of images (unprocessed)
-    ) -> tuple[dict[str, torch.FloatTensor], list[torch.FloatTensor], int]:
-        """
-        Converts raw images and messages into model inputs
-        """
-        assert len(batch_images) == len(
-            batch_messages
-        ), "Number of images and messages should match in the batch dim"
-        # initial forward pass
-        texts_batch = []
-        image_hidden_states = []
-        total_boxes = 0
-        for images, messages in zip(batch_images, batch_messages):
-            # process images
-            bboxes = self.extract_bboxes(images)
-            if len(bboxes) > 0:
-                # get bbox features
-                features = torch.concat([p["features"] for p in bboxes], dim=0)
-                # upscale to text embedding size
-                features_embeds = self.vision_to_text_embedding(features)
-                image_hidden_states.append(features_embeds)
-                # count bboxes total
-                total_boxes += features.shape[0]
-            else:
-                image_hidden_states.append([])
-
-            # process messages
-            text = self.prepare_messages(messages, bboxes)
-            texts_batch.append(text)
-
-        # tokenize text
-        # @NOTE: truncation could cause issues with inventory tokens not matching number of boxes
-        # in that case, we will truncate the boxes from the end, and issue a warning
-        batch = self.tokenizer(
-            texts_batch,
-            truncation=True,
-            padding=True,
-            max_length=16384,
-            return_tensors="pt",
-        )
-        return batch, image_hidden_states, total_boxes
-
-    def forward(
-        self,
-        batch_messages: list[list[dict]] = [],  # list of list of messages (untokenized)
-        batch_images: list[list] = [],  # list of list of images (unprocessed)
-        **kwargs,
-    ):
-        labels = None
-        batch, image_hidden_states, total_boxes = self.process_inputs(
-            batch_messages, batch_images
-        )
-        # move to cuda
-        batch = {k: v.cuda() for k, v in batch.items()}
-        attention_mask = batch["attention_mask"]
-        input_ids = batch["input_ids"]
-
-        labels = input_ids.clone()
-        # remove inventory tokens from labels
-        labels[labels == self.inventory_idx] = -100
-        # sanity check: should have same number of boxes as inventory tokens
-        assert (labels == -100).sum() == total_boxes
-
-        # get text embeddings
-        inputs_embeds = self.text_model.get_input_embeddings()(input_ids)
-        inputs_embeds = self.inputs_merger(
-            input_ids, inputs_embeds, image_hidden_states
-        )
-        # forward pass
-        return self.text_model(
-            inputs_embeds=inputs_embeds,
-            attention_mask=attention_mask,
-            labels=labels,
-            return_dict=True,
-        )
-
-    @torch.no_grad()
-    def generate(
-        self,
-        batch_messages: list[list[dict]],
-        batch_images: list[list],
-        do_sample=True,
-        temperature=0.6,
-        max_new_tokens=32,
-    ):
-        self.tokenizer.padding_side = "left"
-
-        batch, image_hidden_states, _ = self.process_inputs(
-            batch_messages, batch_images
-        )
-        batch = {k: v.cuda() for k, v in batch.items()}
-        attention_mask = batch["attention_mask"]
-        input_ids = batch["input_ids"]
-
-        inputs_embeds = self.text_model.get_input_embeddings()(input_ids)
-        inputs_embeds = self.inputs_merger(
-            input_ids, inputs_embeds, image_hidden_states
-        )
-
-        generated_sequences = self.text_model.generate(
-            inputs_embeds=inputs_embeds,
-            attention_mask=attention_mask,
-            do_sample=do_sample,
-            temperature=temperature,
-            max_new_tokens=max_new_tokens,
-            pad_token_id=self.tokenizer.eos_token_id,
-        )
-
-        # Decode the output
-        text_responses = self.tokenizer.batch_decode(
-            generated_sequences,
-            # generated_sequences[:, prompt_tokens:],
-            skip_special_tokens=False,
-        )
-
-        # remove <|eot_id|> tokens
-        text_responses = [
-            text_response.replace("<|eot_id|>", "") for text_response in text_responses
-        ]
-
-        _, total_tokens_used = generated_sequences.shape
-        return text_responses, total_tokens_used

plancraft/models/react.py

@@ -1,93 +0,0 @@
-from dotenv import load_dotenv
-
-from plancraft.config import EvalConfig
-from plancraft.environments.actions import (
-    NoOp,
-    SymbolicAction,
-)
-from plancraft.models.act import ActModel
-from plancraft.models.utils import (
-    convert_observation_to_message,
-    parse_content_response,
-)
-
-load_dotenv()
-
-
-class ReactModel(ActModel):
-    """
-    Model that does action with interleaved thinking step
-    """
-
-    def __init__(self, cfg: EvalConfig):
-        super().__init__(cfg)
-        self.max_invalid_actions = 3
-
-    def step(self, observation: dict) -> SymbolicAction:
-        # override the step method in ActModel to force thinking step
-
-        self.history.add_observation_to_history(observation)
-        observation_message = convert_observation_to_message(
-            observation,
-            objective=self.history.objective,
-            bbox_model=self.bbox_model,
-            oam_model="oam" in self.llm.model_name,
-            use_text_inventory=self.use_text_inventory,
-            use_multimodal_content_format=self.use_multimodal_content_format,
-            use_images=self.use_images,
-        )
-        # add observation to history
-        self.history.add_message_to_history(content=observation_message, role="user")
-
-        i = 0
-        while i < self.max_invalid_actions:
-            message_window, image_window = self.llm.prepare_messages(
-                history=self.history,
-                max_messages_window=self.max_messages_window,
-                system_prompt=self.system_prompt,
-                prompt_images=self.prompt_images,
-            )
-            think_messages, think_token_used = self.llm.generate_unconstrained(
-                batch_messages=[message_window],
-                images=[image_window],
-                start_messages_generation="think:",
-            )
-            self.history.tokens_used += think_token_used
-            think_message = "think: " + think_messages[0].split("\n")[0].strip()
-            self.history.add_message_to_history(content=think_message, role="assistant")
-
-            # retrieve new message window (with thinking prompt)
-            message_window, image_window = self.llm.prepare_messages(
-                history=self.history,
-                max_messages_window=self.max_messages_window,
-                system_prompt=self.system_prompt,
-                prompt_images=self.prompt_images,
-            )
-            action_messages, action_token_used = self.llm.generate_unconstrained(
-                batch_messages=[message_window],
-                images=[image_window],
-                start_messages_generation="",
-            )
-            self.history.tokens_used += action_token_used
-
-            action_message = action_messages[0].split("\n")[0].strip()
-
-            self.history.add_message_to_history(
-                content=action_message, role="assistant"
-            )
-
-            response = parse_content_response(
-                action_message, valid_actions=self.valid_actions
-            )
-            if not isinstance(response, str):
-                # valid action
-                self.history.add_action_to_history(response)
-                return response
-
-            self.history.add_message_to_history(
-                content=response,
-            )
-            i += 1
-
-        # default move action
-        return NoOp()

plancraft-0.2.0.dist-info/METADATA

@@ -1,74 +0,0 @@
-Metadata-Version: 2.3
-Name: plancraft
-Version: 0.2.0
-Summary: Plancraft: an evaluation dataset for planning with LLM agents
-License: MIT License
-        
-        Copyright (c) 2024 Gautier Dagan
-        
-        Permission is hereby granted, free of charge, to any person obtaining a copy
-        of this software and associated documentation files (the "Software"), to deal
-        in the Software without restriction, including without limitation the rights
-        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-        copies of the Software, and to permit persons to whom the Software is
-        furnished to do so, subject to the following conditions:
-        
-        The above copyright notice and this permission notice shall be included in all
-        copies or substantial portions of the Software.
-        
-        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-        SOFTWARE.
-Requires-Python: >=3.9
-Requires-Dist: hydra-core>=1.3.2
-Requires-Dist: imageio>=2.36.0
-Requires-Dist: loguru
-Requires-Dist: networkx
-Requires-Dist: numpy
-Requires-Dist: openai
-Requires-Dist: pandas
-Requires-Dist: pillow>=8.0.0
-Requires-Dist: pydantic
-Requires-Dist: python-dotenv>=1.0.1
-Requires-Dist: setuptools
-Requires-Dist: tqdm
-Requires-Dist: wandb
-Provides-Extra: full
-Requires-Dist: accelerate; extra == 'full'
-Requires-Dist: einops; extra == 'full'
-Requires-Dist: hf-transfer; extra == 'full'
-Requires-Dist: huggingface-hub; extra == 'full'
-Requires-Dist: matplotlib; extra == 'full'
-Requires-Dist: peft; extra == 'full'
-Requires-Dist: seaborn; extra == 'full'
-Requires-Dist: torch>=2.5.0; extra == 'full'
-Requires-Dist: torchvision>=0.20.0; extra == 'full'
-Requires-Dist: transformers>=4.43.3; extra == 'full'
-Description-Content-Type: text/markdown
-
-# plancraft
-
-[![Test](https://github.com/gautierdag/plancraft/actions/workflows/test.yaml/badge.svg)](https://github.com/gautierdag/plancraft/actions/workflows/test.yaml)
-![Python Version](https://img.shields.io/badge/python-3.9+-blue)
-![Ruff](https://img.shields.io/badge/linter-ruff-blue)
-[![PyPI Version](https://img.shields.io/pypi/v/plancraft)](https://pypi.org/project/plancraft/)
-
-Plancraft is a minecraft environment and agent that innovates on planning LLM agents with a retriever
-
-You can install the package by running the following command:
-
-```bash
-pip install plancraft
-```
-
-Should you need the multimodal version of the package, you will also need a custom [fork](https://github.com/gautierdag/minerl.git) of the minerl package. You can install it by running the following command:
-
-```bash
-pip install git+hhttps://github.com/gautierdag/minerl.git
-```
-
-Note that you may need to follow the same installation instructions as in the [minerl documentation](https://minerl.readthedocs.io/en/latest/tutorials/index.html).