vision-agent 0.0.41__tar.gz → 0.0.43__tar.gz

{vision_agent-0.0.41 → vision_agent-0.0.43}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.0.41
+Version: 0.0.43
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai

{vision_agent-0.0.41 → vision_agent-0.0.43}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "vision-agent"
-version = "0.0.41"
+version = "0.0.43"
 description = "Toolset for Vision Agent"
 authors = ["Landing AI <dev@landing.ai>"]
 readme = "README.md"

{vision_agent-0.0.41 → vision_agent-0.0.43}/vision_agent/agent/vision_agent.py

@@ -256,7 +256,6 @@ def retrieval(
     )
     if tool_id is None:
         return {}, ""
-    _LOGGER.info(f"\t(Tool ID, name): ({tool_id}, {tools[tool_id]['name']})")
 
     tool_instructions = tools[tool_id]
     tool_usage = tool_instructions["usage"]
@@ -265,7 +264,6 @@ def retrieval(
     parameters = choose_parameter(
         model, question, tool_usage, previous_log, reflections
     )
-    _LOGGER.info(f"\tParameters: {parameters} for {tool_name}")
     if parameters is None:
         return {}, ""
     tool_results = {"task": question, "tool_name": tool_name, "parameters": parameters}
@@ -290,7 +288,7 @@ def retrieval(
     tool_results["call_results"] = call_results
 
     call_results_str = str(call_results)
-    _LOGGER.info(f"\tCall Results: {call_results_str}")
+    # _LOGGER.info(f"\tCall Results: {call_results_str}")
     return tool_results, call_results_str
 
 
@@ -344,7 +342,9 @@ def self_reflect(
 
 def parse_reflect(reflect: str) -> bool:
     # GPT-4V has a hard time following directions, so make the criteria less strict
-    return "finish" in reflect.lower() and len(reflect) < 100
+    return (
+        "finish" in reflect.lower() and len(reflect) < 100
+    ) or "finish" in reflect.lower()[-10:]
 
 
 def visualize_result(all_tool_results: List[Dict]) -> List[str]:
@@ -423,10 +423,16 @@ class VisionAgent(Agent):
         verbose: bool = False,
     ):
         self.task_model = (
-            OpenAILLM(json_mode=True) if task_model is None else task_model
+            OpenAILLM(json_mode=True, temperature=0.1)
+            if task_model is None
+            else task_model
+        )
+        self.answer_model = (
+            OpenAILLM(temperature=0.1) if answer_model is None else answer_model
+        )
+        self.reflect_model = (
+            OpenAILMM(temperature=0.1) if reflect_model is None else reflect_model
         )
-        self.answer_model = OpenAILLM() if answer_model is None else answer_model
-        self.reflect_model = OpenAILMM() if reflect_model is None else reflect_model
         self.max_retries = max_retries
 
         self.tools = TOOLS
@@ -466,7 +472,6 @@ class VisionAgent(Agent):
         for _ in range(self.max_retries):
             task_list = create_tasks(self.task_model, question, self.tools, reflections)
 
-            _LOGGER.info(f"Task Dependency: {task_list}")
             task_depend = {"Original Quesiton": question}
             previous_log = ""
             answers = []
@@ -477,7 +482,6 @@ class VisionAgent(Agent):
             for task in task_list:
                 task_str = task["task"]
                 previous_log = str(task_depend)
-                _LOGGER.info(f"\tSubtask: {task_str}")
                 tool_results, call_results = retrieval(
                     self.task_model,
                     task_str,
@@ -492,6 +496,7 @@ class VisionAgent(Agent):
                 tool_results["answer"] = answer
                 all_tool_results.append(tool_results)
 
+                _LOGGER.info(f"\tCall Result: {call_results}")
                 _LOGGER.info(f"\tAnswer: {answer}")
                 answers.append({"task": task_str, "answer": answer})
                 task_depend[task["id"]]["answer"] = answer  # type: ignore
@@ -510,7 +515,7 @@ class VisionAgent(Agent):
                 final_answer,
                 visualized_images[0] if len(visualized_images) > 0 else image,
             )
-            _LOGGER.info(f"\tReflection: {reflection}")
+            _LOGGER.info(f"Reflection: {reflection}")
             if parse_reflect(reflection):
                 break
             else:

{vision_agent-0.0.41 → vision_agent-0.0.43}/vision_agent/llm/llm.py

@@ -1,6 +1,6 @@
 import json
 from abc import ABC, abstractmethod
-from typing import Callable, Dict, List, Mapping, Union, cast
+from typing import Any, Callable, Dict, List, Mapping, Union, cast
 
 from openai import OpenAI
 
@@ -31,30 +31,33 @@ class OpenAILLM(LLM):
     r"""An LLM class for any OpenAI LLM model."""
 
     def __init__(
-        self, model_name: str = "gpt-4-turbo-preview", json_mode: bool = False
+        self,
+        model_name: str = "gpt-4-turbo-preview",
+        json_mode: bool = False,
+        **kwargs: Any
     ):
         self.model_name = model_name
         self.client = OpenAI()
-        self.json_mode = json_mode
+        self.kwargs = kwargs
+        if json_mode:
+            self.kwargs["response_format"] = {"type": "json_object"}
 
     def generate(self, prompt: str) -> str:
-        kwargs = {"response_format": {"type": "json_object"}} if self.json_mode else {}
         response = self.client.chat.completions.create(
             model=self.model_name,
             messages=[
                 {"role": "user", "content": prompt},
             ],
-            **kwargs,  # type: ignore
+            **self.kwargs,
         )
 
         return cast(str, response.choices[0].message.content)
 
     def chat(self, chat: List[Dict[str, str]]) -> str:
-        kwargs = {"response_format": {"type": "json_object"}} if self.json_mode else {}
         response = self.client.chat.completions.create(
             model=self.model_name,
             messages=chat,  # type: ignore
-            **kwargs,
+            **self.kwargs,
         )
 
         return cast(str, response.choices[0].message.content)

{vision_agent-0.0.41 → vision_agent-0.0.43}/vision_agent/lmm/lmm.py

@@ -97,11 +97,15 @@ class OpenAILMM(LMM):
     r"""An LMM class for the OpenAI GPT-4 Vision model."""
 
     def __init__(
-        self, model_name: str = "gpt-4-vision-preview", max_tokens: int = 1024
+        self,
+        model_name: str = "gpt-4-vision-preview",
+        max_tokens: int = 1024,
+        **kwargs: Any,
     ):
         self.model_name = model_name
         self.max_tokens = max_tokens
         self.client = OpenAI()
+        self.kwargs = kwargs
 
     def __call__(
         self,
@@ -123,6 +127,13 @@ class OpenAILMM(LMM):
 
         if image:
             extension = Path(image).suffix
+            if extension.lower() == ".jpeg" or extension.lower() == ".jpg":
+                extension = "jpg"
+            elif extension.lower() == ".png":
+                extension = "png"
+            else:
+                raise ValueError(f"Unsupported image extension: {extension}")
+
             encoded_image = encode_image(image)
             fixed_chat[0]["content"].append(  # type: ignore
                 {
@@ -135,7 +146,7 @@ class OpenAILMM(LMM):
             )
 
         response = self.client.chat.completions.create(
-            model=self.model_name, messages=fixed_chat, max_tokens=self.max_tokens  # type: ignore
+            model=self.model_name, messages=fixed_chat, max_tokens=self.max_tokens, **self.kwargs  # type: ignore
         )
 
         return cast(str, response.choices[0].message.content)
@@ -163,7 +174,7 @@ class OpenAILMM(LMM):
             )
 
         response = self.client.chat.completions.create(
-            model=self.model_name, messages=message, max_tokens=self.max_tokens  # type: ignore
+            model=self.model_name, messages=message, max_tokens=self.max_tokens, **self.kwargs  # type: ignore
         )
         return cast(str, response.choices[0].message.content)
 

vision_agent-0.0.43/vision_agent/tools/tools.json

@@ -0,0 +1,154 @@
+[
+    {
+        "name": "image_question_answering",
+        "description": "answers a question about an image"
+    },
+    {
+        "name": "text_question_answering",
+        "description": "answers a question provided a context in text form"
+    },
+    {
+        "name": "image_captioning",
+        "description": "Generate a caption for the image. It can generate a brief description that can be used for image perception and image generation. For example: a) you can use this tool when you want to know what is it in the image\"; and b) when you want to generate a new image similar or resemble to input.png, you can use `image_captioning` to obtain the description about image input.png."
+    },
+    {
+        "name": "image_to_text",
+        "description": "Generate a description for the image. It can generate a detailed description that can be used for image perception and image generation. For example: a) you can use this tool when you want to know what is it in the image\"; and b) when you want to generate a new image similar or resemble to input.png, you can use `text_to_image` to obtain the description about image input.png."
+    },
+    {
+        "name": "image_to_edge",
+        "description": "get the edge map of the image."
+    },
+    {
+        "name": "image_to_line",
+        "description": "get the line map of the image."
+    },
+    {
+        "name": "image_to_hed",
+        "description": "get the HED map of the image."
+    },
+    {
+        "name": "image_to_scribble",
+        "description": "get the scribble of the image."
+    },
+    {
+        "name": "image_to_pose",
+        "description": "Get the pose of the image. It is usually used in image generation conditioned on pose map from input image."
+    },
+    {
+        "name": "image_to_depth",
+        "description": "get the depth map of the image."
+    },
+    {
+        "name": "image_to_normal",
+        "description": "get the normal map of the image."
+    },
+    {
+        "name": "object_detection",
+        "description": "detect all the objects in the image."
+    },
+    {
+        "name": "image_classification",
+        "description": "classify the objects in the image."
+    },
+    {
+        "name": "closed_set_image_classification",
+        "description": "Given a set of classes as a text prompt, classify the objects in the image based on the given classes."
+    },
+    {
+        "name": "panoptic_segmentation",
+        "description": "segment the common objects in the given image."
+    },
+    {
+        "name": "visual_grounding",
+        "description": "Visual Grounding (VG) aims to locate the most relevant object or region in an image, based on a natural language query. The query can be a phrase, a sentence or even a multi-round dialogue."
+    },
+    {
+        "name": "visual_grounding_segment",
+        "description": "Visual Grounding (VG) aims to locate the most relevant object or region in an image, based on a natural language query. The query can be a phrase, a sentence or even a multi-round dialogue."
+    },
+    {
+        "name": "optical_character_recognition",
+        "description": "Optical Character Recognition (OCR) is the process that converts an image of text into a machine-readable text format."
+    },
+    {
+        "name": "select_category",
+        "description": "select the target classes in category list with the given condition."
+    },
+    {
+        "name": "select_bbox",
+        "description": "select the bounding boxes with the given condition."
+    },
+    {
+        "name": "select_mask",
+        "description": "select the masks with the given condition."
+    },
+    {
+        "name": "count_categories",
+        "description": "count target categories in the given list."
+    },
+    {
+        "name": "count_objects",
+        "description": "count target objects in the given list. It is useful when you want to count the number of objects in the image"
+    },
+    {
+        "name": "count_masks",
+        "description": "count target mask in the given list."
+    },
+    {
+        "name": "video_captioning",
+        "description": "Generate a caption or description for video. It can generate a detailed description that can be used for video perception and video generation. For example: a) you can use this tool when you want to know what happened in the video\"; and b) when you want to generate tags for input video, you can use translate description obtained from `image_captioning` into tags."
+    },
+    {
+        "name": "video_classification",
+        "description": "Classify the video and detect the actions in the video."
+    },
+    {
+        "name": "frame_captioning",
+        "description": "Generate a caption or description for video every n seconds. It can generate a detailed description that can be used for video perception and video generation. For example: a) you can use this tool when you want to know what happened in the video\"; and b) when you want to generate tags for input video, you can use translate description obtained from `image_captioning` into tags."
+    },
+    {
+        "name": "frame_classification",
+        "description": "Classify the video and detect the actions in the  every n seconds."
+    },
+    {
+        "name": "text_to_text_generation",
+        "description": "Text to text generation. It can be used for sentence acceptability judgment, Sentiment analysis, Paraphrasing/sentence similarity, Natural language inference, Sentence completion, Word sense disambiguation, Question answering."
+    },
+    {
+        "name": "openai_chat_model",
+        "description": "Answer the question by Large Language Model. It is useful for tasks such as generating content, answering questions, engaging in conversations and providing explanations. However, it still has some limitations. For example, it can not directly access the up-to-date information like time, weather, etc."
+    },
+    {
+        "name": "summarization",
+        "description": "Summarize sentences, long narratives, articles, papers, textbooks."
+    },
+    {
+        "name": "text_to_tags",
+        "description": "Predict the tags of text, article and papers by using the their textual content as input"
+    },
+    {
+        "name": "sentiment_analysis",
+        "description": "Sentiment analysis is the process of analyzing digital text to determine if the emotional tone of the message is positive, negative, or neutral."
+    }
+]
+
+input_keys: [
+    { "image": "image file"},
+    { "prompt": "text"},
+    { "video": "video file"},
+    { "context": "context for text qa"},
+    { "tool": "tool name mentioned above"},
+]
+
+output_keys: [
+    { "labels": "list of labels for image tasks"},
+    { "scores": "list of scores for image and text tasks"},
+    { "bboxes": "list of bounding boxes for detection tasks"},
+    { "masks": "list of masks for segmentation tasks"},
+    { "text": "list of text for text tasks"},
+    { "frames": "list of frame numbers for video tasks"},
+]
+
+
+

{vision_agent-0.0.41 → vision_agent-0.0.43}/vision_agent/tools/tools.py

@@ -1,5 +1,6 @@
 import logging
 import tempfile
+import os
 from abc import ABC
 from collections import Counter as CounterClass
 from pathlib import Path
@@ -139,7 +140,7 @@ class GroundingDINO(Tool):
         'scores': [0.98, 0.02]}]
     """
 
-    _ENDPOINT = "https://chnicr4kes5ku77niv2zoytggq0qyqlp.lambda-url.us-east-2.on.aws"
+    _ENDPOINT = "https://soi4ewr6fjqqdf5vuss6rrilee0kumxq.lambda-url.us-east-2.on.aws"
 
     name = "grounding_dino_"
     description = "'grounding_dino_' is a tool that can detect arbitrary objects with inputs such as category names or referring expressions."
@@ -182,11 +183,15 @@ class GroundingDINO(Tool):
         image_b64 = convert_to_b64(image)
         data = {
             "prompt": prompt,
-            "images": [image_b64],
+            "images": image_b64,
+            "tool": "visual_grounding",
         }
         res = requests.post(
             self._ENDPOINT,
-            headers={"Content-Type": "application/json"},
+            headers={
+                "Content-Type": "application/json",
+                "Authorization": f"Api-Key {os.environ['BASETEN_API_KEY']}",
+            },
             json=data,
         )
         resp_json: Dict[str, Any] = res.json()
@@ -230,7 +235,7 @@ class GroundingSAM(Tool):
            [1, 1, 1, ..., 1, 1, 1]], dtype=uint8)]}]
     """
 
-    _ENDPOINT = "https://cou5lfmus33jbddl6hoqdfbw7e0qidrw.lambda-url.us-east-2.on.aws"
+    _ENDPOINT = "https://model-owp50nlq.api.baseten.co/production/predict"
 
     name = "grounding_sam_"
     description = "'grounding_sam_' is a tool that can detect and segment arbitrary objects with inputs such as category names or referring expressions."
@@ -274,10 +279,14 @@ class GroundingSAM(Tool):
         data = {
             "classes": prompt,
             "image": image_b64,
+            "tool": "visual_grounding_segment",
         }
         res = requests.post(
             self._ENDPOINT,
-            headers={"Content-Type": "application/json"},
+            headers={
+                "Content-Type": "application/json",
+                "Authorization": f"Api-Key {os.environ['BASETEN_API_KEY']}",
+            },
             json=data,
         )
         resp_json: Dict[str, Any] = res.json()