vision-agent 0.2.155__tar.gz → 0.2.157__tar.gz

{vision_agent-0.2.155 → vision_agent-0.2.157}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.2.155
+Version: 0.2.157
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai
@@ -56,19 +56,23 @@ accomplish the task you want. Vision Agent aims to provide an in-seconds experie
 allowing users to describe their problem in text and have the agent framework generate
 code to solve the task for them. Check out our discord for updates and roadmaps!
 
+## Table of Contents
+- [🚀Quick Start](#quick-start)
+- [📚Documentation](#documentation)
+- [🔍🤖Vision Agent](#vision-agent-basic-usage)
+- [🛠️Tools](#tools)
+- [🤖LMMs](#lmms)
+- [💻🤖Vision Agent Coder](#vision-agent-coder)
+- [🏗️Additional Backends](#additional-backends)
 
-## Web Application
+## Quick Start
+### Web Application
+The fastest way to test out Vision Agent is to use our web application. You can find it
+[here](https://va.landing.ai/).
 
-Try Vision Agent live on (note this may not be running the most up-to-date version) [va.landing.ai](https://va.landing.ai/)
 
-## Documentation
-
-[Vision Agent Library Docs](https://landing-ai.github.io/vision-agent/)
-
-
-## Getting Started
 ### Installation
-To get started, you can install the library using pip:
+To get started with the python library, you can install it using pip:
 
 ```bash
 pip install vision-agent
@@ -82,17 +86,93 @@ export ANTHROPIC_API_KEY="your-api-key"
 export OPENAI_API_KEY="your-api-key"
 ```
 
-### Vision Agent
-There are two agents that you can use. `VisionAgent` is a conversational agent that has
-access to tools that allow it to write an navigate python code and file systems. It can
-converse with the user in natural language. `VisionAgentCoder` is an agent specifically
-for writing code for vision tasks, such as counting people in an image. However, it
-cannot chat with you and can only respond with code. `VisionAgent` can call
-`VisionAgentCoder` to write vision code.
+### Basic Usage
+To get started you can just import the `VisionAgent` and start chatting with it:
+```python
+>>> from vision_agent.agent import VisionAgent
+>>> agent = VisionAgent()
+>>> resp = agent("Hello")
+>>> print(resp)
+[{"role": "user", "content": "Hello"}, {"role": "assistant", "content": "{'thoughts': 'The user has greeted me. I will respond with a greeting and ask how I can assist them.', 'response': 'Hello! How can I assist you today?', 'let_user_respond': True}"}]
+>>> resp.append({"role": "user", "content": "Can you count the number of people in this image?", "media": ["people.jpg"]})
+>>> resp = agent(resp)
+```
+
+The chat messages are similar to `OpenAI`'s format with `role` and `content` keys but
+in addition to those you can add `medai` which is a list of media files that can either
+be images or video files.
+
+## Documentation
+
+[Vision Agent Library Docs](https://landing-ai.github.io/vision-agent/)
+
+## Vision Agent Basic Usage
+### Chatting and Message Formats
+`VisionAgent` is an agent that can chat with you and call other tools or agents to
+write vision code for you. You can interact with it like you would ChatGPT or any other
+chatbot. The agent uses Clause-3.5 for it's LMM and OpenAI for embeddings for searching
+for tools.
+
+The message format is:
+```json
+{
+    "role": "user",
+    "content": "Hello",
+    "media": ["image.jpg"]
+}
+```
+Where `role` can be `user`, `assistant` or `observation` if the agent has executed a 
+function and needs to observe the output. `content` is always the text message and
+`media` is a list of media files that can be images or videos that you want the agent
+to examine.
+
+When the agent responds, inside it's `context` you will find the following data structure:
+```json
+{
+    "thoughts": "The user has greeted me. I will respond with a greeting and ask how I can assist them.",
+    "response": "Hello! How can I assist you today?",
+    "let_user_respond": true
+}
+```
+
+`thoughts` are the thoughts the agent had when processing the message, `response` is the
+response it generated which could contain a python execution, and `let_user_respond` is
+a boolean that tells the agent if it should wait for the user to respond before
+continuing, for example it may want to execute code and look at the output before
+letting the user respond.
 
-#### Basic Usage
-To run the streamlit app locally to chat with `VisionAgent`, you can run the following
-command:
+### Chatting and Artifacts
+If you run `chat_with_code` you will also notice an `Artifact` object. `Artifact`'s
+are a way to sync files between local and remote environments. The agent will read and
+write to the artifact object, which is just a pickle object, when it wants to save or
+load files.
+
+```python
+import vision_agent as va
+from vision_agent.tools.meta_tools import Artifact
+
+artifact = Artifact("artifact.pkl")
+# you can store text files such as code or images in the artifact
+with open("code.py", "r") as f:
+    artifacts["code.py"] = f.read()
+with open("image.png", "rb") as f:
+    artifacts["image.png"] = f.read()
+
+agent = va.agent.VisionAgent()
+response, artifacts = agent.chat_with_code(
+    [
+        {
+            "role": "user",
+            "content": "Can you write code to count the number of people in image.png",
+        }
+    ],
+    artifacts=artifacts,
+)
+```
+
+### Running the Streamlit App
+To test out things quickly, sometimes it's easier to run the streamlit app locally to
+chat with `VisionAgent`, you can run the following command:
 
 ```bash
 pip install -r examples/chat/requirements.txt
@@ -100,25 +180,117 @@ export WORKSPACE=/path/to/your/workspace
 export ZMQ_PORT=5555
 streamlit run examples/chat/app.py
 ```
-You can find more details about the streamlit app [here](examples/chat/).
+You can find more details about the streamlit app [here](examples/chat/), there are
+still some concurrency issues with the streamlit app so if you find it doing weird things
+clear your workspace and restart the app.
+
+## Tools
+There are a variety of tools for the model or the user to use. Some are executed locally
+while others are hosted for you. You can easily access them yourself, for example if
+you want to run `owl_v2_image` and visualize the output you can run:
 
-#### Basic Programmatic Usage
 ```python
->>> from vision_agent.agent import VisionAgent
->>> agent = VisionAgent()
->>> resp = agent("Hello")
->>> print(resp)
-[{"role": "user", "content": "Hello"}, {"role": "assistant", "content": "{'thoughts': 'The user has greeted me. I will respond with a greeting and ask how I can assist them.', 'response': 'Hello! How can I assist you today?', 'let_user_respond': True}"}]
->>> resp.append({"role": "user", "content": "Can you count the number of people in this image?", "media": ["people.jpg"]})
->>> resp = agent(resp)
+import vision_agent.tools as T
+import matplotlib.pyplot as plt
+
+image = T.load_image("dogs.jpg")
+dets = T.owl_v2_image("dogs", image)
+# visualize the owl_v2_ bounding boxes on the image
+viz = T.overlay_bounding_boxes(image, dets)
+
+# plot the image in matplotlib or save it
+plt.imshow(viz)
+plt.show()
+T.save_image(viz, "viz.png")
 ```
 
-`VisionAgent` currently utilizes Claude-3.5 as it's default LMM and uses OpenAI for
-embeddings for tool searching.
+Or if you want to run on video data, for example track sharks and people at 10 FPS:
 
-### Vision Agent Coder
-#### Basic Usage
-You can interact with the agent as you would with any LLM or LMM model:
+```python
+frames_and_ts = T.extract_frames_and_timestamps("sharks.mp4", fps=10)
+# extract only the frames from frames and timestamps
+frames = [f["frame"] for f in frames_and_ts]
+# track the sharks and people in the frames, returns segmentation masks
+track = T.florence2_sam2_video_tracking("shark, person", frames)
+# plot the segmentation masks on the frames
+viz = T.overlay_segmentation_masks(frames, track)
+T.save_video(viz, "viz.mp4")
+```
+
+You can find all available tools in `vision_agent/tools/tools.py`, however the
+`VisionAgent` will only utilizes a subset of tools that have been tested and provide
+the best performance. Those can be found in the same file under the `FUNCION_TOOLS`
+variable inside `tools.py`.
+
+#### Custom Tools
+If you can't find the tool you are looking for you can also add custom tools to the
+agent:
+
+```python
+import vision_agent as va
+import numpy as np
+
+@va.tools.register_tool(imports=["import numpy as np"])
+def custom_tool(image_path: str) -> str:
+    """My custom tool documentation.
+
+    Parameters:
+        image_path (str): The path to the image.
+
+    Returns:
+        str: The result of the tool.
+
+    Example
+    -------
+    >>> custom_tool("image.jpg")
+    """
+
+    return np.zeros((10, 10))
+```
+
+You need to ensure you call `@va.tools.register_tool` with any imports it uses. Global
+variables will not be captured by `register_tool` so you need to include them in the
+function. Make sure the documentation is in the same format above with description,
+`Parameters:`, `Returns:`, and `Example\n-------`. The `VisionAgent` will use your
+documentation when trying to determine when to use your tool. You can find an example
+use case [here](examples/custom_tools/) for adding a custom tool. Note you may need to
+play around with the prompt to ensure the model picks the tool when you want it to.
+
+Can't find the tool you need and want us to host it? Check out our
+[vision-agent-tools](https://github.com/landing-ai/vision-agent-tools) repository where
+we add the source code for all the tools used in `VisionAgent`.
+
+## LMMs
+All of our agents are based off of LMMs or Large Multimodal Models. We provide a thin
+abstraction layer on top of the underlying provider APIs to be able to more easily
+handle media.
+
+
+```python
+from vision_agent.lmm import AnthropicLMM
+
+lmm = AnthropicLMM()
+response = lmm("Describe this image", media=["apple.jpg"])
+>>> "This is an image of an apple."
+```
+
+Or you can use the `OpenAI` chat interaface and pass it other media like videos:
+
+```python
+response = lmm(
+    [
+        {
+            "role": "user",
+            "content": "What's going on in this video?",
+            "media": ["video.mp4"]
+        }
+    ]
+)
+```
+
+## Vision Agent Coder
+Underneath the hood, `VisionAgent` uses `VisionAgentCoder` to generate code to solve
+vision tasks. You can use `VisionAgentCoder` directly to generate code if you want:
 
 ```python
 >>> from vision_agent.agent import VisionAgentCoder
@@ -128,17 +300,17 @@ You can interact with the agent as you would with any LLM or LMM model:
 
 Which produces the following code:
 ```python
-from vision_agent.tools import load_image, grounding_sam
+from vision_agent.tools import load_image, florence2_sam2_image
 
 def calculate_filled_percentage(image_path: str) -> float:
     # Step 1: Load the image
     image = load_image(image_path)
 
     # Step 2: Segment the jar
-    jar_segments = grounding_sam(prompt="jar", image=image)
+    jar_segments = florence2_sam2_image("jar", image)
 
     # Step 3: Segment the coffee beans
-    coffee_beans_segments = grounding_sam(prompt="coffee beans", image=image)
+    coffee_beans_segments = florence2_sam2_image("coffee beans", image)
 
     # Step 4: Calculate the area of the segmented jar
     jar_area = 0
@@ -166,7 +338,7 @@ mode by passing in the verbose argument:
 >>> agent = VisionAgentCoder(verbosity=2)
 ```
 
-#### Detailed Usage
+### Detailed Usage
 You can also have it return more information by calling `chat_with_workflow`. The format
 of the input is a list of dictionaries with the keys `role`, `content`, and `media`:
 
@@ -186,7 +358,7 @@ of the input is a list of dictionaries with the keys `role`, `content`, and `med
 With this you can examine more detailed information such as the testing code, testing
 results, plan or working memory it used to complete the task.
 
-#### Multi-turn conversations
+### Multi-turn conversations
 You can have multi-turn conversations with vision-agent as well, giving it feedback on
 the code and having it update. You just need to add the code as a response from the
 assistant:
@@ -212,60 +384,6 @@ conv.append(
 result = agent.chat_with_workflow(conv)
 ```
 
-### Tools
-There are a variety of tools for the model or the user to use. Some are executed locally
-while others are hosted for you. You can easily access them yourself, for example if
-you want to run `owl_v2_image` and visualize the output you can run:
-
-```python
-import vision_agent.tools as T
-import matplotlib.pyplot as plt
-
-image = T.load_image("dogs.jpg")
-dets = T.owl_v2_image("dogs", image)
-viz = T.overlay_bounding_boxes(image, dets)
-plt.imshow(viz)
-plt.show()
-```
-
-You can find all available tools in `vision_agent/tools/tools.py`, however,
-`VisionAgentCoder` only utilizes a subset of tools that have been tested and provide
-the best performance. Those can be found in the same file under the `TOOLS` variable.
-
-If you can't find the tool you are looking for you can also add custom tools to the
-agent:
-
-```python
-import vision_agent as va
-import numpy as np
-
-@va.tools.register_tool(imports=["import numpy as np"])
-def custom_tool(image_path: str) -> str:
-    """My custom tool documentation.
-
-    Parameters:
-        image_path (str): The path to the image.
-
-    Returns:
-        str: The result of the tool.
-
-    Example
-    -------
-    >>> custom_tool("image.jpg")
-    """
-
-    return np.zeros((10, 10))
-```
-
-You need to ensure you call `@va.tools.register_tool` with any imports it uses. Global
-variables will not be captured by `register_tool` so you need to include them in the
-function. Make sure the documentation is in the same format above with description,
-`Parameters:`, `Returns:`, and `Example\n-------`. You can find an example use case
-[here](examples/custom_tools/) as this is what the agent uses to pick and use the tool.
-
-Can't find the tool you need and want add it to `VisionAgent`? Check out our
-[vision-agent-tools](https://github.com/landing-ai/vision-agent-tools) repository where
-we add the source code for all the tools used in `VisionAgent`.
 
 ## Additional Backends
 ### Anthropic
@@ -370,9 +488,9 @@ agent = va.agent.AzureVisionAgentCoder()
 
 ******************************************************************************************************************************
 
-### Q&A
+## Q&A
 
-#### How to get started with OpenAI API credits
+### How to get started with OpenAI API credits
 
 1. Visit the [OpenAI API platform](https://beta.openai.com/signup/) to sign up for an API key.
 2. Follow the instructions to purchase and manage your API credits.

{vision_agent-0.2.155 → vision_agent-0.2.157}/README.md

@@ -15,19 +15,23 @@ accomplish the task you want. Vision Agent aims to provide an in-seconds experie
 allowing users to describe their problem in text and have the agent framework generate
 code to solve the task for them. Check out our discord for updates and roadmaps!
 
+## Table of Contents
+- [🚀Quick Start](#quick-start)
+- [📚Documentation](#documentation)
+- [🔍🤖Vision Agent](#vision-agent-basic-usage)
+- [🛠️Tools](#tools)
+- [🤖LMMs](#lmms)
+- [💻🤖Vision Agent Coder](#vision-agent-coder)
+- [🏗️Additional Backends](#additional-backends)
 
-## Web Application
+## Quick Start
+### Web Application
+The fastest way to test out Vision Agent is to use our web application. You can find it
+[here](https://va.landing.ai/).
 
-Try Vision Agent live on (note this may not be running the most up-to-date version) [va.landing.ai](https://va.landing.ai/)
 
-## Documentation
-
-[Vision Agent Library Docs](https://landing-ai.github.io/vision-agent/)
-
-
-## Getting Started
 ### Installation
-To get started, you can install the library using pip:
+To get started with the python library, you can install it using pip:
 
 ```bash
 pip install vision-agent
@@ -41,17 +45,93 @@ export ANTHROPIC_API_KEY="your-api-key"
 export OPENAI_API_KEY="your-api-key"
 ```
 
-### Vision Agent
-There are two agents that you can use. `VisionAgent` is a conversational agent that has
-access to tools that allow it to write an navigate python code and file systems. It can
-converse with the user in natural language. `VisionAgentCoder` is an agent specifically
-for writing code for vision tasks, such as counting people in an image. However, it
-cannot chat with you and can only respond with code. `VisionAgent` can call
-`VisionAgentCoder` to write vision code.
+### Basic Usage
+To get started you can just import the `VisionAgent` and start chatting with it:
+```python
+>>> from vision_agent.agent import VisionAgent
+>>> agent = VisionAgent()
+>>> resp = agent("Hello")
+>>> print(resp)
+[{"role": "user", "content": "Hello"}, {"role": "assistant", "content": "{'thoughts': 'The user has greeted me. I will respond with a greeting and ask how I can assist them.', 'response': 'Hello! How can I assist you today?', 'let_user_respond': True}"}]
+>>> resp.append({"role": "user", "content": "Can you count the number of people in this image?", "media": ["people.jpg"]})
+>>> resp = agent(resp)
+```
+
+The chat messages are similar to `OpenAI`'s format with `role` and `content` keys but
+in addition to those you can add `medai` which is a list of media files that can either
+be images or video files.
+
+## Documentation
+
+[Vision Agent Library Docs](https://landing-ai.github.io/vision-agent/)
+
+## Vision Agent Basic Usage
+### Chatting and Message Formats
+`VisionAgent` is an agent that can chat with you and call other tools or agents to
+write vision code for you. You can interact with it like you would ChatGPT or any other
+chatbot. The agent uses Clause-3.5 for it's LMM and OpenAI for embeddings for searching
+for tools.
+
+The message format is:
+```json
+{
+    "role": "user",
+    "content": "Hello",
+    "media": ["image.jpg"]
+}
+```
+Where `role` can be `user`, `assistant` or `observation` if the agent has executed a 
+function and needs to observe the output. `content` is always the text message and
+`media` is a list of media files that can be images or videos that you want the agent
+to examine.
+
+When the agent responds, inside it's `context` you will find the following data structure:
+```json
+{
+    "thoughts": "The user has greeted me. I will respond with a greeting and ask how I can assist them.",
+    "response": "Hello! How can I assist you today?",
+    "let_user_respond": true
+}
+```
+
+`thoughts` are the thoughts the agent had when processing the message, `response` is the
+response it generated which could contain a python execution, and `let_user_respond` is
+a boolean that tells the agent if it should wait for the user to respond before
+continuing, for example it may want to execute code and look at the output before
+letting the user respond.
 
-#### Basic Usage
-To run the streamlit app locally to chat with `VisionAgent`, you can run the following
-command:
+### Chatting and Artifacts
+If you run `chat_with_code` you will also notice an `Artifact` object. `Artifact`'s
+are a way to sync files between local and remote environments. The agent will read and
+write to the artifact object, which is just a pickle object, when it wants to save or
+load files.
+
+```python
+import vision_agent as va
+from vision_agent.tools.meta_tools import Artifact
+
+artifact = Artifact("artifact.pkl")
+# you can store text files such as code or images in the artifact
+with open("code.py", "r") as f:
+    artifacts["code.py"] = f.read()
+with open("image.png", "rb") as f:
+    artifacts["image.png"] = f.read()
+
+agent = va.agent.VisionAgent()
+response, artifacts = agent.chat_with_code(
+    [
+        {
+            "role": "user",
+            "content": "Can you write code to count the number of people in image.png",
+        }
+    ],
+    artifacts=artifacts,
+)
+```
+
+### Running the Streamlit App
+To test out things quickly, sometimes it's easier to run the streamlit app locally to
+chat with `VisionAgent`, you can run the following command:
 
 ```bash
 pip install -r examples/chat/requirements.txt
@@ -59,25 +139,117 @@ export WORKSPACE=/path/to/your/workspace
 export ZMQ_PORT=5555
 streamlit run examples/chat/app.py
 ```
-You can find more details about the streamlit app [here](examples/chat/).
+You can find more details about the streamlit app [here](examples/chat/), there are
+still some concurrency issues with the streamlit app so if you find it doing weird things
+clear your workspace and restart the app.
+
+## Tools
+There are a variety of tools for the model or the user to use. Some are executed locally
+while others are hosted for you. You can easily access them yourself, for example if
+you want to run `owl_v2_image` and visualize the output you can run:
 
-#### Basic Programmatic Usage
 ```python
->>> from vision_agent.agent import VisionAgent
->>> agent = VisionAgent()
->>> resp = agent("Hello")
->>> print(resp)
-[{"role": "user", "content": "Hello"}, {"role": "assistant", "content": "{'thoughts': 'The user has greeted me. I will respond with a greeting and ask how I can assist them.', 'response': 'Hello! How can I assist you today?', 'let_user_respond': True}"}]
->>> resp.append({"role": "user", "content": "Can you count the number of people in this image?", "media": ["people.jpg"]})
->>> resp = agent(resp)
+import vision_agent.tools as T
+import matplotlib.pyplot as plt
+
+image = T.load_image("dogs.jpg")
+dets = T.owl_v2_image("dogs", image)
+# visualize the owl_v2_ bounding boxes on the image
+viz = T.overlay_bounding_boxes(image, dets)
+
+# plot the image in matplotlib or save it
+plt.imshow(viz)
+plt.show()
+T.save_image(viz, "viz.png")
 ```
 
-`VisionAgent` currently utilizes Claude-3.5 as it's default LMM and uses OpenAI for
-embeddings for tool searching.
+Or if you want to run on video data, for example track sharks and people at 10 FPS:
 
-### Vision Agent Coder
-#### Basic Usage
-You can interact with the agent as you would with any LLM or LMM model:
+```python
+frames_and_ts = T.extract_frames_and_timestamps("sharks.mp4", fps=10)
+# extract only the frames from frames and timestamps
+frames = [f["frame"] for f in frames_and_ts]
+# track the sharks and people in the frames, returns segmentation masks
+track = T.florence2_sam2_video_tracking("shark, person", frames)
+# plot the segmentation masks on the frames
+viz = T.overlay_segmentation_masks(frames, track)
+T.save_video(viz, "viz.mp4")
+```
+
+You can find all available tools in `vision_agent/tools/tools.py`, however the
+`VisionAgent` will only utilizes a subset of tools that have been tested and provide
+the best performance. Those can be found in the same file under the `FUNCION_TOOLS`
+variable inside `tools.py`.
+
+#### Custom Tools
+If you can't find the tool you are looking for you can also add custom tools to the
+agent:
+
+```python
+import vision_agent as va
+import numpy as np
+
+@va.tools.register_tool(imports=["import numpy as np"])
+def custom_tool(image_path: str) -> str:
+    """My custom tool documentation.
+
+    Parameters:
+        image_path (str): The path to the image.
+
+    Returns:
+        str: The result of the tool.
+
+    Example
+    -------
+    >>> custom_tool("image.jpg")
+    """
+
+    return np.zeros((10, 10))
+```
+
+You need to ensure you call `@va.tools.register_tool` with any imports it uses. Global
+variables will not be captured by `register_tool` so you need to include them in the
+function. Make sure the documentation is in the same format above with description,
+`Parameters:`, `Returns:`, and `Example\n-------`. The `VisionAgent` will use your
+documentation when trying to determine when to use your tool. You can find an example
+use case [here](examples/custom_tools/) for adding a custom tool. Note you may need to
+play around with the prompt to ensure the model picks the tool when you want it to.
+
+Can't find the tool you need and want us to host it? Check out our
+[vision-agent-tools](https://github.com/landing-ai/vision-agent-tools) repository where
+we add the source code for all the tools used in `VisionAgent`.
+
+## LMMs
+All of our agents are based off of LMMs or Large Multimodal Models. We provide a thin
+abstraction layer on top of the underlying provider APIs to be able to more easily
+handle media.
+
+
+```python
+from vision_agent.lmm import AnthropicLMM
+
+lmm = AnthropicLMM()
+response = lmm("Describe this image", media=["apple.jpg"])
+>>> "This is an image of an apple."
+```
+
+Or you can use the `OpenAI` chat interaface and pass it other media like videos:
+
+```python
+response = lmm(
+    [
+        {
+            "role": "user",
+            "content": "What's going on in this video?",
+            "media": ["video.mp4"]
+        }
+    ]
+)
+```
+
+## Vision Agent Coder
+Underneath the hood, `VisionAgent` uses `VisionAgentCoder` to generate code to solve
+vision tasks. You can use `VisionAgentCoder` directly to generate code if you want:
 
 ```python
 >>> from vision_agent.agent import VisionAgentCoder
@@ -87,17 +259,17 @@ You can interact with the agent as you would with any LLM or LMM model:
 
 Which produces the following code:
 ```python
-from vision_agent.tools import load_image, grounding_sam
+from vision_agent.tools import load_image, florence2_sam2_image
 
 def calculate_filled_percentage(image_path: str) -> float:
     # Step 1: Load the image
     image = load_image(image_path)
 
     # Step 2: Segment the jar
-    jar_segments = grounding_sam(prompt="jar", image=image)
+    jar_segments = florence2_sam2_image("jar", image)
 
     # Step 3: Segment the coffee beans
-    coffee_beans_segments = grounding_sam(prompt="coffee beans", image=image)
+    coffee_beans_segments = florence2_sam2_image("coffee beans", image)
 
     # Step 4: Calculate the area of the segmented jar
     jar_area = 0
@@ -125,7 +297,7 @@ mode by passing in the verbose argument:
 >>> agent = VisionAgentCoder(verbosity=2)
 ```
 
-#### Detailed Usage
+### Detailed Usage
 You can also have it return more information by calling `chat_with_workflow`. The format
 of the input is a list of dictionaries with the keys `role`, `content`, and `media`:
 
@@ -145,7 +317,7 @@ of the input is a list of dictionaries with the keys `role`, `content`, and `med
 With this you can examine more detailed information such as the testing code, testing
 results, plan or working memory it used to complete the task.
 
-#### Multi-turn conversations
+### Multi-turn conversations
 You can have multi-turn conversations with vision-agent as well, giving it feedback on
 the code and having it update. You just need to add the code as a response from the
 assistant:
@@ -171,60 +343,6 @@ conv.append(
 result = agent.chat_with_workflow(conv)
 ```
 
-### Tools
-There are a variety of tools for the model or the user to use. Some are executed locally
-while others are hosted for you. You can easily access them yourself, for example if
-you want to run `owl_v2_image` and visualize the output you can run:
-
-```python
-import vision_agent.tools as T
-import matplotlib.pyplot as plt
-
-image = T.load_image("dogs.jpg")
-dets = T.owl_v2_image("dogs", image)
-viz = T.overlay_bounding_boxes(image, dets)
-plt.imshow(viz)
-plt.show()
-```
-
-You can find all available tools in `vision_agent/tools/tools.py`, however,
-`VisionAgentCoder` only utilizes a subset of tools that have been tested and provide
-the best performance. Those can be found in the same file under the `TOOLS` variable.
-
-If you can't find the tool you are looking for you can also add custom tools to the
-agent:
-
-```python
-import vision_agent as va
-import numpy as np
-
-@va.tools.register_tool(imports=["import numpy as np"])
-def custom_tool(image_path: str) -> str:
-    """My custom tool documentation.
-
-    Parameters:
-        image_path (str): The path to the image.
-
-    Returns:
-        str: The result of the tool.
-
-    Example
-    -------
-    >>> custom_tool("image.jpg")
-    """
-
-    return np.zeros((10, 10))
-```
-
-You need to ensure you call `@va.tools.register_tool` with any imports it uses. Global
-variables will not be captured by `register_tool` so you need to include them in the
-function. Make sure the documentation is in the same format above with description,
-`Parameters:`, `Returns:`, and `Example\n-------`. You can find an example use case
-[here](examples/custom_tools/) as this is what the agent uses to pick and use the tool.
-
-Can't find the tool you need and want add it to `VisionAgent`? Check out our
-[vision-agent-tools](https://github.com/landing-ai/vision-agent-tools) repository where
-we add the source code for all the tools used in `VisionAgent`.
 
 ## Additional Backends
 ### Anthropic
@@ -329,9 +447,9 @@ agent = va.agent.AzureVisionAgentCoder()
 
 ******************************************************************************************************************************
 
-### Q&A
+## Q&A
 
-#### How to get started with OpenAI API credits
+### How to get started with OpenAI API credits
 
 1. Visit the [OpenAI API platform](https://beta.openai.com/signup/) to sign up for an API key.
 2. Follow the instructions to purchase and manage your API credits.

{vision_agent-0.2.155 → vision_agent-0.2.157}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "vision-agent"
-version = "0.2.155"
+version = "0.2.157"
 description = "Toolset for Vision Agent"
 authors = ["Landing AI <dev@landing.ai>"]
 readme = "README.md"

{vision_agent-0.2.155 → vision_agent-0.2.157}/vision_agent/agent/agent_utils.py

@@ -77,3 +77,9 @@ def extract_code(code: str) -> str:
     if code.startswith("python\n"):
         code = code[len("python\n") :]
     return code
+
+
+def remove_installs_from_code(code: str) -> str:
+    pattern = r"\n!pip install.*?(\n|\Z)\n"
+    code = re.sub(pattern, "", code, flags=re.DOTALL)
+    return code

{vision_agent-0.2.155 → vision_agent-0.2.157}/vision_agent/agent/vision_agent.py

@@ -407,8 +407,6 @@ class VisionAgent(Agent):
             code_interpreter.download_file(
                 str(remote_artifacts_path.name), str(self.local_artifacts_path)
             )
-            artifacts.load(self.local_artifacts_path)
-            artifacts.save()
         return orig_chat, artifacts
 
     def streaming_message(self, message: Dict[str, Any]) -> None:

{vision_agent-0.2.155 → vision_agent-0.2.157}/vision_agent/agent/vision_agent_coder.py

@@ -13,7 +13,11 @@ from tabulate import tabulate
 
 import vision_agent.tools as T
 from vision_agent.agent import Agent
-from vision_agent.agent.agent_utils import extract_code, extract_json
+from vision_agent.agent.agent_utils import (
+    extract_code,
+    extract_json,
+    remove_installs_from_code,
+)
 from vision_agent.agent.vision_agent_coder_prompts import (
     CODE,
     FIX_BUG,
@@ -836,8 +840,8 @@ class VisionAgentCoder(Agent):
                 media=media_list,
             )
             success = cast(bool, results["success"])
-            code = cast(str, results["code"])
-            test = cast(str, results["test"])
+            code = remove_installs_from_code(cast(str, results["code"]))
+            test = remove_installs_from_code(cast(str, results["test"]))
             working_memory.extend(results["working_memory"])  # type: ignore
             plan.append({"code": code, "test": test, "plan": plan_i})
 

{vision_agent-0.2.155 → vision_agent-0.2.157}/vision_agent/agent/vision_agent_prompts.py

@@ -28,7 +28,8 @@ Here is the current conversation so far:
 1. **Understand and Clarify**: Make sure you understand the task, ask clarifying questions if the task is not clear.
 2. **Code Generation**: Only use code provided in the Documentation in your <execute_python> tags. Only use `edit_vision_code` to modify code written by `generate_vision_code`.
 3. **Execute**: Do only what the user asked you to do and no more. If you need to ask the user a question, set `let_user_respond` to `true`.
-4. **Output in JSON**: Respond in the following format in JSON:
+4. **Response**: Keep your responses short and concise. Provide the user only with the information they need to continue the conversation.
+5. **Output in JSON**: Respond in the following format in JSON:
 
 ```json
 {{"thoughts": <your thoughts>, "response": <your response to the user>, "let_user_respond": <a boolean whether or not to let the user respond>}}.
@@ -62,7 +63,7 @@ OBSERVATION:
 [{'score': 0.99, 'label': 'dog', 'box': [0.1, 0.2, 0.3, 0.4]}, {'score': 0.23, 'label': 'dog', 'box': [0.2, 0.3, 0.4, 0.5]}]
 
 
-AGENT: {"thoughts": "Two dogs are detected, I will show this to the user and ask them if the result looks good.", "response": "I have written the code to detect dogs and shown the output, do the results look good to you?", "let_user_respond": true}
+AGENT: {"thoughts": "Two dogs are detected, I will show this to the user and ask them if the result looks good.", "response": "The code detectd two dogs, do the results look good to you?", "let_user_respond": true}
 """
 
 EXAMPLES_CODE1_EXTRA = """
@@ -91,7 +92,7 @@ OBSERVATION:
 ----- stdout -----
 [{'score': 0.99, 'label': 'dog', 'box': [0.1, 0.2, 0.3, 0.4]}]
 
-AGENT: {"thoughts": "One dog is detected, I will show this to the user and ask them if the result looks good.", "response": "I have written the code to detect one dog and shown you the output, do the results look good to you?", "let_user_respond": true}
+AGENT: {"thoughts": "One dog is detected, I will show this to the user and ask them if the result looks good.", "response": "The code detected one dog, do these results look good to you?", "let_user_respond": true}
 """
 
 EXAMPLES_CODE2 = """
@@ -157,16 +158,16 @@ OBSERVATION:
 ----- stdout -----
 2
 
-AGENT: {"thoughts": "Two workers with helmets are detected, I will show this to the user and ask them if the result looks good.", "response": "I have written the code to count the workers wearing helmets in code.py and saved the visualization under 'workers_viz.png'.", "let_user_respond": true}
+AGENT: {"thoughts": "Two workers with helmets are detected, I will show this to the user and ask them if the result looks good.", "response": "The code to detect workers with helmets is saved in code.py and the visualization under 'workers_viz.png'.", "let_user_respond": true}
 
 USER: The detections are slightly off. Can you fine tune florence2 using these labels? "[{'image_path': 'image1.png': 'labels': ['worker', 'helmet'], 'bboxes': [[235, 118, 294, 241], [232, 118, 294, 128]]}, {'image_path': 'image2.png': 'labels': ['worker', 'helmet'], 'bboxes': [[235, 118, 294, 241], [232, 118, 294, 128]]}]"
 
-AGENT: {"thoughts": "Because the user has supplied me with labels I can call object_detection_fine_tuning on their behalf to fine tune the model", "response": "I will fine tune florence2 with the labels you provided <execute_python>object_detection_fine_tuning([{'image_path': 'image1.png': 'labels': ['worker', 'helmet'], 'bboxes': [[235, 118, 294, 241], [232, 118, 294, 128]]}, {'image_path': 'image2.png': 'labels': ['worker', 'helmet'], 'bboxes': [[235, 118, 294, 241], [232, 118, 294, 128]]}])</execute_python>", "let_user_respond": false}
+AGENT: {"thoughts": "Because the user has supplied me with labels I can call object_detection_fine_tuning on their behalf to fine tune the model", "response": "<execute_python>object_detection_fine_tuning([{'image_path': 'image1.png': 'labels': ['worker', 'helmet'], 'bboxes': [[235, 118, 294, 241], [232, 118, 294, 128]]}, {'image_path': 'image2.png': 'labels': ['worker', 'helmet'], 'bboxes': [[235, 118, 294, 241], [232, 118, 294, 128]]}])</execute_python>", "let_user_respond": false}
 
 OBSERVATION:
 [Fine tuning id: 23b3b022-5ebf-4798-9373-20ef36429abf]
 
-AGENT: {"thoughts": "The model has finished fine tuning, I will now replace the original florence2_phrase_grounding call with the fine tuning id.", "response": "I will not update the code to use the fine tuned model. <execute_python>use_object_detection_fine_tuning(artifacts, "code.py", "23b3b022-5ebf-4798-9373-20ef36429abf")</execute_python>", "let_user_respond": false}
+AGENT: {"thoughts": "The model has finished fine tuning, I will now replace the original florence2_phrase_grounding call with the fine tuning id.", "response": "<execute_python>use_object_detection_fine_tuning(artifacts, "code.py", "23b3b022-5ebf-4798-9373-20ef36429abf")</execute_python>", "let_user_respond": false}
 
 OBSERVATION:
 [Artifact code.py edits]

{vision_agent-0.2.155 → vision_agent-0.2.157}/vision_agent/tools/__init__.py

@@ -45,7 +45,6 @@ from .tools import (
     loca_zero_shot_counting,
     ocr,
     overlay_bounding_boxes,
-    overlay_counting_results,
     overlay_heat_map,
     overlay_segmentation_masks,
     owl_v2_image,

{vision_agent-0.2.155 → vision_agent-0.2.157}/vision_agent/tools/meta_tools.py

@@ -116,7 +116,9 @@ class Artifacts:
         )
         output_str = "[Artifacts loaded]\n"
         for k in self.artifacts.keys():
-            output_str += f"Artifact {k} loaded to {str(loaded_path / k)}\n"
+            output_str += (
+                f"Artifact name: {k}, loaded to path: {str(loaded_path / k)}\n"
+            )
         output_str += "[End of artifacts]\n"
         print(output_str)
         return output_str

{vision_agent-0.2.155 → vision_agent-0.2.157}/vision_agent/tools/tools.py

@@ -13,7 +13,7 @@ from uuid import UUID
 import cv2
 import numpy as np
 import requests
-from PIL import Image, ImageDraw, ImageEnhance, ImageFont
+from PIL import Image, ImageDraw, ImageFont
 from pillow_heif import register_heif_opener  # type: ignore
 from pytube import YouTube  # type: ignore
 
@@ -1150,10 +1150,10 @@ def florence2_image_caption(image: np.ndarray, detail_caption: bool = True) -> s
 def florence2_phrase_grounding(
     prompt: str, image: np.ndarray, fine_tune_id: Optional[str] = None
 ) -> List[Dict[str, Any]]:
-    """'florence2_phrase_grounding' will run florence2 on a image. It can
-    detect multiple objects given a text prompt which can be object names or caption.
-    You can optionally separate the object names in the text with commas. It returns
-    a list of bounding boxes with normalized coordinates, label names and associated
+    """'florence2_phrase_grounding' is a tool that can detect multiple
+    objects given a text prompt which can be object names or caption. You
+    can optionally separate the object names in the text with commas. It returns a list
+    of bounding boxes with normalized coordinates, label names and associated
     probability scores of 1.0.
 
     Parameters:
@@ -1812,6 +1812,11 @@ def save_image(image: np.ndarray, file_path: str) -> None:
     """
     from IPython.display import display
 
+    if not isinstance(image, np.ndarray) or (
+        image.shape[0] == 0 and image.shape[1] == 0
+    ):
+        raise ValueError("The image is not a valid NumPy array with shape (H, W, C)")
+
     pil_image = Image.fromarray(image.astype(np.uint8)).convert("RGB")
     display(pil_image)
     pil_image.save(file_path)
@@ -1838,6 +1843,15 @@ def save_video(
     if fps <= 0:
         raise ValueError(f"fps must be greater than 0 got {fps}")
 
+    if not isinstance(frames, list) or len(frames) == 0:
+        raise ValueError("Frames must be a list of NumPy arrays")
+
+    for frame in frames:
+        if not isinstance(frame, np.ndarray) or (
+            frame.shape[0] == 0 and frame.shape[1] == 0
+        ):
+            raise ValueError("A frame is not a valid NumPy array with shape (H, W, C)")
+
     if output_video_path is None:
         output_video_path = tempfile.NamedTemporaryFile(
             delete=False, suffix=".mp4"
@@ -1907,30 +1921,36 @@ def overlay_bounding_boxes(
         bboxes = bbox_int[i]
         bboxes = sorted(bboxes, key=lambda x: x["label"], reverse=True)
 
-        width, height = pil_image.size
-        fontsize = max(12, int(min(width, height) / 40))
-        draw = ImageDraw.Draw(pil_image)
-        font = ImageFont.truetype(
-            str(
-                resources.files("vision_agent.fonts").joinpath("default_font_ch_en.ttf")
-            ),
-            fontsize,
-        )
-
-        for elt in bboxes:
-            label = elt["label"]
-            box = elt["bbox"]
-            scores = elt["score"]
-
-            # denormalize the box if it is normalized
-            box = denormalize_bbox(box, (height, width))
-            draw.rectangle(box, outline=color[label], width=4)
-            text = f"{label}: {scores:.2f}"
-            text_box = draw.textbbox((box[0], box[1]), text=text, font=font)
-            draw.rectangle(
-                (box[0], box[1], text_box[2], text_box[3]), fill=color[label]
+        if len(bboxes) > 20:
+            pil_image = _plot_counting(pil_image, bboxes, color)
+        else:
+            width, height = pil_image.size
+            fontsize = max(12, int(min(width, height) / 40))
+            draw = ImageDraw.Draw(pil_image)
+            font = ImageFont.truetype(
+                str(
+                    resources.files("vision_agent.fonts").joinpath(
+                        "default_font_ch_en.ttf"
+                    )
+                ),
+                fontsize,
             )
-            draw.text((box[0], box[1]), text, fill="black", font=font)
+
+            for elt in bboxes:
+                label = elt["label"]
+                box = elt["bbox"]
+                scores = elt["score"]
+
+                # denormalize the box if it is normalized
+                box = denormalize_bbox(box, (height, width))
+                draw.rectangle(box, outline=color[label], width=4)
+                text = f"{label}: {scores:.2f}"
+                text_box = draw.textbbox((box[0], box[1]), text=text, font=font)
+                draw.rectangle(
+                    (box[0], box[1], text_box[2], text_box[3]), fill=color[label]
+                )
+                draw.text((box[0], box[1]), text, fill="black", font=font)
+
         frame_out.append(np.array(pil_image))
     return frame_out[0] if len(frame_out) == 1 else frame_out
 
@@ -2089,39 +2109,19 @@ def overlay_heat_map(
     return np.array(combined)
 
 
-def overlay_counting_results(
-    image: np.ndarray, instances: List[Dict[str, Any]]
-) -> np.ndarray:
-    """'overlay_counting_results' is a utility function that displays counting results on
-    an image.
-
-    Parameters:
-        image (np.ndarray): The image to display the bounding boxes on.
-        instances (List[Dict[str, Any]]): A list of dictionaries containing the bounding
-            box information of each instance
-
-    Returns:
-        np.ndarray: The image with the instance_id dislpayed
-
-    Example
-    -------
-        >>> image_with_bboxes = overlay_counting_results(
-            image, [{'score': 0.99, 'label': 'object', 'bbox': [0.1, 0.11, 0.35, 0.4]}],
-        )
-    """
-    pil_image = Image.fromarray(image.astype(np.uint8)).convert("RGB")
-    color = (158, 218, 229)
-
-    width, height = pil_image.size
+def _plot_counting(
+    image: Image.Image,
+    bboxes: List[Dict[str, Any]],
+    colors: Dict[str, Tuple[int, int, int]],
+) -> Image.Image:
+    width, height = image.size
     fontsize = max(10, int(min(width, height) / 80))
-    pil_image = ImageEnhance.Brightness(pil_image).enhance(0.5)
-    draw = ImageDraw.Draw(pil_image)
+    draw = ImageDraw.Draw(image)
     font = ImageFont.truetype(
         str(resources.files("vision_agent.fonts").joinpath("default_font_ch_en.ttf")),
         fontsize,
     )
-
-    for i, elt in enumerate(instances, 1):
+    for i, elt in enumerate(bboxes, 1):
         label = f"{i}"
         box = elt["bbox"]
 
@@ -2143,7 +2143,7 @@ def overlay_counting_results(
         text_y1 = cy + text_height / 2
 
         # Draw the rectangle encapsulating the text
-        draw.rectangle((text_x0, text_y0, text_x1, text_y1), fill=color)
+        draw.rectangle((text_x0, text_y0, text_x1, text_y1), fill=colors[elt["label"]])
 
         # Draw the text at the center of the bounding box
         draw.text(
@@ -2154,7 +2154,7 @@ def overlay_counting_results(
             anchor="lt",
         )
 
-    return np.array(pil_image)
+    return image
 
 
 FUNCTION_TOOLS = [
@@ -2187,7 +2187,6 @@ UTIL_TOOLS = [
     overlay_bounding_boxes,
     overlay_segmentation_masks,
     overlay_heat_map,
-    overlay_counting_results,
 ]
 
 TOOLS = FUNCTION_TOOLS + UTIL_TOOLS

{vision_agent-0.2.155 → vision_agent-0.2.157}/vision_agent/tools/tool_utils.py

@@ -1,6 +1,6 @@
-import os
 import inspect
 import logging
+import os
 from base64 import b64encode
 from typing import Any, Callable, Dict, List, MutableMapping, Optional, Tuple
 

{vision_agent-0.2.155 → vision_agent-0.2.157}/vision_agent/tools/tools_types.py

@@ -1,6 +1,6 @@
 from enum import Enum
-from uuid import UUID
 from typing import List, Optional, Tuple, Union
+from uuid import UUID
 
 from pydantic import BaseModel, ConfigDict, Field, SerializationInfo, field_serializer