vision-agent 0.2.47__tar.gz → 0.2.78__tar.gz

{vision_agent-0.2.47 → vision_agent-0.2.78}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.2.47
+Version: 0.2.78
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai
@@ -9,8 +9,8 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: e2b (>=0.17.0,<0.18.0)
-Requires-Dist: e2b-code-interpreter (>=0.0.7,<0.0.8)
+Requires-Dist: e2b (>=0.17.1,<0.18.0)
+Requires-Dist: e2b-code-interpreter (==0.0.11a1)
 Requires-Dist: ipykernel (>=6.29.4,<7.0.0)
 Requires-Dist: langsmith (>=0.1.58,<0.2.0)
 Requires-Dist: moviepy (>=1.0.0,<2.0.0)
@@ -21,7 +21,9 @@ Requires-Dist: openai (>=1.0.0,<2.0.0)
 Requires-Dist: opencv-python (>=4.0.0,<5.0.0)
 Requires-Dist: pandas (>=2.0.0,<3.0.0)
 Requires-Dist: pillow (>=10.0.0,<11.0.0)
+Requires-Dist: pillow-heif (>=0.16.0,<0.17.0)
 Requires-Dist: pydantic-settings (>=2.2.1,<3.0.0)
+Requires-Dist: pytube (==15.0.0)
 Requires-Dist: requests (>=2.0.0,<3.0.0)
 Requires-Dist: rich (>=13.7.1,<14.0.0)
 Requires-Dist: scipy (>=1.13.0,<1.14.0)
@@ -38,7 +40,6 @@ Description-Content-Type: text/markdown
     <img alt="vision_agent" height="200px" src="https://github.com/landing-ai/vision-agent/blob/main/assets/logo.jpg?raw=true">
 
 # 🔍🤖 Vision Agent
-
 [![](https://dcbadge.vercel.app/api/server/wPdN8RCYew?compact=true&style=flat)](https://discord.gg/wPdN8RCYew)
 ![ci_status](https://github.com/landing-ai/vision-agent/actions/workflows/ci_cd.yml/badge.svg)
 [![PyPI version](https://badge.fury.io/py/vision-agent.svg)](https://badge.fury.io/py/vision-agent)
@@ -52,9 +53,14 @@ accomplish the task you want. Vision Agent aims to provide an in-seconds experie
 allowing users to describe their problem in text and have the agent framework generate
 code to solve the task for them. Check out our discord for updates and roadmaps!
 
+
+## Web Application
+
+Try Vision Agent live on [va.landing.ai](https://va.landing.ai/)
+
 ## Documentation
 
-- [Vision Agent Library Docs](https://landing-ai.github.io/vision-agent/)
+[Vision Agent Library Docs](https://landing-ai.github.io/vision-agent/)
 
 
 ## Getting Started
@@ -72,7 +78,11 @@ using Azure OpenAI please see the Azure setup section):
 export OPENAI_API_KEY="your-api-key"
 ```
 
+### Important Note on API Usage
+Please be aware that using the API in this project requires you to have API credits (minimum of five US dollars). This is different from the OpenAI subscription used in this chatbot. If you don't have credit, further information can be found [here](https://github.com/landing-ai/vision-agent?tab=readme-ov-file#how-to-get-started-with-openai-api-credits)
+
 ### Vision Agent
+#### Basic Usage
 You can interact with the agent as you would with any LLM or LMM model:
 
 ```python
@@ -88,28 +98,28 @@ from vision_agent.tools import load_image, grounding_sam
 def calculate_filled_percentage(image_path: str) -> float:
     # Step 1: Load the image
     image = load_image(image_path)
-    
+
     # Step 2: Segment the jar
     jar_segments = grounding_sam(prompt="jar", image=image)
-    
+
     # Step 3: Segment the coffee beans
     coffee_beans_segments = grounding_sam(prompt="coffee beans", image=image)
-    
+
     # Step 4: Calculate the area of the segmented jar
     jar_area = 0
     for segment in jar_segments:
         jar_area += segment['mask'].sum()
-    
+
     # Step 5: Calculate the area of the segmented coffee beans
     coffee_beans_area = 0
     for segment in coffee_beans_segments:
         coffee_beans_area += segment['mask'].sum()
-    
+
     # Step 6: Compute the percentage of the jar area that is filled with coffee beans
     if jar_area == 0:
         return 0.0  # To avoid division by zero
     filled_percentage = (coffee_beans_area / jar_area) * 100
-    
+
     # Step 7: Return the computed percentage
     return filled_percentage
 ```
@@ -121,10 +131,12 @@ mode by passing in the verbose argument:
 >>> agent = VisionAgent(verbose=2)
 ```
 
-You can also have it return more information by calling `chat_with_workflow`:
+#### Detailed Usage
+You can also have it return more information by calling `chat_with_workflow`. The format
+of the input is a list of dictionaries with the keys `role`, `content`, and `media`:
 
 ```python
->>> results = agent.chat_with_workflow([{"role": "user", "content": "What percentage of the area of the jar is filled with coffee beans?"}], media="jar.jpg")
+>>> results = agent.chat_with_workflow([{"role": "user", "content": "What percentage of the area of the jar is filled with coffee beans?", "media": ["jar.jpg"]}])
 >>> print(results)
 {
     "code": "from vision_agent.tools import ..."
@@ -135,19 +147,45 @@ You can also have it return more information by calling `chat_with_workflow`:
 }
 ```
 
-With this you can examine more detailed information such as the etesting code, testing
+With this you can examine more detailed information such as the testing code, testing
 results, plan or working memory it used to complete the task.
 
+#### Multi-turn conversations
+You can have multi-turn conversations with vision-agent as well, giving it feedback on
+the code and having it update. You just need to add the code as a response from the
+assistant:
+
+```python
+agent = va.agent.VisionAgent(verbosity=2)
+conv = [
+    {
+        "role": "user",
+        "content": "Are these workers wearing safety gear? Output only a True or False value.",
+        "media": ["workers.png"],
+    }
+]
+result = agent.chat_with_workflow(conv)
+code = result["code"]
+conv.append({"role": "assistant", "content": code})
+conv.append(
+    {
+        "role": "user",
+        "content": "Can you also return the number of workers wearing safety gear?",
+    }
+)
+result = agent.chat_with_workflow(conv)
+```
+
 ### Tools
 There are a variety of tools for the model or the user to use. Some are executed locally
-while others are hosted for you. You can also ask an LLM directly to build a tool for
+while others are hosted for you. You can also ask an LMM directly to build a tool for
 you. For example:
 
 ```python
 >>> import vision_agent as va
->>> llm = va.llm.OpenAILLM()
->>> detector = llm.generate_detector("Can you build a jar detector for me?")
->>> detector("jar.jpg")
+>>> lmm = va.lmm.OpenAILMM()
+>>> detector = lmm.generate_detector("Can you build a jar detector for me?")
+>>> detector(va.tools.load_image("jar.jpg"))
 [{"labels": ["jar",],
   "scores": [0.99],
   "bboxes": [
@@ -185,23 +223,44 @@ ensure the documentation is in the same format above with description, `Paramete
 `Returns:`, and `Example\n-------`. You can find an example use case [here](examples/custom_tools/).
 
 ### Azure Setup
-If you want to use Azure OpenAI models, you can set the environment variable:
+If you want to use Azure OpenAI models, you need to have two OpenAI model deployments:
+
+1. OpenAI GPT-4o model
+2. OpenAI text embedding model
+
+<img width="1201" alt="Screenshot 2024-06-12 at 5 54 48 PM" src="https://github.com/landing-ai/vision-agent/assets/2736300/da125592-b01d-45bc-bc99-d48c9dcdfa32">
+
+Then you can set the following environment variables:
 
 ```bash
 export AZURE_OPENAI_API_KEY="your-api-key"
 export AZURE_OPENAI_ENDPOINT="your-endpoint"
+# The deployment name of your Azure OpenAI chat model
+export AZURE_OPENAI_CHAT_MODEL_DEPLOYMENT_NAME="your_gpt4o_model_deployment_name"
+# The deployment name of your Azure OpenAI text embedding model
+export AZURE_OPENAI_EMBEDDING_MODEL_DEPLOYMENT_NAME="your_embedding_model_deployment_name"
 ```
 
+> NOTE: make sure your Azure model deployment have enough quota (token per minute) to support it. The default value 8000TPM is not enough.
+
 You can then run Vision Agent using the Azure OpenAI models:
 
 ```python
->>> import vision_agent as va
->>> agent = va.agent.VisionAgent(
->>>     planner=va.llm.AzureOpenAILLM(),
->>>     coder=va.lmm.AzureOpenAILMM(),
->>>     tester=va.lmm.AzureOpenAILMM(),
->>>     debugger=va.lmm.AzureOpenAILMM(),
->>> )
+import vision_agent as va
+agent = va.agent.AzureVisionAgent()
 ```
 
+******************************************************************************************************************************
+
+### Q&A
+
+#### How to get started with OpenAI API credits
+
+1. Visit the[OpenAI API platform](https://beta.openai.com/signup/) to sign up for an API key.
+2. Follow the instructions to purchase and manage your API credits.
+3. Ensure your API key is correctly configured in your project settings.
+
+Failure to have sufficient API credits may result in limited or no functionality for the features that rely on the OpenAI API.
+
+For more details on managing your API usage and credits, please refer to the OpenAI API documentation.
 

{vision_agent-0.2.47 → vision_agent-0.2.78}/README.md

@@ -2,7 +2,6 @@
     <img alt="vision_agent" height="200px" src="https://github.com/landing-ai/vision-agent/blob/main/assets/logo.jpg?raw=true">
 
 # 🔍🤖 Vision Agent
-
 [![](https://dcbadge.vercel.app/api/server/wPdN8RCYew?compact=true&style=flat)](https://discord.gg/wPdN8RCYew)
 ![ci_status](https://github.com/landing-ai/vision-agent/actions/workflows/ci_cd.yml/badge.svg)
 [![PyPI version](https://badge.fury.io/py/vision-agent.svg)](https://badge.fury.io/py/vision-agent)
@@ -16,9 +15,14 @@ accomplish the task you want. Vision Agent aims to provide an in-seconds experie
 allowing users to describe their problem in text and have the agent framework generate
 code to solve the task for them. Check out our discord for updates and roadmaps!
 
+
+## Web Application
+
+Try Vision Agent live on [va.landing.ai](https://va.landing.ai/)
+
 ## Documentation
 
-- [Vision Agent Library Docs](https://landing-ai.github.io/vision-agent/)
+[Vision Agent Library Docs](https://landing-ai.github.io/vision-agent/)
 
 
 ## Getting Started
@@ -36,7 +40,11 @@ using Azure OpenAI please see the Azure setup section):
 export OPENAI_API_KEY="your-api-key"
 ```
 
+### Important Note on API Usage
+Please be aware that using the API in this project requires you to have API credits (minimum of five US dollars). This is different from the OpenAI subscription used in this chatbot. If you don't have credit, further information can be found [here](https://github.com/landing-ai/vision-agent?tab=readme-ov-file#how-to-get-started-with-openai-api-credits)
+
 ### Vision Agent
+#### Basic Usage
 You can interact with the agent as you would with any LLM or LMM model:
 
 ```python
@@ -52,28 +60,28 @@ from vision_agent.tools import load_image, grounding_sam
 def calculate_filled_percentage(image_path: str) -> float:
     # Step 1: Load the image
     image = load_image(image_path)
-    
+
     # Step 2: Segment the jar
     jar_segments = grounding_sam(prompt="jar", image=image)
-    
+
     # Step 3: Segment the coffee beans
     coffee_beans_segments = grounding_sam(prompt="coffee beans", image=image)
-    
+
     # Step 4: Calculate the area of the segmented jar
     jar_area = 0
     for segment in jar_segments:
         jar_area += segment['mask'].sum()
-    
+
     # Step 5: Calculate the area of the segmented coffee beans
     coffee_beans_area = 0
     for segment in coffee_beans_segments:
         coffee_beans_area += segment['mask'].sum()
-    
+
     # Step 6: Compute the percentage of the jar area that is filled with coffee beans
     if jar_area == 0:
         return 0.0  # To avoid division by zero
     filled_percentage = (coffee_beans_area / jar_area) * 100
-    
+
     # Step 7: Return the computed percentage
     return filled_percentage
 ```
@@ -85,10 +93,12 @@ mode by passing in the verbose argument:
 >>> agent = VisionAgent(verbose=2)
 ```
 
-You can also have it return more information by calling `chat_with_workflow`:
+#### Detailed Usage
+You can also have it return more information by calling `chat_with_workflow`. The format
+of the input is a list of dictionaries with the keys `role`, `content`, and `media`:
 
 ```python
->>> results = agent.chat_with_workflow([{"role": "user", "content": "What percentage of the area of the jar is filled with coffee beans?"}], media="jar.jpg")
+>>> results = agent.chat_with_workflow([{"role": "user", "content": "What percentage of the area of the jar is filled with coffee beans?", "media": ["jar.jpg"]}])
 >>> print(results)
 {
     "code": "from vision_agent.tools import ..."
@@ -99,19 +109,45 @@ You can also have it return more information by calling `chat_with_workflow`:
 }
 ```
 
-With this you can examine more detailed information such as the etesting code, testing
+With this you can examine more detailed information such as the testing code, testing
 results, plan or working memory it used to complete the task.
 
+#### Multi-turn conversations
+You can have multi-turn conversations with vision-agent as well, giving it feedback on
+the code and having it update. You just need to add the code as a response from the
+assistant:
+
+```python
+agent = va.agent.VisionAgent(verbosity=2)
+conv = [
+    {
+        "role": "user",
+        "content": "Are these workers wearing safety gear? Output only a True or False value.",
+        "media": ["workers.png"],
+    }
+]
+result = agent.chat_with_workflow(conv)
+code = result["code"]
+conv.append({"role": "assistant", "content": code})
+conv.append(
+    {
+        "role": "user",
+        "content": "Can you also return the number of workers wearing safety gear?",
+    }
+)
+result = agent.chat_with_workflow(conv)
+```
+
 ### Tools
 There are a variety of tools for the model or the user to use. Some are executed locally
-while others are hosted for you. You can also ask an LLM directly to build a tool for
+while others are hosted for you. You can also ask an LMM directly to build a tool for
 you. For example:
 
 ```python
 >>> import vision_agent as va
->>> llm = va.llm.OpenAILLM()
->>> detector = llm.generate_detector("Can you build a jar detector for me?")
->>> detector("jar.jpg")
+>>> lmm = va.lmm.OpenAILMM()
+>>> detector = lmm.generate_detector("Can you build a jar detector for me?")
+>>> detector(va.tools.load_image("jar.jpg"))
 [{"labels": ["jar",],
   "scores": [0.99],
   "bboxes": [
@@ -149,22 +185,43 @@ ensure the documentation is in the same format above with description, `Paramete
 `Returns:`, and `Example\n-------`. You can find an example use case [here](examples/custom_tools/).
 
 ### Azure Setup
-If you want to use Azure OpenAI models, you can set the environment variable:
+If you want to use Azure OpenAI models, you need to have two OpenAI model deployments:
+
+1. OpenAI GPT-4o model
+2. OpenAI text embedding model
+
+<img width="1201" alt="Screenshot 2024-06-12 at 5 54 48 PM" src="https://github.com/landing-ai/vision-agent/assets/2736300/da125592-b01d-45bc-bc99-d48c9dcdfa32">
+
+Then you can set the following environment variables:
 
 ```bash
 export AZURE_OPENAI_API_KEY="your-api-key"
 export AZURE_OPENAI_ENDPOINT="your-endpoint"
+# The deployment name of your Azure OpenAI chat model
+export AZURE_OPENAI_CHAT_MODEL_DEPLOYMENT_NAME="your_gpt4o_model_deployment_name"
+# The deployment name of your Azure OpenAI text embedding model
+export AZURE_OPENAI_EMBEDDING_MODEL_DEPLOYMENT_NAME="your_embedding_model_deployment_name"
 ```
 
+> NOTE: make sure your Azure model deployment have enough quota (token per minute) to support it. The default value 8000TPM is not enough.
+
 You can then run Vision Agent using the Azure OpenAI models:
 
 ```python
->>> import vision_agent as va
->>> agent = va.agent.VisionAgent(
->>>     planner=va.llm.AzureOpenAILLM(),
->>>     coder=va.lmm.AzureOpenAILMM(),
->>>     tester=va.lmm.AzureOpenAILMM(),
->>>     debugger=va.lmm.AzureOpenAILMM(),
->>> )
+import vision_agent as va
+agent = va.agent.AzureVisionAgent()
 ```
 
+******************************************************************************************************************************
+
+### Q&A
+
+#### How to get started with OpenAI API credits
+
+1. Visit the[OpenAI API platform](https://beta.openai.com/signup/) to sign up for an API key.
+2. Follow the instructions to purchase and manage your API credits.
+3. Ensure your API key is correctly configured in your project settings.
+
+Failure to have sufficient API credits may result in limited or no functionality for the features that rely on the OpenAI API.
+
+For more details on managing your API usage and credits, please refer to the OpenAI API documentation.

{vision_agent-0.2.47 → vision_agent-0.2.78}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "vision-agent"
-version = "0.2.47"
+version = "0.2.78"
 description = "Toolset for Vision Agent"
 authors = ["Landing AI <dev@landing.ai>"]
 readme = "README.md"
@@ -34,9 +34,11 @@ nbformat = "^5.10.4"
 rich = "^13.7.1"
 langsmith = "^0.1.58"
 ipykernel = "^6.29.4"
-e2b = "^0.17.0"
-e2b-code-interpreter = "^0.0.7"
+e2b = "^0.17.1"
+e2b-code-interpreter = "0.0.11a1"
 tenacity = "^8.3.0"
+pillow-heif = "^0.16.0"
+pytube = "15.0.0"
 
 [tool.poetry.group.dev.dependencies]
 autoflake = "1.*"

vision_agent-0.2.78/vision_agent/__init__.py

@@ -0,0 +1,2 @@
+from .agent import Agent
+from .lmm import LMM, OpenAILMM

vision_agent-0.2.78/vision_agent/agent/__init__.py

@@ -0,0 +1,2 @@
+from .agent import Agent
+from .vision_agent import AzureVisionAgent, VisionAgent

{vision_agent-0.2.47 → vision_agent-0.2.78}/vision_agent/agent/agent.py

@@ -2,12 +2,14 @@ from abc import ABC, abstractmethod
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Union
 
+from vision_agent.lmm import Message
+
 
 class Agent(ABC):
     @abstractmethod
     def __call__(
         self,
-        input: Union[List[Dict[str, str]], str],
+        input: Union[str, List[Message]],
         media: Optional[Union[str, Path]] = None,
     ) -> str:
         pass