vision-agent 0.2.90__py3-none-any.whl → 0.2.92__py3-none-any.whl

vision_agent/utils/execute.py

@@ -40,9 +40,7 @@ _SESSION_TIMEOUT = 600  # 10 minutes
 
 
 class MimeType(str, Enum):
-    """
-    Represents a MIME type.
-    """
+    """Represents a MIME type."""
 
     TEXT_PLAIN = "text/plain"
     TEXT_HTML = "text/html"
@@ -58,7 +56,9 @@ class MimeType(str, Enum):
 
 
 class FileSerializer:
-    """Adaptor class that allows IPython.display.display() to serialize a file to a base64 string representation."""
+    """Adaptor class that allows IPython.display.display() to serialize a file to a
+    base64 string representation.
+    """
 
     def __init__(self, file_uri: str):
         self.video_uri = file_uri
@@ -76,13 +76,15 @@ class FileSerializer:
 
 
 class Result:
-    """
-    Represents the data to be displayed as a result of executing a cell in a Jupyter notebook.
-    The result is similar to the structure returned by ipython kernel: https://ipython.readthedocs.io/en/stable/development/execution.html#execution-semantics
+    """Represents the data to be displayed as a result of executing a cell in a Jupyter
+    notebook. The result is similar to the structure returned by ipython kernel:
+    https://ipython.readthedocs.io/en/stable/development/execution.html#execution-semantics
 
-    The result can contain multiple types of data, such as text, images, plots, etc. Each type of data is represented
-    as a string, and the result can contain multiple types of data. The display calls don't have to have text representation,
-    for the actual result the representation is always present for the result, the other representations are always optional.
+    The result can contain multiple types of data, such as text, images, plots, etc.
+    Each type of data is represented as a string, and the result can contain multiple
+    types of data. The display calls don't have to have text representation, for the
+    actual result the representation is always present for the result, the other
+    representations are always optional.
 
     The class also provides methods to display the data in a Jupyter notebook.
     """
@@ -143,62 +145,43 @@ class Result:
         return str(self.raw)
 
     def _repr_html_(self) -> Optional[str]:
-        """
-        Returns the HTML representation of the data.
-        """
+        """Returns the HTML representation of the data."""
         return self.html
 
     def _repr_markdown_(self) -> Optional[str]:
-        """
-        Returns the Markdown representation of the data.
-        """
+        """Returns the Markdown representation of the data."""
         return self.markdown
 
     def _repr_svg_(self) -> Optional[str]:
-        """
-        Returns the SVG representation of the data.
-        """
+        """Returns the SVG representation of the data."""
         return self.svg
 
     def _repr_png_(self) -> Optional[str]:
-        """
-        Returns the base64 representation of the PNG data.
-        """
+        """Returns the base64 representation of the PNG data."""
         return self.png
 
     def _repr_jpeg_(self) -> Optional[str]:
-        """
-        Returns the base64 representation of the JPEG data.
-        """
+        """Returns the base64 representation of the JPEG data."""
         return self.jpeg
 
     def _repr_pdf_(self) -> Optional[str]:
-        """
-        Returns the PDF representation of the data.
-        """
+        """Returns the PDF representation of the data."""
         return self.pdf
 
     def _repr_latex_(self) -> Optional[str]:
-        """
-        Returns the LaTeX representation of the data.
-        """
+        """Returns the LaTeX representation of the data."""
         return self.latex
 
     def _repr_json_(self) -> Optional[dict]:
-        """
-        Returns the JSON representation of the data.
-        """
+        """Returns the JSON representation of the data."""
         return self.json
 
     def _repr_javascript_(self) -> Optional[str]:
-        """
-        Returns the JavaScript representation of the data.
-        """
+        """Returns the JavaScript representation of the data."""
         return self.javascript
 
     def formats(self) -> Iterable[str]:
-        """
-        Returns all available formats of the result.
+        """Returns all available formats of the result.
 
         :return: All available formats of the result in MIME types.
         """
@@ -239,8 +222,8 @@ class Result:
 
 
 class Logs(BaseModel):
-    """
-    Data printed to stdout and stderr during execution, usually by print statements, logs, warnings, subprocesses, etc.
+    """Data printed to stdout and stderr during execution, usually by print statements,
+    logs, warnings, subprocesses, etc.
     """
 
     stdout: List[str] = []
@@ -257,9 +240,8 @@ class Logs(BaseModel):
 
 
 class Error(BaseModel):
-    """
-    Represents an error that occurred during the execution of a cell.
-    The error contains the name of the error, the value of the error, and the traceback.
+    """Represents an error that occurred during the execution of a cell. The error
+    contains the name of the error, the value of the error, and the traceback.
     """
 
     name: str
@@ -290,9 +272,7 @@ class Error(BaseModel):
 
 
 class Execution(BaseModel):
-    """
-    Represents the result of a cell execution.
-    """
+    """Represents the result of a cell execution."""
 
     class Config:
         arbitrary_types_allowed = True
@@ -305,8 +285,8 @@ class Execution(BaseModel):
     "Error object if an error occurred, None otherwise."
 
     def text(self, include_logs: bool = True) -> str:
-        """
-        Returns the text representation of this object, i.e. including the main result or the error traceback, optionally along with the logs (stdout, stderr).
+        """Returns the text representation of this object, i.e. including the main
+        result or the error traceback, optionally along with the logs (stdout, stderr).
         """
         prefix = str(self.logs) if include_logs else ""
         if self.error:
@@ -330,9 +310,9 @@ class Execution(BaseModel):
         return self.error is None
 
     def get_main_result(self) -> Optional[Result]:
-        """
-        Get the main result of the execution.
-        An execution may have multiple results, e.g. intermediate outputs. The main result is the last output of the cell execution.
+        """Get the main result of the execution. An execution may have multiple
+        results, e.g. intermediate outputs. The main result is the last output of the
+        cell execution.
         """
         if not self.success:
             _LOGGER.info("Result is not available as the execution was not successful.")
@@ -345,16 +325,13 @@ class Execution(BaseModel):
         return main_result
 
     def to_json(self) -> str:
-        """
-        Returns the JSON representation of the Execution object.
-        """
+        """Returns the JSON representation of the Execution object."""
         return self.model_dump_json(exclude_none=True)
 
     @field_serializer("results", when_used="json")
     def serialize_results(results: List[Result]) -> List[Dict[str, Union[str, bool]]]:  # type: ignore
-        """
-        Serializes the results to JSON.
-        This method is used by the Pydantic JSON encoder.
+        """Serializes the results to JSON. This method is used by the Pydantic JSON
+        encoder.
         """
         serialized = []
         for result in results:
@@ -367,9 +344,7 @@ class Execution(BaseModel):
 
     @staticmethod
     def from_exception(exec: Exception, traceback_raw: List[str]) -> "Execution":
-        """
-        Creates an Execution object from an exception.
-        """
+        """Creates an Execution object from an exception."""
         return Execution(
             error=Error(
                 name=exec.__class__.__name__,
@@ -382,9 +357,7 @@ class Execution(BaseModel):
 
     @staticmethod
     def from_e2b_execution(exec: E2BExecution) -> "Execution":  # type: ignore
-        """
-        Creates an Execution object from an E2BResult object.
-        """
+        """Creates an Execution object from an E2BResult object."""
         return Execution(
             results=[Result.from_e2b_result(res) for res in exec.results],
             logs=Logs(stdout=exec.logs.stdout, stderr=exec.logs.stderr),
@@ -545,6 +518,8 @@ nbclient version: {nbclient_version}
 nbformat version: {nbformat.__version__}
 Timeout: {self.timeout}"""
         )
+        sleep(1)
+        self._new_kernel()
 
     def _new_kernel(self) -> None:
         if self.nb_client.kc is None or not run_sync(self.nb_client.kc.is_alive)():  # type: ignore
@@ -557,17 +532,19 @@ Timeout: {self.timeout}"""
             run_sync(self.nb_client.km.shutdown_kernel)(now=True)
             run_sync(self.nb_client.km.cleanup_resources)()
 
-            channels = [
-                self.nb_client.kc.stdin_channel,
-                self.nb_client.kc.hb_channel,
-                self.nb_client.kc.control_channel,
-            ]
+            if self.nb_client.kc is not None:
+                channels = [
+                    self.nb_client.kc.stdin_channel,
+                    self.nb_client.kc.hb_channel,
+                    self.nb_client.kc.control_channel,
+                ]
 
-            for ch in channels:
-                if ch.is_alive():
-                    ch.stop()
+                for ch in channels:
+                    if ch.is_alive():
+                        ch.stop()
+                self.nb_client.kc.stop_channels()
 
-            self.nb_client.kc = None
+                self.nb_client.kc = None
             self.nb_client.km = None
 
     def restart_kernel(self) -> None:
@@ -634,9 +611,8 @@ class CodeInterpreterFactory:
 
 
 def _parse_local_code_interpreter_outputs(outputs: List[Dict[str, Any]]) -> Execution:
-    """
-    Parse notebook cell outputs to Execution object.
-    Output types: https://nbformat.readthedocs.io/en/latest/format_description.html#code-cell-outputs
+    """Parse notebook cell outputs to Execution object. Output types:
+    https://nbformat.readthedocs.io/en/latest/format_description.html#code-cell-outputs
     """
     execution = Execution()
     for data in outputs:

vision_agent/utils/image_utils.py

@@ -232,7 +232,7 @@ def overlay_heat_map(
 
     Parameters:
         image: the input image
-        masks: the heatmap to overlay
+        heat_map: the heatmap to overlay
         alpha: the transparency of the overlay
 
     Returns:

vision_agent/utils/type_defs.py

@@ -14,7 +14,7 @@ class LandingaiAPIKey(BaseSettings):
     """
 
     api_key: str = Field(
-        default="land_sk_IJrojHarPXRjqDj1Fng76mX7yCbzVm1s5rZYxaNXu5v0cNLn0w",
+        default="land_sk_fnmSzD0ksknSfvhyD8UGu9R4ss3bKfLL1Im5gb6tDQTy2z1Oy5",
         alias="LANDINGAI_API_KEY",
         description="The API key of LandingAI.",
     )

{vision_agent-0.2.90.dist-info → vision_agent-0.2.92.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.2.90
+Version: 0.2.92
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai
@@ -57,7 +57,7 @@ code to solve the task for them. Check out our discord for updates and roadmaps!
 
 ## Web Application
 
-Try Vision Agent live on [va.landing.ai](https://va.landing.ai/)
+Try Vision Agent live on (note this may not be running the most up-to-date version) [va.landing.ai](https://va.landing.ai/)
 
 ## Documentation
 
@@ -79,16 +79,44 @@ using Azure OpenAI please see the Azure setup section):
 export OPENAI_API_KEY="your-api-key"
 ```
 
-### Important Note on API Usage
-Please be aware that using the API in this project requires you to have API credits (minimum of five US dollars). This is different from the OpenAI subscription used in this chatbot. If you don't have credit, further information can be found [here](https://github.com/landing-ai/vision-agent?tab=readme-ov-file#how-to-get-started-with-openai-api-credits)
-
 ### Vision Agent
+There are two agents that you can use. Vision Agent is a conversational agent that has
+access to tools that allow it to write an navigate python code and file systems. It can
+converse with the user in natural language. VisionAgentCoder is an agent that can write
+code for vision tasks, such as counting people in an image. However, it cannot converse
+and can only respond with code. VisionAgent can call VisionAgentCoder to write vision
+code.
+
 #### Basic Usage
-You can interact with the agent as you would with any LLM or LMM model:
+To run the streamlit app locally to chat with Vision Agent, you can run the following
+command:
+
+```bash
+pip install -r examples/chat/requirements.txt
+export WORKSPACE=/path/to/your/workspace
+export ZMQ_PORT=5555
+streamlit run examples/chat/app.py
+```
+You can find more details about the streamlit app [here](examples/chat/).
 
+#### Basic Programmatic Usage
 ```python
 >>> from vision_agent.agent import VisionAgent
 >>> agent = VisionAgent()
+>>> resp = agent("Hello")
+>>> print(resp)
+[{"role": "user", "content": "Hello"}, {"role": "assistant", "content": "{'thoughts': 'The user has greeted me. I will respond with a greeting and ask how I can assist them.', 'response': 'Hello! How can I assist you today?', 'let_user_respond': True}"}]
+>>> resp.append({"role": "user", "content": "Can you count the number of people in this image?", "media": ["people.jpg"]})
+>>> resp = agent(resp)
+```
+
+### Vision Agent Coder
+#### Basic Usage
+You can interact with the agent as you would with any LLM or LMM model:
+
+```python
+>>> from vision_agent.agent import VisionAgentCoder
+>>> agent = VisionAgentCoder()
 >>> code = agent("What percentage of the area of the jar is filled with coffee beans?", media="jar.jpg")
 ```
 
@@ -129,7 +157,7 @@ To better understand how the model came up with it's answer, you can run it in d
 mode by passing in the verbose argument:
 
 ```python
->>> agent = VisionAgent(verbose=2)
+>>> agent = VisionAgentCoder(verbose=2)
 ```
 
 #### Detailed Usage
@@ -219,9 +247,11 @@ def custom_tool(image_path: str) -> str:
     return np.zeros((10, 10))
 ```
 
-You need to ensure you call `@va.tools.register_tool` with any imports it might use and
-ensure the documentation is in the same format above with description, `Parameters:`,
-`Returns:`, and `Example\n-------`. You can find an example use case [here](examples/custom_tools/).
+You need to ensure you call `@va.tools.register_tool` with any imports it uses. Global
+variables will not be captured by `register_tool` so you need to include them in the
+function. Make sure the documentation is in the same format above with description,
+`Parameters:`, `Returns:`, and `Example\n-------`. You can find an example use case
+[here](examples/custom_tools/) as this is what the agent uses to pick and use the tool.
 
 ### Azure Setup
 If you want to use Azure OpenAI models, you need to have two OpenAI model deployments:
@@ -248,7 +278,7 @@ You can then run Vision Agent using the Azure OpenAI models:
 
 ```python
 import vision_agent as va
-agent = va.agent.AzureVisionAgent()
+agent = va.agent.AzureVisionAgentCoder()
 ```
 
 ******************************************************************************************************************************
@@ -257,7 +287,7 @@ agent = va.agent.AzureVisionAgent()
 
 #### How to get started with OpenAI API credits
 
-1. Visit the[OpenAI API platform](https://beta.openai.com/signup/) to sign up for an API key.
+1. Visit the [OpenAI API platform](https://beta.openai.com/signup/) to sign up for an API key.
 2. Follow the instructions to purchase and manage your API credits.
 3. Ensure your API key is correctly configured in your project settings.
 

vision_agent-0.2.92.dist-info/RECORD

@@ -0,0 +1,29 @@
+vision_agent/__init__.py,sha256=EAb4-f9iyuEYkBrX4ag1syM8Syx8118_t0R6_C34M9w,57
+vision_agent/agent/__init__.py,sha256=qpduQ9YufJQfMmG6jwKC2xmlbtR2qK8_1eQC1sGA9Ks,135
+vision_agent/agent/agent.py,sha256=Bt8yhjCFXuRdZaHxKEesG40V09nWRt45sZluri1R3AA,575
+vision_agent/agent/agent_utils.py,sha256=JXdl2xz14LKQAmScY-MIW23AD2WBFCsnI0JS6dAyj3Q,1412
+vision_agent/agent/vision_agent.py,sha256=i_rNpc7faqHTifp2c9sQE4Js3qYUKuJeiqauTp90OlE,8417
+vision_agent/agent/vision_agent_coder.py,sha256=Ouq3ws7w5zq8fw550aBrCjerNUrQ0MNvR45dkqTtReE,30321
+vision_agent/agent/vision_agent_coder_prompts.py,sha256=a3R_vHlT2FW3-DSn4OWgzF9zEAx-uKM4ZaTi9Kn-K54,11116
+vision_agent/agent/vision_agent_prompts.py,sha256=hjs-m4ZHR7HE1HtOeX_1rOvTQA2FMEAqEkaBbGPBYDo,6072
+vision_agent/fonts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+vision_agent/fonts/default_font_ch_en.ttf,sha256=1YM0Z3XqLDjSNbF7ihQFSAIUdjF9m1rtHiNC_6QosTE,1594400
+vision_agent/lmm/__init__.py,sha256=YuUZRsMHdn8cMOv6iBU8yUqlIOLrbZQqZl9KPnofsHQ,103
+vision_agent/lmm/lmm.py,sha256=KcS6h-8whGFmwt7t4LNlj0hZ4U-rBojYBLKLmrMsF48,15075
+vision_agent/lmm/types.py,sha256=8TSRoTbXyCKVJiH-wHXI2OiGOMSkYv1vLGYeAXtNpOQ,153
+vision_agent/tools/__init__.py,sha256=UNiaJAOt1C709gaJ-a9h9BzKnY5JmoEUpgKftsOnyPQ,1882
+vision_agent/tools/meta_tools.py,sha256=rmxgVzj-vJKeewHbue3qHru4sYsFLxlSZV-YH-eyH5w,13366
+vision_agent/tools/prompts.py,sha256=V1z4YJLXZuUl_iZ5rY0M5hHc_2tmMEUKr0WocXKGt4E,1430
+vision_agent/tools/tool_utils.py,sha256=XoB-iae8hHrBQgJd3fV6-UjZAkClysobUaOM17IcHuE,4597
+vision_agent/tools/tools.py,sha256=HT8stRTUmhwm2VbpB3QQNXnL1KIxka-BDCkU2tLaFN4,42326
+vision_agent/utils/__init__.py,sha256=CW84HnhqI6XQVuxf2KifkLnSuO7EOhmuL09-gAymAak,219
+vision_agent/utils/exceptions.py,sha256=isVH-SVL4vHj3q5kK4z7cy5_aOapAqHXWkpibfSNbUs,1659
+vision_agent/utils/execute.py,sha256=JPjiXyAHsMSPYFDZ9bmiiX2lgiGroxqDhXhsD8BTv5E,23848
+vision_agent/utils/image_utils.py,sha256=y69wtNla0xHZ1h1x0-vv7nOyKUq69jtjSJBiDCn6EM0,7703
+vision_agent/utils/sim.py,sha256=1HTaiVaBiKeyXIy21IYGXlPw0TipOyw9FPOJDfyLI94,4409
+vision_agent/utils/type_defs.py,sha256=oVFJcicB-s_09lqvn61u0A5ncZsTqZArZledXWbrrg0,1384
+vision_agent/utils/video.py,sha256=rNmU9KEIkZB5-EztZNlUiKYN0mm_55A_2VGUM0QpqLA,8779
+vision_agent-0.2.92.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+vision_agent-0.2.92.dist-info/METADATA,sha256=HFAMIUcz8BQt6I44NYphv28olQx0mVPBC0q1V4XjRtk,10693
+vision_agent-0.2.92.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+vision_agent-0.2.92.dist-info/RECORD,,

vision_agent-0.2.90.dist-info/RECORD

@@ -1,24 +0,0 @@
-vision_agent/__init__.py,sha256=EAb4-f9iyuEYkBrX4ag1syM8Syx8118_t0R6_C34M9w,57
-vision_agent/agent/__init__.py,sha256=IUwfbPMcT8X_rnXMLmI8gJ4ltsHy_XSs9eLiKURJxeY,81
-vision_agent/agent/agent.py,sha256=ZK-5lOtd9-eD9aWcXssJpnOyvZuO7_5hAmnb-6sWVe8,569
-vision_agent/agent/vision_agent.py,sha256=y2bZ1vB4_rfvFF-FgHSnVi3tYD9aZ_g85Dd7HZZdVpk,31351
-vision_agent/agent/vision_agent_prompts.py,sha256=brBV-SmzyzTG5M9nfV3R5xdYT_BUYOKzxNFmTa2Sp-o,11049
-vision_agent/fonts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-vision_agent/fonts/default_font_ch_en.ttf,sha256=1YM0Z3XqLDjSNbF7ihQFSAIUdjF9m1rtHiNC_6QosTE,1594400
-vision_agent/lmm/__init__.py,sha256=j9mQsIXQOYfW6nFd47uTwuBe1ranpEbwW308qLfCWN0,85
-vision_agent/lmm/lmm.py,sha256=035uONyp6_jD3PVdNdSg2PMHOG1voqnpsn2IyybUENs,15147
-vision_agent/tools/__init__.py,sha256=k69hvcy2FWjDqVA0klzybKeoToOH_bom5NTVSliA0Og,1838
-vision_agent/tools/prompts.py,sha256=V1z4YJLXZuUl_iZ5rY0M5hHc_2tmMEUKr0WocXKGt4E,1430
-vision_agent/tools/tool_utils.py,sha256=ZnqaflVbLZB0GmgJJoQsZZs8hWbODXEPH1_Mq1s4bnc,3222
-vision_agent/tools/tools.py,sha256=TkZqNYX-ocwdaCdXd6c6tysSa_HX2y6Nrgl4JKni4IQ,43661
-vision_agent/utils/__init__.py,sha256=CW84HnhqI6XQVuxf2KifkLnSuO7EOhmuL09-gAymAak,219
-vision_agent/utils/exceptions.py,sha256=isVH-SVL4vHj3q5kK4z7cy5_aOapAqHXWkpibfSNbUs,1659
-vision_agent/utils/execute.py,sha256=DxuAoKmKAovgKe8IPkwg1B34osoz9_Ouvl1mi8aPXgE,23923
-vision_agent/utils/image_utils.py,sha256=_cdiS5YrLzqkq_ZgFUO897m5M4_SCIThwUy4lOklfB8,7700
-vision_agent/utils/sim.py,sha256=1HTaiVaBiKeyXIy21IYGXlPw0TipOyw9FPOJDfyLI94,4409
-vision_agent/utils/type_defs.py,sha256=QeQRRIlklZMWzxROcCn5ELxP89nYdXGydy1rAiSpZZw,1384
-vision_agent/utils/video.py,sha256=rNmU9KEIkZB5-EztZNlUiKYN0mm_55A_2VGUM0QpqLA,8779
-vision_agent-0.2.90.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-vision_agent-0.2.90.dist-info/METADATA,sha256=IGFEwRmmrA9JLXtDpunUhiPa7s0bU1ccGGBVSEph4Ck,9477
-vision_agent-0.2.90.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-vision_agent-0.2.90.dist-info/RECORD,,