vision-agent 0.2.134__tar.gz → 0.2.136__tar.gz

{vision_agent-0.2.134 → vision_agent-0.2.136}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.2.134
+Version: 0.2.136
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai

{vision_agent-0.2.134 → vision_agent-0.2.136}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "vision-agent"
-version = "0.2.134"
+version = "0.2.136"
 description = "Toolset for Vision Agent"
 authors = ["Landing AI <dev@landing.ai>"]
 readme = "README.md"

{vision_agent-0.2.134 → vision_agent-0.2.136}/vision_agent/agent/vision_agent.py

@@ -13,7 +13,7 @@ from vision_agent.agent.vision_agent_prompts import (
     VA_CODE,
 )
 from vision_agent.lmm import LMM, Message, OpenAILMM
-from vision_agent.tools import META_TOOL_DOCSTRING, save_image, load_image
+from vision_agent.tools import META_TOOL_DOCSTRING
 from vision_agent.tools.meta_tools import Artifacts, use_extra_vision_agent_args
 from vision_agent.utils import CodeInterpreterFactory
 from vision_agent.utils.execute import CodeInterpreter, Execution
@@ -222,14 +222,7 @@ class VisionAgent(Agent):
             for chat_i in int_chat:
                 if "media" in chat_i:
                     for media in chat_i["media"]:
-                        if type(media) is str and media.startswith(("http", "https")):
-                            # TODO: Ideally we should not call VA.tools here, we should come to revisit how to better support remote image later
-                            file_path = Path(media).name
-                            ndarray = load_image(media)
-                            save_image(ndarray, file_path)
-                            media = file_path
-                        else:
-                            media = cast(str, media)
+                        media = cast(str, media)
                         artifacts.artifacts[Path(media).name] = open(media, "rb").read()
 
                         media_remote_path = (

{vision_agent-0.2.134 → vision_agent-0.2.136}/vision_agent/tools/meta_tools.py

@@ -1,4 +1,5 @@
 import difflib
+import json
 import os
 import pickle as pkl
 import re
@@ -53,25 +54,27 @@ def redisplay_results(execution: Execution) -> None:
     """
     for result in execution.results:
         if result.text is not None:
-            display({MimeType.TEXT_PLAIN: result.text})
+            display({MimeType.TEXT_PLAIN: result.text}, raw=True)
         if result.html is not None:
-            display({MimeType.TEXT_HTML: result.html})
+            display({MimeType.TEXT_HTML: result.html}, raw=True)
         if result.markdown is not None:
-            display({MimeType.TEXT_MARKDOWN: result.markdown})
+            display({MimeType.TEXT_MARKDOWN: result.markdown}, raw=True)
         if result.svg is not None:
-            display({MimeType.IMAGE_SVG: result.svg})
+            display({MimeType.IMAGE_SVG: result.svg}, raw=True)
         if result.png is not None:
-            display({MimeType.IMAGE_PNG: result.png})
+            display({MimeType.IMAGE_PNG: result.png}, raw=True)
         if result.jpeg is not None:
-            display({MimeType.IMAGE_JPEG: result.jpeg})
+            display({MimeType.IMAGE_JPEG: result.jpeg}, raw=True)
         if result.mp4 is not None:
-            display({MimeType.VIDEO_MP4_B64: result.mp4})
+            display({MimeType.VIDEO_MP4_B64: result.mp4}, raw=True)
         if result.latex is not None:
-            display({MimeType.TEXT_LATEX: result.latex})
+            display({MimeType.TEXT_LATEX: result.latex}, raw=True)
         if result.json is not None:
-            display({MimeType.APPLICATION_JSON: result.json})
+            display({MimeType.APPLICATION_JSON: result.json}, raw=True)
+        if result.artifact is not None:
+            display({MimeType.APPLICATION_ARTIFACT: result.artifact}, raw=True)
         if result.extra is not None:
-            display(result.extra)
+            display(result.extra, raw=True)
 
 
 class Artifacts:
@@ -208,7 +211,14 @@ def create_code_artifact(artifacts: Artifacts, name: str) -> str:
         return_str = f"[Artifact {name} created]"
     print(return_str)
 
-    display({MimeType.APPLICATION_JSON: {"last_artifact": name}})
+    display(
+        {
+            MimeType.APPLICATION_ARTIFACT: json.dumps(
+                {"name": name, "content": artifacts[name]}
+            )
+        },
+        raw=True,
+    )
     return return_str
 
 
@@ -292,7 +302,14 @@ def edit_code_artifact(
 
     artifacts[name] = "".join(edited_lines)
 
-    display({MimeType.APPLICATION_JSON: {"last_artifact": name}})
+    display(
+        {
+            MimeType.APPLICATION_ARTIFACT: json.dumps(
+                {"name": name, "content": artifacts[name]}
+            )
+        },
+        raw=True,
+    )
     return open_code_artifact(artifacts, name, cur_line)
 
 
@@ -348,7 +365,10 @@ def generate_vision_code(
     code_lines = code.splitlines(keepends=True)
     total_lines = len(code_lines)
 
-    display({MimeType.APPLICATION_JSON: {"last_artifact": name}})
+    display(
+        {MimeType.APPLICATION_ARTIFACT: json.dumps({"name": name, "content": code})},
+        raw=True,
+    )
     return view_lines(code_lines, 0, total_lines, name, total_lines)
 
 
@@ -413,7 +433,10 @@ def edit_vision_code(
     code_lines = code.splitlines(keepends=True)
     total_lines = len(code_lines)
 
-    display({MimeType.APPLICATION_JSON: {"last_artifact": name}})
+    display(
+        {MimeType.APPLICATION_ARTIFACT: json.dumps({"name": name, "content": code})},
+        raw=True,
+    )
     return view_lines(code_lines, 0, total_lines, name, total_lines)
 
 
@@ -592,6 +615,15 @@ def use_florence2_fine_tuning(
 
     diff = get_diff_with_prompts(name, code, new_code)
     print(diff)
+
+    display(
+        {
+            MimeType.APPLICATION_ARTIFACT: json.dumps(
+                {"name": name, "content": new_code}
+            )
+        },
+        raw=True,
+    )
     return diff
 
 

{vision_agent-0.2.134 → vision_agent-0.2.136}/vision_agent/utils/execute.py

@@ -56,6 +56,7 @@ class MimeType(str, Enum):
     TEXT_LATEX = "text/latex"
     APPLICATION_JSON = "application/json"
     APPLICATION_JAVASCRIPT = "application/javascript"
+    APPLICATION_ARTIFACT = "application/artifact"
 
 
 class FileSerializer:
@@ -103,6 +104,7 @@ class Result:
     latex: Optional[str] = None
     json: Optional[Dict[str, Any]] = None
     javascript: Optional[str] = None
+    artifact_name: Optional[str] = None
     extra: Optional[Dict[str, Any]] = None
     "Extra data that can be included. Not part of the standard types."
 
@@ -127,6 +129,7 @@ class Result:
         self.latex = data.pop(MimeType.TEXT_LATEX, None)
         self.json = data.pop(MimeType.APPLICATION_JSON, None)
         self.javascript = data.pop(MimeType.APPLICATION_JAVASCRIPT, None)
+        self.artifact = data.pop(MimeType.APPLICATION_ARTIFACT, None)
         self.extra = data
         # Only keeping the PNG representation if both PNG and JPEG are present
         if self.png and self.jpeg:
@@ -204,6 +207,8 @@ class Result:
             formats.append("javascript")
         if self.mp4:
             formats.append("mp4")
+        if self.artifact:
+            formats.append("artifact")
         if self.extra:
             formats.extend(iter(self.extra))
         return formats