vision-agent 0.2.133__py3-none-any.whl → 0.2.135__py3-none-any.whl

vision_agent/agent/vision_agent.py

@@ -15,6 +15,7 @@ from vision_agent.agent.vision_agent_prompts import (
 from vision_agent.lmm import LMM, Message, OpenAILMM
 from vision_agent.tools import META_TOOL_DOCSTRING, save_image, load_image
 from vision_agent.tools.meta_tools import Artifacts, use_extra_vision_agent_args
+from vision_agent.tools.tools import extract_frames, save_video
 from vision_agent.utils import CodeInterpreterFactory
 from vision_agent.utils.execute import CodeInterpreter, Execution
 
@@ -224,9 +225,20 @@ class VisionAgent(Agent):
                     for media in chat_i["media"]:
                         if type(media) is str and media.startswith(("http", "https")):
                             # TODO: Ideally we should not call VA.tools here, we should come to revisit how to better support remote image later
-                            file_path = Path(media).name
-                            ndarray = load_image(media)
-                            save_image(ndarray, file_path)
+                            file_path = str(
+                                Path(self.local_artifacts_path).parent
+                                / Path(media).name
+                            )
+                            if file_path.lower().endswith(
+                                ".mp4"
+                            ) or file_path.lower().endswith(".mov"):
+                                video_frames = extract_frames(media)
+                                save_video(
+                                    [frame for frame, _ in video_frames], file_path
+                                )
+                            else:
+                                ndarray = load_image(media)
+                                save_image(ndarray, file_path)
                             media = file_path
                         else:
                             media = cast(str, media)

vision_agent/tools/meta_tools.py

@@ -53,25 +53,27 @@ def redisplay_results(execution: Execution) -> None:
     """
     for result in execution.results:
         if result.text is not None:
-            display({MimeType.TEXT_PLAIN: result.text})
+            display({MimeType.TEXT_PLAIN: result.text}, raw=True)
         if result.html is not None:
-            display({MimeType.TEXT_HTML: result.html})
+            display({MimeType.TEXT_HTML: result.html}, raw=True)
         if result.markdown is not None:
-            display({MimeType.TEXT_MARKDOWN: result.markdown})
+            display({MimeType.TEXT_MARKDOWN: result.markdown}, raw=True)
         if result.svg is not None:
-            display({MimeType.IMAGE_SVG: result.svg})
+            display({MimeType.IMAGE_SVG: result.svg}, raw=True)
         if result.png is not None:
-            display({MimeType.IMAGE_PNG: result.png})
+            display({MimeType.IMAGE_PNG: result.png}, raw=True)
         if result.jpeg is not None:
-            display({MimeType.IMAGE_JPEG: result.jpeg})
+            display({MimeType.IMAGE_JPEG: result.jpeg}, raw=True)
         if result.mp4 is not None:
-            display({MimeType.VIDEO_MP4_B64: result.mp4})
+            display({MimeType.VIDEO_MP4_B64: result.mp4}, raw=True)
         if result.latex is not None:
-            display({MimeType.TEXT_LATEX: result.latex})
+            display({MimeType.TEXT_LATEX: result.latex}, raw=True)
         if result.json is not None:
-            display({MimeType.APPLICATION_JSON: result.json})
+            display({MimeType.APPLICATION_JSON: result.json}, raw=True)
+        if result.artifact_name is not None:
+            display({MimeType.TEXT_ARTIFACT_NAME: result.artifact_name}, raw=True)
         if result.extra is not None:
-            display(result.extra)
+            display(result.extra, raw=True)
 
 
 class Artifacts:
@@ -208,7 +210,7 @@ def create_code_artifact(artifacts: Artifacts, name: str) -> str:
         return_str = f"[Artifact {name} created]"
     print(return_str)
 
-    display({MimeType.APPLICATION_JSON: {"last_artifact": name}})
+    display({MimeType.TEXT_ARTIFACT_NAME: name}, raw=True)
     return return_str
 
 
@@ -292,7 +294,7 @@ def edit_code_artifact(
 
     artifacts[name] = "".join(edited_lines)
 
-    display({MimeType.APPLICATION_JSON: {"last_artifact": name}})
+    display({MimeType.TEXT_ARTIFACT_NAME: name}, raw=True)
     return open_code_artifact(artifacts, name, cur_line)
 
 
@@ -348,7 +350,7 @@ def generate_vision_code(
     code_lines = code.splitlines(keepends=True)
     total_lines = len(code_lines)
 
-    display({MimeType.APPLICATION_JSON: {"last_artifact": name}})
+    display({MimeType.TEXT_ARTIFACT_NAME: name}, raw=True)
     return view_lines(code_lines, 0, total_lines, name, total_lines)
 
 
@@ -413,7 +415,7 @@ def edit_vision_code(
     code_lines = code.splitlines(keepends=True)
     total_lines = len(code_lines)
 
-    display({MimeType.APPLICATION_JSON: {"last_artifact": name}})
+    display({MimeType.TEXT_ARTIFACT_NAME: name}, raw=True)
     return view_lines(code_lines, 0, total_lines, name, total_lines)
 
 
@@ -427,6 +429,7 @@ def write_media_artifact(artifacts: Artifacts, local_path: str) -> str:
     with open(local_path, "rb") as f:
         media = f.read()
     artifacts[Path(local_path).name] = media
+    display({MimeType.TEXT_ARTIFACT_NAME: Path(local_path).name}, raw=True)
     return f"[Media {Path(local_path).name} saved]"
 
 
@@ -592,6 +595,8 @@ def use_florence2_fine_tuning(
 
     diff = get_diff_with_prompts(name, code, new_code)
     print(diff)
+
+    display({MimeType.TEXT_ARTIFACT_NAME: name}, raw=True)
     return diff
 
 

vision_agent/utils/execute.py

@@ -56,6 +56,7 @@ class MimeType(str, Enum):
     TEXT_LATEX = "text/latex"
     APPLICATION_JSON = "application/json"
     APPLICATION_JAVASCRIPT = "application/javascript"
+    TEXT_ARTIFACT_NAME = "text/artifact/name"
 
 
 class FileSerializer:
@@ -103,6 +104,7 @@ class Result:
     latex: Optional[str] = None
     json: Optional[Dict[str, Any]] = None
     javascript: Optional[str] = None
+    artifact_name: Optional[str] = None
     extra: Optional[Dict[str, Any]] = None
     "Extra data that can be included. Not part of the standard types."
 
@@ -127,6 +129,7 @@ class Result:
         self.latex = data.pop(MimeType.TEXT_LATEX, None)
         self.json = data.pop(MimeType.APPLICATION_JSON, None)
         self.javascript = data.pop(MimeType.APPLICATION_JAVASCRIPT, None)
+        self.artifact_name = data.pop(MimeType.TEXT_ARTIFACT_NAME, None)
         self.extra = data
         # Only keeping the PNG representation if both PNG and JPEG are present
         if self.png and self.jpeg:
@@ -204,6 +207,8 @@ class Result:
             formats.append("javascript")
         if self.mp4:
             formats.append("mp4")
+        if self.artifact_name:
+            formats.append("artifact_name")
         if self.extra:
             formats.extend(iter(self.extra))
         return formats
@@ -691,8 +696,9 @@ class CodeInterpreterFactory:
         if not code_sandbox_runtime:
             code_sandbox_runtime = os.getenv("CODE_SANDBOX_RUNTIME", "local")
         if code_sandbox_runtime == "e2b":
+            envs = _get_e2b_env()
             instance: CodeInterpreter = E2BCodeInterpreter(
-                timeout=_SESSION_TIMEOUT, remote_path=remote_path
+                timeout=_SESSION_TIMEOUT, remote_path=remote_path, envs=envs
             )
         elif code_sandbox_runtime == "local":
             instance = LocalCodeInterpreter(
@@ -705,6 +711,20 @@ class CodeInterpreterFactory:
         return instance
 
 
+def _get_e2b_env() -> Union[Dict[str, str], None]:
+    openai_api_key = os.getenv("OPENAI_API_KEY", "")
+    anthropic_api_key = os.getenv("ANTHROPIC_API_KEY", "")
+    if openai_api_key or anthropic_api_key:
+        envs = {}
+        if openai_api_key:
+            envs["OPENAI_API_KEY"] = openai_api_key
+        if anthropic_api_key:
+            envs["ANTHROPIC_API_KEY"] = anthropic_api_key
+    else:
+        envs = None
+    return envs
+
+
 def _parse_local_code_interpreter_outputs(outputs: List[Dict[str, Any]]) -> Execution:
     """Parse notebook cell outputs to Execution object. Output types:
     https://nbformat.readthedocs.io/en/latest/format_description.html#code-cell-outputs

{vision_agent-0.2.133.dist-info → vision_agent-0.2.135.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.2.133
+Version: 0.2.135
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai

{vision_agent-0.2.133.dist-info → vision_agent-0.2.135.dist-info}/RECORD

@@ -2,7 +2,7 @@ vision_agent/__init__.py,sha256=EAb4-f9iyuEYkBrX4ag1syM8Syx8118_t0R6_C34M9w,57
 vision_agent/agent/__init__.py,sha256=TddDT4e3JVc68Dt0zSk0B4OBORx_R2WhAGK71uqEe2w,204
 vision_agent/agent/agent.py,sha256=2cjIOxEuSJrqbfPXYoV0qER5ihXsPFCoEFJa4jpqan0,597
 vision_agent/agent/agent_utils.py,sha256=qOYQn-wJsa4j4YjFOBQ41xyklCg8Y94CIIGw9ZXmgIU,2053
-vision_agent/agent/vision_agent.py,sha256=nfxdY5W5UME7JhwFcsB3j2-L5zsYZzJWdlS2R8U_9lE,13224
+vision_agent/agent/vision_agent.py,sha256=zCgCOPhOBcw9AyoUfyJcJ3HELE0FJvW4X5sWIjd67Bw,13868
 vision_agent/agent/vision_agent_coder.py,sha256=OI95goKTqVaEEPYwkn6bVsHsHZeifoBC8rjG9nD0Znc,36909
 vision_agent/agent/vision_agent_coder_prompts.py,sha256=a7P19QscKNiaweke0zHPCfi5GQImpG-ZGKv_kXz0seg,13452
 vision_agent/agent/vision_agent_prompts.py,sha256=-fXiIIb48duXVljWYcJ0Y4ZzfNnRFi3C5cKdF4SdDo8,10075
@@ -15,19 +15,19 @@ vision_agent/lmm/__init__.py,sha256=YuUZRsMHdn8cMOv6iBU8yUqlIOLrbZQqZl9KPnofsHQ,
 vision_agent/lmm/lmm.py,sha256=soWmEjtleQUSH2G3tYZWxOmteIqkgMVcmuZfx4mxszU,16838
 vision_agent/lmm/types.py,sha256=ZEXR_ptBL0ZwDMTDYkgxUCmSZFmBYPQd2jreNzr_8UY,221
 vision_agent/tools/__init__.py,sha256=nufZNzbcLTuXwxFmvZNj99qE8EO2qtEPT8wFsuI9vyE,2397
-vision_agent/tools/meta_tools.py,sha256=qbf_dzVmhf4zhv-xY1zaqRFshDlvj_7ilFQtSr70hdQ,21213
+vision_agent/tools/meta_tools.py,sha256=Oz-wbsVSjVIH2LkTg5E4Yt2jN2N5W4DbvGkx-yVi0H0,21549
 vision_agent/tools/prompts.py,sha256=V1z4YJLXZuUl_iZ5rY0M5hHc_2tmMEUKr0WocXKGt4E,1430
 vision_agent/tools/tool_utils.py,sha256=ZYqzcw_e937reoNr7gJgyKjQ7Gudxz1ttfIyo7F65w8,7758
 vision_agent/tools/tools.py,sha256=WKeB99ED0o_ISS_vZc-ch_1Dc8_Fl2fhnGlfVNwNouc,70024
 vision_agent/tools/tools_types.py,sha256=rLpCUODPY0yI65SLOTJOxfHFfqWM3WjOq-AYX25Chjk,2356
 vision_agent/utils/__init__.py,sha256=7fMgbZiEwbNS0fBOS_hJI5PuEYBblw36zLi_UjUzvj4,244
 vision_agent/utils/exceptions.py,sha256=booSPSuoULF7OXRr_YbC4dtKt6gM_HyiFQHBuaW86C4,2052
-vision_agent/utils/execute.py,sha256=7MW9GY0gwq1VjOIV07ds4xO11voPZ0Iu_RSfb8K98Y0,27263
+vision_agent/utils/execute.py,sha256=QY1GFwRDghecue_lz6s2IiRzcG1y8BrYrBohipYs7l4,27982
 vision_agent/utils/image_utils.py,sha256=zTTOJFOieMzwIquTFnW7T6ssx9o6XfoZ0Unqyk7GJrg,10746
 vision_agent/utils/sim.py,sha256=ebE9Cs00pVEDI1HMjAzUBk88tQQmc2U-yAzIDinnekU,5572
 vision_agent/utils/type_defs.py,sha256=BE12s3JNQy36QvauXHjwyeffVh5enfcvd4vTzSwvEZI,1384
 vision_agent/utils/video.py,sha256=hOjfEOZNcddYdoa0CoviXA4Vo9kwURKuojIJgLLJdp0,4745
-vision_agent-0.2.133.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-vision_agent-0.2.133.dist-info/METADATA,sha256=A95zjuoBeW_yaDMLZUU0aITAjXHXSiDBXXvfrmpfhGo,12252
-vision_agent-0.2.133.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-vision_agent-0.2.133.dist-info/RECORD,,
+vision_agent-0.2.135.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+vision_agent-0.2.135.dist-info/METADATA,sha256=oL3jpDgWSw8X5Bp_nEzTy93CrKPf0rdz_C7w3KQPc8I,12252
+vision_agent-0.2.135.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+vision_agent-0.2.135.dist-info/RECORD,,