vision-agent 0.2.143__py3-none-any.whl → 0.2.145__py3-none-any.whl

vision_agent/agent/vision_agent.py

@@ -229,7 +229,7 @@ class VisionAgent(Agent):
         ) as code_interpreter:
             orig_chat = copy.deepcopy(chat)
             int_chat = copy.deepcopy(chat)
-            last_user_message_content = chat[-1].get("content")
+            last_user_message = chat[-1]
             media_list = []
             for chat_i in int_chat:
                 if "media" in chat_i:
@@ -278,32 +278,9 @@ class VisionAgent(Agent):
             orig_chat.append({"role": "observation", "content": artifacts_loaded})
             self.streaming_message({"role": "observation", "content": artifacts_loaded})
 
-            if int_chat[-1]["role"] == "user":
-                last_user_message_content = cast(str, int_chat[-1].get("content", ""))
-                user_code_action = parse_execution(last_user_message_content, False)
-                if user_code_action is not None:
-                    user_result, user_obs = run_code_action(
-                        user_code_action, code_interpreter, str(remote_artifacts_path)
-                    )
-                    if self.verbosity >= 1:
-                        _LOGGER.info(user_obs)
-                    int_chat.append({"role": "observation", "content": user_obs})
-                    orig_chat.append(
-                        {
-                            "role": "observation",
-                            "content": user_obs,
-                            "execution": user_result,
-                        }
-                    )
-                    self.streaming_message(
-                        {
-                            "role": "observation",
-                            "content": user_obs,
-                            "execution": user_result,
-                            "finished": True,
-                        }
-                    )
-                    finished = True
+            finished = self.execute_user_code_action(
+                last_user_message, code_interpreter, remote_artifacts_path
+            )
 
             while not finished and iterations < self.max_iterations:
                 response = run_conversation(self.agent, int_chat)
@@ -315,11 +292,6 @@ class VisionAgent(Agent):
                 # sometimes it gets stuck in a loop, so we force it to exit
                 if last_response == response:
                     response["let_user_respond"] = True
-                    self.streaming_message(
-                        {"role": "assistant", "error": "Stuck in loop"}
-                    )
-                else:
-                    self.streaming_message({"role": "assistant", "content": response})
 
                 finished = response["let_user_respond"]
 
@@ -327,6 +299,28 @@ class VisionAgent(Agent):
                     response["response"], test_multi_plan, customized_tool_names
                 )
 
+                if last_response == response:
+                    self.streaming_message(
+                        {
+                            "role": "assistant",
+                            "content": "{}",
+                            "error": {
+                                "name": "Error when running conversation agent",
+                                "value": "Agent is stuck in conversation loop, exited",
+                                "traceback_raw": [],
+                            },
+                            "finished": finished and code_action is None,
+                        }
+                    )
+                else:
+                    self.streaming_message(
+                        {
+                            "role": "assistant",
+                            "content": response,
+                            "finished": finished and code_action is None,
+                        }
+                    )
+
                 if code_action is not None:
                     result, obs = run_code_action(
                         code_action, code_interpreter, str(remote_artifacts_path)
@@ -353,6 +347,7 @@ class VisionAgent(Agent):
                             "role": "observation",
                             "content": obs,
                             "execution": result,
+                            "finished": finished,
                         }
                     )
 
@@ -367,6 +362,34 @@ class VisionAgent(Agent):
             artifacts.save()
         return orig_chat, artifacts
 
+    def execute_user_code_action(
+        self,
+        last_user_message: Message,
+        code_interpreter: CodeInterpreter,
+        remote_artifacts_path: Path,
+    ) -> bool:
+        if last_user_message["role"] != "user":
+            return False
+        user_code_action = parse_execution(
+            cast(str, last_user_message.get("content", "")), False
+        )
+        if user_code_action is not None:
+            user_result, user_obs = run_code_action(
+                user_code_action, code_interpreter, str(remote_artifacts_path)
+            )
+            if self.verbosity >= 1:
+                _LOGGER.info(user_obs)
+            self.streaming_message(
+                {
+                    "role": "observation",
+                    "content": user_obs,
+                    "execution": user_result,
+                    "finished": True,
+                }
+            )
+            return True
+        return False
+
     def streaming_message(self, message: Dict[str, Any]) -> None:
         if self.callback_message:
             self.callback_message(message)

vision_agent/tools/meta_tools.py

@@ -425,6 +425,7 @@ def edit_vision_code(
 
     agent = va.agent.VisionAgentCoder()
     if name not in artifacts:
+        print(f"[Artifact {name} does not exist]")
         return f"[Artifact {name} does not exist]"
 
     code = artifacts[name]

vision_agent/tools/tools.py

@@ -1181,7 +1181,12 @@ def florence2_phrase_grounding(
             fine_tuning=FineTuning(job_id=UUID(fine_tune_id)),
         )
         data = data_obj.model_dump(by_alias=True)
-        detections = send_inference_request(data, "tools", v2=False)
+        detections = send_inference_request(
+            data,
+            "tools",
+            v2=False,
+            metadata_payload={"function_name": "florence2_phrase_grounding"},
+        )
     else:
         data = {
             "image": image_b64,

{vision_agent-0.2.143.dist-info → vision_agent-0.2.145.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.2.143
+Version: 0.2.145
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai

{vision_agent-0.2.143.dist-info → vision_agent-0.2.145.dist-info}/RECORD

@@ -2,7 +2,7 @@ vision_agent/__init__.py,sha256=EAb4-f9iyuEYkBrX4ag1syM8Syx8118_t0R6_C34M9w,57
 vision_agent/agent/__init__.py,sha256=NF2LABqHixLvbsOIO-fe-VKZ7awvShLtcT0oQT4eWtI,235
 vision_agent/agent/agent.py,sha256=2cjIOxEuSJrqbfPXYoV0qER5ihXsPFCoEFJa4jpqan0,597
 vision_agent/agent/agent_utils.py,sha256=PEUHqvnHmFL4np_TeFmKMwr5s_dWfdfJz6TF_ogd1dU,2353
-vision_agent/agent/vision_agent.py,sha256=k1bUmvoz0KjVEu62PYA9djnq3pqzv2S1UsW6gLnTd7w,17023
+vision_agent/agent/vision_agent.py,sha256=Fp2uSbroRzGrxEwbb9srGdl0h31awkzDFm2tTfn28GI,17587
 vision_agent/agent/vision_agent_coder.py,sha256=4bbebV1sKE10vsxcZR-R8P54X2HjLeU9lDt7ylIZAT4,38429
 vision_agent/agent/vision_agent_coder_prompts.py,sha256=YWK4C--YRS1Kuab11Gn-AXBzar1j_GNnTnxi_nnaPRY,14901
 vision_agent/agent/vision_agent_prompts.py,sha256=e_ASPeRFU1yZsQhCkK_bIBG-eyIWyWXmN64lFk-r7e0,10897
@@ -15,10 +15,10 @@ vision_agent/lmm/__init__.py,sha256=jyY1sJb_tYKg5-Wzs3p1lvwFkc-aUNZfMcLy3TOC4Zg,
 vision_agent/lmm/lmm.py,sha256=B5ClgwvbybVCWkf9opDMLjTtJZemUU4KUkQoRxGh43I,16787
 vision_agent/lmm/types.py,sha256=ZEXR_ptBL0ZwDMTDYkgxUCmSZFmBYPQd2jreNzr_8UY,221
 vision_agent/tools/__init__.py,sha256=zUv3aVPN1MXfyQiQi5To4rkQGtG7mxLQ1NjLI3pxM80,2412
-vision_agent/tools/meta_tools.py,sha256=XO5Ahe5ZauomynxgDcBuzmm0ocXwTnmZ0wjfgvOzDWc,23426
+vision_agent/tools/meta_tools.py,sha256=iHvMeBktWcVi-0DOrSMak1gsZrM_VKJlAq1mAFbBemE,23477
 vision_agent/tools/prompts.py,sha256=V1z4YJLXZuUl_iZ5rY0M5hHc_2tmMEUKr0WocXKGt4E,1430
 vision_agent/tools/tool_utils.py,sha256=5ukuDMxbEH4iKetYR9I7twzsA8ECyP4tVwYXQq54mxI,8020
-vision_agent/tools/tools.py,sha256=dD_8AmAQb0oKVZHg2w2kSKlvWrG9yaKRbaHTz_kHgjA,73648
+vision_agent/tools/tools.py,sha256=lLGZ-GgORSbFCTaBWI07Om1L8gnO00d8MN600YC7s_o,73773
 vision_agent/tools/tools_types.py,sha256=JUOZWGW2q-dlJ85CHr9gvo9KQk_rXyjJhi-iwPNn4eM,2397
 vision_agent/utils/__init__.py,sha256=7fMgbZiEwbNS0fBOS_hJI5PuEYBblw36zLi_UjUzvj4,244
 vision_agent/utils/exceptions.py,sha256=booSPSuoULF7OXRr_YbC4dtKt6gM_HyiFQHBuaW86C4,2052
@@ -27,7 +27,7 @@ vision_agent/utils/image_utils.py,sha256=rm9GfXvD4JrjnqKrP_f2gfq4SzmqYC0IdC1kKwd
 vision_agent/utils/sim.py,sha256=ebE9Cs00pVEDI1HMjAzUBk88tQQmc2U-yAzIDinnekU,5572
 vision_agent/utils/type_defs.py,sha256=BE12s3JNQy36QvauXHjwyeffVh5enfcvd4vTzSwvEZI,1384
 vision_agent/utils/video.py,sha256=xbMEoRk13l4fHeQlbvMQhLCn8RNndYmsDhUf01TUeR8,4781
-vision_agent-0.2.143.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-vision_agent-0.2.143.dist-info/METADATA,sha256=JPDcTSZk7U0j20CaJnw-WzKG7s5EjX0axntfS58264I,13758
-vision_agent-0.2.143.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-vision_agent-0.2.143.dist-info/RECORD,,
+vision_agent-0.2.145.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+vision_agent-0.2.145.dist-info/METADATA,sha256=Z7gVyhZAtrpb3zTrFl2LZli9hw6f2VW_0p5_MQtpURM,13758
+vision_agent-0.2.145.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+vision_agent-0.2.145.dist-info/RECORD,,