vision-agent 0.2.241__py3-none-any.whl → 0.2.243__py3-none-any.whl

vision_agent/agent/agent.py

@@ -5,6 +5,7 @@ from typing import Any, Dict, List, Optional, Union
 from vision_agent.models import (
     AgentMessage,
     CodeContext,
+    ErrorContext,
     InteractionContext,
     Message,
     PlanContext,
@@ -36,7 +37,7 @@ class AgentCoder(Agent):
         chat: List[AgentMessage],
         max_steps: Optional[int] = None,
         code_interpreter: Optional[CodeInterpreter] = None,
-    ) -> Union[CodeContext, InteractionContext]:
+    ) -> Union[CodeContext, InteractionContext, ErrorContext]:
         pass
 
     @abstractmethod
@@ -56,5 +57,5 @@ class AgentPlanner(Agent):
         chat: List[AgentMessage],
         max_steps: Optional[int] = None,
         code_interpreter: Optional[CodeInterpreter] = None,
-    ) -> Union[PlanContext, InteractionContext]:
+    ) -> Union[PlanContext, InteractionContext, ErrorContext]:
         pass

vision_agent/agent/vision_agent_coder_v2.py

@@ -13,6 +13,7 @@ from vision_agent.lmm import LMM
 from vision_agent.models import (
     AgentMessage,
     CodeContext,
+    ErrorContext,
     InteractionContext,
     Message,
     PlanContext,
@@ -365,6 +366,8 @@ class VisionAgentCoderV2(AgentCoder):
         code_or_interaction = self.generate_code(input_msg)
         if isinstance(code_or_interaction, InteractionContext):
             return code_or_interaction.chat[-1].content
+        elif isinstance(code_or_interaction, ErrorContext):
+            return code_or_interaction.error
         return code_or_interaction.code
 
     def generate_code(
@@ -372,7 +375,7 @@ class VisionAgentCoderV2(AgentCoder):
         chat: List[AgentMessage],
         max_steps: Optional[int] = None,
         code_interpreter: Optional[CodeInterpreter] = None,
-    ) -> Union[CodeContext, InteractionContext]:
+    ) -> Union[CodeContext, InteractionContext, ErrorContext]:
         """Generate vision code from a conversation.
 
         Parameters:
@@ -404,6 +407,8 @@ class VisionAgentCoderV2(AgentCoder):
             # the planner needs an interaction, so return before generating code
             if isinstance(plan_context, InteractionContext):
                 return plan_context
+            elif isinstance(plan_context, ErrorContext):
+                return plan_context
 
             code_context = self.generate_code_from_plan(
                 orig_chat,

vision_agent/agent/vision_agent_planner_v2.py

@@ -24,7 +24,13 @@ from vision_agent.agent.vision_agent_planner_prompts_v2 import (
 )
 from vision_agent.configs import Config
 from vision_agent.lmm import LMM
-from vision_agent.models import AgentMessage, InteractionContext, Message, PlanContext
+from vision_agent.models import (
+    AgentMessage,
+    ErrorContext,
+    InteractionContext,
+    Message,
+    PlanContext,
+)
 from vision_agent.tools.planner_tools import check_function_call
 from vision_agent.utils.agent import (
     add_media_to_chat,
@@ -322,7 +328,7 @@ def create_finalize_plan(
     model: LMM,
     chat: List[AgentMessage],
     verbose: bool = False,
-) -> Tuple[List[AgentMessage], PlanContext]:
+) -> Tuple[List[AgentMessage], Union[PlanContext, ErrorContext]]:
     # if we're in the middle of an interaction, don't finalize the plan
     if chat[-1].role == "interaction":
         return [], PlanContext(plan="", instructions=[], code="")
@@ -337,11 +343,19 @@ def create_finalize_plan(
     return_chat = [AgentMessage(role="planner", content=plan_str, media=None)]
 
     plan_json = extract_tag(plan_str, "json")
-    plan = (
-        extract_json(plan_json)
-        if plan_json is not None
-        else {"plan": plan_str, "instructions": [], "code": ""}
-    )
+
+    # sometimes the planner model will refuse to answer a question becuase of some
+    # safety concern, we then wont be able to parse the response so we have to send
+    # it back to the user/conversation agent
+    try:
+        plan = (
+            extract_json(plan_json)
+            if plan_json is not None
+            else {"plan": plan_str, "instructions": [], "code": ""}
+        )
+    except json.JSONDecodeError:
+        return return_chat, ErrorContext(error=plan_str)
+
     code_snippets = extract_tag(plan_str, "code")
     plan["code"] = code_snippets if code_snippets is not None else ""
     if verbose:
@@ -473,14 +487,17 @@ class VisionAgentPlannerV2(AgentPlanner):
         plan_or_interaction = self.generate_plan(input_msg)
         if isinstance(plan_or_interaction, InteractionContext):
             return plan_or_interaction.chat[-1].content
-        return plan_or_interaction.plan
+        elif isinstance(plan_or_interaction, PlanContext):
+            return plan_or_interaction.plan
+        else:
+            return plan_or_interaction.error
 
     def generate_plan(
         self,
         chat: List[AgentMessage],
         max_steps: Optional[int] = None,
         code_interpreter: Optional[CodeInterpreter] = None,
-    ) -> Union[PlanContext, InteractionContext]:
+    ) -> Union[PlanContext, InteractionContext, ErrorContext]:
         """Generate a plan to solve a vision task.
 
         Parameters:
@@ -571,7 +588,7 @@ class VisionAgentPlannerV2(AgentPlanner):
                 for chat_elt in updated_chat:
                     self.update_callback(chat_elt.model_dump())
 
-            context: Union[PlanContext, InteractionContext]
+            context: Union[PlanContext, InteractionContext, ErrorContext]
             if interaction:
                 context = InteractionContext(chat=int_chat)
             else:

vision_agent/agent/vision_agent_prompts_v2.py

@@ -16,17 +16,29 @@ AGENT: <response>Yes, I can help you with that. I will write the code to detect
 OBSERVATION:
 <final_code>
 from vision_agent.tools import load_image, owl_v2_image
-def detect_dogs(image_path: str):
+def detect_dogs(image_path: str) -> int:
     image = load_image(image_path)
     dogs = owl_v2_image(image)
-    return dogs
+    return len(dogs)
 </final_code>
 <final_test>
 def test_detect_dogs():
     dogs = detect_dogs("images/dogs.jpg")
-    assert len(dogs) > 0
+    assert isinstance(dogs, int)
+    print(f"Number of dogs detected: {{dogs}}")
+    return dogs
 </final_test>
 
+OBSERVATION: ----- stdout -----
+Number of dogs detected: 8
+
+----- stderr -----
+
+----- Intermediate output-----
+None
+----- Final output -----
+8
+
 AGENT: <response>Here is the code to detect dogs in the image.</response>
 --- END EXAMPLE1 ---
 

vision_agent/agent/vision_agent_v2.py

@@ -11,6 +11,7 @@ from vision_agent.lmm import LMM
 from vision_agent.models import (
     AgentMessage,
     CodeContext,
+    ErrorContext,
     InteractionContext,
     Message,
     PlanContext,
@@ -27,7 +28,9 @@ CONFIG = Config()
 
 
 def extract_conversation(
-    chat: List[AgentMessage], include_conv: bool = False
+    chat: List[AgentMessage],
+    include_conv: bool = False,
+    include_errors: bool = False,
 ) -> Tuple[List[AgentMessage], Optional[str]]:
     chat = copy.deepcopy(chat)
 
@@ -43,13 +46,18 @@ def extract_conversation(
         elif chat_i.role == "coder":
             if "<final_code>" in chat_i.content:
                 extracted_chat.append(chat_i)
+        elif chat_i.role == "final_observation":
+            extracted_chat.append(chat_i)
         elif include_conv and chat_i.role == "conversation":
             extracted_chat.append(chat_i)
+        elif include_errors and chat_i.role == "error_observation":
+            extracted_chat.append(chat_i)
 
-    # only keep the last <final_code> and <final_test>
+    # only keep the last <final_code>, <final_test>
     final_code = None
     extracted_chat_strip_code: List[AgentMessage] = []
-    for chat_i in reversed(extracted_chat):
+    for chat_i in reversed((extracted_chat)):
+        # don't check role here because user could send updated <final_code>
         if "<final_code>" in chat_i.content and final_code is None:
             extracted_chat_strip_code = [chat_i] + extracted_chat_strip_code
             final_code = extract_tag(chat_i.content, "final_code")
@@ -66,7 +74,12 @@ def extract_conversation(
 
 
 def run_conversation(agent: LMM, chat: List[AgentMessage]) -> str:
-    extracted_chat, _ = extract_conversation(chat, include_conv=True)
+    # Include conversation and error messages. The error messages can come from one of
+    # the agents refusing to write a correctly formatted message, want to inform the
+    # conversation agent of this.
+    extracted_chat, _ = extract_conversation(
+        chat, include_conv=True, include_errors=True
+    )
 
     conv = format_conversation(extracted_chat)
     prompt = CONVERSATION.format(
@@ -101,7 +114,9 @@ def maybe_run_action(
         if isinstance(context, CodeContext):
             return [
                 AgentMessage(role="coder", content=format_code_context(context)),
-                AgentMessage(role="observation", content=context.test_result.text()),
+                AgentMessage(
+                    role="final_observation", content=context.test_result.text()
+                ),
             ]
         elif isinstance(context, InteractionContext):
             return [
@@ -110,6 +125,10 @@ def maybe_run_action(
                     content=json.dumps([elt.model_dump() for elt in context.chat]),
                 )
             ]
+        elif isinstance(context, ErrorContext):
+            return [
+                AgentMessage(role="error_observation", content=context.error),
+            ]
     elif action == "edit_code":
         # We don't want to pass code in plan_context.code so the coder will generate
         # new code from plan_context.plan
@@ -129,7 +148,7 @@ def maybe_run_action(
         )
         return [
             AgentMessage(role="coder", content=format_code_context(context)),
-            AgentMessage(role="observation", content=context.test_result.text()),
+            AgentMessage(role="final_observation", content=context.test_result.text()),
         ]
     elif action == "view_image":
         pass

vision_agent/models/__init__.py

@@ -1,4 +1,10 @@
-from .agent_types import AgentMessage, CodeContext, InteractionContext, PlanContext
+from .agent_types import (
+    AgentMessage,
+    CodeContext,
+    ErrorContext,
+    InteractionContext,
+    PlanContext,
+)
 from .lmm_types import Message, TextOrImage
 from .tools_types import (
     BboxInput,

vision_agent/models/agent_types.py

@@ -29,11 +29,15 @@ class AgentMessage(BaseModel):
         Literal["user"],
         Literal["assistant"],  # planner, coder and conversation are of type assistant
         Literal["observation"],
+        Literal["final_observation"],  # the observation from the final code output
+        Literal["error_observation"],  # the observation from the error message
         Literal["interaction"],
         Literal["interaction_response"],
         Literal["conversation"],
         Literal["planner"],
-        Literal["planner_update"],
+        Literal[
+            "planner_update"
+        ],  # an intermediate update from the planner to show partial information
         Literal["coder"],
     ]
     content: str
@@ -75,3 +79,14 @@ class InteractionContext(BaseModel):
     """
 
     chat: List[AgentMessage]
+
+
+class ErrorContext(BaseModel):
+    """ErrorContext is a data model that represents an error message. These errors can
+    happen in the planning phase when a model does not output correctly formatted
+    messages (often because it considers some response to be a safety issue).
+
+    error: The error message.
+    """
+
+    error: str

vision_agent/utils/agent.py

@@ -159,11 +159,12 @@ def format_conversation(chat: List[AgentMessage]) -> str:
     chat = copy.deepcopy(chat)
     prompt = ""
     for chat_i in chat:
-        if chat_i.role == "user" or chat_i.role == "coder":
-            if "<final_code>" in chat_i.content:
-                prompt += f"OBSERVATION: {chat_i.content}\n\n"
-            elif chat_i.role == "user":
+        # we want to print user messages, final code, final code observations or errors
+        if chat_i.role in ["user", "coder", "final_observation", "error_observation"]:
+            if chat_i.role == "user":
                 prompt += f"USER: {chat_i.content}\n\n"
+            else:
+                prompt += f"OBSERVATION: {chat_i.content}\n\n"
         elif chat_i.role == "conversation":
             prompt += f"AGENT: {chat_i.content}\n\n"
     return prompt

{vision_agent-0.2.241.dist-info → vision_agent-0.2.243.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.2.241
+Version: 0.2.243
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai

{vision_agent-0.2.241.dist-info → vision_agent-0.2.243.dist-info}/RECORD

@@ -3,19 +3,19 @@ vision_agent/.sim_tools/embs.npy,sha256=pi7h3NHlrKncIGNR-oPn_XoTe2PzBb9-aFMi7qK0
 vision_agent/__init__.py,sha256=EAb4-f9iyuEYkBrX4ag1syM8Syx8118_t0R6_C34M9w,57
 vision_agent/agent/README.md,sha256=Q4w7FWw38qaWosQYAZ7NqWx8Q5XzuWrlv7nLhjUd1-8,5527
 vision_agent/agent/__init__.py,sha256=M8CffavdIh8Zh-skznLHIaQkYGCGK7vk4dq1FaVkbs4,617
-vision_agent/agent/agent.py,sha256=RoS7kMfXYILv0zuPpcxqQIlaHGa3K-qw_5EwgsEJTPQ,1530
+vision_agent/agent/agent.py,sha256=o1Zuhl6h2R7uVwvUur0Aj38kak8U08plfeFWPst_ErM,1576
 vision_agent/agent/vision_agent.py,sha256=4LqvwPTSsiuJEDwBbMx9Dg9ALJwNR6x1c63TZvOMm8A,23486
 vision_agent/agent/vision_agent_coder.py,sha256=Ry6AiyAj3hsSeYPu_5guMcTzf2E4SoebPzpHyJtSPbQ,27360
 vision_agent/agent/vision_agent_coder_prompts.py,sha256=D4RJxTWoxpl-WtYRvHNxaLSdWVHsdYb0jJIQ2ZCGU0A,12277
 vision_agent/agent/vision_agent_coder_prompts_v2.py,sha256=53b_DhQtffX5wxLuCbNQ83AJhB0P_3wEnuKr-v5bx-o,4866
-vision_agent/agent/vision_agent_coder_v2.py,sha256=bWAUyk7-lYKwLIjkL_wUTeYv06zIIroJE1yIDRaGUHw,17059
+vision_agent/agent/vision_agent_coder_v2.py,sha256=I4gWrneFIqhX6W-MxiaNyPKGk5tRKgC8xryV-YdeSZU,17289
 vision_agent/agent/vision_agent_planner.py,sha256=rp_atRMDg35WFXNKOTkjUpGPrpSCsiMhcfZtqK-DIV4,18668
 vision_agent/agent/vision_agent_planner_prompts.py,sha256=rYRdJthc-sQN57VgCBKrF09Sd73BSxcBdjNe6C4WNZ8,6837
 vision_agent/agent/vision_agent_planner_prompts_v2.py,sha256=TiiF5BGnFVraFlQnDaeRU67927LvszvpcMUOgVgo0ps,35843
-vision_agent/agent/vision_agent_planner_v2.py,sha256=IqEP5ded5P4ESkLEur81gUvJtTmTdlKAx9uQyLyIwPc,21212
+vision_agent/agent/vision_agent_planner_v2.py,sha256=GOhaTsVCh02X09IKkC4k9z79lsmU4VgRW7WJLKjdG1k,21755
 vision_agent/agent/vision_agent_prompts.py,sha256=KaJwYPUP7_GvQsCPPs6Fdawmi3AQWmWajBUuzj7gTG4,13812
-vision_agent/agent/vision_agent_prompts_v2.py,sha256=Wyxa15NOe75PefAfw3_RRwvgjg8YVqCrU7WvvWoYJpk,2733
-vision_agent/agent/vision_agent_v2.py,sha256=O070_QdgsqNzex5eRtye8QmJgCtHcf_B7zeteWpw3LM,10895
+vision_agent/agent/vision_agent_prompts_v2.py,sha256=jTfu_heNTBaHj1UNI0XIyyFDgDOjPTPP83vrS-g3A1U,2961
+vision_agent/agent/vision_agent_v2.py,sha256=QPAyDjnRRHUCD4Pw4TQYffWkucbn4WkEjYn8dBIWll4,11682
 vision_agent/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vision_agent/clients/http.py,sha256=k883i6M_4nl7zwwHSI-yP5sAgQZIDPM1nrKD6YFJ3Xs,2009
 vision_agent/configs/__init__.py,sha256=Iu75-w9_nlPmnB_qKA7nYaaaHf7xtTrDmK8N4v2WV34,27
@@ -27,8 +27,8 @@ vision_agent/fonts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 vision_agent/fonts/default_font_ch_en.ttf,sha256=1YM0Z3XqLDjSNbF7ihQFSAIUdjF9m1rtHiNC_6QosTE,1594400
 vision_agent/lmm/__init__.py,sha256=4qX2lmGnKWHeKftXueEi9xj_ieK2nQh_ipHf72nKGFk,84
 vision_agent/lmm/lmm.py,sha256=XYp1frrqQ-6q-0y2IWwM8-EIH5UrFZ21SAhkcM32J9w,19355
-vision_agent/models/__init__.py,sha256=qAdygB-0EsmxMHNzYTPNM6tAF8Fym95gm9bsHJafdgE,287
-vision_agent/models/agent_types.py,sha256=dIdxATH_PP76pD5Wfo0oofWt6iPQh0vpf48QbEQSzhs,2472
+vision_agent/models/__init__.py,sha256=eIP0pD5dYog8zUA7uuTmUxCF6SIutbLRLRE0cmuCJgQ,326
+vision_agent/models/agent_types.py,sha256=vBZ9-ns5lHDdFMO7ulCGGeZ6OwRo3gK4O3vN0814IWc,3064
 vision_agent/models/lmm_types.py,sha256=v04h-NjbczHOIN8UWa1vvO5-1BDuZ4JQhD2mge1cXmw,305
 vision_agent/models/tools_types.py,sha256=8hYf2OZhI58gvf65KGaeGkt4EQ56nwLFqIQDPHioOBc,2339
 vision_agent/sim/__init__.py,sha256=Aouz6HEPPTYcLxR5_0fTYCL1OvPKAH1RMWAF90QXAlA,135
@@ -39,7 +39,7 @@ vision_agent/tools/planner_tools.py,sha256=orBTdJQz2NKoLuX9WE6XixaYuG305xz0UBYvZ
 vision_agent/tools/prompts.py,sha256=V1z4YJLXZuUl_iZ5rY0M5hHc_2tmMEUKr0WocXKGt4E,1430
 vision_agent/tools/tools.py,sha256=uhvgPeAzhOV2vfBa216vq-JVItqgzIRKs1JMBezj2Es,107631
 vision_agent/utils/__init__.py,sha256=mANUs_84VL-3gpZbXryvV2mWU623eWnRlJCSUHtMjuw,122
-vision_agent/utils/agent.py,sha256=QGKcbzpAjcVj0958bXYLv07-d2i1GU7-bXVG7bTGRMA,14619
+vision_agent/utils/agent.py,sha256=8z4Ei0q397lVWUga8v9nQKuenGAsh2wfkAKQOB8CwpI,14701
 vision_agent/utils/exceptions.py,sha256=zis8smCbdEylBVZBTVfEUfAh7Rb7cWV3MSPambu6FsQ,1837
 vision_agent/utils/execute.py,sha256=vOEP5Ys7S2lc0_7pOJbgk7OaWi85hrCNu9_8Bo3zk6I,29356
 vision_agent/utils/image_utils.py,sha256=bJM2mEvB6E__M9pxi74yQYzAiZ7mu3KE2ptyVrp5vzQ,12533
@@ -48,7 +48,7 @@ vision_agent/utils/tools_doc.py,sha256=yFue6KSXoa_Z1ngCdBEc4SdPZOWF1rVLeaHu02I8W
 vision_agent/utils/type_defs.py,sha256=BE12s3JNQy36QvauXHjwyeffVh5enfcvd4vTzSwvEZI,1384
 vision_agent/utils/video.py,sha256=rjsQ1sKKisaQ6AVjJz0zd_G4g-ovRweS_rs4JEhenoI,5340
 vision_agent/utils/video_tracking.py,sha256=eMIiWOG24bgXbqOy1DTtepO2gPo1ClW6Y0tdbEF_14k,12227
-vision_agent-0.2.241.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-vision_agent-0.2.241.dist-info/METADATA,sha256=kIO6XLDr55qrZEThNPu0egm6rjliM945mOSpD4FGFrE,5712
-vision_agent-0.2.241.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-vision_agent-0.2.241.dist-info/RECORD,,
+vision_agent-0.2.243.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+vision_agent-0.2.243.dist-info/METADATA,sha256=R1mAlqsjgXjmE8Okex12-on706FU16JeP4jNM3ZlFBQ,5712
+vision_agent-0.2.243.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+vision_agent-0.2.243.dist-info/RECORD,,