vision-agent 0.2.31__py3-none-any.whl → 0.2.32__py3-none-any.whl

vision_agent/agent/agent_coder.py

@@ -67,11 +67,17 @@ def parse_file_name(s: str) -> str:
     return "".join([p for p in s.split(" ") if p.endswith(".png")])
 
 
-def write_program(question: str, feedback: str, model: LLM) -> str:
+def write_program(
+    question: str, feedback: str, model: LLM, media: Optional[Union[str, Path]] = None
+) -> str:
     prompt = PROGRAM.format(
         docstring=TOOL_DOCSTRING, question=question, feedback=feedback
     )
-    completion = model(prompt)
+    if isinstance(model, OpenAILMM):
+        completion = model(prompt, images=[media] if media else None)
+    else:
+        completion = model(prompt)
+
     return preprocess_data(completion)
 
 
@@ -168,7 +174,7 @@ class AgentCoder(Agent):
         code = ""
         feedback = ""
         for _ in range(self.max_turns):
-            code = write_program(question, feedback, self.coder_agent)
+            code = write_program(question, feedback, self.coder_agent, media=media)
             if self.verbose:
                 _CONSOLE.print(
                     Syntax(code, "python", theme="gruvbox-dark", line_numbers=True)

vision_agent/agent/vision_agent.py

@@ -21,6 +21,7 @@ from vision_agent.agent.vision_agent_prompts import (
     USER_REQ,
 )
 from vision_agent.llm import LLM, OpenAILLM
+from vision_agent.lmm import LMM, OpenAILMM
 from vision_agent.tools import TOOL_DESCRIPTIONS, TOOLS_DF, UTILITIES_DOCSTRING
 from vision_agent.utils import Execute
 from vision_agent.utils.sim import Sim
@@ -77,7 +78,8 @@ def write_plan(
     chat: List[Dict[str, str]],
     tool_desc: str,
     working_memory: str,
-    model: LLM,
+    model: Union[LLM, LMM],
+    media: Optional[List[Union[str, Path]]] = None,
 ) -> List[Dict[str, str]]:
     chat = copy.deepcopy(chat)
     if chat[-1]["role"] != "user":
@@ -87,7 +89,10 @@ def write_plan(
     context = USER_REQ.format(user_request=user_request)
     prompt = PLAN.format(context=context, tool_desc=tool_desc, feedback=working_memory)
     chat[-1]["content"] = prompt
-    return extract_json(model.chat(chat))["plan"]  # type: ignore
+    if isinstance(model, OpenAILMM):
+        return extract_json(model.chat(chat, images=media))["plan"]  # type: ignore
+    else:
+        return extract_json(model.chat(chat))["plan"]  # type: ignore
 
 
 def reflect(
@@ -324,7 +329,7 @@ class VisionAgent(Agent):
             input = [{"role": "user", "content": input}]
         results = self.chat_with_workflow(input, media)
         results.pop("working_memory")
-        return results["code"]  # type: ignore
+        return results  # type: ignore
 
     def chat_with_workflow(
         self,
@@ -363,7 +368,11 @@ class VisionAgent(Agent):
 
         while not success and retries < self.max_retries:
             plan_i = write_plan(
-                chat, TOOL_DESCRIPTIONS, format_memory(working_memory), self.planner
+                chat,
+                TOOL_DESCRIPTIONS,
+                format_memory(working_memory),
+                self.planner,
+                media=[media] if media else None,
             )
             plan_i_str = "\n-".join([e["instructions"] for e in plan_i])
             if self.verbosity >= 1:

vision_agent/agent/vision_agent_prompts.py

@@ -169,11 +169,13 @@ This is the documentation for the functions you have access to. You may call any
 1. Verify the fundamental functionality under normal conditions.
 2. Ensure each test case is well-documented with comments explaining the scenario it covers.
 3. Your test case MUST run only on the given image which is {media}
-4. DO NOT use any non-existent or dummy image or video files that are not provided by the user's instructions.
-5. DO NOT mock any functions, you must test their functionality as is.
-6. DO NOT assert the output value, run the code and verify it runs without any errors and assert only the output format or data structure.
-7. DO NOT import the testing function as it will available in the testing environment.
-8. Print the output of the function that is being tested.
+4. Your test case MUST run only with the given values which is available in the question - {question}
+5. DO NOT use any non-existent or dummy image or video files that are not provided by the user's instructions.
+6. DO NOT mock any functions, you must test their functionality as is.
+7. DO NOT assert the output value, run the code and assert only the output format or data structure.
+8. DO NOT use try except block to handle the error, let the error be raised if the code is incorrect.
+9. DO NOT import the testing function as it will available in the testing environment.
+10. Print the output of the function that is being tested.
 """
 
 

{vision_agent-0.2.31.dist-info → vision_agent-0.2.32.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.2.31
+Version: 0.2.32
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai

{vision_agent-0.2.31.dist-info → vision_agent-0.2.32.dist-info}/RECORD

@@ -1,7 +1,7 @@
 vision_agent/__init__.py,sha256=GVLHCeK_R-zgldpbcPmOzJat-BkadvkuRCMxDvTIcXs,108
 vision_agent/agent/__init__.py,sha256=iiC5eknTQnv87iSwAoHqBthJ3g2Zm6D0dWbYPDfuQ7A,245
 vision_agent/agent/agent.py,sha256=TXh93MOwmArNRieOkYrhliq1rf7wIkhxvCdTiGhTqFs,538
-vision_agent/agent/agent_coder.py,sha256=MQw8SPeNy1D9tUvB-u60H9ab1eLXnrpV0Ggn7Eq_mIo,6988
+vision_agent/agent/agent_coder.py,sha256=o66bpopKwo1kRDxiW6SujueMvLZ3NC5dawQKFrI3e_w,7171
 vision_agent/agent/agent_coder_prompts.py,sha256=CJe3v7xvHQ32u3RQAXQga_Tk_4UgU64RBAMHZ3S70KY,5538
 vision_agent/agent/data_interpreter.py,sha256=YCREEHWiyTYpKT8hibotylEkx1kF5AH0k9wnmymwPBY,15143
 vision_agent/agent/data_interpreter_prompts.py,sha256=RDJggOfXwGaEoIcTYGX41ZEayCgYei1AootDOc_SN2g,6134
@@ -11,8 +11,8 @@ vision_agent/agent/easytool_v2.py,sha256=CjY-sSj3abxnSq3ZHZMt-7YvRWDXEZsC6RN8FFI
 vision_agent/agent/easytool_v2_prompts.py,sha256=MZSIwovYgB-f-kdJ6btaNDVXptJn47bfOL3-Zn6NiC0,8573
 vision_agent/agent/reflexion.py,sha256=AlM5AvBJvCslXlYQdZiadq4oVHsNBm3IF_03DglTxRo,10506
 vision_agent/agent/reflexion_prompts.py,sha256=G7UAeNz_g2qCb2yN6OaIC7bQVUkda4m3z42EG8wAyfE,9342
-vision_agent/agent/vision_agent.py,sha256=5Bfxif2sqRKS1ZUlQ4yT468EfevI9CQ6V7_Y6xRbbq0,14992
-vision_agent/agent/vision_agent_prompts.py,sha256=s6T5UnyrKIAcaKqcMudWQOBCHt6Obn9QpX3QtqiDv2I,8034
+vision_agent/agent/vision_agent.py,sha256=gvGNPSyZ1WpIzBcxz03xAe96cSdVBI0zCYIfrPs9jw0,15326
+vision_agent/agent/vision_agent_prompts.py,sha256=0YbiS59IEWbiE43gCvOqfWrpudIAhTn8FHzXW0Y-Gaw,8201
 vision_agent/fonts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vision_agent/fonts/default_font_ch_en.ttf,sha256=1YM0Z3XqLDjSNbF7ihQFSAIUdjF9m1rtHiNC_6QosTE,1594400
 vision_agent/llm/__init__.py,sha256=BoUm_zSAKnLlE8s-gKTSQugXDqVZKPqYlWwlTLdhcz4,48
@@ -30,7 +30,7 @@ vision_agent/utils/image_utils.py,sha256=_cdiS5YrLzqkq_ZgFUO897m5M4_SCIThwUy4lOk
 vision_agent/utils/sim.py,sha256=oUZ-6eu8Io-UNt9GXJ0XRKtP-Wc0sPWVzYGVpB2yDFk,3001
 vision_agent/utils/type_defs.py,sha256=BlI8ywWHAplC7kYWLvt4AOdnKpEW3qWEFm-GEOSkrFQ,1792
 vision_agent/utils/video.py,sha256=xTElFSFp1Jw4ulOMnk81Vxsh-9dTxcWUO6P9fzEi3AM,7653
-vision_agent-0.2.31.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-vision_agent-0.2.31.dist-info/METADATA,sha256=tsCUD6WuSXUt5XLCmOD89DMzDTAxyrCPiA0cAES85AI,5942
-vision_agent-0.2.31.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-vision_agent-0.2.31.dist-info/RECORD,,
+vision_agent-0.2.32.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+vision_agent-0.2.32.dist-info/METADATA,sha256=okG2Wf1LavPw4agq2iH7tY_rLlrabKgOOCL43pzfEMY,5942
+vision_agent-0.2.32.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+vision_agent-0.2.32.dist-info/RECORD,,