vision-agent 0.2.230__py3-none-any.whl → 0.2.231__py3-none-any.whl

vision_agent/agent/types.py

@@ -33,6 +33,7 @@ class AgentMessage(BaseModel):
         Literal["interaction_response"],
         Literal["conversation"],
         Literal["planner"],
+        Literal["planner_update"],
         Literal["coder"],
     ]
     content: str

vision_agent/agent/vision_agent_planner_v2.py

@@ -513,6 +513,7 @@ class VisionAgentPlannerV2(AgentPlanner):
                 code = extract_tag(response, "execute_python")
                 finalize_plan = extract_tag(response, "finalize_plan")
                 finished = finalize_plan is not None
+                self.update_callback({"role": "planner_update", "content": response})
 
                 if self.verbose:
                     _CONSOLE.print(

vision_agent/configs/anthropic_config.py

@@ -81,7 +81,7 @@ class Config(BaseModel):
     tool_tester_kwargs: dict = Field(
         default_factory=lambda: {
             "model_name": "claude-3-5-sonnet-20241022",
-            "temperature": 1.0,
+            "temperature": 0.0,
             "image_size": 768,
         }
     )
@@ -111,7 +111,7 @@ class Config(BaseModel):
     vqa_kwargs: dict = Field(
         default_factory=lambda: {
             "model_name": "claude-3-5-sonnet-20241022",
-            "temperature": 1.0,
+            "temperature": 0.0,
             "image_size": 768,
         }
     )

vision_agent/configs/openai_config.py

@@ -98,7 +98,7 @@ class Config(BaseModel):
     tool_chooser_kwargs: dict = Field(
         default_factory=lambda: {
             "model_name": "gpt-4o-2024-08-06",
-            "temperature": 0.0,
+            "temperature": 1.0,
             "image_size": 768,
             "image_detail": "low",
         }
@@ -109,7 +109,7 @@ class Config(BaseModel):
     suggester_kwargs: dict = Field(
         default_factory=lambda: {
             "model_name": "gpt-4o-2024-08-06",
-            "temperature": 0.0,
+            "temperature": 1.0,
             "image_size": 768,
             "image_detail": "low",
         }

vision_agent/tools/__init__.py

@@ -23,6 +23,9 @@ from .tools import (
     TOOLS_INFO,
     UTIL_TOOLS,
     UTILITIES_DOCSTRING,
+    agentic_object_detection,
+    agentic_sam2_instance_segmentation,
+    agentic_sam2_video_tracking,
     claude35_text_extraction,
     closest_box_distance,
     closest_mask_distance,
@@ -30,6 +33,7 @@ from .tools import (
     countgd_sam2_instance_segmentation,
     countgd_sam2_video_tracking,
     countgd_visual_prompt_object_detection,
+    custom_object_detection,
     depth_anything_v2,
     detr_segmentation,
     document_extraction,
@@ -63,10 +67,6 @@ from .tools import (
     video_temporal_localization,
     vit_image_classification,
     vit_nsfw_classification,
-    custom_object_detection,
-    agentic_object_detection,
-    agentic_sam2_instance_segmentation,
-    agentic_sam2_video_tracking,
 )
 
 __new_tools__ = [

vision_agent/tools/planner_tools.py

@@ -10,12 +10,7 @@ from IPython.display import display
 from PIL import Image
 
 import vision_agent.tools as T
-from vision_agent.agent.agent_utils import (
-    DefaultImports,
-    extract_code,
-    extract_json,
-    extract_tag,
-)
+from vision_agent.agent.agent_utils import DefaultImports, extract_json, extract_tag
 from vision_agent.agent.vision_agent_planner_prompts_v2 import (
     CATEGORIZE_TOOL_REQUEST,
     FINALIZE_PLAN,
@@ -36,6 +31,9 @@ from vision_agent.utils.image_utils import convert_to_b64
 from vision_agent.utils.sim import get_tool_recommender
 
 TOOL_FUNCTIONS = {tool.__name__: tool for tool in T.TOOLS}
+LOAD_TOOLS_DOCSTRING = T.get_tool_documentation(
+    [T.load_image, T.extract_frames_and_timestamps]
+)
 
 CONFIG = Config()
 _LOGGER = logging.getLogger(__name__)
@@ -179,6 +177,7 @@ def run_tool_testing(
                 cleaned_tool_docs.append(tool_doc)
         tool_docs = cleaned_tool_docs
     tool_docs_str = "\n".join([e["doc"] for e in tool_docs])
+    tool_docs_str += "\n" + LOAD_TOOLS_DOCSTRING
 
     prompt = TEST_TOOLS.format(
         tool_docs=tool_docs_str,
@@ -217,8 +216,15 @@ def run_tool_testing(
             examples=EXAMPLES,
             media=str(image_paths),
         )
-        code = extract_code(lmm.generate(prompt, media=image_paths))  # type: ignore
-        code = process_code(code)
+        response = cast(str, lmm.generate(prompt, media=image_paths))
+        code = extract_tag(response, "code")
+        if code is None:
+            code = response
+
+        try:
+            code = process_code(code)
+        except Exception as e:
+            _LOGGER.error(f"Error processing code: {e}")
         tool_output = code_interpreter.exec_isolation(
             DefaultImports.prepend_imports(code)
         )

vision_agent/tools/tool_utils.py

@@ -318,6 +318,9 @@ def single_nms(
 def nms(
     all_preds: List[List[Dict[str, Any]]], iou_threshold: float
 ) -> List[List[Dict[str, Any]]]:
+    if not isinstance(all_preds[0], List):
+        all_preds = [all_preds]
+
     return_preds = []
     for frame_preds in all_preds:
         frame_preds = single_nms(frame_preds, iou_threshold)

vision_agent/tools/tools.py

@@ -222,7 +222,7 @@ def sam2(
     ret = _sam2(image, detections, image_size)
     _display_tool_trace(
         sam2.__name__,
-        {},
+        {"detections": detections},
         ret["display_data"],
         ret["files"],
     )
@@ -314,18 +314,29 @@ def od_sam2_video_tracking(
 
     # Process each segment and collect detections
     detections_per_segment: List[Any] = []
-    for segment_index, segment in enumerate(segments):
-        segment_detections = process_segment(
-            segment_frames=segment,
-            od_model=od_model,
-            prompt=prompt,
-            fine_tune_id=fine_tune_id,
-            chunk_length=chunk_length,
-            image_size=image_size,
-            segment_index=segment_index,
-            object_detection_tool=_apply_object_detection,
-        )
-        detections_per_segment.append(segment_detections)
+    with ThreadPoolExecutor() as executor:
+        futures = {
+            executor.submit(
+                process_segment,
+                segment_frames=segment,
+                od_model=od_model,
+                prompt=prompt,
+                fine_tune_id=fine_tune_id,
+                chunk_length=chunk_length,
+                image_size=image_size,
+                segment_index=segment_index,
+                object_detection_tool=_apply_object_detection,
+            ): segment_index
+            for segment_index, segment in enumerate(segments)
+        }
+
+        for future in as_completed(futures):
+            segment_index = futures[future]
+            detections_per_segment.append((segment_index, future.result()))
+
+    detections_per_segment = [
+        x[1] for x in sorted(detections_per_segment, key=lambda x: x[0])
+    ]
 
     merged_detections = merge_segments(detections_per_segment)
     post_processed = post_process(merged_detections, image_size)
@@ -390,7 +401,7 @@ def _owlv2_object_detection(
         {
             "label": bbox["label"],
             "bbox": normalize_bbox(bbox["bounding_box"], image_size),
-            "score": bbox["score"],
+            "score": round(bbox["score"], 2),
         }
         for bbox in bboxes
     ]
@@ -398,7 +409,7 @@ def _owlv2_object_detection(
         {
             "label": bbox["label"],
             "bbox": bbox["bounding_box"],
-            "score": bbox["score"],
+            "score": round(bbox["score"], 2),
         }
         for bbox in bboxes
     ]
@@ -582,7 +593,7 @@ def owlv2_sam2_video_tracking(
     )
     _display_tool_trace(
         owlv2_sam2_video_tracking.__name__,
-        {},
+        {"prompt": prompt, "chunk_length": chunk_length},
         ret["display_data"],
         ret["files"],
     )
@@ -1681,7 +1692,7 @@ def video_temporal_localization(
     prompt: str,
     frames: List[np.ndarray],
     model: str = "qwen2vl",
-    chunk_length_frames: Optional[int] = 2,
+    chunk_length_frames: int = 2,
 ) -> List[float]:
     """'video_temporal_localization' will run qwen2vl on each chunk_length_frames
     value selected for the video. It can detect multiple objects independently per
@@ -1695,7 +1706,7 @@ def video_temporal_localization(
         frames (List[np.ndarray]): The reference frames used for the question
         model (str): The model to use for the inference. Valid values are
             'qwen2vl', 'gpt4o'.
-        chunk_length_frames (Optional[int]): length of each chunk in frames
+        chunk_length_frames (int): length of each chunk in frames
 
     Returns:
         List[float]: A list of floats with a value of 1.0 if the objects to be found
@@ -1714,8 +1725,7 @@ def video_temporal_localization(
         "model": model,
         "function_name": "video_temporal_localization",
     }
-    if chunk_length_frames is not None:
-        payload["chunk_length_frames"] = chunk_length_frames
+    payload["chunk_length_frames"] = chunk_length_frames
 
     data = send_inference_request(
         payload, "video-temporal-localization", files=files, v2=True
@@ -1726,7 +1736,13 @@ def video_temporal_localization(
         data,
         files,
     )
-    return [cast(float, value) for value in data]
+    chunked_data = [cast(float, value) for value in data]
+
+    full_data = []
+    for value in chunked_data:
+        full_data.extend([value] * chunk_length_frames)
+
+    return full_data[: len(frames)]
 
 
 def vit_image_classification(image: np.ndarray) -> Dict[str, Any]:
@@ -2150,7 +2166,7 @@ def siglip_classification(image: np.ndarray, labels: List[str]) -> Dict[str, Any
     return response
 
 
-# agentic od tools
+# Agentic OD Tools
 
 
 def _agentic_object_detection(
@@ -2646,7 +2662,7 @@ def save_image(image: np.ndarray, file_path: str) -> None:
 
 
 def save_video(
-    frames: List[np.ndarray], output_video_path: Optional[str] = None, fps: float = 1
+    frames: List[np.ndarray], output_video_path: Optional[str] = None, fps: float = 5
 ) -> str:
     """'save_video' is a utility function that saves a list of frames as a mp4 video file on disk.
 

vision_agent/utils/sim.py

@@ -98,10 +98,12 @@ class Sim:
             raise ValueError("key is required if no column 'embs' is present.")
 
         if sim_key is not None:
-            self.df["embs"] = self.df[sim_key].apply(
-                lambda x: get_embedding(
-                    self.emb_call,
-                    x,
+            self.df = self.df.assign(
+                embs=self.df[sim_key].apply(
+                    lambda x: get_embedding(
+                        self.emb_call,
+                        x,
+                    )
                 )
             )
 
@@ -141,7 +143,9 @@ class Sim:
 
         df_load = pd.read_csv(load_dir / "df.csv")
         if platform.system() == "Windows":
-            df_load["doc"] = df_load["doc"].apply(lambda x: x.replace("\r", ""))
+            df_load = df_load.assign(
+                doc=df_load.doc.apply(lambda x: x.replace("\r", ""))
+            )
         return df.equals(df_load)  # type: ignore
 
     @lru_cache(maxsize=256)
@@ -166,7 +170,9 @@ class Sim:
             self.emb_call,
             query,
         )
-        self.df["sim"] = self.df.embs.apply(lambda x: 1 - cosine(x, embedding))
+        self.df = self.df.assign(
+            sim=self.df.embs.apply(lambda x: 1 - cosine(x, embedding))
+        )
         res = self.df.sort_values("sim", ascending=False).head(k)
         if thresh is not None:
             res = res[res.sim > thresh]
@@ -214,8 +220,13 @@ class AzureSim(Sim):
             raise ValueError("key is required if no column 'embs' is present.")
 
         if sim_key is not None:
-            self.df["embs"] = self.df[sim_key].apply(
-                lambda x: get_embedding(self.emb_call, x)
+            self.df = self.df.assign(
+                embs=self.df[sim_key].apply(
+                    lambda x: get_embedding(
+                        self.emb_call,
+                        x,
+                    )
+                )
             )
 
 
@@ -245,8 +256,13 @@ class OllamaSim(Sim):
             raise ValueError("key is required if no column 'embs' is present.")
 
         if sim_key is not None:
-            self.df["embs"] = self.df[sim_key].apply(
-                lambda x: get_embedding(emb_call, x)
+            self.df = self.df.assign(
+                embs=self.df[sim_key].apply(
+                    lambda x: get_embedding(
+                        self.emb_call,
+                        x,
+                    )
+                )
             )
 
 
@@ -267,8 +283,13 @@ class StellaSim(Sim):
             raise ValueError("key is required if no column 'embs' is present.")
 
         if sim_key is not None:
-            self.df["embs"] = self.df[sim_key].apply(
-                lambda x: get_embedding(emb_call, x)
+            self.df = self.df.assign(
+                embs=self.df[sim_key].apply(
+                    lambda x: get_embedding(
+                        self.emb_call,
+                        x,
+                    )
+                )
             )
 
     @staticmethod

{vision_agent-0.2.230.dist-info → vision_agent-0.2.231.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.2.230
+Version: 0.2.231
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai
@@ -71,12 +71,7 @@ export ANTHROPIC_API_KEY="your-api-key"
 export OPENAI_API_KEY="your-api-key"
 ```
 
----
-**NOTE**
-We found using both Anthropic Claude-3.5 and OpenAI o1 to be provide the best performance
-for VisionAgent. If you want to use a different LLM provider or only one, see
-'Using Other LLM Providers' below.
----
+> **_NOTE:_** We found using both Anthropic Claude-3.5 and OpenAI o1 to be provide the best performance for VisionAgent. If you want to use a different LLM provider or only one, see 'Using Other LLM Providers' below.
 
 ## Documentation
 
@@ -149,8 +144,5 @@ directory. For example to change to Anthropic simply just run:
 cp vision_agent/configs/anthropic_config.py vision_agent/configs/config.py
 ```
 
-**NOTE**
-VisionAgent moves fast and we are constantly updating and changing the library. If you
-have any questions or need help, please reach out to us on our discord channel.
----
+> **_NOTE:_** VisionAgent moves fast and we are constantly updating and changing the library. If you have any questions or need help, please reach out to us on our discord channel.
 

{vision_agent-0.2.230.dist-info → vision_agent-0.2.231.dist-info}/RECORD

@@ -5,7 +5,7 @@ vision_agent/agent/README.md,sha256=Q4w7FWw38qaWosQYAZ7NqWx8Q5XzuWrlv7nLhjUd1-8,
 vision_agent/agent/__init__.py,sha256=M8CffavdIh8Zh-skznLHIaQkYGCGK7vk4dq1FaVkbs4,617
 vision_agent/agent/agent.py,sha256=_1tHWAs7Jm5tqDzEcPfCRvJV3uRRveyh4n9_9pd6I1w,1565
 vision_agent/agent/agent_utils.py,sha256=IXxN9XruaeNTreUrdztb3kWJhimpsdH6hjv6xT4jg1Q,14062
-vision_agent/agent/types.py,sha256=DkFm3VMMrKlhYyfxEmZx4keppD72Ov3wmLCbM2J2o10,2437
+vision_agent/agent/types.py,sha256=dIdxATH_PP76pD5Wfo0oofWt6iPQh0vpf48QbEQSzhs,2472
 vision_agent/agent/vision_agent.py,sha256=fH9NOLk7twL1fPr9vLSqkaYhah-gfDWfTOVF2FfMyzI,23461
 vision_agent/agent/vision_agent_coder.py,sha256=flUxOibyGZK19BCSK5mhaD3HjCxHw6c6FtKom6N2q1E,27359
 vision_agent/agent/vision_agent_coder_prompts.py,sha256=_kkPLezUVnBXieNPlxMQab_6J6P7F-aa6ItF5NhZZsM,12281
@@ -14,7 +14,7 @@ vision_agent/agent/vision_agent_coder_v2.py,sha256=ZR2PQoMqNM6yK3vn_0rrCJf_EplRK
 vision_agent/agent/vision_agent_planner.py,sha256=fFzjNkZBKkh8Y_oS06ATI4qz31xmIJvixb_tV1kX8KA,18590
 vision_agent/agent/vision_agent_planner_prompts.py,sha256=rYRdJthc-sQN57VgCBKrF09Sd73BSxcBdjNe6C4WNZ8,6837
 vision_agent/agent/vision_agent_planner_prompts_v2.py,sha256=5xTx93lNpoyT4eAD9jicwDyDAkuW7eQqicr17zCjrQw,33337
-vision_agent/agent/vision_agent_planner_v2.py,sha256=Vbfe_QrhHVViFoTYk8UTkXiZ6cptAiMexQClmRucQeA,20482
+vision_agent/agent/vision_agent_planner_v2.py,sha256=7hBQdg9y4oCLDiQ54Kh12_uIMywedKKNPWiKPRA01cQ,20568
 vision_agent/agent/vision_agent_prompts.py,sha256=KaJwYPUP7_GvQsCPPs6Fdawmi3AQWmWajBUuzj7gTG4,13812
 vision_agent/agent/vision_agent_prompts_v2.py,sha256=AW_bW1boGiCLyLFd3h4GQenfDACttQagDHwpBkSW4Xo,2518
 vision_agent/agent/vision_agent_v2.py,sha256=335VT0hk0jkB14y4W3cJo5ueEu1wY_jjN-R_m2xaQ30,10752
@@ -22,31 +22,31 @@ vision_agent/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
 vision_agent/clients/http.py,sha256=k883i6M_4nl7zwwHSI-yP5sAgQZIDPM1nrKD6YFJ3Xs,2009
 vision_agent/clients/landing_public_api.py,sha256=lU2ev6E8NICmR8DMUljuGcVFy5VNJQ4WQkWC8WnnJEc,1503
 vision_agent/configs/__init__.py,sha256=Iu75-w9_nlPmnB_qKA7nYaaaHf7xtTrDmK8N4v2WV34,27
-vision_agent/configs/anthropic_config.py,sha256=qYpl03wcM6wYcI24rl-9Y8Cyt8STbtUYR0IR4e5YFsU,4298
+vision_agent/configs/anthropic_config.py,sha256=T1UuESgiY8913A6wA42P7-cg8FTk9-LkJpyywo7OnIQ,4298
 vision_agent/configs/anthropic_openai_config.py,sha256=YQjFxmlxppn5L55dJjK_v1myBJQ_V5J4q25pmUtwTOU,4310
 vision_agent/configs/config.py,sha256=YQjFxmlxppn5L55dJjK_v1myBJQ_V5J4q25pmUtwTOU,4310
-vision_agent/configs/openai_config.py,sha256=hCn-e8rRzg4I3OQV3zG0NTWVSULg-_9KAKx1D1IdKOU,4498
+vision_agent/configs/openai_config.py,sha256=v2_AIY89d7LKWn4uqA2G047U2IdmnqZrGH2Iww9gRIw,4498
 vision_agent/fonts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vision_agent/fonts/default_font_ch_en.ttf,sha256=1YM0Z3XqLDjSNbF7ihQFSAIUdjF9m1rtHiNC_6QosTE,1594400
 vision_agent/lmm/__init__.py,sha256=xk2Rn8Zgpy2xwYaOGHzy4tXxnxo2aj6SkpNjeJ8yxcY,111
 vision_agent/lmm/lmm.py,sha256=arwfYPWme_RxCxSpEQ0ZkpHO22GFPCwVeoSvXqLPOAk,19288
 vision_agent/lmm/types.py,sha256=ZEXR_ptBL0ZwDMTDYkgxUCmSZFmBYPQd2jreNzr_8UY,221
-vision_agent/tools/__init__.py,sha256=8VpAC8zEk8OwcMLcTn7gEAfw6ihqlsEfzjEaW5yd5-4,2897
+vision_agent/tools/__init__.py,sha256=zopUrANPx7p0NGy6BxmEaYhDrj8DX8w7BLfgmCbz-mU,2897
 vision_agent/tools/meta_tools.py,sha256=TPeS7QWnc_PmmU_ndiDT03dXbQ5yDSP33E7U8cSj7Ls,28660
-vision_agent/tools/planner_tools.py,sha256=VL9bv7i2FS0v3wMi9eoqgWczoG8vJ-MW5de6JXnbcdA,14354
+vision_agent/tools/planner_tools.py,sha256=Mk3N-I-Qs4ezeyv8EL9BxdxmJG5oWiH5bFkvgwJKB0s,14660
 vision_agent/tools/prompts.py,sha256=V1z4YJLXZuUl_iZ5rY0M5hHc_2tmMEUKr0WocXKGt4E,1430
-vision_agent/tools/tool_utils.py,sha256=kXB0F-HwmiChpQgKk7tMo-Acsl3UXxjaJV9mYo_q6n4,10076
-vision_agent/tools/tools.py,sha256=FgoWy2rHxo7z0Gj3gq7-seE7I5ss4i9qNiTuyCJZg-4,105471
+vision_agent/tools/tool_utils.py,sha256=xJRWF96Ge9RvhhVHrOtifjUYoc4HIJ2y7c2VOQ2Lp8s,10152
+vision_agent/tools/tools.py,sha256=3B3xWFVA3qfAO6ySSQ2yUPUAiTrgJomL48hLO_VP6RQ,106015
 vision_agent/tools/tools_types.py,sha256=8hYf2OZhI58gvf65KGaeGkt4EQ56nwLFqIQDPHioOBc,2339
 vision_agent/utils/__init__.py,sha256=QKk4zVjMwGxQI0MQ-aZZA50N-qItxRY4EB9CwQkZ2HY,185
 vision_agent/utils/exceptions.py,sha256=booSPSuoULF7OXRr_YbC4dtKt6gM_HyiFQHBuaW86C4,2052
 vision_agent/utils/execute.py,sha256=vOEP5Ys7S2lc0_7pOJbgk7OaWi85hrCNu9_8Bo3zk6I,29356
 vision_agent/utils/image_utils.py,sha256=z_ONgcza125B10NkoGwPOzXnL470bpTWZbkB16NeeH0,12188
-vision_agent/utils/sim.py,sha256=qr-6UWAxxGwtwIAKZjZCY_pu9VwBI_TTB8bfrGsaABg,9282
+vision_agent/utils/sim.py,sha256=DYya76dYVtifFyXilMLxBzGgyfyeqhEwU4RJ4894lCI,9796
 vision_agent/utils/type_defs.py,sha256=BE12s3JNQy36QvauXHjwyeffVh5enfcvd4vTzSwvEZI,1384
 vision_agent/utils/video.py,sha256=e1VwKhXzzlC5LcFMyrcQYrPnpnX4wxDpnQ-76sB4jgM,6001
 vision_agent/utils/video_tracking.py,sha256=wK5dOutqV2t2aeaxedstCBa7xy-NNQE0-QZqKu1QUds,9498
-vision_agent-0.2.230.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-vision_agent-0.2.230.dist-info/METADATA,sha256=RU0v0XPXdyFA43SLZjk4sKgXB_peosxwCodQ1F9p4wA,5762
-vision_agent-0.2.230.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-vision_agent-0.2.230.dist-info/RECORD,,
+vision_agent-0.2.231.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+vision_agent-0.2.231.dist-info/METADATA,sha256=N8t9F4hZ4bgyZeDhrVepMZzO5dtRmzRB8VI6fq1fFAA,5760
+vision_agent-0.2.231.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+vision_agent-0.2.231.dist-info/RECORD,,