vision-agent 1.1.11__py3-none-any.whl → 1.1.13__py3-none-any.whl

vision_agent/.sim_tools/df.csv

@@ -718,4 +718,4 @@ desc,doc,name
                     [0, 0, 0, ..., 0, 0, 0],
                     [0, 0, 0, ..., 0, 0, 0]], dtype=uint8),
             }],
-        )",overlay_segmentation_masks
+        )",overlay_segmentation_masks

vision_agent/lmm/lmm.py

@@ -3,12 +3,16 @@ import os
 from abc import ABC, abstractmethod
 from pathlib import Path
 from typing import Any, Dict, Iterator, List, Optional, Sequence, Union, cast
+import base64
 
 import anthropic
 import requests
 from anthropic.types import ImageBlockParam, MessageParam, TextBlockParam
 from openai import AzureOpenAI, OpenAI
 
+from google import genai  # type: ignore
+from google.genai import types  # type: ignore
+
 from vision_agent.models import Message
 from vision_agent.utils.image_utils import encode_media
 
@@ -516,28 +520,152 @@ class AnthropicLMM(LMM):
             return cast(str, response.content[0].text)
 
 
-class GoogleLMM(OpenAILMM):
+class GoogleLMM(LMM):
     r"""An LMM class for the Google LMMs."""
 
     def __init__(
         self,
+        model_name: str = "gemini-2.5-pro-preview-03-25",
         api_key: Optional[str] = None,
-        model_name: str = "gemini-2.0-flash-exp",
-        max_tokens: int = 4096,
-        image_detail: str = "low",
         image_size: int = 768,
+        image_detail: str = "low",
         **kwargs: Any,
     ):
-        base_url = "https://generativelanguage.googleapis.com/v1beta/openai/"
         if not api_key:
-            api_key = os.environ.get("GEMINI_API_KEY")
-
-        self.client = OpenAI(api_key=api_key, base_url=base_url)
+            api_key = os.environ.get("GOOGLE_API_KEY")
 
+        # Create the client using the Google Genai client
+        self.client = genai.Client(api_key=api_key)
         self.model_name = model_name
         self.image_size = image_size
         self.image_detail = image_detail
-
-        if "max_tokens" not in kwargs:
-            kwargs["max_tokens"] = max_tokens
         self.kwargs = kwargs
+
+    def __call__(
+        self,
+        input: Union[str, Sequence[Dict[str, Any]]],
+        **kwargs: Any,
+    ) -> Union[str, Iterator[Optional[str]]]:
+        if isinstance(input, str):
+            return self.generate(input, **kwargs)
+        return self.chat(input, **kwargs)
+
+    def chat(
+        self,
+        chat: Sequence[Dict[str, Any]],
+        **kwargs: Any,
+    ) -> Union[str, Iterator[Optional[str]]]:
+        prompt_parts = []
+        for message in chat:
+            if message["role"] != "user":
+                continue  # Gemini expects only user input
+            prompt_parts.extend(self._convert_message_parts(message, **kwargs))
+
+        tmp_kwargs = self.kwargs | kwargs
+        generation_config = self._create_generation_config(tmp_kwargs)
+
+        if tmp_kwargs.get("stream"):
+
+            def f() -> Iterator[Optional[str]]:
+                # Use the client to stream content
+                response_stream = self.client.models.generate_content_stream(
+                    model=self.model_name,
+                    contents=prompt_parts,
+                    config=generation_config,
+                )
+                for chunk in response_stream:
+                    if chunk.text:
+                        yield chunk.text
+
+            return f()
+        else:
+            # Use the client for non-streaming
+            response = self.client.models.generate_content(
+                model=self.model_name,
+                contents=prompt_parts,
+                config=generation_config,
+            )
+            return cast(str, response.text)
+
+    def generate(
+        self,
+        prompt: str,
+        media: Optional[Sequence[Union[str, Path]]] = None,
+        **kwargs: Any,
+    ) -> Union[str, Iterator[Optional[str]]]:
+        prompt_parts = [{"text": prompt}]
+        if media:
+            for m in media:
+                prompt_parts.append(self._convert_media_part(m, **kwargs))
+
+        tmp_kwargs = self.kwargs | kwargs
+        generation_config = self._create_generation_config(tmp_kwargs)
+
+        if tmp_kwargs.get("stream"):
+
+            def f() -> Iterator[Optional[str]]:
+                response_stream = self.client.models.generate_content_stream(
+                    model=self.model_name,
+                    contents=prompt_parts,
+                    config=generation_config,
+                )
+                for chunk in response_stream:
+                    if chunk.text:
+                        yield chunk.text
+
+            return f()
+        else:
+            response = self.client.models.generate_content(
+                model=self.model_name,
+                contents=prompt_parts,
+                config=generation_config,
+            )
+            return cast(str, response.text)
+
+    def _convert_message_parts(
+        self, message: Dict[str, Any], **kwargs: Any
+    ) -> List[Any]:
+        parts = [{"text": message["content"]}]
+        if "media" in message:
+            for media_path in message["media"]:
+                parts.append(self._convert_media_part(media_path, **kwargs))
+        return parts
+
+    def _convert_media_part(self, media: Union[str, Path], **kwargs: Any) -> types.Part:
+        resize = kwargs.get("resize", self.image_size)
+        encoded_media = encode_media(str(media), resize=resize)
+
+        if encoded_media.startswith("data:image/"):
+            encoded_media = encoded_media.split(",", 1)[-1]
+
+        binary_data = base64.b64decode(encoded_media)
+
+        return types.Part.from_bytes(
+            data=binary_data,
+            mime_type="image/png",
+        )
+
+    def _create_generation_config(
+        self, kwargs: Dict[str, Any]
+    ) -> types.GenerateContentConfig:
+        # Extract generation-specific parameters
+        config_params = {}
+
+        # Handle known parameters
+        for param in [
+            "max_output_tokens",
+            "temperature",
+            "top_p",
+            "top_k",
+            "response_mime_type",
+            "stop_sequences",
+            "candidate_count",
+            "seed",
+            "safety_settings",
+            "system_instruction",
+        ]:
+            if param in kwargs:
+                config_params[param] = kwargs[param]
+
+        # Create a GenerateContentConfig object
+        return types.GenerateContentConfig(**config_params)

vision_agent/tools/tools.py

@@ -2959,13 +2959,10 @@ def gemini_image_generation(
             return image
         else:
             try:
-                _LOGGER.warning("All retries failed; prompting for fresh generation.")
-                time.sleep(10)
-                output_image_bytes = try_generate_content(
-                    types.Content(parts=[types.Part(text="Generate an image.")]),
-                    num_retries=1,
-                )
-
+                current_dir = os.path.dirname(os.path.abspath(__file__))
+                img_path = os.path.join(current_dir, "../../assets/gemini.png")
+                with open(img_path, "rb") as img_file:
+                    output_image_bytes = img_file.read()
             except Exception as e:
                 raise ValueError(f"Fallback generation failed: {str(e)}")
 

{vision_agent-1.1.11.dist-info → vision_agent-1.1.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vision-agent
-Version: 1.1.11
+Version: 1.1.13
 Summary: Toolset for Vision Agent
 Project-URL: Homepage, https://landing.ai
 Project-URL: repository, https://github.com/landing-ai/vision-agent

{vision_agent-1.1.11.dist-info → vision_agent-1.1.13.dist-info}/RECORD

@@ -1,5 +1,5 @@
 vision_agent/__init__.py,sha256=EAb4-f9iyuEYkBrX4ag1syM8Syx8118_t0R6_C34M9w,57
-vision_agent/.sim_tools/df.csv,sha256=e87wnyPtPNo2lw6kPQmU9RnzGq81Q7-YXvrPox1GRYA,41876
+vision_agent/.sim_tools/df.csv,sha256=fLh8HN76ezbOXZUoZbnkhNi5vvjYif2jSblHtRdY8dY,41875
 vision_agent/.sim_tools/embs.npy,sha256=uUPZ6QuCAr8JAtFa1L9ndAag5ycptIeJ2I8P9U8Y6YY,245888
 vision_agent/agent/README.md,sha256=3XSPG_VO7-6y6P8COvcgSSonWj5uvfgvfmOkBpfKK8Q,5527
 vision_agent/agent/__init__.py,sha256=_-nGLHhRTLViXxBSb9D4OwLTqk9HXKPEkTBkvK8c7OU,206
@@ -19,7 +19,7 @@ vision_agent/configs/openai_config.py,sha256=Bw7ElBYmBcaZttyRBoNpcy3uTkqg5qADk8L
 vision_agent/fonts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vision_agent/fonts/default_font_ch_en.ttf,sha256=1YM0Z3XqLDjSNbF7ihQFSAIUdjF9m1rtHiNC_6QosTE,1594400
 vision_agent/lmm/__init__.py,sha256=4qX2lmGnKWHeKftXueEi9xj_ieK2nQh_ipHf72nKGFk,84
-vision_agent/lmm/lmm.py,sha256=utGJMeGEKImqHrY0q9kGu0uK3owG8wKyDustwrDrLto,19421
+vision_agent/lmm/lmm.py,sha256=w23nWSmUiW1rxfRC-Td44-UR3-8k0ey80-0SVZraeqA,23681
 vision_agent/models/__init__.py,sha256=eIP0pD5dYog8zUA7uuTmUxCF6SIutbLRLRE0cmuCJgQ,326
 vision_agent/models/agent_types.py,sha256=vBZ9-ns5lHDdFMO7ulCGGeZ6OwRo3gK4O3vN0814IWc,3064
 vision_agent/models/lmm_types.py,sha256=v04h-NjbczHOIN8UWa1vvO5-1BDuZ4JQhD2mge1cXmw,305
@@ -30,7 +30,7 @@ vision_agent/tools/__init__.py,sha256=PRUka2eqHwPWJxwfpLj-O2Ab7hXG_dsE1Aov3TE6te
 vision_agent/tools/meta_tools.py,sha256=9iJilpGYEiXW0nYPTYAWHa7l23wGN8IM5KbE7mWDOT0,6798
 vision_agent/tools/planner_tools.py,sha256=iQWtTgXdomn0IWrbmvXXM-y8Q_RSEOxyP04HIRLrgWI,19576
 vision_agent/tools/prompts.py,sha256=V1z4YJLXZuUl_iZ5rY0M5hHc_2tmMEUKr0WocXKGt4E,1430
-vision_agent/tools/tools.py,sha256=K2QW-0esNBygtlRcyUwVRRhV1FV5UaznHYhFkFZkL-I,130851
+vision_agent/tools/tools.py,sha256=A1YpJuarR1P9ZLnCuakxLiUUtYsnlrvfwlUrkBey_FU,130803
 vision_agent/utils/__init__.py,sha256=mANUs_84VL-3gpZbXryvV2mWU623eWnRlJCSUHtMjuw,122
 vision_agent/utils/agent.py,sha256=2ifTP5QElItnr4YHOJR6L5P1PUzV0GhChTTqVxuVyQg,15153
 vision_agent/utils/exceptions.py,sha256=zis8smCbdEylBVZBTVfEUfAh7Rb7cWV3MSPambu6FsQ,1837
@@ -40,7 +40,7 @@ vision_agent/utils/tools.py,sha256=Days0dETPRQLSDamMKPnXFsc5g5IKX9QJcPPNmSHNdM,8
 vision_agent/utils/tools_doc.py,sha256=PKcXXbJktiuPi9q6Q1zXzFx24Dh229SNgWBDtZ2fQSQ,2730
 vision_agent/utils/video.py,sha256=rjsQ1sKKisaQ6AVjJz0zd_G4g-ovRweS_rs4JEhenoI,5340
 vision_agent/utils/video_tracking.py,sha256=DZLFpNCuzuPJQzbQoVNcp-m4dKxgiKdCNM5QTh_zURE,12245
-vision_agent-1.1.11.dist-info/METADATA,sha256=23g2wi-mLe8lh7zchrhAruxqsyJ1nKD8ILKO4s11Kfw,12673
-vision_agent-1.1.11.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-vision_agent-1.1.11.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-vision_agent-1.1.11.dist-info/RECORD,,
+vision_agent-1.1.13.dist-info/METADATA,sha256=1LVRyxXfxT_eGGfpgK5fioWESB6FWx4LDm_xylNpZdY,12673
+vision_agent-1.1.13.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+vision_agent-1.1.13.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+vision_agent-1.1.13.dist-info/RECORD,,