vision-agent 0.0.52__tar.gz → 0.1.1__tar.gz

{vision_agent-0.0.52 → vision_agent-0.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.0.52
+Version: 0.1.1
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai
@@ -16,6 +16,7 @@ Requires-Dist: openai (>=1.0.0,<2.0.0)
 Requires-Dist: opencv-python-headless (>=4.0.0,<5.0.0)
 Requires-Dist: pandas (>=2.0.0,<3.0.0)
 Requires-Dist: pillow (>=10.0.0,<11.0.0)
+Requires-Dist: pydantic-settings (>=2.2.1,<3.0.0)
 Requires-Dist: requests (>=2.0.0,<3.0.0)
 Requires-Dist: sentence-transformers (>=2.0.0,<3.0.0)
 Requires-Dist: tabulate (>=0.9.0,<0.10.0)

{vision_agent-0.0.52 → vision_agent-0.1.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "vision-agent"
-version = "0.0.52"
+version = "0.1.1"
 description = "Toolset for Vision Agent"
 authors = ["Landing AI <dev@landing.ai>"]
 readme = "README.md"
@@ -31,6 +31,7 @@ typing_extensions = "4.*"
 moviepy = "1.*"
 opencv-python-headless = "4.*"
 tabulate = "^0.9.0"
+pydantic-settings = "^2.2.1"
 
 [tool.poetry.group.dev.dependencies]
 autoflake = "1.*"
@@ -49,6 +50,7 @@ mkdocs = "^1.5.3"
 mkdocstrings = {extras = ["python"], version = "^0.23.0"}
 mkdocs-material = "^9.4.2"
 types-tabulate = "^0.9.0.20240106"
+scikit-image = "<0.23.1"
 
 [tool.pytest.ini_options]
 log_cli = true

{vision_agent-0.0.52 → vision_agent-0.1.1}/vision_agent/agent/vision_agent.py

@@ -351,7 +351,7 @@ class VisionAgent(Agent):
         task_model: Optional[Union[LLM, LMM]] = None,
         answer_model: Optional[Union[LLM, LMM]] = None,
         reflect_model: Optional[Union[LLM, LMM]] = None,
-        max_retries: int = 3,
+        max_retries: int = 2,
         verbose: bool = False,
         report_progress_callback: Optional[Callable[[str], None]] = None,
     ):
@@ -476,7 +476,7 @@ class VisionAgent(Agent):
                 reflections += "\n" + reflection
         # '<END>' is a symbol to indicate the end of the chat, which is useful for streaming logs.
         self.log_progress(
-            f"The Vision Agent has concluded this chat. <ANSWER>{final_answer}</<ANSWER>"
+            f"The Vision Agent has concluded this chat. <ANSWER>{final_answer}</ANSWER>"
         )
 
         if visualize_output:

{vision_agent-0.0.52 → vision_agent-0.1.1}/vision_agent/tools/__init__.py

@@ -9,6 +9,7 @@ from .tools import (
     ExtractFrames,
     GroundingDINO,
     GroundingSAM,
+    ImageCaption,
     SegArea,
     SegIoU,
     Tool,

{vision_agent-0.0.52 → vision_agent-0.1.1}/vision_agent/tools/tools.py

@@ -12,8 +12,11 @@ from PIL.Image import Image as ImageType
 
 from vision_agent.image_utils import convert_to_b64, get_image_size
 from vision_agent.tools.video import extract_frames_from_video
+from vision_agent.type_defs import LandingaiAPIKey
 
 _LOGGER = logging.getLogger(__name__)
+_LND_API_KEY = LandingaiAPIKey().api_key
+_LND_API_URL = "https://api.dev.landing.ai/v1/agent"
 
 
 def normalize_bbox(
@@ -80,8 +83,6 @@ class CLIP(Tool):
         [{"labels": ["red line", "yellow dot"], "scores": [0.98, 0.02]}]
     """
 
-    _ENDPOINT = "https://soi4ewr6fjqqdf5vuss6rrilee0kumxq.lambda-url.us-east-2.on.aws"
-
     name = "clip_"
     description = "'clip_' is a tool that can classify any image given a set of input names or tags. It returns a list of the input names along with their probability scores."
     usage = {
@@ -125,23 +126,63 @@ class CLIP(Tool):
             "image": image_b64,
             "tool": "closed_set_image_classification",
         }
-        res = requests.post(
-            self._ENDPOINT,
-            headers={"Content-Type": "application/json"},
-            json=data,
-        )
-        resp_json: Dict[str, Any] = res.json()
-        if (
-            "statusCode" in resp_json and resp_json["statusCode"] != 200
-        ) or "statusCode" not in resp_json:
-            _LOGGER.error(f"Request failed: {resp_json}")
-            raise ValueError(f"Request failed: {resp_json}")
-
-        resp_json["data"]["scores"] = [
-            round(prob, 4) for prob in resp_json["data"]["scores"]
-        ]
+        resp_data = _send_inference_request(data, "tools")
+        resp_data["scores"] = [round(prob, 4) for prob in resp_data["scores"]]
+        return resp_data
+
+
+class ImageCaption(Tool):
+    r"""ImageCaption is a tool that can caption an image based on its contents
+    or tags.
+
+    Example
+    -------
+        >>> import vision_agent as va
+        >>> caption = va.tools.ImageCaption()
+        >>> caption("image1.jpg")
+        {'text': ['a box of orange and white socks']}
+    """
+
+    name = "image_caption_"
+    description = "'image_caption_' is a tool that can caption an image based on its contents or tags. It returns a text describing the image"
+    usage = {
+        "required_parameters": [
+            {"name": "image", "type": "str"},
+        ],
+        "examples": [
+            {
+                "scenario": "Can you describe this image ? Image name: cat.jpg",
+                "parameters": {"image": "cat.jpg"},
+            },
+            {
+                "scenario": "Can you caption this image with their main contents ? Image name: cat_dog.jpg",
+                "parameters": {"image": "cat_dog.jpg"},
+            },
+            {
+                "scenario": "Can you build me a image captioning tool ? Image name: shirts.jpg",
+                "parameters": {
+                    "image": "shirts.jpg",
+                },
+            },
+        ],
+    }
 
-        return resp_json["data"]  # type: ignore
+    # TODO: Add support for input multiple images, which aligns with the output type.
+    def __call__(self, image: Union[str, ImageType]) -> Dict:
+        """Invoke the Image captioning model.
+
+        Parameters:
+            image: the input image to caption.
+
+        Returns:
+            A list of dictionaries containing the labels and scores. Each dictionary contains the classification result for an image. E.g. [{"labels": ["red line", "yellow dot"], "scores": [0.98, 0.02]}]
+        """
+        image_b64 = convert_to_b64(image)
+        data = {
+            "image": image_b64,
+            "tool": "image_captioning",
+        }
+        return _send_inference_request(data, "tools")
 
 
 class GroundingDINO(Tool):
@@ -158,8 +199,6 @@ class GroundingDINO(Tool):
         'scores': [0.98, 0.02]}]
     """
 
-    _ENDPOINT = "https://soi4ewr6fjqqdf5vuss6rrilee0kumxq.lambda-url.us-east-2.on.aws"
-
     name = "grounding_dino_"
     description = "'grounding_dino_' is a tool that can detect arbitrary objects with inputs such as category names or referring expressions. It returns a list of bounding boxes, label names and associated probability scores."
     usage = {
@@ -222,24 +261,13 @@ class GroundingDINO(Tool):
             "tool": "visual_grounding",
             "kwargs": {"box_threshold": box_threshold, "iou_threshold": iou_threshold},
         }
-        res = requests.post(
-            self._ENDPOINT,
-            headers={"Content-Type": "application/json"},
-            json=request_data,
-        )
-        resp_json: Dict[str, Any] = res.json()
-        if (
-            "statusCode" in resp_json and resp_json["statusCode"] != 200
-        ) or "statusCode" not in resp_json:
-            _LOGGER.error(f"Request failed: {resp_json}")
-            raise ValueError(f"Request failed: {resp_json}")
-        data: Dict[str, Any] = resp_json["data"]
+        data: Dict[str, Any] = _send_inference_request(request_data, "tools")
         if "bboxes" in data:
             data["bboxes"] = [normalize_bbox(box, image_size) for box in data["bboxes"]]
         if "scores" in data:
             data["scores"] = [round(score, 2) for score in data["scores"]]
         if "labels" in data:
-            data["labels"] = [label for label in data["labels"]]
+            data["labels"] = list(data["labels"])
         data["size"] = (image_size[1], image_size[0])
         return data
 
@@ -267,8 +295,6 @@ class GroundingSAM(Tool):
            [1, 1, 1, ..., 1, 1, 1]], dtype=uint8)]}]
     """
 
-    _ENDPOINT = "https://soi4ewr6fjqqdf5vuss6rrilee0kumxq.lambda-url.us-east-2.on.aws"
-
     name = "grounding_sam_"
     description = "'grounding_sam_' is a tool that can detect arbitrary objects with inputs such as category names or referring expressions. It returns a list of bounding boxes, label names and masks file names and associated probability scores."
     usage = {
@@ -331,18 +357,7 @@ class GroundingSAM(Tool):
             "tool": "visual_grounding_segment",
             "kwargs": {"box_threshold": box_threshold, "iou_threshold": iou_threshold},
         }
-        res = requests.post(
-            self._ENDPOINT,
-            headers={"Content-Type": "application/json"},
-            json=request_data,
-        )
-        resp_json: Dict[str, Any] = res.json()
-        if (
-            "statusCode" in resp_json and resp_json["statusCode"] != 200
-        ) or "statusCode" not in resp_json:
-            _LOGGER.error(f"Request failed: {resp_json}")
-            raise ValueError(f"Request failed: {resp_json}")
-        data: Dict[str, Any] = resp_json["data"]
+        data: Dict[str, Any] = _send_inference_request(request_data, "tools")
         ret_pred: Dict[str, List] = {"labels": [], "bboxes": [], "masks": []}
         if "bboxes" in data:
             ret_pred["bboxes"] = [
@@ -631,6 +646,7 @@ TOOLS = {
         [
             NoOp,
             CLIP,
+            ImageCaption,
             GroundingDINO,
             AgentGroundingSAM,
             ExtractFrames,
@@ -645,3 +661,20 @@ TOOLS = {
     )
     if (hasattr(c, "name") and hasattr(c, "description") and hasattr(c, "usage"))
 }
+
+
+def _send_inference_request(
+    payload: Dict[str, Any], endpoint_name: str
+) -> Dict[str, Any]:
+    res = requests.post(
+        f"{_LND_API_URL}/model/{endpoint_name}",
+        headers={
+            "Content-Type": "application/json",
+            "apikey": _LND_API_KEY,
+        },
+        json=payload,
+    )
+    if res.status_code != 200:
+        _LOGGER.error(f"Request failed: {res.text}")
+        raise ValueError(f"Request failed: {res.text}")
+    return res.json()["data"]  # type: ignore

vision_agent-0.1.1/vision_agent/type_defs.py

@@ -0,0 +1,48 @@
+from pydantic import Field, field_validator
+from pydantic_settings import BaseSettings
+
+
+class LandingaiAPIKey(BaseSettings):
+    """The API key of a user in a particular organization in LandingLens.
+    It supports loading from environment variables or .env files.
+    The supported name of the environment variables are (case-insensitive):
+    - LANDINGAI_API_KEY
+
+    Environment variables will always take priority over values loaded from a dotenv file.
+    """
+
+    api_key: str = Field(
+        default="land_sk_hw34v3tyEc35OAhP8F7hnGnrDv2C8hD2ycMyq0aMkVS1H40D22",
+        alias="LANDINGAI_API_KEY",
+        description="The API key of LandingAI.",
+    )
+
+    @field_validator("api_key")
+    @classmethod
+    def is_api_key_valid(cls, key: str) -> str:
+        """Check if the API key is a v2 key."""
+        if not key:
+            raise InvalidApiKeyError(f"LandingAI API key is required, but it's {key}")
+        if not key.startswith("land_sk_"):
+            raise InvalidApiKeyError(
+                f"LandingAI API key (v2) must start with 'land_sk_' prefix, but it's {key}. See https://support.landing.ai/docs/api-key for more information."
+            )
+        return key
+
+    class Config:
+        env_file = ".env"
+        env_prefix = "landingai_"
+        case_sensitive = False
+        extra = "ignore"
+
+
+class InvalidApiKeyError(Exception):
+    """Exception raised when the an invalid API key is provided. This error could be raised from any SDK code, not limited to a HTTP client."""
+
+    def __init__(self, message: str):
+        self.message = f"""{message}
+For more information, see https://landing-ai.github.io/landingai-python/landingai.html#manage-api-credentials"""
+        super().__init__(self.message)
+
+    def __str__(self) -> str:
+        return self.message