vision-agent 0.2.152__tar.gz → 0.2.154__tar.gz

{vision_agent-0.2.152 → vision_agent-0.2.154}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.2.152
+Version: 0.2.154
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai

{vision_agent-0.2.152 → vision_agent-0.2.154}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "vision-agent"
-version = "0.2.152"
+version = "0.2.154"
 description = "Toolset for Vision Agent"
 authors = ["Landing AI <dev@landing.ai>"]
 readme = "README.md"

{vision_agent-0.2.152 → vision_agent-0.2.154}/vision_agent/tools/__init__.py

@@ -25,6 +25,7 @@ from .tools import (
     florence2_image_caption,
     florence2_ocr,
     florence2_phrase_grounding,
+    florence2_phrase_grounding_video,
     florence2_roberta_vqa,
     florence2_sam2_image,
     florence2_sam2_video_tracking,

{vision_agent-0.2.152 → vision_agent-0.2.154}/vision_agent/tools/meta_tools.py

@@ -671,6 +671,10 @@ def use_object_detection_fine_tuning(
             r'florence2_phrase_grounding\(\s*["\']([^"\']+)["\']\s*,\s*([^,]+)(?:,\s*["\'][^"\']+["\'])?\s*\)',
             lambda match: f'florence2_phrase_grounding("{match.group(1)}", {match.group(2)}, "{fine_tune_id}")',
         ),
+        (
+            r'florence2_phrase_grounding_video\(\s*["\']([^"\']+)["\']\s*,\s*([^,]+)(?:,\s*["\'][^"\']+["\'])?\s*\)',
+            lambda match: f'florence2_phrase_grounding_video("{match.group(1)}", {match.group(2)}, "{fine_tune_id}")',
+        ),
         (
             r'owl_v2_image\(\s*["\']([^"\']+)["\']\s*,\s*([^,]+)(?:,\s*["\'][^"\']+["\'])?\s*\)',
             lambda match: f'owl_v2_image("{match.group(1)}", {match.group(2)}, "{fine_tune_id}")',

{vision_agent-0.2.152 → vision_agent-0.2.154}/vision_agent/tools/tool_utils.py

@@ -1,6 +1,6 @@
+import os
 import inspect
 import logging
-import os
 from base64 import b64encode
 from typing import Any, Callable, Dict, List, MutableMapping, Optional, Tuple
 
@@ -37,8 +37,9 @@ def send_inference_request(
     files: Optional[List[Tuple[Any, ...]]] = None,
     v2: bool = False,
     metadata_payload: Optional[Dict[str, Any]] = None,
+    is_form: bool = False,
 ) -> Any:
-    # TODO: runtime_tag and function_name should be metadata_payload and now included
+    # TODO: runtime_tag and function_name should be metadata_payload and not included
     # in the service payload
     if runtime_tag := os.environ.get("RUNTIME_TAG", ""):
         payload["runtime_tag"] = runtime_tag
@@ -64,7 +65,7 @@ def send_inference_request(
     elif metadata_payload is not None and "function_name" in metadata_payload:
         function_name = metadata_payload["function_name"]
 
-    response = _call_post(url, payload, session, files, function_name)
+    response = _call_post(url, payload, session, files, function_name, is_form)
 
     # TODO: consider making the response schema the same between below two sources
     return response if "TOOL_ENDPOINT_AUTH" in os.environ else response["data"]
@@ -75,6 +76,7 @@ def send_task_inference_request(
     task_name: str,
     files: Optional[List[Tuple[Any, ...]]] = None,
     metadata: Optional[Dict[str, Any]] = None,
+    is_form: bool = False,
 ) -> Any:
     url = f"{_LND_API_URL_v2}/{task_name}"
     headers = {"apikey": _LND_API_KEY}
@@ -87,7 +89,7 @@ def send_task_inference_request(
     function_name = "unknown"
     if metadata is not None and "function_name" in metadata:
         function_name = metadata["function_name"]
-    response = _call_post(url, payload, session, files, function_name)
+    response = _call_post(url, payload, session, files, function_name, is_form)
     return response["data"]
 
 
@@ -203,6 +205,7 @@ def _call_post(
     session: Session,
     files: Optional[List[Tuple[Any, ...]]] = None,
     function_name: str = "unknown",
+    is_form: bool = False,
 ) -> Any:
     files_in_b64 = None
     if files:
@@ -210,6 +213,8 @@ def _call_post(
     try:
         if files is not None:
             response = session.post(url, data=payload, files=files)
+        elif is_form:
+            response = session.post(url, data=payload)
         else:
             response = session.post(url, json=payload)
 

{vision_agent-0.2.152 → vision_agent-0.2.154}/vision_agent/tools/tools.py

@@ -1,3 +1,4 @@
+import base64
 import io
 import json
 import logging
@@ -28,7 +29,6 @@ from vision_agent.tools.tool_utils import (
     send_task_inference_request,
 )
 from vision_agent.tools.tools_types import (
-    FineTuning,
     Florence2FtRequest,
     JobStatus,
     ODResponseData,
@@ -194,20 +194,26 @@ def owl_v2_image(
         data_obj = Florence2FtRequest(
             image=image_b64,
             task=PromptTask.PHRASE_GROUNDING,
-            tool="florencev2_fine_tuning",
             prompt=prompt,
-            fine_tuning=FineTuning(job_id=UUID(fine_tune_id)),
+            job_id=UUID(fine_tune_id),
         )
-        data = data_obj.model_dump(by_alias=True)
-        detections = send_inference_request(data, "tools", v2=False)
-        detections = detections["<CAPTION_TO_PHRASE_GROUNDING>"]
+        data = data_obj.model_dump(by_alias=True, exclude_none=True)
+        detections = send_inference_request(
+            data,
+            "florence2-ft",
+            v2=True,
+            is_form=True,
+            metadata_payload={"function_name": "owl_v2_image"},
+        )
+        # get the first frame
+        detection = detections[0]
         bboxes_formatted = [
             ODResponseData(
-                label=detections["labels"][i],
-                bbox=normalize_bbox(detections["bboxes"][i], image_size),
+                label=detection["labels"][i],
+                bbox=normalize_bbox(detection["bboxes"][i], image_size),
                 score=1.0,
             )
-            for i in range(len(detections["bboxes"]))
+            for i in range(len(detection["bboxes"]))
         ]
         return [bbox.model_dump() for bbox in bboxes_formatted]
 
@@ -419,25 +425,30 @@ def florence2_sam2_image(
         req_data_obj = Florence2FtRequest(
             image=image_b64,
             task=PromptTask.PHRASE_GROUNDING,
-            tool="florencev2_fine_tuning",
             prompt=prompt,
-            fine_tuning=FineTuning(
-                job_id=UUID(fine_tune_id),
-                postprocessing="sam2",
-            ),
+            postprocessing="sam2",
+            job_id=UUID(fine_tune_id),
+        )
+        req_data = req_data_obj.model_dump(by_alias=True, exclude_none=True)
+        detections_ft = send_inference_request(
+            req_data,
+            "florence2-ft",
+            v2=True,
+            is_form=True,
+            metadata_payload={"function_name": "florence2_sam2_image"},
         )
-        req_data = req_data_obj.model_dump(by_alias=True)
-        detections_ft = send_inference_request(req_data, "tools", v2=False)
-        detections_ft = detections_ft["<CAPTION_TO_PHRASE_GROUNDING>"]
+        # get the first frame
+        detection = detections_ft[0]
         return_data = []
-        all_masks = np.array(detections_ft["masks"])
-        for i in range(len(detections_ft["bboxes"])):
+        for i in range(len(detection["bboxes"])):
             return_data.append(
                 {
                     "score": 1.0,
-                    "label": detections_ft["labels"][i],
-                    "bbox": detections_ft["bboxes"][i],
-                    "mask": all_masks[i, :, :].astype(np.uint8),
+                    "label": detection["labels"][i],
+                    "bbox": normalize_bbox(
+                        detection["bboxes"][i], detection["masks"][i]["size"]
+                    ),
+                    "mask": rle_decode_array(detection["masks"][i]),
                 }
             )
         return return_data
@@ -451,6 +462,7 @@ def florence2_sam2_image(
     detections: Dict[str, Any] = send_inference_request(
         payload, "florence2-sam2", files=files, v2=True
     )
+
     return_data = []
     for _, data_i in detections["0"].items():
         mask = rle_decode_array(data_i["mask"])
@@ -688,22 +700,18 @@ def countgd_counting(
             {'score': 0.98, 'label': 'flower', 'bbox': [0.44, 0.24, 0.49, 0.58},
         ]
     """
-    buffer_bytes = numpy_to_bytes(image)
-    files = [("image", buffer_bytes)]
+    image_b64 = convert_to_b64(image)
     prompt = prompt.replace(", ", " .")
-    payload = {"prompts": [prompt], "model": "countgd"}
+    payload = {"prompt": prompt, "image": image_b64}
     metadata = {"function_name": "countgd_counting"}
-    resp_data = send_task_inference_request(
-        payload, "text-to-object-detection", files=files, metadata=metadata
-    )
-    bboxes_per_frame = resp_data[0]
+    resp_data = send_task_inference_request(payload, "countgd", metadata=metadata)
     bboxes_formatted = [
         ODResponseData(
             label=bbox["label"],
-            bbox=list(map(lambda x: round(x, 2), bbox["bounding_box"])),
+            bbox=list(map(lambda x: round(x, 2), bbox["bbox"])),
             score=round(bbox["score"], 2),
         )
-        for bbox in bboxes_per_frame
+        for bbox in resp_data
     ]
     filtered_bboxes = filter_bboxes_by_threshold(bboxes_formatted, box_threshold)
     return [bbox.model_dump() for bbox in filtered_bboxes]
@@ -887,7 +895,10 @@ def ixc25_temporal_localization(prompt: str, frames: List[np.ndarray]) -> List[b
         "function_name": "ixc25_temporal_localization",
     }
     data: List[int] = send_inference_request(
-        payload, "video-temporal-localization", files=files, v2=True
+        payload,
+        "video-temporal-localization?model=internlm-xcomposer",
+        files=files,
+        v2=True,
     )
     chunk_size = round(len(frames) / len(data))
     data_explode = [[elt] * chunk_size for elt in data]
@@ -1135,10 +1146,10 @@ def florence2_image_caption(image: np.ndarray, detail_caption: bool = True) -> s
 def florence2_phrase_grounding(
     prompt: str, image: np.ndarray, fine_tune_id: Optional[str] = None
 ) -> List[Dict[str, Any]]:
-    """'florence2_phrase_grounding' is a tool that can detect multiple
-    objects given a text prompt which can be object names or caption. You
-    can optionally separate the object names in the text with commas. It returns a list
-    of bounding boxes with normalized coordinates, label names and associated
+    """'florence2_phrase_grounding' will run florence2 on a image. It can
+    detect multiple objects given a text prompt which can be object names or caption.
+    You can optionally separate the object names in the text with commas. It returns
+    a list of bounding boxes with normalized coordinates, label names and associated
     probability scores of 1.0.
 
     Parameters:
@@ -1176,17 +1187,19 @@ def florence2_phrase_grounding(
         data_obj = Florence2FtRequest(
             image=image_b64,
             task=PromptTask.PHRASE_GROUNDING,
-            tool="florencev2_fine_tuning",
             prompt=prompt,
-            fine_tuning=FineTuning(job_id=UUID(fine_tune_id)),
+            job_id=UUID(fine_tune_id),
         )
-        data = data_obj.model_dump(by_alias=True)
+        data = data_obj.model_dump(by_alias=True, exclude_none=True)
         detections = send_inference_request(
             data,
-            "tools",
-            v2=False,
+            "florence2-ft",
+            v2=True,
+            is_form=True,
             metadata_payload={"function_name": "florence2_phrase_grounding"},
         )
+        # get the first frame
+        detection = detections[0]
     else:
         data = {
             "image": image_b64,
@@ -1195,20 +1208,107 @@ def florence2_phrase_grounding(
             "function_name": "florence2_phrase_grounding",
         }
         detections = send_inference_request(data, "florence2", v2=True)
+        detection = detections["<CAPTION_TO_PHRASE_GROUNDING>"]
 
-    detections = detections["<CAPTION_TO_PHRASE_GROUNDING>"]
     return_data = []
-    for i in range(len(detections["bboxes"])):
+    for i in range(len(detection["bboxes"])):
         return_data.append(
             ODResponseData(
-                label=detections["labels"][i],
-                bbox=normalize_bbox(detections["bboxes"][i], image_size),
+                label=detection["labels"][i],
+                bbox=normalize_bbox(detection["bboxes"][i], image_size),
                 score=1.0,
             )
         )
     return [bbox.model_dump() for bbox in return_data]
 
 
+def florence2_phrase_grounding_video(
+    prompt: str, frames: List[np.ndarray], fine_tune_id: Optional[str] = None
+) -> List[List[Dict[str, Any]]]:
+    """'florence2_phrase_grounding_video' will run florence2 on each frame of a video.
+    It can detect multiple objects given a text prompt which can be object names or
+    caption. You can optionally separate the object names in the text with commas.
+    It returns a list of lists where each inner list contains bounding boxes with
+    normalized coordinates, label names and associated probability scores of 1.0.
+
+    Parameters:
+        prompt (str): The prompt to ground to the video.
+        frames (List[np.ndarray]): The list of frames to detect objects.
+        fine_tune_id (Optional[str]): If you have a fine-tuned model, you can pass the
+            fine-tuned model ID here to use it.
+
+    Returns:
+        List[List[Dict[str, Any]]]: A list of lists of dictionaries containing the score,
+            label, and bounding box of the detected objects with normalized coordinates
+            between 0 and 1 (xmin, ymin, xmax, ymax). xmin and ymin are the coordinates
+            of the top-left and xmax and ymax are the coordinates of the bottom-right of
+            the bounding box. The scores are always 1.0 and cannot be thresholded.
+
+    Example
+    -------
+        >>> florence2_phrase_grounding_video('person looking at a coyote', frames)
+        [
+            [
+                {'score': 1.0, 'label': 'person', 'bbox': [0.1, 0.11, 0.35, 0.4]},
+                {'score': 1.0, 'label': 'coyote', 'bbox': [0.34, 0.21, 0.85, 0.5},
+            ],
+            ...
+        ]
+    """
+    if len(frames) == 0:
+        raise ValueError("No frames provided")
+
+    image_size = frames[0].shape[:2]
+    buffer_bytes = frames_to_bytes(frames)
+    files = [("video", buffer_bytes)]
+
+    if fine_tune_id is not None:
+        landing_api = LandingPublicAPI()
+        status = landing_api.check_fine_tuning_job(UUID(fine_tune_id))
+        if status is not JobStatus.SUCCEEDED:
+            raise FineTuneModelIsNotReady(
+                f"Fine-tuned model {fine_tune_id} is not ready yet"
+            )
+
+        data_obj = Florence2FtRequest(
+            task=PromptTask.PHRASE_GROUNDING,
+            prompt=prompt,
+            job_id=UUID(fine_tune_id),
+        )
+
+        data = data_obj.model_dump(by_alias=True, exclude_none=True, mode="json")
+        detections = send_inference_request(
+            data,
+            "florence2-ft",
+            v2=True,
+            files=files,
+            metadata_payload={"function_name": "florence2_phrase_grounding_video"},
+        )
+    else:
+        data = {
+            "prompt": prompt,
+            "task": "<CAPTION_TO_PHRASE_GROUNDING>",
+            "function_name": "florence2_phrase_grounding_video",
+            "video": base64.b64encode(buffer_bytes).decode("utf-8"),
+        }
+        detections = send_inference_request(data, "florence2", v2=True)
+        detections = [d["<CAPTION_TO_PHRASE_GROUNDING>"] for d in detections]
+
+    bboxes_formatted = []
+    for frame_data in detections:
+        bboxes_formatted_per_frame = []
+        for idx in range(len(frame_data["bboxes"])):
+            bboxes_formatted_per_frame.append(
+                ODResponseData(
+                    label=frame_data["labels"][idx],
+                    bbox=normalize_bbox(frame_data["bboxes"][idx], image_size),
+                    score=1.0,
+                )
+            )
+        bboxes_formatted.append(bboxes_formatted_per_frame)
+    return [[bbox.model_dump() for bbox in frame] for frame in bboxes_formatted]
+
+
 def florence2_ocr(image: np.ndarray) -> List[Dict[str, Any]]:
     """'florence2_ocr' is a tool that can detect text and text regions in an image.
     Each text region contains one line of text. It returns a list of detected text,
@@ -1220,7 +1320,7 @@ def florence2_ocr(image: np.ndarray) -> List[Dict[str, Any]]:
 
     Returns:
         List[Dict[str, Any]]: A list of dictionaries containing the detected text, bbox
-            with nornmalized coordinates, and confidence score.
+            with normalized coordinates, and confidence score.
 
     Example
     -------

{vision_agent-0.2.152 → vision_agent-0.2.154}/vision_agent/tools/tools_types.py

@@ -1,6 +1,6 @@
 from enum import Enum
-from typing import List, Optional, Tuple, Union
 from uuid import UUID
+from typing import List, Optional, Tuple, Union
 
 from pydantic import BaseModel, ConfigDict, Field, SerializationInfo, field_serializer
 
@@ -24,27 +24,22 @@ class PromptTask(str, Enum):
     PHRASE_GROUNDING = "<CAPTION_TO_PHRASE_GROUNDING>"
 
 
-class FineTuning(BaseModel):
+class Florence2FtRequest(BaseModel):
     model_config = ConfigDict(populate_by_name=True)
 
-    job_id: UUID = Field(alias="jobId")
+    image: Optional[str] = None
+    video: Optional[bytes] = None
+    task: PromptTask
+    prompt: Optional[str] = ""
+    chunk_length_frames: Optional[int] = None
     postprocessing: Optional[str] = None
+    job_id: Optional[UUID] = Field(None, alias="jobId")
 
     @field_serializer("job_id")
     def serialize_job_id(self, job_id: UUID, _info: SerializationInfo) -> str:
         return str(job_id)
 
 
-class Florence2FtRequest(BaseModel):
-    model_config = ConfigDict(populate_by_name=True)
-
-    image: str
-    task: PromptTask
-    tool: str
-    prompt: Optional[str] = ""
-    fine_tuning: Optional[FineTuning] = Field(None, alias="fineTuning")
-
-
 class JobStatus(str, Enum):
     """The status of a fine-tuning job.