roboflow-workflows 0.2.7__tar.gz → 0.2.8__tar.gz

{roboflow_workflows-0.2.7/roboflow_workflows.egg-info → roboflow_workflows-0.2.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: roboflow-workflows
-Version: 0.2.7
+Version: 0.2.8
 Summary: Roboflow Workflows execution engine and block library
 License: LICENSE.core (Apache 2.0) applies to all files in this repository
         except for files in or under any directory that contains a superseding

{roboflow_workflows-0.2.7 → roboflow_workflows-0.2.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "roboflow-workflows"
-version = "0.2.7"
+version = "0.2.8"
 description = "Roboflow Workflows execution engine and block library"
 readme = "README.md"
 license = {file = "LICENSE"}

roboflow_workflows-0.2.8/roboflow_workflows/core_steps/common/vlm_decoding/argon_segmentation.py

@@ -0,0 +1,233 @@
+"""Decode Argon normalized YX components and holes into RLE instance masks."""
+
+import math
+from typing import Any, List, Tuple
+from uuid import uuid4
+
+import numpy as np
+import supervision as sv
+from PIL import Image, ImageDraw
+from pycocotools import mask as mask_utils
+from roboflow_workflows._compat_names import get_logger
+from roboflow_workflows.core_steps.common.utils import (
+    attach_parents_coordinates_to_sv_detections,
+)
+from roboflow_workflows.core_steps.common.vlm_decoding.json_extraction import (
+    extract_json,
+)
+from roboflow_workflows.core_steps.common.vlm_decoding.segmentation import (
+    extract_segmentation_entries,
+)
+from roboflow_workflows.core_steps.common.vlm_decoding.tensor_native import (
+    to_tensor_native_predictions,
+)
+from roboflow_workflows.core_steps.common.vlm_decoding.utils import create_classes_index
+from roboflow_workflows.environment import (
+    WORKFLOWS_VLM_SEGMENTATION_MAX_POLYGON_VERTICES,
+)
+from roboflow_workflows.execution_engine.constants import (
+    DETECTION_ID_KEY,
+    IMAGE_DIMENSIONS_KEY,
+    INFERENCE_ID_KEY,
+    PREDICTION_TYPE_KEY,
+    RLE_MASK_KEY_IN_SV_DETECTIONS,
+)
+from roboflow_workflows.execution_engine.entities.base import WorkflowImageData
+from supervision.config import CLASS_NAME_DATA_FIELD
+
+logger = get_logger(__name__)
+PREDICTION_TYPE = "instance-segmentation"
+
+ARGON_SEGMENTATION_PROMPT = "Segment every visible instance of these classes: {class_list}. Treat each distinct visible physical object as one instance. Segment visible surfaces only; do not infer hidden portions. Exclude cast shadows, reflections and background. Return only a JSON list. Each item must contain label (use the exact class name), box_2d [ymin,xmin,ymax,xmax] normalized to 0-1000 relative to the whole image, and mask, a list of components. Each component has outer, a polygon list of [y,x] vertices, and holes, a list of polygon lists of [y,x] vertices; all coordinates are normalized to 0-1000 relative to the WHOLE image. Follow the visible object boundary closely. Do not output bounding rectangles as masks. Keep separate physical objects separate. If one physical object has separate visible components because of occlusion, put each visible component in a separate component object inside that same object mask list; do not connect across the occluder. For each component, outer encloses its visible region. Use holes for enclosed openings or enclosed occluders that are not part of the object; use an empty holes list when none exist. All mask points use [y,x], vertical first and horizontal second. Subtract holes only from their own component, then union the components of one physical object."
+
+ARGON_SEGMENTATION_SCHEMA = {
+    "type": "array",
+    "items": {
+        "type": "object",
+        "properties": {
+            "label": {"type": "string"},
+            "box_2d": {
+                "type": "array",
+                "items": {"type": "integer"},
+                "minItems": 4,
+                "maxItems": 4,
+            },
+            "mask": {
+                "type": "array",
+                "minItems": 1,
+                "items": {
+                    "type": "object",
+                    "properties": {
+                        "outer": {
+                            "type": "array",
+                            "items": {
+                                "type": "array",
+                                "items": {"type": "number"},
+                                "minItems": 2,
+                                "maxItems": 2,
+                            },
+                            "minItems": 3,
+                        },
+                        "holes": {
+                            "type": "array",
+                            "items": {
+                                "type": "array",
+                                "items": {
+                                    "type": "array",
+                                    "items": {"type": "number"},
+                                    "minItems": 2,
+                                    "maxItems": 2,
+                                },
+                                "minItems": 3,
+                            },
+                        },
+                    },
+                    "required": ["outer", "holes"],
+                },
+            },
+        },
+        "required": ["label", "box_2d", "mask"],
+    },
+}
+
+
+def _read_ring(raw: object) -> list:
+    if not isinstance(raw, list) or len(raw) < 3:
+        raise ValueError("Each mask boundary requires at least three YX points")
+    points = []
+    for point in raw:
+        if not isinstance(point, list) or len(point) != 2:
+            raise ValueError("Mask vertices must be [y, x] pairs")
+        if any(
+            isinstance(v, bool)
+            or not isinstance(v, (int, float))
+            or not math.isfinite(v)
+            or not 0 <= v <= 1000
+            for v in point
+        ):
+            raise ValueError("Mask coordinates must be finite numbers in [0, 1000]")
+        points.append(point)
+    return points
+
+
+def _read_components(entry: dict) -> list:
+    raw = entry.get("mask")
+    if not isinstance(raw, list) or not raw:
+        raise ValueError("An instance requires at least one mask component")
+    components = []
+    vertex_count = 0
+    for component in raw:
+        if not isinstance(component, dict) or not isinstance(
+            component.get("holes"), list
+        ):
+            raise ValueError("Each component requires outer and holes")
+        outer = _read_ring(component.get("outer"))
+        holes = [_read_ring(hole) for hole in component["holes"]]
+        vertex_count += len(outer) + sum(len(hole) for hole in holes)
+        if vertex_count > WORKFLOWS_VLM_SEGMENTATION_MAX_POLYGON_VERTICES:
+            raise ValueError("Instance exceeds the segmentation vertex limit")
+        components.append((outer, holes))
+    return components
+
+
+def _components_to_rle(components: list, *, width: int, height: int) -> dict:
+    union = Image.new("1", (width, height))
+    for outer, holes in components:
+        component = Image.new("1", (width, height))
+        draw = ImageDraw.Draw(component)
+        draw.polygon([(x * width / 1000, y * height / 1000) for y, x in outer], fill=1)
+        for hole in holes:
+            draw.polygon(
+                [(x * width / 1000, y * height / 1000) for y, x in hole], fill=0
+            )
+        union.paste(1, mask=component)
+    encoded = mask_utils.encode(np.asfortranarray(np.asarray(union, dtype=np.uint8)))
+    return encoded
+
+
+def decode_argon_segmentations(
+    raw_output: str,
+    *,
+    image: WorkflowImageData,
+    classes: List[str],
+    inference_id: str,
+) -> Tuple[bool, Any]:
+    """Decode Argon multipart masks, preserving holes and instance identity.
+
+    Args:
+        raw_output (str): Model JSON with normalized YX mask components.
+        image (WorkflowImageData): Original image and parent metadata.
+        classes (List[str]): Requested labels used to assign class IDs.
+        inference_id (str): Identifier attached to decoded predictions.
+
+    Returns:
+        Tuple[bool, Any]: Error flag and RLE predictions,
+            converted to the tensor carrier when enabled. Invalid instances are
+            skipped; wholly malformed nonempty responses return an error.
+    """
+    error, parsed = extract_json(raw_output)
+    if error:
+        return True, None
+
+    try:
+        entries = extract_segmentation_entries(parsed)
+        image_height, image_width = image._read_shape_without_materialization()
+        class_name2id = create_classes_index(classes=classes)
+        xyxy, rle_masks, class_id, class_name = [], [], [], []
+        for entry in entries:
+            try:
+                label = entry.get("label")
+                if not isinstance(label, str):
+                    raise ValueError("Instance requires a text label")
+                components = _read_components(entry)
+                rle = _components_to_rle(
+                    components, width=image_width, height=image_height
+                )
+                if mask_utils.area(rle) == 0:
+                    raise ValueError("Instance covers no pixels")
+            except (ValueError, TypeError, KeyError) as error:
+                logger.warning("Skipping invalid Argon mask: %s", error)
+                continue
+            x, y, w, h = mask_utils.toBbox(rle)
+            xyxy.append([x, y, x + w, y + h])
+            rle_masks.append(rle)
+            class_id.append(class_name2id.get(label, -1))
+            class_name.append(label)
+        if entries and not xyxy:
+            raise ValueError("No valid Argon masks in nonempty response")
+        count = len(xyxy)
+        data = {
+            CLASS_NAME_DATA_FIELD: np.array(class_name) if count else np.empty(0),
+            IMAGE_DIMENSIONS_KEY: np.array([[image_height, image_width]] * count),
+            INFERENCE_ID_KEY: np.array([inference_id] * count),
+            DETECTION_ID_KEY: np.array([str(uuid4()) for _ in range(count)]),
+            PREDICTION_TYPE_KEY: np.array([PREDICTION_TYPE] * count),
+            RLE_MASK_KEY_IN_SV_DETECTIONS: np.array(rle_masks, dtype=object),
+        }
+        detections = sv.Detections(
+            xyxy=np.array(xyxy, dtype=float) if count else np.empty((0, 4)),
+            # The prompt asks for no confidence; downstream filters see 1.0.
+            confidence=np.ones(count) if count else np.empty(0),
+            class_id=np.array(class_id).astype(int) if count else np.empty(0),
+            mask=None,
+            tracker_id=None,
+            data=data,
+        )
+        if count == 0:
+            # Per-row data is empty, so keep the frame size where
+            # `empty_detections_with_image_metadata` keeps it.
+            detections.metadata[IMAGE_DIMENSIONS_KEY] = [image_height, image_width]
+        detections = attach_parents_coordinates_to_sv_detections(
+            detections=detections,
+            image=image,
+        )
+        predictions = to_tensor_native_predictions(
+            predictions=detections,
+            image=image,
+            classes=classes,
+            inference_id=inference_id,
+        )
+        return False, predictions
+    except Exception as error:
+        logger.warning("Could not decode Argon segmentation: %s", error)
+        return True, None

{roboflow_workflows-0.2.7 → roboflow_workflows-0.2.8}/roboflow_workflows/core_steps/models/foundation/google_gemini/v6.py

@@ -11,6 +11,8 @@ normalized to 0-1000 - the shared ``yxyx_0_1000`` coordinate contract.
 """
 
 import base64
+import copy
+import io
 import json
 import re
 from functools import partial
@@ -18,6 +20,7 @@ from typing import Any, Dict, List, Literal, Optional, Tuple, Type, Union
 from uuid import uuid4
 
 import requests
+from PIL import Image
 from pydantic import ConfigDict, Field, field_validator, model_validator
 from requests import Response
 from roboflow_workflows.core_steps.common.reasoning import (
@@ -36,6 +39,11 @@ from roboflow_workflows.core_steps.common.vlm_decoding import (
     decode_vlm_output,
     describe_vlm_prediction_outputs,
 )
+from roboflow_workflows.core_steps.common.vlm_decoding.argon_segmentation import (
+    ARGON_SEGMENTATION_PROMPT,
+    ARGON_SEGMENTATION_SCHEMA,
+    decode_argon_segmentations,
+)
 from roboflow_workflows.core_steps.common.vlms import VLM_TASKS_METADATA
 from roboflow_workflows.environment import (
     WORKFLOWS_REMOTE_EXECUTION_MAX_STEP_CONCURRENT_REQUESTS,
@@ -90,6 +98,12 @@ MODEL_ALIASES = {
 # `minimal`. 2.5 pages contradict (`thinking_level` vs `thinking_budget` only);
 # we keep 2.5 unsupported.
 GEMINI_MODELS = [
+    {
+        "id": "gemini-4-argon",
+        "name": "Gemini 4 Argon",
+        "thinking_levels": ["low", "high"],
+        "supports_native_code_execution": False,
+    },
     {
         "id": "gemini-3.8-flash",
         "name": "Gemini 3.8 Flash",
@@ -199,6 +213,7 @@ SUPPORTED_TASK_TYPES_LIST = [
     "caption",
     "detailed-caption",
     "object-detection",
+    "instance-segmentation",
 ]
 SUPPORTED_TASK_TYPES = set(SUPPORTED_TASK_TYPES_LIST)
 
@@ -211,6 +226,10 @@ RELEVANT_TASKS_METADATA["object-detection"] = {
     "returning a Gemini-native `box_2d` JSON list (y_min, x_min, y_max, x_max "
     "integers normalized to 0-1000), decoded into predictions by the block.",
 }
+RELEVANT_TASKS_METADATA["instance-segmentation"] = {
+    "name": "Instance Segmentation",
+    "description": "Gemini 4 Argon only: predict masks with detached components and holes, decoded to RLE predictions.",
+}
 RELEVANT_TASKS_DOCS_DESCRIPTION = "\n\n".join(
     f"* **{v['name']}** (`{k}`) - {v['description']}"
     for k, v in RELEVANT_TASKS_METADATA.items()
@@ -225,12 +244,17 @@ You can specify arbitrary text prompts or predefined ones, the block supports th
 
 ## Version Differences
 
+Instance segmentation is available only for `gemini-4-argon`. It uses lossless PNG
+images up to 1600 pixels on the long edge and normalized YX polygons with
+detached components and holes. Predictions use RLE masks; bounding boxes are
+derived from the masks. Other Gemini models reject this task.
+
 This version (v6) decodes the model answer inside the block, adding
 `predictions`, `error_status` and `inference_id` outputs next to the raw
 `output` string:
 
-* `predictions` holds object detections for the `object-detection` task and a
-classification prediction for the `classification` /
+* `predictions` holds object detections for `object-detection`, RLE instance
+masks for `instance-segmentation`, and a classification prediction for `classification` /
 `multi-label-classification` tasks - the kind of the output follows the
 selected task type.
 * `predictions` is `None` for every other task (unconstrained prompting, OCR,
@@ -271,6 +295,7 @@ TASKS_REQUIRING_CLASSES = {
     "classification",
     "multi-label-classification",
     "object-detection",
+    "instance-segmentation",
 }
 
 TASKS_REQUIRING_OUTPUT_STRUCTURE = {
@@ -444,6 +469,14 @@ class BlockManifest(WorkflowBlockManifest):
 
     @model_validator(mode="after")
     def validate(self) -> "BlockManifest":
+        if (
+            self.task_type == "instance-segmentation"
+            and self.model_version != "gemini-4-argon"
+            and not self.model_version.startswith("$")
+        ):
+            raise ValueError(
+                "Instance segmentation is supported only for gemini-4-argon"
+            )
         if self.task_type in TASKS_REQUIRING_PROMPT and self.prompt is None:
             raise ValueError(
                 f"`prompt` parameter required to be set for task `{self.task_type}`"
@@ -475,7 +508,7 @@ class BlockManifest(WorkflowBlockManifest):
     def describe_outputs(cls) -> List[OutputDefinition]:
         return [
             *cls._describe_raw_outputs(),
-            *describe_vlm_prediction_outputs(),
+            *describe_vlm_prediction_outputs(SUPPORTED_TASK_TYPES_LIST),
         ]
 
     def get_actual_outputs(self) -> List[OutputDefinition]:
@@ -573,14 +606,22 @@ class GoogleGeminiBlockV6(WorkflowBlock):
         results = []
         for image, (content, input_tokens, output_tokens) in zip(images, raw_outputs):
             inference_id = str(uuid4())
-            error_status, predictions = decode_vlm_output(
-                task_type=task_type,
-                raw_output=content,
-                image=image,
-                classes=classes,
-                inference_id=inference_id,
-                box_format=DETECTION_BOX_FORMAT,
-            )
+            if task_type == "instance-segmentation":
+                error_status, predictions = decode_argon_segmentations(
+                    content,
+                    image=image,
+                    classes=classes,
+                    inference_id=inference_id,
+                )
+            else:
+                error_status, predictions = decode_vlm_output(
+                    task_type=task_type,
+                    raw_output=content,
+                    image=image,
+                    classes=classes,
+                    inference_id=inference_id,
+                    box_format=DETECTION_BOX_FORMAT,
+                )
             results.append(
                 {
                     "output": content,
@@ -613,12 +654,25 @@ def run_gemini_prompting(
 ) -> List[Tuple[str, Optional[int], Optional[int]]]:
     if task_type not in PROMPT_BUILDERS:
         raise ValueError(f"Task type: {task_type} not supported.")
+    if task_type == "instance-segmentation" and model_version != "gemini-4-argon":
+        raise ValueError("Instance segmentation is supported only for gemini-4-argon")
+    if task_type == "instance-segmentation" and google_code_execution:
+        raise ValueError(
+            "Code execution is not supported with segmentation JSON output"
+        )
     gemini_prompts = []
     for image in images:
         loaded_image, _ = load_image(image)
-        base64_image = base64.b64encode(
-            encode_image_to_jpeg_bytes(loaded_image)
-        ).decode("ascii")
+        if task_type == "instance-segmentation":
+            uploaded = Image.fromarray(loaded_image[:, :, ::-1])
+            uploaded.thumbnail((1600, 1600))
+            buffer = io.BytesIO()
+            uploaded.save(buffer, format="PNG")
+            base64_image = base64.b64encode(buffer.getvalue()).decode("ascii")
+        else:
+            base64_image = base64.b64encode(
+                encode_image_to_jpeg_bytes(loaded_image)
+            ).decode("ascii")
         generated_prompt = PROMPT_BUILDERS[task_type](
             base64_image=base64_image,
             prompt=prompt,
@@ -1166,6 +1220,62 @@ def prepare_object_detection_prompt(
     }
 
 
+def prepare_instance_segmentation_prompt(
+    base64_image: str,
+    classes: List[str],
+    model_version: str,
+    temperature: Optional[float],
+    thinking_level: Optional[str],
+    max_tokens: Optional[int],
+    **kwargs,
+) -> dict:
+    """Build an Argon-only structured request for components and holes.
+
+    Args:
+        base64_image (str): Lossless PNG image encoded as base64.
+        classes (List[str]): Allowed instance labels.
+        model_version (str): Must be gemini-4-argon.
+        temperature (Optional[float]): Sampling temperature.
+        thinking_level (Optional[str]): Model reasoning level.
+        max_tokens (Optional[int]): Output token budget.
+        **kwargs: Unused shared prompt-builder arguments.
+
+    Returns:
+        dict: Gemini generateContent payload with a constrained mask schema.
+
+    Raises:
+        ValueError: If model or classes do not support segmentation.
+    """
+    if model_version != "gemini-4-argon" or not classes:
+        raise ValueError("Argon segmentation requires gemini-4-argon and classes")
+
+    config = prepare_generation_config(
+        max_tokens=max_tokens,
+        temperature=temperature,
+        thinking_level=thinking_level,
+        model_version=model_version,
+        response_mime_type="application/json",
+    )
+    if temperature is not None:
+        config["temperature"] = temperature
+    config["response_schema"] = copy.deepcopy(ARGON_SEGMENTATION_SCHEMA)
+    payload = {
+        "contents": {
+            "role": "user",
+            "parts": [
+                {"inline_data": {"mime_type": "image/png", "data": base64_image}},
+                {
+                    "text": ARGON_SEGMENTATION_PROMPT.format(
+                        class_list=", ".join(classes)
+                    )
+                },
+            ],
+        },
+        "generationConfig": config,
+    }
+    return payload
+
+
 def prepare_generation_config(
     max_tokens: Optional[int],
     temperature: Optional[float],
@@ -1225,4 +1335,5 @@ PROMPT_BUILDERS = {
     "multi-label-classification": prepare_multi_label_classification_prompt,
     "structured-answering": prepare_structured_answering_prompt,
     "object-detection": prepare_object_detection_prompt,
+    "instance-segmentation": prepare_instance_segmentation_prompt,
 }

{roboflow_workflows-0.2.7 → roboflow_workflows-0.2.8/roboflow_workflows.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: roboflow-workflows
-Version: 0.2.7
+Version: 0.2.8
 Summary: Roboflow Workflows execution engine and block library
 License: LICENSE.core (Apache 2.0) applies to all files in this repository
         except for files in or under any directory that contains a superseding

{roboflow_workflows-0.2.7 → roboflow_workflows-0.2.8}/roboflow_workflows.egg-info/SOURCES.txt

@@ -176,6 +176,7 @@ roboflow_workflows/core_steps/common/query_language/operations/strings/base.py
 roboflow_workflows/core_steps/common/query_language/operations/timestamps/__init__.py
 roboflow_workflows/core_steps/common/query_language/operations/timestamps/base.py
 roboflow_workflows/core_steps/common/vlm_decoding/__init__.py
+roboflow_workflows/core_steps/common/vlm_decoding/argon_segmentation.py
 roboflow_workflows/core_steps/common/vlm_decoding/classification.py
 roboflow_workflows/core_steps/common/vlm_decoding/detection_formats.py
 roboflow_workflows/core_steps/common/vlm_decoding/detections.py