gimlet-api 0.0.7__py3-none-any.whl → 0.0.8__py3-none-any.whl

gml/hf.py

@@ -15,14 +15,23 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import glob
+import math
 import tempfile
+import warnings
 from collections.abc import Iterable
 from pathlib import Path
 from typing import Any, BinaryIO, Dict, List, Optional, TextIO, Tuple
 
-import gml.proto.src.api.corepb.v1.model_exec_pb2 as modelexecpb
 import torch
 import transformers
+from transformers import (
+    BaseImageProcessor,
+    Pipeline,
+    PreTrainedModel,
+    PreTrainedTokenizer,
+)
+
+import gml.proto.src.api.corepb.v1.model_exec_pb2 as modelexecpb
 from gml.asset_manager import AssetManager
 from gml.model import GenerationConfig, Model, TorchModel
 from gml.preprocessing import (
@@ -34,10 +43,12 @@ from gml.preprocessing import (
 )
 from gml.tensor import (
     AttentionKeyValueCacheTensorSemantics,
+    AttentionMaskDimension,
     BatchDimension,
     BoundingBoxFormat,
     DetectionNumCandidatesDimension,
     DetectionOutputDimension,
+    DimensionSemantics,
     ImageChannelDimension,
     ImageHeightDimension,
     ImageWidthDimension,
@@ -46,12 +57,8 @@ from gml.tensor import (
     TokensDimension,
     VocabLogitsDimension,
 )
-from transformers import (
-    BaseImageProcessor,
-    Pipeline,
-    PreTrainedModel,
-    PreTrainedTokenizer,
-)
+
+FALLBACK_RESIZE_SIZE = 512
 
 
 class HuggingFaceTokenizer(Model):
@@ -77,7 +84,6 @@ class HuggingFaceTokenizer(Model):
 
 
 class HuggingFaceGenerationConfig(GenerationConfig):
-
     def __init__(self, model: PreTrainedModel):
         config = model.generation_config
         eos_tokens = config.eos_token_id
@@ -242,25 +248,34 @@ class HuggingFaceTextGenerationPipeline:
 
 
 class HuggingFaceImageProcessor:
-
     def __init__(
         self,
         model: PreTrainedModel,
         processor: BaseImageProcessor,
+        image_size_override: Optional[Tuple[int, int]] = None,
     ):
         self.model = model
         self.processor = processor
+        self.image_size_override = image_size_override
 
     def input_spec(self) -> Dict[str, Any]:
         target_size = None
         image_preprocessing_steps = []
-        if (
-            hasattr(self.processor, "do_resize")
-            and self.processor.do_resize
-            and hasattr(self.processor, "size")
-        ):
+        has_do_resize = (
+            hasattr(self.processor, "do_resize") and self.processor.do_resize
+        )
+        has_do_pad = hasattr(self.processor, "do_pad") and self.processor.do_pad
+        # NOTE: it is possible for both do_resize and do_pad to be set, in which case we only use do_resize.
+        if has_do_resize:
             target_size, preprocessing_step = self._convert_resize()
             image_preprocessing_steps.append(preprocessing_step)
+        elif has_do_pad:
+            target_size, preprocessing_step = self._convert_pad()
+            image_preprocessing_steps.append(preprocessing_step)
+        else:
+            raise ValueError(
+                "could not determine target size for resize from model config"
+            )
 
         if (
             hasattr(self.processor, "do_rescale")
@@ -291,7 +306,7 @@ class HuggingFaceImageProcessor:
         # TODO(james): figure out if this is specified anywhere in the huggingface pipeline.
         channel_format = "rgb"
 
-        dimensions = [
+        dimensions: list[DimensionSemantics] = [
             BatchDimension(),
         ]
         input_shape = [1]
@@ -342,21 +357,38 @@ class HuggingFaceImageProcessor:
             "class_labels": labels,
         }
 
-    def output_spec_object_detection(self) -> Dict[str, Any]:
+    def output_spec_depth(self) -> Dict[str, Any]:
+        dimensions = [
+            BatchDimension(),
+            ImageHeightDimension(),
+            ImageWidthDimension(),
+        ]
+        output_tensor_semantics = [
+            TensorSemantics(dimensions),
+        ]
+        return {
+            "output_tensor_semantics": output_tensor_semantics,
+        }
+
+    def output_spec_object_detection(self, zero_shot=False) -> Dict[str, Any]:
         if not hasattr(self.processor, "post_process_object_detection"):
             raise NotImplementedError(
                 "processor must have post_process_object_detection set"
             )
 
-        id_to_label = self.model.config.id2label
-        max_id = max(id_to_label)
-        labels = []
-        for i in range(max_id):
-            if i not in id_to_label:
-                labels.append("")
-                continue
-            labels.append(id_to_label[i])
-        num_classes = max_id + 1
+        if zero_shot:
+            num_classes = -1
+            labels = []
+        else:
+            id_to_label = self.model.config.id2label
+            max_id = max(id_to_label)
+            labels = []
+            for i in range(max_id):
+                if i not in id_to_label:
+                    labels.append("")
+                    continue
+                labels.append(id_to_label[i])
+            num_classes = max_id + 1
 
         # TODO(james): verify assumptions made here apply broadly.
         output_tensor_semantics = []
@@ -366,7 +398,7 @@ class HuggingFaceImageProcessor:
             DetectionNumCandidatesDimension(is_nms=False),
             DetectionOutputDimension(
                 scores_range=(0, num_classes),
-                scores_are_logits=True,
+                scores_are_logits=not zero_shot,
             ),
         ]
         output_tensor_semantics.append(TensorSemantics(logits_dimensions))
@@ -385,12 +417,45 @@ class HuggingFaceImageProcessor:
             "class_labels": labels,
         }
 
+    def _get_size(self) -> Dict[str, int]:
+        size = None
+        if self.image_size_override:
+            size = {
+                "height": self.image_size_override[0],
+                "width": self.image_size_override[1],
+            }
+        elif hasattr(self.processor, "size") and self.processor.size is not None:
+            size = self.processor.size
+        elif (
+            hasattr(self.model.config, "image_size")
+            and self.model.config.image_size is not None
+        ):
+            size = {
+                "height": self.model.config.image_size,
+                "width": self.model.config.image_size,
+            }
+        else:
+            warnings.warn(
+                f"using fallback resize size of {FALLBACK_RESIZE_SIZE} for model",
+                stacklevel=1,
+            )
+            size = {
+                "width": FALLBACK_RESIZE_SIZE,
+                "height": FALLBACK_RESIZE_SIZE,
+            }
+        return size
+
     def _convert_resize(self) -> Tuple[Tuple[int, int], ImagePreprocessingStep]:
-        size = self.processor.size
+        size = self._get_size()
+        size_divisor: int | None = None
+        if hasattr(self.processor, "size_divisor"):
+            size_divisor = self.processor.size_divisor
+
         target_size = None
         preprocess_step = None
+
         if "height" in size and "width" in size:
-            target_size = [size["height"], size["width"]]
+            target_size = (size["height"], size["width"])
             preprocess_step = ResizeImage()
         elif (
             "shortest_edge" in size
@@ -410,12 +475,55 @@ class HuggingFaceImageProcessor:
                 if not min_size or edge_size < min_size:
                     min_size = edge_size
 
-            target_size = [min_size, min_size]
+            if min_size is None:
+                raise ValueError(
+                    "could not determine target size for resize from model config"
+                )
+            target_size = (min_size, min_size)
+            preprocess_step = LetterboxImage()
+        else:
+            raise ValueError(
+                "could not determine target size for resize from model config"
+            )
+        if size_divisor:
+            target_size = (
+                math.ceil(target_size[0] / size_divisor) * size_divisor,
+                math.ceil(target_size[1] / size_divisor) * size_divisor,
+            )
+        return target_size, preprocess_step
+
+    def _convert_pad(self) -> Tuple[Tuple[int, int], ImagePreprocessingStep]:
+        # NOTE: There is a wide variety of ways that huggingface pads images.
+        # We found at least 3 different ways to pad images in the codebase:
+        # 1. Center pad (pad top,left, bottom, right) to match target size
+        # https://github.com/huggingface/transformers/blob/70b07d97cf2c5f61fff55700b65528a1b6845cd2/src/transformers/models/dpt/image_processing_dpt.py#L231
+        # 2. Right/Top pad (pad top, and right) to match target size
+        # https://github.com/huggingface/transformers/blob/174890280b340b89c5bfa092f6b4fb0e2dc2d7fc/src/transformers/models/conditional_detr/image_processing_conditional_detr.py#L846
+        # 3. Pad to nearest multiple of size_divisor
+        # https://github.com/huggingface/transformers/blob/70b07d97cf2c5f61fff55700b65528a1b6845cd2/src/transformers/models/llava_onevision/image_processing_llava_onevision.py#L177-179
+        #
+        # We decided to simply implement padding with LetterBoxImage(),
+        # because we assume the models won't be that sensitive to the type of padding,
+        # but this may need to be revisited in the future.
+        size = self._get_size()
+        size_divisor: int | None = None
+        if hasattr(self.processor, "size_divisor"):
+            size_divisor = self.processor.size_divisor
+
+        target_size = None
+        preprocess_step = None
+        if "height" in size and "width" in size:
+            target_size = (size["height"], size["width"])
             preprocess_step = LetterboxImage()
         else:
             raise ValueError(
                 "could not determine target size for resize from model config"
             )
+        if size_divisor:
+            target_size = (
+                math.ceil(target_size[0] / size_divisor) * size_divisor,
+                math.ceil(target_size[1] / size_divisor) * size_divisor,
+            )
         return target_size, preprocess_step
 
 
@@ -424,11 +532,13 @@ class HuggingFaceImageSegmentationPipeline:
         self,
         pipeline: Pipeline,
         name: Optional[str] = None,
+        image_size_override: Optional[Tuple[int, int]] = None,
     ):
         self.pipeline = pipeline
         if name is None:
             name = pipeline.model.name_or_path
 
+        self.image_size_override = image_size_override
         self.model = TorchModel(
             name,
             torch_module=self.pipeline.model,
@@ -446,7 +556,9 @@ class HuggingFaceImageSegmentationPipeline:
             )
 
         image_processor = HuggingFaceImageProcessor(
-            self.pipeline.model, self.pipeline.image_processor
+            self.pipeline.model,
+            self.pipeline.image_processor,
+            image_size_override=self.image_size_override,
         )
         spec = image_processor.input_spec()
         spec.update(image_processor.output_spec_segmentation())
@@ -471,11 +583,13 @@ class HuggingFaceObjectDetectionPipeline:
         self,
         pipeline: Pipeline,
         name: Optional[str] = None,
+        image_size_override: Optional[Tuple[int, int]] = None,
     ):
         self.pipeline = pipeline
         if name is None:
             name = pipeline.model.name_or_path
 
+        self.image_size_override = image_size_override
         self.model = TorchModel(
             name,
             torch_module=ObjectDetectionWrapper(self.pipeline.model),
@@ -493,7 +607,9 @@ class HuggingFaceObjectDetectionPipeline:
             )
 
         image_processor = HuggingFaceImageProcessor(
-            self.pipeline.model, self.pipeline.image_processor
+            self.pipeline.model,
+            self.pipeline.image_processor,
+            image_size_override=self.image_size_override,
         )
         spec = image_processor.input_spec()
         spec.update(image_processor.output_spec_object_detection())
@@ -503,6 +619,141 @@ class HuggingFaceObjectDetectionPipeline:
         return [self.model]
 
 
+class ZeroShotObjectDetectionWrapper(torch.nn.Module):
+    def __init__(self, model: PreTrainedModel):
+        super().__init__()
+        self.model = model
+
+    def forward(self, image, tokens, attention_mask):
+        outputs = self.model(
+            input_ids=tokens, pixel_values=image, attention_mask=attention_mask
+        )
+        return torch.sigmoid(outputs.logits), outputs.pred_boxes
+
+
+class HuggingFaceZeroShotObjectDetectionPipeline:
+    def __init__(
+        self,
+        pipeline: Pipeline,
+        name: Optional[str] = None,
+        tokenizer_name: Optional[str] = None,
+        image_size_override: Optional[Tuple[int, int]] = None,
+    ):
+        self.pipeline = pipeline
+        if name is None:
+            name = pipeline.model.name_or_path
+
+        self.tokenizer_model = HuggingFaceTokenizer(
+            self.pipeline.tokenizer, tokenizer_name
+        )
+
+        self.image_size_override = image_size_override
+        self.detection_model = TorchModel(
+            name,
+            torch_module=ZeroShotObjectDetectionWrapper(self.pipeline.model),
+            **self._guess_model_spec(),
+        )
+
+    def _add_zero_shot_inputs(self, spec: Dict):
+        example_inputs = spec["example_inputs"]
+        if "dynamic_shapes" not in spec:
+            spec["dynamic_shapes"] = [{} for _ in example_inputs]
+
+        max_length = self.pipeline.model.config.text_config.max_length
+        example_inputs.extend(
+            [
+                torch.randint(200, [2, max_length]).to(torch.int32),
+                torch.ones([2, max_length]).to(torch.int32),
+            ]
+        )
+
+        input_tensor_semantics = spec["input_tensor_semantics"]
+        input_tensor_semantics.extend(
+            [
+                TensorSemantics(
+                    [
+                        BatchDimension(),
+                        TokensDimension(),
+                    ]
+                ),
+                TensorSemantics(
+                    [
+                        BatchDimension(),
+                        AttentionMaskDimension(),
+                    ]
+                ),
+            ]
+        )
+
+        spec["dynamic_shapes"].extend(
+            [
+                {0: "num_labels"},
+                {0: "num_labels"},
+            ]
+        )
+
+    def _guess_model_spec(self) -> Dict:
+        if self.pipeline.image_processor is None:
+            raise ValueError(
+                "Could not determine image preprocessing for pipeline with image_processor=None"
+            )
+
+        image_processor = HuggingFaceImageProcessor(
+            self.pipeline.model,
+            self.pipeline.image_processor,
+            image_size_override=self.image_size_override,
+        )
+        spec = image_processor.input_spec()
+        self._add_zero_shot_inputs(spec)
+        spec.update(image_processor.output_spec_object_detection(zero_shot=True))
+        return spec
+
+    def models(self) -> List[Model]:
+        return [self.detection_model, self.tokenizer_model]
+
+
+class HuggingFaceDepthEstimationPipeline:
+    def __init__(
+        self,
+        pipeline: Pipeline,
+        name: Optional[str] = None,
+        image_size_override: Optional[Tuple[int, int]] = None,
+    ):
+        self.pipeline = pipeline
+        if name is None:
+            name = pipeline.model.name_or_path
+
+        self.image_size_override = image_size_override
+
+        self.model = TorchModel(
+            name,
+            torch_module=self.pipeline.model,
+            **self._guess_model_spec(),
+        )
+
+    def _guess_model_spec(self) -> Dict:
+        if self.pipeline.image_processor is None:
+            raise ValueError(
+                "Could not determine image preprocessing for pipeline with image_processor=None"
+            )
+        if self.pipeline.tokenizer is not None:
+            raise NotImplementedError(
+                "HuggingFaceDepthEstimationPipeline does not yet support token inputs"
+            )
+
+        image_processor = HuggingFaceImageProcessor(
+            self.pipeline.model,
+            self.pipeline.image_processor,
+            image_size_override=self.image_size_override,
+        )
+        spec = image_processor.input_spec()
+        spec.update(image_processor.output_spec_depth())
+        return spec
+
+    def models(self) -> List[Model]:
+        return [self.model]
+
+
 def import_huggingface_pipeline(pipeline: Pipeline, **kwargs) -> List[Model]:
     if pipeline.framework != "pt":
         raise ValueError(
@@ -517,8 +768,19 @@ def import_huggingface_pipeline(pipeline: Pipeline, **kwargs) -> List[Model]:
         return HuggingFaceImageSegmentationPipeline(pipeline, **kwargs).models()
     elif pipeline.task == "object-detection":
         return HuggingFaceObjectDetectionPipeline(pipeline, **kwargs).models()
+    elif pipeline.task == "zero-shot-object-detection":
+        return HuggingFaceZeroShotObjectDetectionPipeline(pipeline, **kwargs).models()
+    elif pipeline.task == "depth-estimation":
+        return HuggingFaceDepthEstimationPipeline(pipeline, **kwargs).models()
     raise ValueError(
         "unimplemented: hugging face pipeline task: {} (supported tasks: [{}])".format(
-            pipeline.task, ["text-generation", "image-segmentation", "object-detection"]
+            pipeline.task,
+            [
+                "text-generation",
+                "image-segmentation",
+                "object-detection",
+                "zero-shot-object-detection",
+                "depth-estimation",
+            ],
         )
     )

gml/model.py

@@ -21,8 +21,9 @@ import io
 from pathlib import Path
 from typing import BinaryIO, Dict, List, Literal, Optional, Sequence, TextIO, Tuple
 
-import gml.proto.src.api.corepb.v1.model_exec_pb2 as modelexecpb
 import torch
+
+import gml.proto.src.api.corepb.v1.model_exec_pb2 as modelexecpb
 from gml.asset_manager import AssetManager, TempFileAssetManager
 from gml.compile import to_torch_mlir
 from gml.preprocessing import ImagePreprocessingStep

gml/pipelines.py

@@ -15,7 +15,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import abc
-from typing import List
+from typing import List, Optional
 
 import gml.proto.src.api.corepb.v1.model_exec_pb2 as modelexecpb
 from gml.model import Model
@@ -41,12 +41,44 @@ class SingleModelPipeline(Pipeline):
 
 
 class SimpleDetectionPipeline(SingleModelPipeline):
-    def __init__(self, add_tracking_id: bool = False):
-        self.add_tracking_id = add_tracking_id
+    def __init__(
+        self,
+        track_objects: Optional[bool] = None,
+        add_tracking_id: Optional[bool] = None,
+    ):
+        self.track_objects = False
+        if add_tracking_id is not None:
+            import warnings
+
+            warnings.warn(
+                "The 'add_tracking_id' parameter is deprecated and will be removed in a future version.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+            self.track_objects = add_tracking_id
+
+        if track_objects is not None:
+            self.track_objects = track_objects
+
+        if track_objects is not None and add_tracking_id is not None:
+            raise ValueError(
+                "'track_objects' and 'add_tracking_id' cannot be set simultaneously."
+            )
 
     def _to_yaml(self, model_name: str, org_name: str):
-        add_tracking_id = "true" if self.add_tracking_id else "false"
         # editorconfig-checker-disable
+        video_stream_detections = ".detect.detections"
+        track_node = ""
+        if self.track_objects:
+            track_node = """
+- name: track
+  kind: Track
+  inputs:
+    detections: .detect.detections
+  outputs:
+  - tracked_detections
+"""
+            video_stream_detections = ".track.tracked_detections"
         return f"""---
 nodes:
 - name: camera_source
@@ -56,7 +88,6 @@ nodes:
 - name: detect
   kind: Detect
   attributes:
-    add_tracking_id: {add_tracking_id}
     model:
       model:
         name: {model_name}
@@ -66,13 +97,14 @@ nodes:
     frame: .camera_source.frame
   outputs:
   - detections
+{track_node}
 - name: video_stream_sink
   kind: VideoStreamSink
   attributes:
     frame_rate_limit: 30
   inputs:
     frame: .camera_source.frame
-    detections: .detect.detections
+    detections: {video_stream_detections}
 """
 
 
@@ -110,6 +142,36 @@ nodes:
 """
 
 
+class SimpleDepthEstimationPipeline(SingleModelPipeline):
+    def _to_yaml(self, model_name: str, org_name: str):
+        # editorconfig-checker-disable
+        return f"""---
+nodes:
+- name: camera_source
+  kind: CameraSource
+  outputs:
+  - frame
+- name: estimate_depth
+  kind: EstimateDepth
+  attributes:
+    model:
+      model:
+        name: {model_name}
+        org: {org_name}
+    frame_rate_limit: 30
+  inputs:
+    frame: .camera_source.frame
+  outputs:
+  - depth
+- name: video_stream_sink
+  kind: VideoStreamSink
+  attributes:
+    frame_rate_limit: 30
+  inputs:
+    frame: .estimate_depth.depth
+"""
+
+
 class LiveChatPipeline(Pipeline):
     def to_yaml(self, models: List[Model], org_name: str) -> str:
         if len(models) != 2:
@@ -173,4 +235,61 @@ nodes:
 """
 
 
+class ZeroShotObjectDetectionPipeline(Pipeline):
+    def __init__(self, conf_threshold=0.1):
+        self.conf_threshold = conf_threshold
+
+    def to_yaml(self, models: List[Model], org_name: str) -> str:
+        if len(models) != 2:
+            raise ValueError(
+                "ZeroShotObjectDetectionPipeline expects two models (a detection model and a tokenizer)"
+            )
+        tokenizer = None
+        detect = None
+        for m in models:
+            if m.storage_format == modelexecpb.ModelInfo.MODEL_STORAGE_FORMAT_OPAQUE:
+                tokenizer = m
+            else:
+                detect = m
+        if tokenizer is None or detect is None:
+            raise ValueError(
+                "ZeroShotObjectDetectionPipeline expects both a tokenizer model and a detection model)"
+            )
+        return f"""---
+nodes:
+- name: camera_source
+  kind: CameraSource
+  outputs:
+  - frame
+- name: text_source
+  kind: TextStreamSource
+  outputs:
+  - prompt
+- name: detect
+  kind: Detect
+  attributes:
+    model:
+      model:
+        name: {detect.name}
+        org: {org_name}
+    tokenizer:
+      model:
+        name: {tokenizer.name}
+        org: {org_name}
+    conf_threshold: {self.conf_threshold}
+  inputs:
+    frame: .camera_source.frame
+    prompt: .text_source.prompt
+  outputs:
+  - detections
+- name: video_stream_sink
+  kind: VideoStreamSink
+  attributes:
+    frame_rate_limit: 30
+  inputs:
+    frame: .camera_source.frame
+    detections: .detect.detections
+"""
+
+
 # editorconfig-checker-enable

gml/preprocessing.py

@@ -17,9 +17,10 @@
 import abc
 from typing import List
 
-import gml.proto.src.api.corepb.v1.model_exec_pb2 as modelexecpb
 import google.protobuf.wrappers_pb2 as wrapperspb
 
+import gml.proto.src.api.corepb.v1.model_exec_pb2 as modelexecpb
+
 
 class ImagePreprocessingStep(abc.ABC):
     @abc.abstractmethod