inference-models 0.18.3__py3-none-any.whl

inference_models/models/perception_encoder/vision_encoder/transforms.py

@@ -0,0 +1,33 @@
+import torchvision.transforms as T
+
+from inference_models.models.perception_encoder.vision_encoder.tokenizer import (
+    SimpleTokenizer,
+)
+
+
+def get_image_transform(
+    image_size: int,
+    center_crop: bool = False,
+    interpolation: T.InterpolationMode = T.InterpolationMode.BILINEAR,  # We used bilinear during training
+):
+    if center_crop:
+        crop = [
+            T.Resize(image_size, interpolation=interpolation),
+            T.CenterCrop(image_size),
+        ]
+    else:
+        # "Squash": most versatile
+        crop = [T.Resize((image_size, image_size), interpolation=interpolation)]
+
+    return T.Compose(
+        crop
+        + [
+            T.Lambda(lambda x: x.convert("RGB")),
+            T.ToTensor(),
+            T.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5], inplace=True),
+        ]
+    )
+
+
+def get_text_tokenizer(context_length: int):
+    return SimpleTokenizer(context_length=context_length)

inference_models/models/qwen25vl/__init__.py

@@ -0,0 +1 @@
+# This file makes the qwen25vl directory a Python package

inference_models/models/qwen25vl/qwen25vl_hf.py

@@ -0,0 +1,285 @@
+import os
+from typing import List, Optional, Tuple, Union
+
+import numpy as np
+import torch
+from peft import PeftModel
+from transformers import (
+    BitsAndBytesConfig,
+    Qwen2_5_VLForConditionalGeneration,
+    Qwen2_5_VLProcessor,
+)
+
+from inference_models.configuration import DEFAULT_DEVICE
+from inference_models.entities import ColorFormat
+from inference_models.models.common.roboflow.model_packages import (
+    InferenceConfig,
+    ResizeMode,
+    parse_inference_config,
+)
+from inference_models.models.common.roboflow.pre_processing import (
+    pre_process_network_input,
+)
+
+
+class Qwen25VLHF:
+    @classmethod
+    def from_pretrained(
+        cls,
+        model_name_or_path: str,
+        device: torch.device = DEFAULT_DEVICE,
+        trust_remote_code: bool = False,
+        local_files_only: bool = True,
+        quantization_config: Optional[BitsAndBytesConfig] = None,
+        disable_quantization: bool = False,
+        **kwargs,
+    ) -> "Qwen25VLHF":
+        adapter_config_path = os.path.join(model_name_or_path, "adapter_config.json")
+        inference_config_path = os.path.join(
+            model_name_or_path, "inference_config.json"
+        )
+        inference_config = None
+        if os.path.exists(inference_config_path):
+            inference_config = parse_inference_config(
+                config_path=inference_config_path,
+                allowed_resize_modes={
+                    ResizeMode.STRETCH_TO,
+                    ResizeMode.LETTERBOX,
+                    ResizeMode.CENTER_CROP,
+                    ResizeMode.LETTERBOX_REFLECT_EDGES,
+                },
+            )
+        if (
+            quantization_config is None
+            and device.type == "cuda"
+            and not disable_quantization
+        ):
+            quantization_config = BitsAndBytesConfig(
+                load_in_4bit=True,
+                bnb_4bit_compute_dtype=torch.float16,
+                bnb_4bit_quant_type="nf4",
+            )
+        if os.path.exists(adapter_config_path):
+            base_model_path = os.path.join(model_name_or_path, "base")
+            model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
+                base_model_path,
+                dtype="auto",
+                trust_remote_code=trust_remote_code,
+                local_files_only=local_files_only,
+                quantization_config=quantization_config,
+            )
+            model = PeftModel.from_pretrained(model, model_name_or_path)
+            if quantization_config is None:
+                model.merge_and_unload()
+            model.to(device)
+            processor = Qwen2_5_VLProcessor.from_pretrained(
+                model_name_or_path,
+                trust_remote_code=trust_remote_code,
+                local_files_only=local_files_only,
+                use_fast=True,
+            )
+        else:
+            model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
+                model_name_or_path,
+                dtype="auto",
+                device_map=device,
+                trust_remote_code=trust_remote_code,
+                local_files_only=local_files_only,
+                quantization_config=quantization_config,
+            ).eval()
+            Qwen2_5_VLProcessor.image_processor_class = "Qwen2VLImageProcessor"
+            processor = Qwen2_5_VLProcessor.from_pretrained(
+                model_name_or_path,
+                trust_remote_code=trust_remote_code,
+                local_files_only=local_files_only,
+                use_fast=True,
+            )
+        return cls(
+            model=model,
+            processor=processor,
+            inference_config=inference_config,
+            device=device,
+        )
+
+    def __init__(
+        self,
+        model: Qwen2_5_VLForConditionalGeneration,
+        processor: Qwen2_5_VLProcessor,
+        inference_config: Optional[InferenceConfig],
+        device: torch.device,
+    ):
+        self._model = model
+        self._processor = processor
+        self._inference_config = inference_config
+        self._device = device
+        self.default_system_prompt = (
+            "You are a Qwen2.5-VL model that can answer questions about any image."
+        )
+
+    def prompt(
+        self,
+        images: Union[torch.Tensor, List[torch.Tensor], np.ndarray, List[np.ndarray]],
+        prompt: str = None,
+        input_color_format: ColorFormat = None,
+        max_new_tokens: int = 512,
+        do_sample: bool = False,
+        skip_special_tokens: bool = False,
+        **kwargs,
+    ) -> List[str]:
+        inputs = self.pre_process_generation(
+            images=images, prompt=prompt, input_color_format=input_color_format
+        )
+        generated_ids = self.generate(
+            inputs=inputs,
+            max_new_tokens=max_new_tokens,
+            do_sample=do_sample,
+        )
+        return self.post_process_generation(
+            generated_ids=generated_ids,
+            skip_special_tokens=skip_special_tokens,
+        )
+
+    def pre_process_generation(
+        self,
+        images: Union[torch.Tensor, List[torch.Tensor], np.ndarray, List[np.ndarray]],
+        prompt: str = None,
+        input_color_format: ColorFormat = None,
+        image_size: Optional[Tuple[int, int]] = None,
+        **kwargs,
+    ) -> dict:
+        def _to_tensor(image: Union[np.ndarray, torch.Tensor]) -> torch.Tensor:
+            is_numpy = isinstance(image, np.ndarray)
+            if is_numpy:
+                tensor_image = torch.from_numpy(image.copy()).permute(2, 0, 1)
+            else:
+                tensor_image = image
+            if input_color_format == "bgr" or (is_numpy and input_color_format is None):
+                tensor_image = tensor_image[[2, 1, 0], :, :]
+            return tensor_image
+
+        if self._inference_config is None:
+            if isinstance(images, torch.Tensor) and images.ndim > 3:
+                image_list = [_to_tensor(img) for img in images]
+            elif not isinstance(images, list):
+                image_list = [_to_tensor(images)]
+            else:
+                image_list = [_to_tensor(img) for img in images]
+        else:
+            images = pre_process_network_input(
+                images=images,
+                image_pre_processing=self._inference_config.image_pre_processing,
+                network_input=self._inference_config.network_input,
+                target_device=self._device,
+                input_color_format=input_color_format,
+                image_size_wh=image_size,
+            )[0]
+            image_list = [e[0] for e in torch.split(images, 1, dim=0)]
+        # Handle prompt and system prompt parsing logic from original implementation
+        if prompt is None:
+            prompt = ""
+            system_prompt = self.default_system_prompt
+        else:
+            split_prompt = prompt.split("<system_prompt>")
+            if len(split_prompt) == 1:
+                prompt = split_prompt[0]
+                system_prompt = self.default_system_prompt
+            else:
+                prompt = split_prompt[0]
+                system_prompt = split_prompt[1]
+
+        # Construct conversation following original implementation structure
+        conversation = [
+            {
+                "role": "system",
+                "content": [{"type": "text", "text": system_prompt}],
+            },
+            {
+                "role": "user",
+                "content": [
+                    {"type": "image"},  # Processor will handle the actual image
+                    {"type": "text", "text": prompt},
+                ],
+            },
+        ]
+
+        # Apply chat template
+        text_input = self._processor.apply_chat_template(
+            conversation, tokenize=False, add_generation_prompt=True
+        )
+
+        # Process inputs - processor will handle tensor/array inputs directly
+        model_inputs = self._processor(
+            text=text_input,
+            images=image_list,
+            return_tensors="pt",
+            padding=True,
+        )
+
+        # Move inputs to device
+        model_inputs = {
+            k: v.to(self._device)
+            for k, v in model_inputs.items()
+            if isinstance(v, torch.Tensor)
+        }
+
+        return model_inputs
+
+    def generate(
+        self,
+        inputs: dict,
+        max_new_tokens: int = 512,
+        do_sample: bool = False,
+        **kwargs,
+    ) -> torch.Tensor:
+        input_len = inputs["input_ids"].shape[-1]
+
+        with torch.inference_mode():
+            generation = self._model.generate(
+                **inputs,
+                max_new_tokens=max_new_tokens,
+                do_sample=do_sample,
+                pad_token_id=self._processor.tokenizer.pad_token_id,
+                eos_token_id=self._processor.tokenizer.eos_token_id,
+                bos_token_id=self._processor.tokenizer.bos_token_id,
+            )
+
+        # Return only the newly generated tokens
+        return generation[:, input_len:]
+
+    def post_process_generation(
+        self,
+        generated_ids: torch.Tensor,
+        skip_special_tokens: bool = False,
+        **kwargs,
+    ) -> List[str]:
+        # Decode the generated tokens
+        decoded = self._processor.batch_decode(
+            generated_ids,
+            skip_special_tokens=skip_special_tokens,
+        )
+
+        # Apply the same post-processing as original implementation
+        result = []
+        for text in decoded:
+            text = text.replace("assistant\n", "")
+            text = text.replace(" addCriterion\n", "")
+            result.append(text.strip())
+
+        return result
+
+
+def adjust_lora_model_state_dict(state_dict: dict) -> dict:
+    return {
+        refactor_adapter_weights_key(key=key): value
+        for key, value in state_dict.items()
+    }
+
+
+def refactor_adapter_weights_key(key: str) -> str:
+    if ".language_model." in key:
+        return key
+    return (
+        key.replace("model.layers", "model.language_model.layers")
+        .replace(".weight", ".default.weight")
+        .replace(".lora_magnitude_vector", ".lora_magnitude_vector.default.weight")
+    )

inference_models/models/resnet/resnet_classification_onnx.py

@@ -0,0 +1,330 @@
+from threading import Lock
+from typing import List, Optional, Tuple, Union
+
+import numpy as np
+import torch
+
+from inference_models import (
+    ClassificationModel,
+    ClassificationPrediction,
+    MultiLabelClassificationModel,
+    MultiLabelClassificationPrediction,
+)
+from inference_models.configuration import DEFAULT_DEVICE
+from inference_models.entities import ColorFormat
+from inference_models.errors import (
+    CorruptedModelPackageError,
+    EnvironmentConfigurationError,
+    MissingDependencyError,
+)
+from inference_models.models.base.types import PreprocessedInputs
+from inference_models.models.common.model_packages import get_model_package_contents
+from inference_models.models.common.onnx import (
+    run_session_with_batch_size_limit,
+    set_execution_provider_defaults,
+)
+from inference_models.models.common.roboflow.model_packages import (
+    InferenceConfig,
+    ResizeMode,
+    parse_class_names_file,
+    parse_inference_config,
+)
+from inference_models.models.common.roboflow.pre_processing import (
+    pre_process_network_input,
+)
+from inference_models.utils.onnx_introspection import (
+    get_selected_onnx_execution_providers,
+)
+
+try:
+    import onnxruntime
+except ImportError as import_error:
+    raise MissingDependencyError(
+        message=f"Could not import ResNet model with ONNX backend - this error means that some additional dependencies "
+        f"are not installed in the environment. If you run the `inference-models` library directly in your Python "
+        f"program, make sure the following extras of the package are installed: \n"
+        f"\t* `onnx-cpu` - when you wish to use library with CPU support only\n"
+        f"\t* `onnx-cu12` - for running on GPU with Cuda 12 installed\n"
+        f"\t* `onnx-cu118` - for running on GPU with Cuda 11.8 installed\n"
+        f"\t* `onnx-jp6-cu126` - for running on Jetson with Jetpack 6\n"
+        f"If you see this error using Roboflow infrastructure, make sure the service you use does support the model. "
+        f"You can also contact Roboflow to get support.",
+        help_url="https://todo",
+    ) from import_error
+
+
+class ResNetForClassificationOnnx(ClassificationModel[torch.Tensor, torch.Tensor]):
+
+    @classmethod
+    def from_pretrained(
+        cls,
+        model_name_or_path: str,
+        onnx_execution_providers: Optional[List[Union[str, tuple]]] = None,
+        default_onnx_trt_options: bool = True,
+        device: torch.device = DEFAULT_DEVICE,
+        **kwargs,
+    ) -> "ResNetForClassificationOnnx":
+        if onnx_execution_providers is None:
+            onnx_execution_providers = get_selected_onnx_execution_providers()
+        if not onnx_execution_providers:
+            raise EnvironmentConfigurationError(
+                message=f"Could not initialize model - selected backend is ONNX which requires execution provider to "
+                f"be specified - explicitly in `from_pretrained(...)` method or via env variable "
+                f"`ONNXRUNTIME_EXECUTION_PROVIDERS`. If you run model locally - adjust your setup, otherwise "
+                f"contact the platform support.",
+                help_url="https://todo",
+            )
+        onnx_execution_providers = set_execution_provider_defaults(
+            providers=onnx_execution_providers,
+            model_package_path=model_name_or_path,
+            device=device,
+            default_onnx_trt_options=default_onnx_trt_options,
+        )
+        model_package_content = get_model_package_contents(
+            model_package_dir=model_name_or_path,
+            elements=[
+                "class_names.txt",
+                "inference_config.json",
+                "weights.onnx",
+            ],
+        )
+        class_names = parse_class_names_file(
+            class_names_path=model_package_content["class_names.txt"]
+        )
+        inference_config = parse_inference_config(
+            config_path=model_package_content["inference_config.json"],
+            allowed_resize_modes={
+                ResizeMode.STRETCH_TO,
+                ResizeMode.LETTERBOX,
+                ResizeMode.CENTER_CROP,
+                ResizeMode.LETTERBOX_REFLECT_EDGES,
+            },
+        )
+        if inference_config.post_processing.type != "softmax":
+            raise CorruptedModelPackageError(
+                message="Expected Softmax to be the post-processing",
+                help_url="https://todo",
+            )
+        session = onnxruntime.InferenceSession(
+            path_or_bytes=model_package_content["weights.onnx"],
+            providers=onnx_execution_providers,
+        )
+        input_shape = session.get_inputs()[0].shape
+        input_batch_size = input_shape[0]
+        if isinstance(input_batch_size, str):
+            input_batch_size = None
+        input_name = session.get_inputs()[0].name
+        return cls(
+            session=session,
+            input_name=input_name,
+            inference_config=inference_config,
+            class_names=class_names,
+            device=device,
+            input_batch_size=input_batch_size,
+        )
+
+    def __init__(
+        self,
+        session: onnxruntime.InferenceSession,
+        input_name: str,
+        inference_config: InferenceConfig,
+        class_names: List[str],
+        device: torch.device,
+        input_batch_size: Optional[int],
+    ):
+        self._session = session
+        self._input_name = input_name
+        self._inference_config = inference_config
+        self._class_names = class_names
+        self._device = device
+        self._input_batch_size = input_batch_size
+        self._session_thread_lock = Lock()
+
+    @property
+    def class_names(self) -> List[str]:
+        return self._class_names
+
+    def pre_process(
+        self,
+        images: Union[torch.Tensor, List[torch.Tensor], np.ndarray, List[np.ndarray]],
+        input_color_format: Optional[ColorFormat] = None,
+        image_size: Optional[Tuple[int, int]] = None,
+        **kwargs,
+    ) -> torch.Tensor:
+        return pre_process_network_input(
+            images=images,
+            image_pre_processing=self._inference_config.image_pre_processing,
+            network_input=self._inference_config.network_input,
+            target_device=self._device,
+            input_color_format=input_color_format,
+            image_size_wh=image_size,
+        )[0]
+
+    def forward(
+        self, pre_processed_images: PreprocessedInputs, **kwargs
+    ) -> torch.Tensor:
+        with self._session_thread_lock:
+            return run_session_with_batch_size_limit(
+                session=self._session,
+                inputs={self._input_name: pre_processed_images},
+                min_batch_size=self._input_batch_size,
+                max_batch_size=self._input_batch_size,
+            )[0]
+
+    def post_process(
+        self,
+        model_results: torch.Tensor,
+        **kwargs,
+    ) -> ClassificationPrediction:
+        if self._inference_config.post_processing.fused:
+            confidence = model_results
+        else:
+            confidence = torch.nn.functional.softmax(model_results, dim=-1)
+        return ClassificationPrediction(
+            class_id=confidence.argmax(dim=-1),
+            confidence=confidence,
+        )
+
+
+class ResNetForMultiLabelClassificationOnnx(
+    MultiLabelClassificationModel[torch.Tensor, torch.Tensor]
+):
+
+    @classmethod
+    def from_pretrained(
+        cls,
+        model_name_or_path: str,
+        onnx_execution_providers: Optional[List[Union[str, tuple]]] = None,
+        default_onnx_trt_options: bool = True,
+        device: torch.device = DEFAULT_DEVICE,
+        **kwargs,
+    ) -> "ResNetForMultiLabelClassificationOnnx":
+        if onnx_execution_providers is None:
+            onnx_execution_providers = get_selected_onnx_execution_providers()
+        if not onnx_execution_providers:
+            raise EnvironmentConfigurationError(
+                message=f"Could not initialize model - selected backend is ONNX which requires execution provider to "
+                f"be specified - explicitly in `from_pretrained(...)` method or via env variable "
+                f"`ONNXRUNTIME_EXECUTION_PROVIDERS`. If you run model locally - adjust your setup, otherwise "
+                f"contact the platform support.",
+                help_url="https://todo",
+            )
+        onnx_execution_providers = set_execution_provider_defaults(
+            providers=onnx_execution_providers,
+            model_package_path=model_name_or_path,
+            device=device,
+            default_onnx_trt_options=default_onnx_trt_options,
+        )
+        model_package_content = get_model_package_contents(
+            model_package_dir=model_name_or_path,
+            elements=[
+                "class_names.txt",
+                "inference_config.json",
+                "weights.onnx",
+            ],
+        )
+        class_names = parse_class_names_file(
+            class_names_path=model_package_content["class_names.txt"]
+        )
+        inference_config = parse_inference_config(
+            config_path=model_package_content["inference_config.json"],
+            allowed_resize_modes={
+                ResizeMode.STRETCH_TO,
+                ResizeMode.LETTERBOX,
+                ResizeMode.CENTER_CROP,
+                ResizeMode.LETTERBOX_REFLECT_EDGES,
+            },
+        )
+        if inference_config.post_processing.type != "sigmoid":
+            raise CorruptedModelPackageError(
+                message="Expected sigmoid to be the post-processing",
+                help_url="https://todo",
+            )
+        session = onnxruntime.InferenceSession(
+            path_or_bytes=model_package_content["weights.onnx"],
+            providers=onnx_execution_providers,
+        )
+        input_shape = session.get_inputs()[0].shape
+        input_batch_size = input_shape[0]
+        if isinstance(input_batch_size, str):
+            input_batch_size = None
+        input_name = session.get_inputs()[0].name
+        return cls(
+            session=session,
+            input_name=input_name,
+            inference_config=inference_config,
+            class_names=class_names,
+            device=device,
+            input_batch_size=input_batch_size,
+        )
+
+    def __init__(
+        self,
+        session: onnxruntime.InferenceSession,
+        input_name: str,
+        inference_config: InferenceConfig,
+        class_names: List[str],
+        device: torch.device,
+        input_batch_size: Optional[int],
+    ):
+        self._session = session
+        self._input_name = input_name
+        self._inference_config = inference_config
+        self._class_names = class_names
+        self._device = device
+        self._input_batch_size = input_batch_size
+        self._session_thread_lock = Lock()
+
+    @property
+    def class_names(self) -> List[str]:
+        return self._class_names
+
+    def pre_process(
+        self,
+        images: Union[torch.Tensor, List[torch.Tensor], np.ndarray, List[np.ndarray]],
+        input_color_format: Optional[ColorFormat] = None,
+        image_size: Optional[Tuple[int, int]] = None,
+        **kwargs,
+    ) -> torch.Tensor:
+        return pre_process_network_input(
+            images=images,
+            image_pre_processing=self._inference_config.image_pre_processing,
+            network_input=self._inference_config.network_input,
+            target_device=self._device,
+            input_color_format=input_color_format,
+            image_size_wh=image_size,
+        )[0]
+
+    def forward(
+        self, pre_processed_images: PreprocessedInputs, **kwargs
+    ) -> torch.Tensor:
+        with self._session_thread_lock:
+            return run_session_with_batch_size_limit(
+                session=self._session,
+                inputs={self._input_name: pre_processed_images},
+                min_batch_size=self._input_batch_size,
+                max_batch_size=self._input_batch_size,
+            )[0]
+
+    def post_process(
+        self,
+        model_results: torch.Tensor,
+        confidence: float = 0.5,
+        **kwargs,
+    ) -> List[MultiLabelClassificationPrediction]:
+        if self._inference_config.post_processing.fused:
+            model_results = model_results
+        else:
+            model_results = torch.nn.functional.sigmoid(model_results)
+        results = []
+        for batch_element_confidence in model_results:
+            predicted_classes = torch.argwhere(
+                batch_element_confidence >= confidence
+            ).squeeze(dim=-1)
+            results.append(
+                MultiLabelClassificationPrediction(
+                    class_ids=predicted_classes,
+                    confidence=batch_element_confidence,
+                )
+            )
+        return results