inference-models 0.18.3__py3-none-any.whl

inference_models/__init__.py

@@ -0,0 +1,36 @@
+import os
+
+if os.environ.get("PYTORCH_ENABLE_MPS_FALLBACK") is None:
+    os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
+if os.environ.get("TOKENIZERS_PARALLELISM") is None:
+    os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+from inference_models.entities import ColorFormat
+from inference_models.model_pipelines.auto_loaders.core import AutoModelPipeline
+from inference_models.models.auto_loaders.core import AutoModel
+from inference_models.models.base.classification import (
+    ClassificationModel,
+    ClassificationPrediction,
+    MultiLabelClassificationModel,
+    MultiLabelClassificationPrediction,
+)
+from inference_models.models.base.depth_estimation import DepthEstimationModel
+from inference_models.models.base.documents_parsing import (
+    StructuredOCRModel,
+    TextOnlyOCRModel,
+)
+from inference_models.models.base.embeddings import TextImageEmbeddingModel
+from inference_models.models.base.instance_segmentation import (
+    InstanceDetections,
+    InstanceSegmentationModel,
+)
+from inference_models.models.base.keypoints_detection import (
+    KeyPoints,
+    KeyPointsDetectionModel,
+)
+from inference_models.models.base.object_detection import (
+    Detections,
+    ObjectDetectionModel,
+    OpenVocabularyObjectDetectionModel,
+)
+from inference_models.models.base.semantic_segmentation import SemanticSegmentationModel

inference_models/configuration.py

@@ -0,0 +1,72 @@
+import os
+
+import torch
+
+from inference_models.utils.environment import parse_comma_separated_values, str2bool
+
+ONNXRUNTIME_EXECUTION_PROVIDERS = parse_comma_separated_values(
+    values=os.getenv(
+        "ONNXRUNTIME_EXECUTION_PROVIDERS",
+        "CUDAExecutionProvider,OpenVINOExecutionProvider,CoreMLExecutionProvider,CPUExecutionProvider",
+    )
+    .strip("[")
+    .strip("]")
+)
+DEFAULT_DEVICE_STR = os.getenv(
+    "DEFAULT_DEVICE",
+    ("cuda" if torch.cuda.is_available() else "cpu"),
+)
+DEFAULT_DEVICE = torch.device(DEFAULT_DEVICE_STR)
+ROBOFLOW_API_KEY = os.getenv("ROBOFLOW_API_KEY")
+API_CALLS_TIMEOUT = int(os.getenv("API_CALLS_TIMEOUT", "5"))
+API_CALLS_MAX_TRIES = int(os.getenv("API_CALLS_MAX_TRIES", "3"))
+IDEMPOTENT_API_REQUEST_CODES_TO_RETRY = set(
+    int(e.strip())
+    for e in os.getenv(
+        "IDEMPOTENT_API_REQUEST_CODES_TO_RETRY", "408,429,502,503,504"
+    ).split(",")
+)
+ROBOFLOW_ENVIRONMENT = os.getenv("ROBOFLOW_ENVIRONMENT", "prod")
+ROBOFLOW_API_HOST = os.getenv(
+    "ROBOFLOW_API_HOST",
+    (
+        "https://api.roboflow.com"
+        if ROBOFLOW_ENVIRONMENT.lower() == "prod"
+        else "https://api.roboflow.one"
+    ),
+)
+RUNNING_ON_JETSON = os.getenv("RUNNING_ON_JETSON")
+L4T_VERSION = os.getenv("L4T_VERSION")
+INFERENCE_HOME = os.getenv("INFERENCE_HOME", "/tmp/cache")
+DISABLE_INTERACTIVE_PROGRESS_BARS = str2bool(
+    os.getenv("DISABLE_INTERACTIVE_PROGRESS_BARS", "False")
+)
+LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING")
+VERBOSE_LOG_LEVEL = os.getenv("VERBOSE_LOG_LEVEL", "INFO")
+DISABLE_VERBOSE_LOGGER = str2bool(os.getenv("DISABLE_VERBOSE_LOGGER", "False"))
+AUTO_LOADER_CACHE_EXPIRATION_MINUTES = int(
+    os.getenv("AUTO_LOADER_CACHE_EXPIRATION_MINUTES", "1440")
+)
+ALLOW_URL_INPUT = str2bool(os.getenv("ALLOW_URL_INPUT", True))
+ALLOW_NON_HTTPS_URL_INPUT = str2bool(os.getenv("ALLOW_NON_HTTPS_URL_INPUT", False))
+ALLOW_URL_INPUT_WITHOUT_FQDN = str2bool(
+    os.getenv("ALLOW_URL_INPUT_WITHOUT_FQDN", False)
+)
+WHITELISTED_DESTINATIONS_FOR_URL_INPUT = os.getenv(
+    "WHITELISTED_DESTINATIONS_FOR_URL_INPUT"
+)
+if WHITELISTED_DESTINATIONS_FOR_URL_INPUT is not None:
+    WHITELISTED_DESTINATIONS_FOR_URL_INPUT = parse_comma_separated_values(
+        WHITELISTED_DESTINATIONS_FOR_URL_INPUT
+    )
+
+BLACKLISTED_DESTINATIONS_FOR_URL_INPUT = os.getenv(
+    "BLACKLISTED_DESTINATIONS_FOR_URL_INPUT"
+)
+if BLACKLISTED_DESTINATIONS_FOR_URL_INPUT is not None:
+    BLACKLISTED_DESTINATIONS_FOR_URL_INPUT = parse_comma_separated_values(
+        BLACKLISTED_DESTINATIONS_FOR_URL_INPUT
+    )
+ALLOW_LOCAL_STORAGE_ACCESS_FOR_REFERENCE_DATA = os.getenv(
+    "ALLOW_LOCAL_STORAGE_ACCESS_FOR_REFERENCE_DATA"
+)

inference_models/constants.py

@@ -0,0 +1,2 @@
+HTTP_CODES_TO_RETRY = {408, 429, 502, 503, 504}
+DOWNLOAD_CHUNK_SIZE = 8 * 1024

inference_models/entities.py

@@ -0,0 +1,5 @@
+from collections import namedtuple
+from typing import Literal
+
+ImageDimensions = namedtuple("ImageDimensions", ["height", "width"])
+ColorFormat = Literal["rgb", "bgr"]

inference_models/errors.py

@@ -0,0 +1,137 @@
+from typing import Optional
+
+
+class BaseInferenceError(Exception):
+
+    def __init__(self, message: str, help_url: Optional[str] = None):
+        super().__init__(message)
+        self._help_url = help_url
+
+    @property
+    def help_url(self) -> Optional[str]:
+        return self._help_url
+
+    def __str__(self) -> str:
+        if self._help_url is None:
+            return super().__str__()
+        return f"{super().__str__()} - VISIT {self._help_url} FOR FURTHER SUPPORT"
+
+
+class AssumptionError(BaseInferenceError):
+    pass
+
+
+class EnvironmentConfigurationError(BaseInferenceError):
+    pass
+
+
+class ModelRuntimeError(BaseInferenceError):
+    pass
+
+
+class ModelInputError(BaseInferenceError):
+    pass
+
+
+class RetryError(BaseInferenceError):
+    pass
+
+
+class ModelRetrievalError(BaseInferenceError):
+    pass
+
+
+class UntrustedFileError(BaseInferenceError):
+    pass
+
+
+class FileHashSumMissmatch(BaseInferenceError):
+    pass
+
+
+class UnauthorizedModelAccessError(ModelRetrievalError):
+    pass
+
+
+class ModelMetadataConsistencyError(ModelRetrievalError):
+    pass
+
+
+class ModelMetadataHandlerNotImplementedError(ModelRetrievalError):
+    pass
+
+
+class InvalidEnvVariable(BaseInferenceError):
+    pass
+
+
+class ModelPackageNegotiationError(BaseInferenceError):
+    pass
+
+
+class UnknownBackendTypeError(ModelPackageNegotiationError):
+    pass
+
+
+class UnknownQuantizationError(ModelPackageNegotiationError):
+    pass
+
+
+class InvalidRequestedBatchSizeError(ModelPackageNegotiationError):
+    pass
+
+
+class RuntimeIntrospectionError(ModelPackageNegotiationError):
+    pass
+
+
+class JetsonTypeResolutionError(RuntimeIntrospectionError):
+    pass
+
+
+class NoModelPackagesAvailableError(ModelPackageNegotiationError):
+    pass
+
+
+class AmbiguousModelPackageResolutionError(ModelPackageNegotiationError):
+    pass
+
+
+class ModelLoadingError(BaseInferenceError):
+    pass
+
+
+class InsecureModelIdentifierError(ModelLoadingError):
+    pass
+
+
+class DirectLocalStorageAccessError(ModelLoadingError):
+    pass
+
+
+class ModelImplementationLoaderError(ModelLoadingError):
+    pass
+
+
+class CorruptedModelPackageError(ModelLoadingError):
+    pass
+
+
+class MissingDependencyError(BaseInferenceError):
+    pass
+
+
+class InvalidParameterError(BaseInferenceError):
+    pass
+
+
+class DependencyModelParametersValidationError(ModelLoadingError):
+    pass
+
+
+class ModelPipelineInitializationError(ModelLoadingError):
+    pass
+
+
+class ModelPipelineNotFound(ModelPipelineInitializationError):
+    pass

inference_models/logger.py

@@ -0,0 +1,52 @@
+import logging
+
+from inference_models.configuration import (
+    DISABLE_VERBOSE_LOGGER,
+    LOG_LEVEL,
+    VERBOSE_LOG_LEVEL,
+)
+
+
+def configure_log_level(
+    logger: logging.Logger, log_level: str, fallback_level: int
+) -> None:
+    log_level = getattr(logging, log_level, fallback_level)
+    logger.setLevel(log_level)
+    if not logger.handlers:
+        handler = logging.StreamHandler()
+        formatter = logging.Formatter("%(message)s")
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+    for handler in logger.handlers:
+        handler.setLevel(log_level)
+    logger.propagate = False
+
+
+LOGGER = logging.getLogger("inference-models")
+configure_log_level(logger=LOGGER, log_level=LOG_LEVEL, fallback_level=logging.WARNING)
+VERBOSE_LOGGER = logging.getLogger("inference-models-verbose")
+configure_log_level(
+    logger=VERBOSE_LOGGER, log_level=VERBOSE_LOG_LEVEL, fallback_level=logging.INFO
+)
+
+
+def verbose_info(
+    message: str,
+    verbose_requested: bool = True,
+) -> None:
+    if DISABLE_VERBOSE_LOGGER:
+        return None
+    if not verbose_requested:
+        return None
+    VERBOSE_LOGGER.info(message)
+
+
+def verbose_debug(
+    message: str,
+    verbose_requested: bool = True,
+) -> None:
+    if DISABLE_VERBOSE_LOGGER:
+        return None
+    if not verbose_requested:
+        return None
+    VERBOSE_LOGGER.debug(message)

inference_models/model_pipelines/auto_loaders/core.py

@@ -0,0 +1,120 @@
+from typing import List, Optional, Union
+
+from rich.console import Console
+from rich.tree import Tree
+
+from inference_models.errors import ModelPipelineInitializationError
+from inference_models.logger import verbose_info
+from inference_models.model_pipelines.auto_loaders.pipelines_registry import (
+    REGISTERED_PIPELINES,
+    get_default_pipeline_parameters,
+    resolve_pipeline_class,
+)
+from inference_models.models.auto_loaders.access_manager import ModelAccessManager
+from inference_models.models.auto_loaders.auto_resolution_cache import (
+    AutoResolutionCache,
+)
+from inference_models.models.auto_loaders.core import AutoModel
+from inference_models.models.auto_loaders.dependency_models import (
+    DependencyModelParameters,
+    prepare_dependency_model_parameters,
+)
+from inference_models.models.auto_loaders.entities import AnyModel
+
+
+class AutoModelPipeline:
+
+    @classmethod
+    def list_available_pipelines(cls) -> None:
+        console = Console()
+        tree = Tree("Available Model Pipelines:")
+        for pipeline_id in sorted(REGISTERED_PIPELINES):
+            tree.add(pipeline_id)
+        console.print(tree)
+
+    @classmethod
+    def from_pretrained(
+        cls,
+        pipline_id: str,
+        models_parameters: Optional[
+            List[Optional[Union[str, dict, DependencyModelParameters]]]
+        ] = None,
+        weights_provider: str = "roboflow",
+        api_key: Optional[str] = None,
+        max_package_loading_attempts: Optional[int] = None,
+        verbose: bool = False,
+        model_download_file_lock_acquire_timeout: int = 10,
+        allow_untrusted_packages: bool = False,
+        trt_engine_host_code_allowed: bool = True,
+        allow_local_code_packages: bool = True,
+        verify_hash_while_download: bool = True,
+        download_files_without_hash: bool = False,
+        use_auto_resolution_cache: bool = True,
+        auto_resolution_cache: Optional[AutoResolutionCache] = None,
+        allow_direct_local_storage_loading: bool = True,
+        model_access_manager: Optional[ModelAccessManager] = None,
+        **kwargs,
+    ) -> AnyModel:
+        pipeline_class = resolve_pipeline_class(pipline_id=pipline_id)
+        models = []
+        verbose_info(
+            message=f"Initializing models for pipeline `{pipline_id}`",
+            verbose_requested=verbose,
+        )
+        default_parameters = get_default_pipeline_parameters(pipline_id=pipline_id)
+        if models_parameters is None and default_parameters is None:
+            raise ModelPipelineInitializationError(
+                message=f"Could not initialize model pipeline `{pipline_id}` - models parameters not provided and "
+                f"default values not registered in the library. If you run locally, please verify your "
+                f"integration - it must specify the models to be used by the pipeline. If you use Roboflow "
+                f"hosted solution, contact us to get help.",
+                help_url="https://todo",
+            )
+        if models_parameters is None:
+            models_parameters = default_parameters
+        if default_parameters is None:
+            default_parameters = [None] * len(models_parameters)
+        for idx, model_parameters in enumerate(models_parameters):
+            if model_parameters is None:
+                parameters_to_be_used = (
+                    default_parameters[idx] if idx < len(default_parameters) else None
+                )
+            else:
+                parameters_to_be_used = model_parameters
+            resolved_model_parameters = prepare_dependency_model_parameters(
+                model_parameters=parameters_to_be_used
+            )
+            verbose_info(
+                message=f"Initializing model: `{resolved_model_parameters.model_id_or_path}`",
+                verbose_requested=verbose,
+            )
+            model = AutoModel.from_pretrained(
+                model_id_or_path=resolved_model_parameters.model_id_or_path,
+                weights_provider=weights_provider,
+                api_key=api_key,
+                model_package_id=resolved_model_parameters.model_package_id,
+                backend=resolved_model_parameters.backend,
+                batch_size=resolved_model_parameters.batch_size,
+                quantization=resolved_model_parameters.quantization,
+                onnx_execution_providers=resolved_model_parameters.onnx_execution_providers,
+                device=resolved_model_parameters.device,
+                default_onnx_trt_options=resolved_model_parameters.default_onnx_trt_options,
+                max_package_loading_attempts=max_package_loading_attempts,
+                verbose=verbose,
+                model_download_file_lock_acquire_timeout=model_download_file_lock_acquire_timeout,
+                allow_untrusted_packages=allow_untrusted_packages,
+                trt_engine_host_code_allowed=trt_engine_host_code_allowed,
+                allow_local_code_packages=allow_local_code_packages,
+                verify_hash_while_download=verify_hash_while_download,
+                download_files_without_hash=download_files_without_hash,
+                use_auto_resolution_cache=use_auto_resolution_cache,
+                auto_resolution_cache=auto_resolution_cache,
+                allow_direct_local_storage_loading=allow_direct_local_storage_loading,
+                model_access_manager=model_access_manager,
+                nms_fusion_preferences=resolved_model_parameters.nms_fusion_preferences,
+                model_type=resolved_model_parameters.model_type,
+                task_type=resolved_model_parameters.task_type,
+                **resolved_model_parameters.kwargs,
+            )
+            models.append(model)
+        return pipeline_class.with_models(models, **kwargs)

inference_models/model_pipelines/auto_loaders/pipelines_registry.py

@@ -0,0 +1,36 @@
+from typing import Dict, List, Optional, Union
+
+from inference_models.errors import ModelPipelineNotFound
+from inference_models.utils.imports import LazyClass
+
+REGISTERED_PIPELINES: Dict[str, LazyClass] = {
+    "face-and-gaze-detection": LazyClass(
+        module_name="inference_models.model_pipelines.face_and_gaze_detection.mediapipe_l2cs",
+        class_name="FaceAndGazeDetectionMPAndL2CS",
+    )
+}
+
+DEFAULT_PIPELINES_PARAMETERS: Dict[str, List[Union[str, dict]]] = {
+    "face-and-gaze-detection": [
+        "mediapipe/face-detector",
+        "l2cs-net/rn50",
+    ]
+}
+
+
+def resolve_pipeline_class(pipline_id: str) -> type:
+    if pipline_id not in REGISTERED_PIPELINES:
+        raise ModelPipelineNotFound(
+            message=f"Could not find model pipeline with id: `{pipline_id}`. "
+            f"Registered pipelines: {list(REGISTERED_PIPELINES.keys())}. This error ma be caused by typo "
+            f"in the identifier, or pipeline is not registered / not supported in the environment you try to "
+            f"run it.",
+            help_url="https://todo",
+        )
+    return REGISTERED_PIPELINES[pipline_id].resolve()
+
+
+def get_default_pipeline_parameters(
+    pipline_id: str,
+) -> Optional[List[Union[str, dict]]]:
+    return DEFAULT_PIPELINES_PARAMETERS.get(pipline_id)

inference_models/model_pipelines/face_and_gaze_detection/mediapipe_l2cs.py

@@ -0,0 +1,200 @@
+from typing import Any, List, Optional, Tuple, Union
+
+import numpy as np
+import torch
+
+from inference_models import Detections, KeyPoints
+from inference_models.configuration import DEFAULT_DEVICE
+from inference_models.entities import ColorFormat
+from inference_models.errors import ModelPipelineInitializationError, ModelRuntimeError
+from inference_models.models.l2cs.l2cs_onnx import (
+    DEFAULT_GAZE_MAX_BATCH_SIZE,
+    L2CSGazeDetection,
+    L2CSNetOnnx,
+)
+from inference_models.models.mediapipe_face_detection.face_detection import (
+    MediaPipeFaceDetector,
+)
+
+
+class FaceAndGazeDetectionMPAndL2CS:
+
+    @classmethod
+    def with_models(
+        cls, models: List[Any], **kwargs
+    ) -> "FaceAndGazeDetectionMPAndL2CS":
+        if len(models) != 2:
+            raise ModelPipelineInitializationError(
+                message="Model pipeline `face-and-gaze-detection` requires two models tu run - face detector "
+                "and gaze detector. If you run `inference` locally, verify the parameter of pipeline loader "
+                "to make sure that two models parameters' are provided. If you use Roboflow hosted solution, "
+                "contact us to get help.",
+                help_url="https://todo",
+            )
+        face_detector, gaze_detector = models
+        if not isinstance(face_detector, MediaPipeFaceDetector):
+            raise ModelPipelineInitializationError(
+                message="Model pipeline `face-and-gaze-detection` requires first model to be `MediaPipeFaceDetector` - "
+                "if you run `inference` locally, make sure that you initialized the pipeline pointing model of "
+                "matching type.",
+                help_url="https://todo",
+            )
+        if not isinstance(gaze_detector, L2CSNetOnnx):
+            raise ModelPipelineInitializationError(
+                message="Model pipeline `face-and-gaze-detection` requires second model to be `L2CSNet` - "
+                "if you run `inference` locally, make sure that you initialized the pipeline pointing model of "
+                "matching type.",
+                help_url="https://todo",
+            )
+        return FaceAndGazeDetectionMPAndL2CS.from_pretrained(
+            face_detector=face_detector, gaze_detector=gaze_detector, **kwargs
+        )
+
+    @classmethod
+    def from_pretrained(
+        cls,
+        face_detector: Union[str, MediaPipeFaceDetector],
+        gaze_detector: Union[str, L2CSNetOnnx],
+        onnx_execution_providers: Optional[List[Union[str, tuple]]] = None,
+        default_onnx_trt_options: bool = True,
+        device: torch.device = DEFAULT_DEVICE,
+        max_batch_size: int = DEFAULT_GAZE_MAX_BATCH_SIZE,
+        **kwargs,
+    ) -> "FaceAndGazeDetectionMPAndL2CS":
+        if isinstance(face_detector, str):
+            face_detector = MediaPipeFaceDetector.from_pretrained(
+                model_name_or_path=face_detector
+            )
+        if isinstance(gaze_detector, str):
+            gaze_detector = L2CSNetOnnx.from_pretrained(
+                model_name_or_path=gaze_detector,
+                onnx_execution_providers=onnx_execution_providers,
+                default_onnx_trt_options=default_onnx_trt_options,
+                device=device,
+                max_batch_size=max_batch_size,
+            )
+        return cls(
+            face_detector=face_detector,
+            gaze_detector=gaze_detector,
+        )
+
+    def __init__(
+        self,
+        face_detector: MediaPipeFaceDetector,
+        gaze_detector: L2CSNetOnnx,
+    ):
+        self._face_detector = face_detector
+        self._gaze_detector = gaze_detector
+
+    @property
+    def class_names(self) -> List[str]:
+        return self._face_detector.class_names
+
+    @property
+    def key_points_classes(self) -> List[List[str]]:
+        return self._face_detector.key_points_classes
+
+    @property
+    def skeletons(self) -> List[List[Tuple[int, int]]]:
+        return [[(5, 1), (1, 2), (4, 0), (0, 2), (2, 3)]]
+
+    def infer(
+        self,
+        images: Union[torch.Tensor, List[torch.Tensor], np.ndarray, List[np.ndarray]],
+        input_color_format: Optional[ColorFormat] = None,
+        conf_threshold: float = 0.25,
+        **kwargs,
+    ) -> Tuple[List[KeyPoints], List[Detections], List[L2CSGazeDetection]]:
+        key_points, detections = self._face_detector(
+            images,
+            input_color_format=input_color_format,
+            conf_thresh=conf_threshold,
+            **kwargs,
+        )
+        crops, crops_images_bounds = crop_images_to_detections(
+            images=images,
+            detections=detections,
+            device=self._gaze_detector.device,
+        )
+        gaze_detections = self._gaze_detector(crops, input_color_format="rgb", **kwargs)
+        gaze_detections_dispatched = []
+        for start, end in crops_images_bounds:
+            gaze_detections_dispatched.append(
+                L2CSGazeDetection(
+                    yaw=gaze_detections.yaw[start:end],
+                    pitch=gaze_detections.pitch[start:end],
+                )
+            )
+        return key_points, detections, gaze_detections_dispatched
+
+    def __call__(
+        self,
+        images: Union[torch.Tensor, List[torch.Tensor], np.ndarray, List[np.ndarray]],
+        input_color_format: Optional[ColorFormat] = None,
+        conf_threshold: float = 0.25,
+        **kwargs,
+    ) -> Tuple[List[KeyPoints], List[Detections], List[L2CSGazeDetection]]:
+        return self.infer(
+            images=images,
+            input_color_format=input_color_format,
+            conf_threshold=conf_threshold,
+            **kwargs,
+        )
+
+
+def crop_images_to_detections(
+    images: Union[torch.Tensor, List[torch.Tensor], np.ndarray, List[np.ndarray]],
+    detections: List[Detections],
+    device: torch.device,
+    input_color_format: Optional[ColorFormat] = None,
+) -> Tuple[List[torch.Tensor], List[Tuple[int, int]]]:
+    if isinstance(images, np.ndarray):
+        input_color_format = input_color_format or "bgr"
+        if input_color_format != "rgb":
+            images = np.ascontiguousarray(images[:, :, ::-1])
+        prepared_images = [torch.from_numpy(images).permute(2, 0, 1).to(device)]
+    elif isinstance(images, torch.Tensor):
+        input_color_format = input_color_format or "rgb"
+        images = images.to(device)
+        if len(images.shape) == 3:
+            images = images.unsqueeze(dim=0)
+        if input_color_format != "rgb":
+            images = images[:, [2, 1, 0], :, :]
+        prepared_images = [i for i in images]
+    elif isinstance(images, list) and len(images) == 0:
+        raise ModelRuntimeError(
+            message="Detected empty input to the model",
+            help_url="https://todo",
+        )
+    elif isinstance(images, list) and isinstance(images[0], np.ndarray):
+        prepared_images = []
+        input_color_format = input_color_format or "bgr"
+        for image in images:
+            if input_color_format != "rgb":
+                image = np.ascontiguousarray(image[:, :, ::-1])
+            prepared_images.append(torch.from_numpy(image).permute(2, 0, 1).to(device))
+    elif isinstance(images, list) and isinstance(images[0], torch.Tensor):
+        prepared_images = []
+        input_color_format = input_color_format or "rgb"
+        for image in images:
+            if input_color_format != "rgb":
+                image = image[[2, 1, 0], :, :]
+            prepared_images.append(image.to(device))
+    else:
+        raise ModelRuntimeError(
+            message=f"Detected unknown input batch element: {type(images)}",
+            help_url="https://todo",
+        )
+    crops = []
+    crops_images_bounds = []
+    for image, image_detections in zip(prepared_images, detections):
+        start_bound = len(crops)
+        for xyxy in image_detections.xyxy:
+            x_min, y_min, x_max, y_max = xyxy.tolist()
+            crop = image[:, y_min:y_max, x_min:x_max]
+            if crop.numel() == 0:
+                continue
+            crops.append(crop)
+        end_bound = len(crops)
+        crops_images_bounds.append((start_bound, end_bound))
+    return crops, crops_images_bounds