onnxruntime_extensions 0.14.0__cp313-cp313-macosx_11_0_x86_64.whl

onnxruntime_extensions/pp_api.py

@@ -0,0 +1,83 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+###############################################################################
+
+import os
+from . import _extensions_pydll as _C
+if not hasattr(_C, "delete_object"):
+    raise ImportError(
+        "onnxruntime_extensions is not built with pre-processing C API\n"
+        "To enable it, please build the package with --ortx-user-option=pp_api")
+
+create_processor = _C.create_processor
+load_images = _C.load_images
+image_pre_process = _C.image_pre_process
+tensor_result_get_at = _C.tensor_result_get_at
+
+create_tokenizer = _C.create_tokenizer
+batch_tokenize = _C.batch_tokenize
+batch_detokenize = _C.batch_detokenize
+
+delete_object = _C.delete_object
+
+
+class Tokenizer:
+    def __init__(self, tokenizer_dir):
+        self.tokenizer = None
+        if os.path.isdir(tokenizer_dir):
+            self.tokenizer = create_tokenizer(tokenizer_dir)
+        else:
+            try:
+                from transformers.utils import cached_file
+                resolved_full_file = cached_file(
+                    tokenizer_dir, "tokenizer.json")
+                resolved_config_file = cached_file(
+                    tokenizer_dir, "tokenizer_config.json")
+            except ImportError:
+                raise ValueError(
+                    f"Directory '{tokenizer_dir}' not found and transformers is not available")
+            if not os.path.exists(resolved_full_file):
+                raise FileNotFoundError(
+                    f"Downloaded HF file '{resolved_full_file}' cannot be found")
+            if (os.path.dirname(resolved_full_file) != os.path.dirname(resolved_config_file)):
+                raise FileNotFoundError(
+                    f"Downloaded HF files '{resolved_full_file}' "
+                    f"and '{resolved_config_file}' are not in the same directory")
+
+            tokenizer_dir = os.path.dirname(resolved_full_file)
+            self.tokenizer = create_tokenizer(tokenizer_dir)
+
+    def tokenize(self, text):
+        if isinstance(text, (list, tuple)):
+            return batch_tokenize(self.tokenizer, text)
+        return batch_tokenize(self.tokenizer, [text])[0]
+
+    def detokenize(self, tokens):
+        return batch_detokenize(self.tokenizer, [tokens])
+
+    def __del__(self):
+        if delete_object and self.tokenizer:
+            delete_object(self.tokenizer)
+        self.tokenizer = None
+
+
+class ImageProcessor:
+    def __init__(self, processor_json):
+        self.processor = create_processor(processor_json)
+
+    def pre_process(self, images):
+        if isinstance(images, str):
+            images = [images]
+        if isinstance(images, list):
+            images = load_images(images)
+        return image_pre_process(self.processor, images)
+
+    @staticmethod
+    def to_numpy(result, idx):
+        return tensor_result_get_at(result, idx)
+
+    def __del__(self):
+        if delete_object and self.processor:
+            delete_object(self.processor)
+        self.processor = None

onnxruntime_extensions/tools/add_HuggingFace_CLIPImageProcessor_to_model.py

@@ -0,0 +1,171 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+import argparse
+import os
+from pathlib import Path
+import onnx
+
+from .pre_post_processing import *
+
+
+class Dict2Class(object):
+    '''
+    Convert dict to class 
+    '''
+    def __init__(self, my_dict):
+        for key in my_dict:
+            setattr(self, key, my_dict[key])
+
+def image_processor(args: argparse.Namespace):
+    # support user providing encoded image bytes
+    steps = [
+        ConvertImageToBGR(),  # custom op to convert jpg/png to BGR (output is HWC)
+    ]  # Normalization params are for RGB ordering
+    if args.do_convert_rgb:
+        steps.append(ReverseAxis(axis=2, dim_value=3, name="BGR_to_RGB"))
+
+    if args.do_resize:
+        to_size = args.size
+        steps.append(Resize(to_size))
+
+    if args.do_center_crop:
+        to_size = args.crop_size
+        to_size = (to_size, to_size)
+        steps.append(CenterCrop(*to_size))
+
+    if args.do_rescale:
+        steps.append(ImageBytesToFloat(args.rescale_factor))
+
+    steps.append(ChannelsLastToChannelsFirst())
+
+    if args.do_normalize:
+        mean_std = list(zip(args.image_mean, args.image_std))
+        layout = 'CHW'
+        steps.append(Normalize(mean_std, layout=layout))
+
+    steps.append(Unsqueeze([0]))  # add batch dim
+
+    return steps
+
+
+def clip_image_processor(model_file: Path, output_file: Path, **kwargs):
+    """
+    Used for models like stable-diffusion. should be compatible with 
+    https://github.com/huggingface/transformers/blob/main/src/transformers/models/clip/image_processing_clip.py
+
+    It's similar to 'CLIP' image processor, and aligns with the HuggingFace class name.
+
+    A typical usage example is used in Stable diffusion.
+
+    :param model_file: The input model file path.
+    :param output_file: The output file path, where the finalized model saved to.
+    :param Kwargs
+        onnx_opset: The opset version of onnx model, default(18).
+        do_convert_rgb: Convert image from BGR to RGB. default(True)
+        do_resize: Resize the image's (height, width) dimensions to the specified `size`. default(True)
+        size: The shortest edge of the image is resized to size. Default(224)
+        resample: An optional resampling filter. Default(cubic)
+        do_center_crop: Whether to center crop the image to the specified `crop_size`. Default(True)
+        crop_size: Size of the output image after applying `center_crop`. Default(224)
+        do_rescale: Whether to rescale the image by the specified scale (rescale_factor). Default(True)
+        rescale_factor: Scale factor to use if rescaling the image. Default(1/255)
+        do_normalize: Whether to normalize the image. Default(True)
+        image_mean: Mean values for image normalization. Default([0.485, 0.456, 0.406])
+        image_std: Standard deviation values for image normalization. Default([0.229, 0.224, 0.225])
+
+
+    """
+    args = Dict2Class(kwargs)
+    # Load model
+    model = onnx.load(str(model_file.resolve(strict=True)))
+    inputs = [create_named_value("image", onnx.TensorProto.UINT8, ["num_bytes"])]
+
+    pipeline = PrePostProcessor(inputs, args.opset)
+
+    preprocessing = image_processor(args)
+
+    pipeline.add_pre_processing(preprocessing)
+
+    new_model = pipeline.run(model)
+    onnx.save_model(new_model, str(output_file.resolve()))
+    print(f"Updated model saved to {output_file}")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        os.path.basename(__file__),
+        description="""Add CLIPImageProcessor to a model.
+
+        The updated model will be written in the same location as the original model, 
+        with '.onnx' updated to '.with_clip_processor.onnx'
+
+        Example usage:
+            object detection:
+            - python -m onnxruntime_extensions.tools.add_HuggingFace_CLIPImageProcessor_to_model model.onnx  
+        """,
+    )
+
+    parser.add_argument(
+        "--opset", type=int, required=False, default=18,
+        help="ONNX opset to use. Minimum allowed is 16. Opset 18 is required for Resize with anti-aliasing.",
+    )
+    
+    parser.add_argument(
+        "--do_resize", type=bool, required=False, default=True,
+        help="Whether to resize the image's (height, width) dimensions to the specified `size`. default(True)",
+    )
+    parser.add_argument(
+        "--size", type=int, required=False, default=224,
+        help="The shortest edge of the image is resized to size. Default(224)",
+    )
+    parser.add_argument(
+        "--resample", type=str, default="cubic", choices=["cubic", "nearest","linear"],
+        help="Whether to resize the image's (height, width) dimensions to the specified `size`. Default(cubic)",
+    )
+    parser.add_argument(
+        "--do_center_crop", type=bool, default=True,
+        help="Whether to center crop the image to the specified `crop_size`. Default(True)",
+    )
+    parser.add_argument(
+        "--crop_size", type=int, default=224,
+        help="Size of the output image after applying `center_crop`. Default(224)",
+    )
+    parser.add_argument(
+        "--do_rescale", type=bool, default=True,
+        help="Whether to rescale the image by the specified scale (rescale_factor). Default(True)",
+    )
+    parser.add_argument(
+        "--rescale_factor", type=float, default=1/255,
+        help="Scale factor to use if rescaling the image. Default(1/255)",
+    )
+    parser.add_argument(
+        "--do_normalize", type=bool, default=True,
+        help="Whether to normalize the image. Default(True)",
+    )
+    parser.add_argument(
+        "--image_mean", type=str, default="[0.48145466, 0.4578275, 0.40821073]",
+        help=" Mean to use if normalizing the image, default([0.48145466, 0.4578275, 0.40821073])",
+    )
+    parser.add_argument(
+        "--image_std", type=str, default="[0.26862954, 0.26130258, 0.27577711]",
+        help="Image standard deviation., default([0.26862954, 0.26130258, 0.27577711]).",
+    )
+    parser.add_argument(
+        "--do_convert_rgb", type=bool, default=True,
+        help="Convert image from BGR to RGB. Default(True)",
+    )
+    parser.add_argument("model", type=Path, help="Provide path to ONNX model to update.")
+
+    args = parser.parse_args()
+
+    args.image_mean = [float(x) for x in args.image_mean.replace('[','').replace(']','').split(",")]
+    args.image_std = [float(x) for x in args.image_std.replace('[','').replace(']','').split(",")]  
+
+    model_path = args.model.resolve(strict=True)
+    new_model_path = model_path.with_suffix(".with_clip_processor.onnx")
+
+    clip_image_processor(model_path, new_model_path, **vars(args))
+
+
+if __name__ == "__main__":
+    main()