onnxruntime_extensions 0.12.0__cp39-cp39-win_amd64.whl → 0.14.0__cp39-cp39-win_amd64.whl

onnxruntime_extensions/_cuops.py

@@ -364,8 +364,15 @@ class SentencepieceDecoder(CustomOp):
     @classmethod
     def get_inputs(cls):
         return [
-            cls.io_def("ids", onnx.TensorProto.INT64, [None])
+            cls.io_def("ids", onnx.TensorProto.INT64, [None]),
+            cls.io_def('fairseq', onnx_proto.TensorProto.BOOL, [None])
         ]
+    
+    @classmethod
+    def input_default_values(cls):
+        return {
+            'fairseq': [False]
+        }
 
     @classmethod
     def get_outputs(cls):
@@ -491,6 +498,16 @@ class StftNorm(CustomOp):
         ]
 
 
+class HfJsonTokenizer(CustomOp):
+    @classmethod
+    def get_inputs(cls):
+        return [cls.io_def('str', onnx_proto.TensorProto.STRING, ['N'])]
+
+    @classmethod
+    def get_outputs(cls):
+        return [cls.io_def("ids", onnx.TensorProto.INT64, ['N', None])]
+
+
 # TODO: have a C++ impl.
 def _argsort_op(x, dim):
     d = numpy.argsort(x, dim)
@@ -544,4 +561,4 @@ class SingleOpGraph:
 
     @staticmethod
     def get_op_class(op_type):
-        return globals()[op_type]
+        return globals()[op_type]

onnxruntime_extensions/_hf_cvt.py

@@ -48,8 +48,9 @@ class HFTokenizerConverter(CustomOpConverter):
             model_dir = hf_tokenizer.name_or_path
         else:
             model_dir = os.path.dirname(vocab_file)
-        tokenizer_json = json.load(
-            open(os.path.join(model_dir, tokenizer_file), "r", encoding="utf-8"))
+        f = open(os.path.join(model_dir, tokenizer_file), "r", encoding="utf-8")
+        tokenizer_json = json.load(f)
+        f.close()
         # get vocab object from json file
         vocab = tokenizer_json.get("model", {}).get("vocab", {})
         sorted_merges = tokenizer_json.get("model", {}).get("merges", [])
@@ -167,7 +168,8 @@ class HFTokenizerConverter(CustomOpConverter):
 TokenOpParam = namedtuple("TokenOpParam",
                           ["pre_op", "pre_attribute_cvt",
                            "post_op", "post_attribute_cvt",
-                           "default_inputs"],
+                           "default_encoder_inputs",
+                           "default_decoder_inputs"],
                           defaults=(None, None, None, None, None))
 
 # Some tokenizers can be added by this table
@@ -175,35 +177,36 @@ TokenOpParam = namedtuple("TokenOpParam",
 # @formatter:off
 _PROCESSOR_DICT = {
     "BertTokenizer":        TokenOpParam('BertTokenizer',   HFTokenizerConverter.bert_tokenizer,
-                                         'BertDecoder',     HFTokenizerConverter.bpe_decoder, None),
+                                         'BertDecoder',     HFTokenizerConverter.bpe_decoder, None, None),
     "DistilBertTokenizer":  TokenOpParam('BertTokenizer',   HFTokenizerConverter.bert_tokenizer,
-                                         'BertDecoder',     HFTokenizerConverter.bpe_decoder, None),
+                                         'BertDecoder',     HFTokenizerConverter.bpe_decoder, None, None),
     "GPT2Tokenizer":        TokenOpParam('GPT2Tokenizer',   HFTokenizerConverter.bpe_tokenizer,
-                                         'BpeDecoder',      HFTokenizerConverter.bpe_decoder, None),
+                                         'BpeDecoder',      HFTokenizerConverter.bpe_decoder, None, None),
     "CodeGenTokenizer":     TokenOpParam('GPT2Tokenizer',   HFTokenizerConverter.bpe_tokenizer,
-                                         'BpeDecoder',      HFTokenizerConverter.bpe_decoder, None),
+                                         'BpeDecoder',      HFTokenizerConverter.bpe_decoder, None, None),
     "CLIPTokenizer":        TokenOpParam('CLIPTokenizer',   HFTokenizerConverter.clip_tokenizer,
-                                         'BpeDecoder',      HFTokenizerConverter.bpe_decoder, None),
+                                         'BpeDecoder',      HFTokenizerConverter.bpe_decoder, None, None),
     "RobertaTokenizer":     TokenOpParam('RobertaTokenizer',        HFTokenizerConverter.roberta_tokenizer,
-                                         'BpeDecoder',              HFTokenizerConverter.bpe_decoder, None),
+                                         'BpeDecoder',              HFTokenizerConverter.bpe_decoder, None, None),
     "BartTokenizer":        TokenOpParam('RobertaTokenizer',        HFTokenizerConverter.roberta_tokenizer,
-                                         'BpeDecoder',              HFTokenizerConverter.bpe_decoder, None),
+                                         'BpeDecoder',              HFTokenizerConverter.bpe_decoder, None, None),
     "LayoutLMv3Tokenizer":  TokenOpParam('RobertaTokenizer',        HFTokenizerConverter.roberta_tokenizer,
-                                         'BpeDecoder',              HFTokenizerConverter.bpe_decoder, None),
+                                         'BpeDecoder',              HFTokenizerConverter.bpe_decoder, None, None),
     "LongformerTokenizer":  TokenOpParam('RobertaTokenizer',        HFTokenizerConverter.roberta_tokenizer,
-                                         'BpeDecoder',              HFTokenizerConverter.bpe_decoder, None),
+                                         'BpeDecoder',              HFTokenizerConverter.bpe_decoder, None, None),
     "LEDTokenizer":         TokenOpParam('RobertaTokenizer',        HFTokenizerConverter.roberta_tokenizer,
-                                         'BpeDecoder',              HFTokenizerConverter.bpe_decoder, None),
+                                         'BpeDecoder',              HFTokenizerConverter.bpe_decoder, None, None),
     "MvpTokenizer":         TokenOpParam('RobertaTokenizer',        HFTokenizerConverter.roberta_tokenizer,
-                                         'BpeDecoder',              HFTokenizerConverter.bpe_decoder, None),
+                                         'BpeDecoder',              HFTokenizerConverter.bpe_decoder, None, None),
     "T5Tokenizer":          TokenOpParam('SentencepieceTokenizer',  HFTokenizerConverter.spm_tokenizer,
                                          'SentencepieceDecoder',    HFTokenizerConverter.spm_decoder,
-                                         default_inputs={'add_eos': [True]}),
+                                         default_encoder_inputs={'add_eos': [True]}, default_decoder_inputs=None),
     "LlamaTokenizer":       TokenOpParam('SpmTokenizer',            HFTokenizerConverter.bpe_tokenizer,
-                                         'BpeDecoder',              HFTokenizerConverter.bpe_decoder, None),
+                                         'BpeDecoder',              HFTokenizerConverter.bpe_decoder, None, None),
     "XLMRobertaTokenizer":  TokenOpParam('SentencepieceTokenizer',  HFTokenizerConverter.spm_tokenizer,
                                          'SentencepieceDecoder',    HFTokenizerConverter.spm_decoder,
-                                         default_inputs={'add_bos': [True], 'add_eos': [True], 'fairseq': [True]}),
+                                         default_encoder_inputs={'add_bos': [True], 'add_eos': [True], 'fairseq': [True]},
+                                         default_decoder_inputs={'fairseq': [True]}),
 }
 # @formatter:on
 
@@ -245,8 +248,8 @@ class HFTokenizerOnnxGraph:
 
         # add default_inputs into initializers to simplify the model input
         n_inputs = len(default_inputs)
-        if self.cvt_quadruple.default_inputs is not None:
-            default_inputs.update(self.cvt_quadruple.default_inputs)
+        if self.cvt_quadruple.default_encoder_inputs is not None:
+            default_inputs.update(self.cvt_quadruple.default_encoder_inputs)
             if len(default_inputs) != n_inputs:
                 raise ValueError(
                     "Op: {} does not have the inputs from its TokenOpParam.".format(_cvt_op))
@@ -286,7 +289,43 @@ class HFTokenizerOnnxGraph:
         return g
 
     def post_processing(self, **kwargs):
+        with_default_inputs = kwargs.pop("WITH_DEFAULT_INPUTS", True)
+
         _cvt_op = self.cvt_quadruple.post_op
         _cvt_func = self.cvt_quadruple.post_attribute_cvt
         cvt = partial(_cvt_func, self.cvt_obj)
-        return SingleOpGraph.build_graph(_cvt_op, cvt=cvt, **kwargs)
+        g = SingleOpGraph.build_graph(_cvt_op, cvt=cvt, **kwargs)
+
+        default_inputs = {}
+        if with_default_inputs:
+            op_class = SingleOpGraph.get_op_class(_cvt_op)
+            default_inputs = op_class.input_default_values()
+            if default_inputs is None:
+                encoder_inputs = self.cvt_quadruple.default_encoder_inputs
+                if encoder_inputs is not None and encoder_inputs["fairseq"]:
+                    default_inputs = {} # need to set to empty dict to call .update later
+                else:
+                    return g
+
+        # add default_inputs into initializers to simplify the model input
+        if self.cvt_quadruple.default_decoder_inputs is not None:
+            default_inputs.update(self.cvt_quadruple.default_decoder_inputs)
+
+        new_initializers = []
+
+        for k, v in default_inputs.items():
+            input_value_info = next((i for i in g.input if i.name == k), None)
+            if input_value_info is None:
+                raise ValueError(
+                    "The input {} is not found in the graph".format(k))
+
+            np_dtype = onnx.helper.tensor_dtype_to_np_dtype(
+                input_value_info.type.tensor_type.elem_type)
+            value = nparray(v, np_dtype)
+            new_initializers.append(onnx.numpy_helper.from_array(value, k))
+        g.initializer.extend(new_initializers)
+        new_inputs = [i for i in g.input if i.name not in default_inputs]
+        g.ClearField("input")
+        g.input.extend(new_inputs)
+
+        return g

onnxruntime_extensions/_version.py

@@ -1,2 +1,2 @@
 # Generated by setup.py, DON'T MANUALLY UPDATE IT!
-__version__ = "0.12.0"
+__version__ = "0.14.0"

onnxruntime_extensions/cvt.py

@@ -12,6 +12,24 @@ from typing import Union
 from ._hf_cvt import HFTokenizerConverter, HFTokenizerOnnxGraph  # noqa
 from ._ortapi2 import make_onnx_model, SingleOpGraph
 
+import os
+import numpy as np
+import tempfile
+import shutil
+
+# edit environment variables to avoid protobuf version mismatch
+os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
+
+from transformers.convert_slow_tokenizer import SpmConverter  # noqa: E402
+from transformers import AutoTokenizer  # noqa: E402
+from tokenizers import decoders, normalizers, pre_tokenizers, Regex  # noqa: E402
+
+
+OrtxTokenizer = None
+try:
+    from onnxruntime_extensions.pp_api import Tokenizer as OrtxTokenizer
+except ImportError:
+    pass
 
 _is_torch_available = False
 try:
@@ -24,11 +42,150 @@ except ImportError:
 
 _PRE_POST_PAIR = {'TrieTokenizer': "TrieDetokenizer"}
 
+def _get_prepend_scheme(add_prefix_space: bool, original_tokenizer) -> str:
+    if add_prefix_space:
+        prepend_scheme = "always"
+        if not getattr(original_tokenizer, "legacy", True):
+            prepend_scheme = "first"
+    else:
+        prepend_scheme = "never"
+    return prepend_scheme
+
+
+class Baichuan2Converter(SpmConverter):
+    handle_byte_fallback = True
+
+    def __init__(self, original_tokenizer):
+        super().__init__(original_tokenizer)
+        original_tokenizer.add_prefix_space = False
+
+    def vocab(self, proto):
+        vocab = [
+            (self.original_tokenizer.convert_ids_to_tokens(0), 0.0),
+            (self.original_tokenizer.convert_ids_to_tokens(1), 0.0),
+            (self.original_tokenizer.convert_ids_to_tokens(2), 0.0),
+        ]
+        vocab += [(piece.piece, piece.score) for piece in proto.pieces[3:]]
+        return vocab
+
+    def unk_id(self, proto):
+        unk_id = 0
+        return unk_id
+
+    def decoder(self, replacement, add_prefix_space):
+        sequence = [
+            decoders.Replace("▁", " "),
+            decoders.ByteFallback(),
+            decoders.Fuse(),
+        ]
+        if add_prefix_space:
+            sequence += [decoders.Strip(content=" ", left=1)]
+        return decoders.Sequence(sequence)
+
+    def normalizer(self, proto):
+        if getattr(self.original_tokenizer, "legacy", True):
+            sequence = []
+            if getattr(self.original_tokenizer, "add_prefix_space", True):
+                sequence += [normalizers.Prepend(prepend="▁")]
+            sequence += [normalizers.Replace(pattern=" ", content="▁")]
+            return normalizers.Sequence(sequence)
+        return None  # non-legacy, no normalizer
+
+    def pre_tokenizer(self, replacement, add_prefix_space):
+        if not getattr(self.original_tokenizer, "legacy", True):  # non-legacy, we need a replace
+            prepend_scheme = _get_prepend_scheme(add_prefix_space, self.original_tokenizer)
+            return pre_tokenizers.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme, split=False)
+        else:
+            return super().pre_tokenizer(replacement, add_prefix_space)
+
+
+class ChatGlmConverter(SpmConverter):
+    def normalizer(self, proto):
+        precompiled_charsmap = proto.normalizer_spec.precompiled_charsmap
+        _normalizers = [
+            normalizers.Strip(left=False, right=True),  # stripping is important
+            normalizers.Replace(Regex(" {2,}"), "▁"),
+        ]
+        return normalizers.Sequence([normalizers.Precompiled(precompiled_charsmap)] + _normalizers)
+
+    def pre_tokenizer(self, replacement, add_prefix_space):
+        prepend_scheme = "always"
+        if hasattr(self.original_tokenizer, "legacy") and not self.original_tokenizer.legacy:
+            prepend_scheme = "first"
+        return pre_tokenizers.Metaspace(
+            replacement=replacement, add_prefix_space=add_prefix_space, prepend_scheme=prepend_scheme
+        )
+
+
+JSON_TOKEN_CONVERTERS = {
+    "BaichuanTokenizer": Baichuan2Converter,
+    "ChatGLMTokenizer": ChatGlmConverter,
+}
+
+# Save tokenizer JSON files using HuggingFace AutoTokenizer
+def convert_tokenizer(model_path, output_dir):
+    tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
+    if output_dir is None:
+        if os.path.isdir(model_path):
+            output_dir = model_path
+        else:
+            # create a temporary directory
+            output_dir = tempfile.mkdtemp()
+            tokenizer.save_pretrained(output_dir)
+        json_path = os.path.join(output_dir, "tokenizer.json")
+
+    if type(tokenizer).__name__ in JSON_TOKEN_CONVERTERS:
+        GenericSpmConverter = JSON_TOKEN_CONVERTERS[type(tokenizer).__name__]
+
+    converted = GenericSpmConverter(tokenizer).converted()
+    converted.save(json_path)
+    print(f"**Tokenizer saved to {json_path}")
+    return output_dir
+
+# Validate tokenizer files downloaded from memory
+def validate_tokenizer(model_path, output_dir):
+    test_sentence = "I like walking my cute dog\n and\x17 then, 生活的真谛是   \t\t\t\t \n\n61"
+    if OrtxTokenizer is None:
+        print("onnxruntime_extensions package was built with C API enabled, skipping tokenization test")
+    ortx_tokenizer = OrtxTokenizer(output_dir)
+    tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True, use_fast=False)
+    expected_ids = tokenizer(test_sentence, return_tensors="np")["input_ids"]
+    ortx_ids = np.asarray(ortx_tokenizer.tokenize(test_sentence))
+    assert np.array_equal(expected_ids[0], ortx_ids), f"Tokenization mismatch: {expected_ids[0]} != {ortx_ids}"
+    print("Tokenization test passed")
+
+# Download tokenizer JSON files from memory
+def download_tokenizer(tokenizer_dir, output_dir):
+    try:
+        from transformers.utils import cached_file
+
+        resolved_full_file = cached_file(tokenizer_dir, "tokenizer.json")
+        resolved_config_file = cached_file(tokenizer_dir, "tokenizer_config.json")
+    except ImportError:
+        raise ValueError(f"Directory '{tokenizer_dir}' not found and transformers is not available")
+    if not os.path.exists(resolved_full_file):
+        raise FileNotFoundError(f"Downloaded HF file '{resolved_full_file}' cannot be found")
+    if os.path.dirname(resolved_full_file) != os.path.dirname(resolved_config_file):
+        raise FileNotFoundError(
+            f"Downloaded HF files '{resolved_full_file}' " f"and '{resolved_config_file}' are not in the same directory"
+        )
+
+    if output_dir is None or len(output_dir) == 0:
+        output_dir = os.path.dirname(resolved_full_file)
+        print(f"Using {output_dir} as output directory")
+        return output_dir
+    else:
+        # copy the files to the output directory
+        shutil.copy(resolved_full_file, output_dir)
+        shutil.copy(resolved_config_file, output_dir)
+        return output_dir
+
 
 def gen_processing_models(processor: Union[str, object],
                           pre_kwargs: dict = None,
                           post_kwargs: dict = None,
                           opset: int = None,
+                          schema_v2: bool = False,
                           **kwargs):
     """
     Generate the pre- and post-processing ONNX model, basing on the name or HF class.
@@ -47,6 +204,9 @@ def gen_processing_models(processor: Union[str, object],
         Keyword arguments for generating the post-processing model
     opset: int
         the target opset version of the model
+    schema_v2: bool
+        the flag for using embedded tokenizer files; this option leverages the blob-loading functionality
+        which loads HF tokenizers from memory rather than using the tokenizer files in HF JSON format.
     kwargs:
         The additional arguments for generating models
 
@@ -58,11 +218,42 @@ def gen_processing_models(processor: Union[str, object],
     if pre_kwargs is None and post_kwargs is None:
         raise ValueError(
             "Either pre_kwargs or post_kwargs should be provided. None means no processing graph output.")
-    if isinstance(processor, str):
+
+    # If true, we get the tokenizer JSON files by either downloading from cache or using HuggingFace AutoTokenizer
+    # to convert them, and then create an ONNX model with the JSON files as strings in the model attributes (attrs).
+    if schema_v2:
+        model_name = processor if isinstance(processor, str) else type(processor).__name__
+
+        converted_tokenizer = {"Baichuan2", "chatglm"}
+        need_convert = False
+        for token in converted_tokenizer:
+            if model_name.find(token) != -1:
+                need_convert = True
+                break
+
+        if need_convert:
+            model_dir = convert_tokenizer(model_name)
+            validate_tokenizer(model_name, None)
+        else:
+            model_dir = download_tokenizer(model_name, None)
+
+        # Load the content of tokenizer.json into a string
+        with open(f"{model_dir}/tokenizer.json", "r", encoding="utf-8") as f:
+            tokenizer_vocab = f.read()
+
+        # Load the content of tokenizer_config.json into a string
+        with open(f"{model_dir}/tokenizer_config.json", "r", encoding="utf-8") as f:
+            tokenizer_config = f.read()
+
+        # Create an ONNX model with these JSON file strings in attrs
         g_pre, g_post = (None, None)
-        if pre_kwargs:
-            g_pre = SingleOpGraph.build_graph(processor, **pre_kwargs)
-        if post_kwargs:
+        if pre_kwargs is not None:
+            # Add tokenizer_vocab and tokenizer_config to the kwargs
+            # so they are added to attrs in build_graph
+            pre_kwargs['tokenizer_vocab'] = tokenizer_vocab
+            pre_kwargs['tokenizer_config'] = tokenizer_config
+            g_pre = SingleOpGraph.build_graph("HfJsonTokenizer", **pre_kwargs)
+        if post_kwargs is not None:
             if pre_kwargs is None:
                 cls_name = processor
             else:
@@ -70,27 +261,46 @@ def gen_processing_models(processor: Union[str, object],
                     raise RuntimeError(
                         f"Cannot locate the post processing operator name from {processor}")
                 cls_name = _PRE_POST_PAIR[processor]
+            # Add tokenizer_vocab and tokenizer_config to the kwargs
+            # so they are added to attrs in build_graph
+            post_kwargs['tokenizer_vocab'] = tokenizer_vocab
+            post_kwargs['tokenizer_config'] = tokenizer_config
             g_post = SingleOpGraph.build_graph(cls_name, **post_kwargs)
         return make_onnx_model(g_pre) if g_pre else None, make_onnx_model(g_post) if g_post else None
-
-    cls_name = type(processor).__name__
-    if cls_name == "WhisperProcessor":
-        if WhisperDataProcGraph is None:
-            raise ValueError(
-                "The Whisper processor needs torch.onnx support, please install pytorch 2.0 and above")
-        _converter = WhisperDataProcGraph(processor, opset=opset, **kwargs)
-        pre_m = _converter.pre_processing(
-            **pre_kwargs) if pre_kwargs is not None else None
-        post_m = _converter.post_processing(
-            **post_kwargs) if post_kwargs is not None else None
-        return pre_m, post_m
-    elif HFTokenizerOnnxGraph.is_supported(processor):
-        _converter = HFTokenizerOnnxGraph(processor)
-        pre_g = _converter.pre_processing(
-            **pre_kwargs) if pre_kwargs is not None else None
-        post_g = _converter.post_processing(
-            **post_kwargs) if post_kwargs is not None else None
-        return make_onnx_model(pre_g) if pre_g else None, \
-            make_onnx_model(post_g) if post_g else None
     else:
-        raise ValueError(f"Unsupported processor/tokenizer: {cls_name}")
+        if isinstance(processor, str):
+            g_pre, g_post = (None, None)
+            if pre_kwargs:
+                g_pre = SingleOpGraph.build_graph(processor, **pre_kwargs)
+            if post_kwargs:
+                if pre_kwargs is None:
+                    cls_name = processor
+                else:
+                    if processor not in _PRE_POST_PAIR:
+                        raise RuntimeError(
+                            f"Cannot locate the post processing operator name from {processor}")
+                    cls_name = _PRE_POST_PAIR[processor]
+                g_post = SingleOpGraph.build_graph(cls_name, **post_kwargs)
+            return make_onnx_model(g_pre) if g_pre else None, make_onnx_model(g_post) if g_post else None
+
+        cls_name = type(processor).__name__
+        if cls_name == "WhisperProcessor":
+            if WhisperDataProcGraph is None:
+                raise ValueError(
+                    "The Whisper processor needs torch.onnx support, please install pytorch 2.0 and above")
+            _converter = WhisperDataProcGraph(processor, opset=opset, **kwargs)
+            pre_m = _converter.pre_processing(
+                **pre_kwargs) if pre_kwargs is not None else None
+            post_m = _converter.post_processing(
+                **post_kwargs) if post_kwargs is not None else None
+            return pre_m, post_m
+        elif HFTokenizerOnnxGraph.is_supported(processor):
+            _converter = HFTokenizerOnnxGraph(processor)
+            pre_g = _converter.pre_processing(
+                **pre_kwargs) if pre_kwargs is not None else None
+            post_g = _converter.post_processing(
+                **post_kwargs) if post_kwargs is not None else None
+            return make_onnx_model(pre_g) if pre_g else None, \
+                make_onnx_model(post_g) if post_g else None
+        else:
+            raise ValueError(f"Unsupported processor/tokenizer: {cls_name}")

onnxruntime_extensions/pp_api.py

@@ -3,11 +3,81 @@
 # license information.
 ###############################################################################
 
+import os
 from . import _extensions_pydll as _C
-if not hasattr(_C, "create_processor"):
-    raise ImportError("onnxruntime_extensions is not built with pre-processing API")
+if not hasattr(_C, "delete_object"):
+    raise ImportError(
+        "onnxruntime_extensions is not built with pre-processing C API\n"
+        "To enable it, please build the package with --ortx-user-option=pp_api")
 
 create_processor = _C.create_processor
 load_images = _C.load_images
 image_pre_process = _C.image_pre_process
 tensor_result_get_at = _C.tensor_result_get_at
+
+create_tokenizer = _C.create_tokenizer
+batch_tokenize = _C.batch_tokenize
+batch_detokenize = _C.batch_detokenize
+
+delete_object = _C.delete_object
+
+
+class Tokenizer:
+    def __init__(self, tokenizer_dir):
+        self.tokenizer = None
+        if os.path.isdir(tokenizer_dir):
+            self.tokenizer = create_tokenizer(tokenizer_dir)
+        else:
+            try:
+                from transformers.utils import cached_file
+                resolved_full_file = cached_file(
+                    tokenizer_dir, "tokenizer.json")
+                resolved_config_file = cached_file(
+                    tokenizer_dir, "tokenizer_config.json")
+            except ImportError:
+                raise ValueError(
+                    f"Directory '{tokenizer_dir}' not found and transformers is not available")
+            if not os.path.exists(resolved_full_file):
+                raise FileNotFoundError(
+                    f"Downloaded HF file '{resolved_full_file}' cannot be found")
+            if (os.path.dirname(resolved_full_file) != os.path.dirname(resolved_config_file)):
+                raise FileNotFoundError(
+                    f"Downloaded HF files '{resolved_full_file}' "
+                    f"and '{resolved_config_file}' are not in the same directory")
+
+            tokenizer_dir = os.path.dirname(resolved_full_file)
+            self.tokenizer = create_tokenizer(tokenizer_dir)
+
+    def tokenize(self, text):
+        if isinstance(text, (list, tuple)):
+            return batch_tokenize(self.tokenizer, text)
+        return batch_tokenize(self.tokenizer, [text])[0]
+
+    def detokenize(self, tokens):
+        return batch_detokenize(self.tokenizer, [tokens])
+
+    def __del__(self):
+        if delete_object and self.tokenizer:
+            delete_object(self.tokenizer)
+        self.tokenizer = None
+
+
+class ImageProcessor:
+    def __init__(self, processor_json):
+        self.processor = create_processor(processor_json)
+
+    def pre_process(self, images):
+        if isinstance(images, str):
+            images = [images]
+        if isinstance(images, list):
+            images = load_images(images)
+        return image_pre_process(self.processor, images)
+
+    @staticmethod
+    def to_numpy(result, idx):
+        return tensor_result_get_at(result, idx)
+
+    def __del__(self):
+        if delete_object and self.processor:
+            delete_object(self.processor)
+        self.processor = None

onnxruntime_extensions/tools/add_pre_post_processing_to_model.py

@@ -163,7 +163,8 @@ def superresolution(model_file: Path, output_file: Path, output_format: str, onn
 
 
 def yolo_detection(model_file: Path, output_file: Path, output_format: str = 'jpg',
-                   onnx_opset: int = 16, num_classes: int = 80, input_shape: List[int] = None):
+                   onnx_opset: int = 16, num_classes: int = 80, input_shape: List[int] = None,
+                   output_as_image: bool = True):
     """
     SSD-like model and Faster-RCNN-like model are including NMS inside already, You can find it from onnx model zoo.
 
@@ -185,6 +186,7 @@ def yolo_detection(model_file: Path, output_file: Path, output_format: str = 'jp
     :param onnx_opset: The opset version of onnx model, default(16).
     :param num_classes: The number of classes, default(80).
     :param input_shape: The shape of input image (height,width), default will be asked from model input.
+    :param output_as_image: The flag that means that the model should have the image with boxes instead of the coordinates of the boxess
     """
     model = onnx.load(str(model_file.resolve(strict=True)))
     inputs = [create_named_value("image", onnx.TensorProto.UINT8, ["num_bytes"])]
@@ -284,19 +286,23 @@ Because we need to execute the model to determine the output shape in order to a
             utils.IoMapEntry("Resize", producer_idx=0, consumer_idx=2),
             utils.IoMapEntry("LetterBox", producer_idx=0, consumer_idx=3),
         ]),
-        # DrawBoundingBoxes on the original image
-        # Model imported from pytorch has CENTER_XYWH format
-        # two mode for how to color box,
-        #   1. colour_by_classes=True, (colour_by_classes), 2. colour_by_classes=False,(colour_by_confidence)
-        (DrawBoundingBoxes(mode='CENTER_XYWH', num_classes=num_classes, colour_by_classes=True),
-         [
-            utils.IoMapEntry("ConvertImageToBGR", producer_idx=0, consumer_idx=0),
-            utils.IoMapEntry("ScaleBoundingBoxes", producer_idx=0, consumer_idx=1),
-        ]),
-        # Encode to jpg/png
-        ConvertBGRToImage(image_format=output_format),
     ]
 
+    if output_as_image:
+        post_processing_steps += [
+            # DrawBoundingBoxes on the original image
+            # Model imported from pytorch has CENTER_XYWH format
+            # two mode for how to color box,
+            #   1. colour_by_classes=True, (colour_by_classes), 2. colour_by_classes=False,(colour_by_confidence)
+            (DrawBoundingBoxes(mode='CENTER_XYWH', num_classes=num_classes, colour_by_classes=True),
+            [
+                utils.IoMapEntry("ConvertImageToBGR", producer_idx=0, consumer_idx=0),
+                utils.IoMapEntry("ScaleBoundingBoxes", producer_idx=0, consumer_idx=1),
+            ]),
+            # Encode to jpg/png
+            ConvertBGRToImage(image_format=output_format),
+        ]
+
     pipeline.add_post_processing(post_processing_steps)
 
     new_model = pipeline.run(model)

{onnxruntime_extensions-0.12.0.dist-info → onnxruntime_extensions-0.14.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: onnxruntime_extensions
-Version: 0.12.0
+Version: 0.14.0
 Summary: ONNXRuntime Extensions
 Home-page: https://github.com/microsoft/onnxruntime-extensions
 Author: Microsoft Corporation
@@ -18,6 +18,14 @@ Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: License :: OSI Approved :: MIT License
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: summary
 
 # ONNXRuntime-Extensions
 
@@ -25,29 +33,17 @@ License-File: LICENSE
 
 ## What's ONNXRuntime-Extensions
 
-Introduction: ONNXRuntime-Extensions is a library that extends the capability of the ONNX models and inference with ONNX Runtime, via ONNX Runtime Custom Operator ABIs. It includes a set of [ONNX Runtime Custom Operator](https://onnxruntime.ai/docs/reference/operators/add-custom-op.html) to support the common pre- and post-processing operators for vision, text, and nlp models. And it supports multiple languages and platforms, like Python on Windows/Linux/macOS, some mobile platforms like Android and iOS, and Web-Assembly etc. The basic workflow is to enhance a ONNX model firstly and then do the model inference with ONNX Runtime and ONNXRuntime-Extensions package.
+Introduction: ONNXRuntime-Extensions is a C/C++ library that extends the capability of the ONNX models and inference with ONNX Runtime, via ONNX Runtime Custom Operator ABIs. It includes a set of [ONNX Runtime Custom Operator](https://onnxruntime.ai/docs/reference/operators/add-custom-op.html) to support the common pre- and post-processing operators for vision, text, and nlp models. And it supports multiple languages and platforms, like Python on Windows/Linux/macOS, some mobile platforms like Android and iOS, and Web-Assembly etc. The basic workflow is to enhance a ONNX model firstly and then do the model inference with ONNX Runtime and ONNXRuntime-Extensions package.
 
 
 ## Quickstart
+The library can be utilized as either a C/C++ library or other advance language packages like Python, Java, C#, etc. To build it as a shared library, you can use the `build.bat` or `build.sh` scripts located in the root folder. The CMake build definition is available in the `CMakeLists.txt` file and can be modified by appending options to `build.bat` or `build.sh`, such as `build.bat -DOCOS_BUILD_SHARED_LIB=OFF`. For more details, please refer to the [C API documentation](./docs/c_api.md).
 
 ### **Python installation**
 ```bash
 pip install onnxruntime-extensions
 ````
-
-
-### **Nightly Build**
-
-#### <strong>on Windows</strong>
-```cmd
-pip install --index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/pypi/simple/ onnxruntime-extensions
-```
-Please ensure that you have met the prerequisites of onnxruntime-extensions (e.g., onnx and onnxruntime) in your Python environment.
-#### <strong>on Linux/macOS</strong>
-Please make sure the compiler toolkit like gcc(later than g++ 8.0) or clang are installed before the following command
-```bash
-python -m pip install git+https://github.com/microsoft/onnxruntime-extensions.git
-```
+The nightly build is also available for the latest features, please refer to [nightly build](./docs/development.md#nightly-build)
 
 
 ## Usage

{onnxruntime_extensions-0.12.0.dist-info → onnxruntime_extensions-0.14.0.dist-info}/RECORD

@@ -1,15 +1,15 @@
 onnxruntime_extensions/__init__.py,sha256=GMnMIHJ-uqvJGPn5fpCZOi7OG16kFVpfOTTO88kYJWY,2387
-onnxruntime_extensions/_cuops.py,sha256=SUD2NhEWHeMem8ylCtCGBKutSuZQs4WMj1ke65-52vA,16193
-onnxruntime_extensions/_extensions_pydll.cp39-win_amd64.pyd,sha256=IVAfa-rN8jRjKNbd3v1_agIsU7lCFBLinSXi03RAfQc,3323392
+onnxruntime_extensions/_cuops.py,sha256=W4hmBNoNvFk84V4UAUpltGNFjzcf0fju3iCeuatqXHE,16661
+onnxruntime_extensions/_extensions_pydll.cp39-win_amd64.pyd,sha256=0BtiZfy7lditfJwj-cOFbyntPzDZcERTbsHiVD7tiko,1900032
 onnxruntime_extensions/_extensions_pydll.pyi,sha256=mYXkqNaCgAbs161RDKgDjxIX9vWdYdVPDC-0X9cieco,1070
-onnxruntime_extensions/_hf_cvt.py,sha256=HJwpcdc02aYV9qgAYkrtSYbkargYi0xTqf7Ye60D84A,14062
+onnxruntime_extensions/_hf_cvt.py,sha256=7-nV40_lCydWHBMXUkfe3oaJSI7l0SDQdLT92yZG2oc,15945
 onnxruntime_extensions/_ocos.py,sha256=OlDOlCH_vWFOBkjbp6Pujgw6rgk8Fd3_2Mi5ev1eeS0,4193
 onnxruntime_extensions/_ortapi2.py,sha256=Tfrf9fQMQ0e7Wa4R8s4SHdwMNBdmj33wH3y5vMkVVQE,9951
 onnxruntime_extensions/_torch_cvt.py,sha256=hGOiw24QuFpK_3CLjg8Fs2GD_cCdM049xcJxkHVRbAk,10185
-onnxruntime_extensions/_version.py,sha256=gjGFlfLnfAn2djJsgIststV0PRkvMcSSr51ENp79FDA,76
+onnxruntime_extensions/_version.py,sha256=u5KwYLG4_oeOTmNuRw2dLiPJ5hByZa12xh0VGidbJMU,76
 onnxruntime_extensions/cmd.py,sha256=eIiNNY0ohbUCPgmr9RwOfi0Gzw7nWL17i625L-ZKezI,2428
-onnxruntime_extensions/cvt.py,sha256=XMz0CZXBJQ9IwnixjzJwz-utKyu9HREIEUCviZg6v8A,3977
-onnxruntime_extensions/pp_api.py,sha256=-Qty5kyN0stBft6vecPucGnjQLZXQd_8PzaCvcQM6ys,571
+onnxruntime_extensions/cvt.py,sha256=2cPsKj4weGDveV36mtoQ9yVUfjtqmFNUpFghrsppXOg,13409
+onnxruntime_extensions/pp_api.py,sha256=Fk1iEMPwcnr84V9ALhr-zuMPNi_fyIMPTrKPeOQooZs,3157
 onnxruntime_extensions/util.py,sha256=KxNFY0-5CG1i9HADcCc4V33PNukTO46Os_KIL8pj-l8,7394
 onnxruntime_extensions/onnxprocess/__init__.py,sha256=BnveHXnu2nTQNbCLeZujZgZwO9A3yWFbQGTDthCFbIc,534
 onnxruntime_extensions/onnxprocess/_builder.py,sha256=L_afKeE7Wc4mWJ47eVXQ2stvmal_37QVTQZgKmt0ZK8,1844
@@ -27,7 +27,7 @@ onnxruntime_extensions/pnp/_unifier.py,sha256=FPQYL1Z6f1Tv2qRsnhW_is9k7-GmCYhf6Z
 onnxruntime_extensions/pnp/_utils.py,sha256=xBh7-_VstgqXlhBaQ_6E5GV6341ywCRQsrJZZZtYaCc,13061
 onnxruntime_extensions/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 onnxruntime_extensions/tools/add_HuggingFace_CLIPImageProcessor_to_model.py,sha256=iNGAd9Ym0iKDQkXdWdka-R3S47TT3hMTihdGXg0uHL0,6786
-onnxruntime_extensions/tools/add_pre_post_processing_to_model.py,sha256=yNXxriAqPqRQzEi7fcK4CCcyCnhubU-wiO2d2rGIOxw,23996
+onnxruntime_extensions/tools/add_pre_post_processing_to_model.py,sha256=M2dSO2FdLo1Hs0GDVqYmKxmWDj7BsKCiyhpqxmCdDWg,24301
 onnxruntime_extensions/tools/pre_post_processing/__init__.py,sha256=YKxCtG2McBExYYmcf1tbqDquqIS1iTs4iPx86MBcfRo,125
 onnxruntime_extensions/tools/pre_post_processing/pre_post_processor.py,sha256=lnQ4TUKkZ-TvVC8U_ov3Nsz9gzES0ktnmD-DPTzutPA,19635
 onnxruntime_extensions/tools/pre_post_processing/step.py,sha256=SYFxtrDmXyFpnnlPl4c49Yg1THFZvh5Y9NwuvquHTVg,9394
@@ -36,8 +36,8 @@ onnxruntime_extensions/tools/pre_post_processing/steps/__init__.py,sha256=pdVRZB
 onnxruntime_extensions/tools/pre_post_processing/steps/general.py,sha256=fF_XVFSKOCu482Sqjp-nVPbs-ZVGpPal2ekbO1gUO_4,13781
 onnxruntime_extensions/tools/pre_post_processing/steps/nlp.py,sha256=ZCxRNxqfANplxCe0I-6BfHziM1jDYJsNQKbHdM3Y1I0,15173
 onnxruntime_extensions/tools/pre_post_processing/steps/vision.py,sha256=BM6CGylOSu4l6UarPfW0I2tgkJDa1Q-gYz__CxZle-k,53183
-onnxruntime_extensions-0.12.0.dist-info/LICENSE,sha256=mQaUD2Gx8LUz-n2ZuvVReLKAj74RPqUd-_rYVyzNXys,1162
-onnxruntime_extensions-0.12.0.dist-info/METADATA,sha256=ulMfUUUKIosCAUeqozJAJ5LQbpnQsU6HvsvbA-_tTXA,4452
-onnxruntime_extensions-0.12.0.dist-info/WHEEL,sha256=4qHc_4HH-JGeVqXiOhLz9XdiROeRsdeB9MhbJiO4SkE,99
-onnxruntime_extensions-0.12.0.dist-info/top_level.txt,sha256=XyAgQDKyXsf6_0MJb58kRdHwigpTn7A7kl9diBEjs8M,23
-onnxruntime_extensions-0.12.0.dist-info/RECORD,,
+onnxruntime_extensions-0.14.0.dist-info/LICENSE,sha256=mQaUD2Gx8LUz-n2ZuvVReLKAj74RPqUd-_rYVyzNXys,1162
+onnxruntime_extensions-0.14.0.dist-info/METADATA,sha256=udEUfhboh5qT4rtnEBg94FkkPmjjAYRJx9rWfg5ZrJg,4657
+onnxruntime_extensions-0.14.0.dist-info/WHEEL,sha256=agy-BJge3afXwWznUXANATmKFW4eqelqRR0uf608A_0,99
+onnxruntime_extensions-0.14.0.dist-info/top_level.txt,sha256=XyAgQDKyXsf6_0MJb58kRdHwigpTn7A7kl9diBEjs8M,23
+onnxruntime_extensions-0.14.0.dist-info/RECORD,,

{onnxruntime_extensions-0.12.0.dist-info → onnxruntime_extensions-0.14.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (72.2.0)
+Generator: setuptools (75.8.0)
 Root-Is-Purelib: false
 Tag: cp39-cp39-win_amd64