truss 0.10.0rc1__py3-none-any.whl → 0.60.0__py3-none-any.whl

truss/templates/trtllm/packages/schema.py

@@ -1,216 +0,0 @@
-from enum import Enum
-from pathlib import Path
-from typing import Optional
-
-import numpy as np
-import tritonclient
-import tritonclient.grpc.aio as grpcclient
-from pydantic import BaseModel, ConfigDict, PrivateAttr
-
-
-class ModelInput:
-    def __init__(
-        self,
-        prompt: str,
-        request_id: int,
-        max_tokens: int = 50,
-        beam_width: int = 1,
-        bad_words_list: Optional[list] = None,
-        stop_words_list: Optional[list] = None,
-        repetition_penalty: float = 1.0,
-        ignore_eos: bool = False,
-        stream: bool = True,
-        eos_token_id: int = None,  # type: ignore
-    ) -> None:
-        self.stream = stream
-        self.request_id = request_id
-        self._prompt = prompt
-        self._max_tokens = max_tokens
-        self._beam_width = beam_width
-        self._bad_words_list = [""] if bad_words_list is None else bad_words_list
-        self._stop_words_list = [""] if stop_words_list is None else stop_words_list
-        self._repetition_penalty = repetition_penalty
-        self._eos_token_id = eos_token_id
-        self._ignore_eos = ignore_eos
-
-    def _prepare_grpc_tensor(
-        self, name: str, input_data: np.ndarray
-    ) -> grpcclient.InferInput:
-        tensor = grpcclient.InferInput(
-            name,
-            input_data.shape,
-            tritonclient.utils.np_to_triton_dtype(input_data.dtype),
-        )
-        tensor.set_data_from_numpy(input_data)
-        return tensor
-
-    def to_tensors(self):
-        if self._eos_token_id is None and self._ignore_eos:
-            raise ValueError("eos_token_id is required when ignore_eos is True")
-
-        prompt_data = np.array([[self._prompt]], dtype=object)
-        output_len_data = np.ones_like(prompt_data, dtype=np.uint32) * self._max_tokens
-        bad_words_data = np.array([self._bad_words_list], dtype=object)
-        stop_words_data = np.array([self._stop_words_list], dtype=object)
-        stream_data = np.array([[self.stream]], dtype=bool)
-        beam_width_data = np.array([[self._beam_width]], dtype=np.uint32)
-        repetition_penalty_data = np.array(
-            [[self._repetition_penalty]], dtype=np.float32
-        )
-
-        inputs = [
-            self._prepare_grpc_tensor("text_input", prompt_data),
-            self._prepare_grpc_tensor("max_tokens", output_len_data),
-            self._prepare_grpc_tensor("bad_words", bad_words_data),
-            self._prepare_grpc_tensor("stop_words", stop_words_data),
-            self._prepare_grpc_tensor("stream", stream_data),
-            self._prepare_grpc_tensor("beam_width", beam_width_data),
-            self._prepare_grpc_tensor("repetition_penalty", repetition_penalty_data),
-        ]
-
-        if not self._ignore_eos:
-            end_id_data = np.array([[self._eos_token_id]], dtype=np.uint32)
-            inputs.append(self._prepare_grpc_tensor("end_id", end_id_data))
-
-        return inputs
-
-
-class Quant(Enum):
-    NO_QUANT = "no_quant"
-    WEIGHTS_ONLY = "weights_only"
-    WEIGHTS_KV_INT8 = "weights_kv_int8"
-    SMOOTH_QUANT = "smooth_quant"
-
-
-class EngineType(Enum):
-    LLAMA = "llama"
-    MISTRAL = "mistral"
-
-
-class ArgsConfig(BaseModel):
-    max_input_len: Optional[int] = None
-    max_output_len: Optional[int] = None
-    max_batch_size: Optional[int] = None
-    tp_size: Optional[int] = None
-    pp_size: Optional[int] = None
-    world_size: Optional[int] = None
-    gather_all_token_logits: Optional[bool] = None
-    multi_block_mode: Optional[bool] = None
-    remove_input_padding: Optional[bool] = None
-    use_gpt_attention_plugin: Optional[str] = None
-    paged_kv_cache: Optional[bool] = None
-    use_inflight_batching: Optional[bool] = None
-    enable_context_fmha: Optional[bool] = None
-    use_gemm_plugin: Optional[str] = None
-    use_weight_only: Optional[bool] = None
-    output_dir: Optional[str] = None
-    model_dir: Optional[str] = None
-    ft_model_dir: Optional[str] = None
-    dtype: Optional[str] = None
-    int8_kv_cache: Optional[bool] = None
-    use_smooth_quant: Optional[bool] = None
-    per_token: Optional[bool] = None
-    per_channel: Optional[bool] = None
-    parallel_build: Optional[bool] = None
-
-    # to disable warning because `model_dir` starts with `model_` prefix
-    model_config = ConfigDict(protected_namespaces=())  # type: ignore
-
-    def as_command_arguments(self) -> list:
-        non_bool_args = [
-            element
-            for arg, value in self.dict().items()
-            for element in [f"--{arg}", str(value)]
-            if value is not None and not isinstance(value, bool)
-        ]
-        bool_args = [
-            f"--{arg}"
-            for arg, value in self.dict().items()
-            if isinstance(value, bool) and value
-        ]
-        return non_bool_args + bool_args
-
-
-class CalibrationConfig(BaseModel):
-    kv_cache: Optional[bool] = None  # either to calibrate kv cache
-    sq_alpha: Optional[float] = None
-
-    def cache_path(self) -> Path:
-        if self.kv_cache is not None:
-            return Path("kv_cache")
-        else:
-            return Path(f"sq_{self.sq_alpha}")
-
-
-class EngineBuildArgs(BaseModel, use_enum_values=True):
-    repo: Optional[str] = None
-    args: Optional[ArgsConfig] = None
-    quant: Optional[Quant] = None
-    calibration: Optional[CalibrationConfig] = None
-    engine_type: Optional[EngineType] = None
-
-
-class TrussBuildConfig(BaseModel):
-    """
-    This is a spec for what the config.yaml looks like to take advantage of TRT-LLM + TRT-LLM builds. We structure the
-    configuration with the below top-level keys.
-
-    Example (for building an engine)
-    ```
-    build:
-        model_server: TRT_LLM
-        arguments:
-            tokenizer_repository: "mistralai/mistral-v2-instruct"
-            arguments:
-                max_input_len: 1024
-                max_output_len: 1024
-                max_batch_size: 64
-            quant: "weights_kv_int8"
-            tensor_parallel_count: 2
-            pipeline_parallel_count: 1
-    ```
-
-    Example (for using an existing engine)
-    ```
-    build:
-        model_server: TRT_LLM
-        arguments:
-            engine_repository: "baseten/mistral-v2-32k"
-            tensor_parallel_count: 2
-            pipeline_parallel_count: 1
-    ```
-
-    """
-
-    tokenizer_repository: str
-    quant: Quant = Quant.NO_QUANT
-    pipeline_parallel_count: int = 1
-    tensor_parallel_count: int = 1
-    arguments: Optional[ArgsConfig] = None
-    engine_repository: Optional[str] = None
-    calibration: Optional[CalibrationConfig] = None
-    engine_type: Optional[EngineType] = None
-    _engine_build_args: Optional[EngineBuildArgs] = PrivateAttr(default=None)
-
-    @property
-    def engine_build_args(self) -> EngineBuildArgs:
-        if self._engine_build_args is None:
-            repo = self.tokenizer_repository
-            quant = self.quant
-            calibration = self.calibration
-            engine_type = self.engine_type
-            args = self.arguments or ArgsConfig()
-            args.tp_size = self.tensor_parallel_count
-            args.pp_size = self.pipeline_parallel_count
-            self._engine_build_args = EngineBuildArgs(
-                repo=repo,
-                quant=quant,
-                calibration=calibration,
-                engine_type=engine_type,
-                args=args,
-            )
-        return self._engine_build_args
-
-    @property
-    def requires_build(self):
-        return self.engine_repository is None

truss/templates/trtllm/packages/tensorrt_llm_model_repository/ensemble/config.pbtxt

@@ -1,246 +0,0 @@
-# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#  * Redistributions of source code must retain the above copyright
-#    notice, this list of conditions and the following disclaimer.
-#  * Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-#  * Neither the name of NVIDIA CORPORATION nor the names of its
-#    contributors may be used to endorse or promote products derived
-#    from this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
-# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
-# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-name: "ensemble"
-platform: "ensemble"
-max_batch_size: 2048
-input [
-  {
-    name: "text_input"
-    data_type: TYPE_STRING
-    dims: [ -1 ]
-  },
-  {
-    name: "max_tokens"
-    data_type: TYPE_UINT32
-    dims: [ -1 ]
-  },
-  {
-   name: "bad_words"
-   data_type: TYPE_STRING
-   dims: [ -1 ]
-  },
-  {
-   name: "stop_words"
-   data_type: TYPE_STRING
-   dims: [ -1 ]
-  },
-  {
-    name: "end_id"
-    data_type: TYPE_UINT32
-    dims: [ 1 ]
-    optional: true
-  },
-  {
-    name: "pad_id"
-    data_type: TYPE_UINT32
-    dims: [ 1 ]
-    optional: true
-  },
-  {
-    name: "top_k"
-    data_type: TYPE_UINT32
-    dims: [ 1 ]
-    optional: true
-  },
-  {
-    name: "top_p"
-    data_type: TYPE_FP32
-    dims: [ 1 ]
-    optional: true
-  },
-  {
-    name: "temperature"
-    data_type: TYPE_FP32
-    dims: [ 1 ]
-    optional: true
-  },
-  {
-    name: "length_penalty"
-    data_type: TYPE_FP32
-    dims: [ 1 ]
-    optional: true
-  },
-  {
-    name: "repetition_penalty"
-    data_type: TYPE_FP32
-    dims: [ 1 ]
-    optional: true
-  },
-  {
-    name: "min_length"
-    data_type: TYPE_UINT32
-    dims: [ 1 ]
-    optional: true
-  },
-  {
-    name: "presence_penalty"
-    data_type: TYPE_FP32
-    dims: [ 1 ]
-    optional: true
-  },
-  {
-    name: "random_seed"
-    data_type: TYPE_UINT64
-    dims: [ 1 ]
-    optional: true
-  },
-  {
-    name: "beam_width"
-    data_type: TYPE_UINT32
-    dims: [ 1 ]
-    optional: true
-  },
-  {
-    name: "stream"
-    data_type: TYPE_BOOL
-    dims: [ 1 ]
-    optional: true
-  }
-]
-output [
-  {
-    name: "text_output"
-    data_type: TYPE_STRING
-    dims: [ -1, -1 ]
-  }
-]
-ensemble_scheduling {
-  step [
-    {
-      model_name: "preprocessing"
-      model_version: -1
-      input_map {
-        key: "QUERY"
-        value: "text_input"
-      }
-      input_map {
-        key: "REQUEST_OUTPUT_LEN"
-        value: "max_tokens"
-      }
-      input_map {
-        key: "BAD_WORDS_DICT"
-        value: "bad_words"
-      }
-      input_map {
-        key: "STOP_WORDS_DICT"
-        value: "stop_words"
-      }
-      output_map {
-        key: "REQUEST_INPUT_LEN"
-        value: "_REQUEST_INPUT_LEN"
-      }
-      output_map {
-        key: "INPUT_ID"
-        value: "_INPUT_ID"
-      }
-      output_map {
-        key: "REQUEST_OUTPUT_LEN"
-        value: "_REQUEST_OUTPUT_LEN"
-      }
-    },
-    {
-      model_name: "tensorrt_llm"
-      model_version: -1
-      input_map {
-        key: "input_ids"
-        value: "_INPUT_ID"
-      }
-      input_map {
-        key: "input_lengths"
-        value: "_REQUEST_INPUT_LEN"
-      }
-      input_map {
-        key: "request_output_len"
-        value: "_REQUEST_OUTPUT_LEN"
-      }
-      input_map {
-          key: "end_id"
-          value: "end_id"
-      }
-      input_map {
-          key: "pad_id"
-          value: "pad_id"
-      }
-      input_map {
-          key: "runtime_top_k"
-          value: "top_k"
-      }
-      input_map {
-          key: "runtime_top_p"
-          value: "top_p"
-      }
-      input_map {
-          key: "temperature"
-          value: "temperature"
-      }
-      input_map {
-          key: "len_penalty"
-          value: "length_penalty"
-      }
-      input_map {
-          key: "repetition_penalty"
-          value: "repetition_penalty"
-      }
-      input_map {
-          key: "min_length"
-          value: "min_length"
-      }
-      input_map {
-          key: "presence_penalty"
-          value: "presence_penalty"
-      }
-      input_map {
-          key: "random_seed"
-          value: "random_seed"
-      }
-      input_map {
-          key: "beam_width"
-          value: "beam_width"
-      }
-      input_map {
-          key: "streaming"
-          value: "stream"
-      }
-      output_map {
-        key: "output_ids"
-        value: "_TOKENS_BATCH"
-      }
-    },
-    {
-      model_name: "postprocessing"
-      model_version: -1
-      input_map {
-        key: "TOKENS_BATCH"
-        value: "_TOKENS_BATCH"
-      }
-      output_map {
-        key: "OUTPUT"
-        value: "text_output"
-      }
-    }
-  ]
-}

truss/templates/trtllm/packages/tensorrt_llm_model_repository/postprocessing/1/model.py

@@ -1,181 +0,0 @@
-# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#  * Redistributions of source code must retain the above copyright
-#    notice, this list of conditions and the following disclaimer.
-#  * Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-#  * Neither the name of NVIDIA CORPORATION nor the names of its
-#    contributors may be used to endorse or promote products derived
-#    from this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
-# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
-# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-import json
-import os
-from collections import OrderedDict
-
-import numpy as np
-import triton_python_backend_utils as pb_utils
-from transformers import AutoTokenizer, LlamaTokenizer, T5Tokenizer
-
-
-class TritonPythonModel:
-    """Your Python model must use the same class name. Every Python model
-    that is created must have "TritonPythonModel" as the class name.
-    """
-
-    def initialize(self, args):
-        """`initialize` is called only once when the model is being loaded.
-        Implementing `initialize` function is optional. This function allows
-        the model to initialize any state associated with this model.
-        Parameters
-        ----------
-        args : dict
-          Both keys and values are strings. The dictionary keys and values are:
-          * model_config: A JSON string containing the model configuration
-          * model_instance_kind: A string containing model instance kind
-          * model_instance_device_id: A string containing model instance device ID
-          * model_repository: Model repository path
-          * model_version: Model version
-          * model_name: Model name
-        """
-        # Parse model configs
-        model_config = json.loads(args["model_config"])
-        # NOTE: Keep this in sync with the truss model.py variable
-        tokenizer_dir = os.environ["TRITON_TOKENIZER_REPOSITORY"]
-        tokenizer_type = model_config["parameters"]["tokenizer_type"]["string_value"]
-
-        if tokenizer_type == "t5":
-            self.tokenizer = T5Tokenizer(vocab_file=tokenizer_dir, padding_side="left")
-        elif tokenizer_type == "auto":
-            self.tokenizer = AutoTokenizer.from_pretrained(
-                tokenizer_dir, padding_side="left"
-            )
-        elif tokenizer_type == "llama":
-            self.tokenizer = LlamaTokenizer.from_pretrained(
-                tokenizer_dir, legacy=False, padding_side="left"
-            )
-        else:
-            raise AttributeError(f"Unexpected tokenizer type: {tokenizer_type}")
-        self.tokenizer.pad_token = self.tokenizer.eos_token
-
-        # Parse model output configs
-        output_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT")
-        # Convert Triton types to numpy types
-        self.output_dtype = pb_utils.triton_string_to_numpy(output_config["data_type"])
-
-        self.state_dict = OrderedDict()
-        # TODO(pankaj) This should come from the batch size
-        self.cache_size = 2048
-
-    def execute(self, requests):
-        """`execute` must be implemented in every Python model. `execute`
-        function receives a list of pb_utils.InferenceRequest as the only
-        argument. This function is called when an inference is requested
-        for this model. Depending on the batching configuration (e.g. Dynamic
-        Batching) used, `requests` may contain multiple requests. Every
-        Python model, must create one pb_utils.InferenceResponse for every
-        pb_utils.InferenceRequest in `requests`. If there is an error, you can
-        set the error argument when creating a pb_utils.InferenceResponse.
-        Parameters
-        ----------
-        requests : list
-          A list of pb_utils.InferenceRequest
-        Returns
-        -------
-        list
-          A list of pb_utils.InferenceResponse. The length of this list must
-          be the same as `requests`
-        """
-
-        responses = []
-
-        # Every Python backend must iterate over everyone of the requests
-        # and create a pb_utils.InferenceResponse for each of them.
-        for idx, request in enumerate(requests):
-            # Get request ID
-            request_id = request.request_id()
-
-            # Get input tensors
-            tokens_batch = (
-                pb_utils.get_input_tensor_by_name(request, "TOKENS_BATCH")
-                .as_numpy()
-                .flatten()
-            )
-            if len(tokens_batch) == 0:
-                continue
-
-            # Postprocess output data
-            prev_token = self._get_prev_token(request_id)
-            self._store_prev_token(request_id, tokens_batch[-1])
-            if prev_token is None:
-                delta = self.tokenizer.decode(tokens_batch)
-            else:
-                # TODO(pankaj) Figure out how to make tokenizer.decode not
-                # ignore initial whitespace so we can avoid this hack.
-                # Get string with and without previous token and diff. This hack
-                # is needed because tokenizer.decode strips initial whitespace.
-                old_string = self.tokenizer.decode([prev_token])
-                with_prev_token = np.concatenate(([prev_token], tokens_batch))
-                new_string = self.tokenizer.decode(with_prev_token)
-                delta = self._compute_delta(old_string, new_string)
-
-            # Create output tensor
-            output_tensor = pb_utils.Tensor(
-                "OUTPUT", np.array([delta]).astype(self.output_dtype)
-            )
-            inference_response = pb_utils.InferenceResponse(
-                output_tensors=[output_tensor]
-            )
-            responses.append(inference_response)
-
-        return responses
-
-    def finalize(self):
-        print("Cleaning up...")
-
-    def _store_prev_token(self, request_id, token):
-        if request_id in self.state_dict:
-            self.state_dict[request_id]["prev_token"] = token
-
-            # Move request ID to end of queue to prevent it from being evicted
-            self.state_dict.move_to_end(request_id)
-        else:
-            # Evict least recently used item if cache is full
-            if len(self.state_dict) > self.cache_size:
-                self.state_dict.popitem(last=False)
-
-            self.state_dict[request_id] = {"prev_token": token}
-
-    def _get_prev_token(self, request_id):
-        if request_id in self.state_dict:
-            return self.state_dict[request_id]["prev_token"]
-        return None
-
-    def _compute_delta(self, prev_str, new_str):
-        delta = "".join(
-            [
-                char
-                for index, char in enumerate(new_str)
-                if index >= len(prev_str) or char != prev_str[index]
-            ]
-        )
-        return delta
-
-    def _postprocessing(self, tokens):
-        decoded_tokens = self.tokenizer.decode(tokens)
-        return decoded_tokens

truss/templates/trtllm/packages/tensorrt_llm_model_repository/postprocessing/config.pbtxt

@@ -1,64 +0,0 @@
-# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#  * Redistributions of source code must retain the above copyright
-#    notice, this list of conditions and the following disclaimer.
-#  * Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-#  * Neither the name of NVIDIA CORPORATION nor the names of its
-#    contributors may be used to endorse or promote products derived
-#    from this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
-# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
-# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-name: "postprocessing"
-backend: "python"
-max_batch_size: 2048
-input [
-  {
-    name: "TOKENS_BATCH"
-    data_type: TYPE_INT32
-    dims: [ -1, -1 ]
-  }
-]
-output [
-  {
-    name: "OUTPUT"
-    data_type: TYPE_STRING
-    dims: [ -1, -1 ]
-  }
-]
-
-parameters {
-  key: "tokenizer_dir"
-  value: {
-    string_value: "NousResearch/Llama-2-7b-hf"
-  }
-}
-
-parameters {
-  key: "tokenizer_type"
-  value: {
-    string_value: "auto"
-  }
-}
-
-instance_group [
-    {
-        count: 1
-        kind: KIND_CPU
-    }
-]