tritonclient 2.34.0__py3-none-any.whl → 2.36.0__py3-none-any.whl

{tritonclient-2.34.0.data → tritonclient-2.36.0.data}/purelib/tritonclient/grpc/_client.py

@@ -1532,6 +1532,7 @@ class InferenceServerClient(InferenceServerClientBase):
                            sequence_id=0,
                            sequence_start=False,
                            sequence_end=False,
+                           enable_empty_final_response=False,
                            priority=0,
                            timeout=None,
                            parameters=None):
@@ -1571,6 +1572,18 @@ class InferenceServerClient(InferenceServerClientBase):
             Indicates whether the request being added marks the end of the
             sequence. Default value is False. This argument is ignored if
             'sequence_id' is 0 or "".
+        enable_empty_final_response: bool
+            Indicates whether "empty" responses should be generated and sent
+            back to the client from the server during streaming inference when
+            they contain the TRITONSERVER_RESPONSE_COMPLETE_FINAL flag. 
+            This strictly relates to the case of models/backends that send
+            flags-only responses (use TRITONBACKEND_ResponseFactorySendFlags(TRITONSERVER_RESPONSE_COMPLETE_FINAL)
+            or InferenceResponseSender.send(flags=TRITONSERVER_RESPONSE_COMPLETE_FINAL))
+            Currently, this only occurs for decoupled models, and can be
+            used to communicate to the client when a request has received
+            its final response from the model. If the backend sends the final
+            flag along with a non-empty response, this arg is not needed.
+            Default value is False.
         priority : int
             Indicates the priority of the request. Priority value zero
             indicates that the default priority level should be used
@@ -1616,6 +1629,13 @@ class InferenceServerClient(InferenceServerClientBase):
                                          priority=priority,
                                          timeout=timeout,
                                          parameters=parameters)
+
+        # Unique to streaming inference as it only pertains to decoupled models
+        # Only attach the parameter if True, no need to send/parse when False.
+        if enable_empty_final_response:
+            request.parameters[
+                'triton_enable_empty_final_response'].bool_param = True
+
         if self._verbose:
             print("async_stream_infer\n{}".format(request))
         # Enqueues the request to the stream

{tritonclient-2.34.0.data → tritonclient-2.36.0.data}/purelib/tritonclient/grpc/_utils.py

@@ -85,7 +85,7 @@ def _get_inference_request(model_name, inputs, model_version, request_id,
         request.parameters['sequence_start'].bool_param = sequence_start
         request.parameters['sequence_end'].bool_param = sequence_end
     if priority != 0:
-        request.parameters['priority'].int64_param = priority
+        request.parameters['priority'].uint64_param = priority
     if timeout is not None:
         request.parameters['timeout'].int64_param = timeout
 

{tritonclient-2.34.0.data → tritonclient-2.36.0.data}/purelib/tritonclient/grpc/aio/__init__.py

@@ -24,9 +24,12 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import base64
+
+import rapidjson as json
+from google.protobuf.json_format import MessageToJson
 from tritonclient.grpc import *
 from .._utils import _get_inference_request, _grpc_compression_type
-import base64
 from ..._client import InferenceServerClientBase
 from ..._request import Request
 from ..._plugin import InferenceServerClientPlugin

{tritonclient-2.34.0.data → tritonclient-2.36.0.data}/purelib/tritonclient/grpc/model_config_pb2.py

@@ -15,7 +15,7 @@ _sym_db = _symbol_database.Default()
 
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x12model_config.proto\x12\tinference\"\x96\x01\n\x10ModelRateLimiter\x12\x37\n\tresources\x18\x01 \x03(\x0b\x32$.inference.ModelRateLimiter.Resource\x12\x10\n\x08priority\x18\x02 \x01(\r\x1a\x37\n\x08Resource\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06global\x18\x02 \x01(\x08\x12\r\n\x05\x63ount\x18\x03 \x01(\r\"\x87\x04\n\x12ModelInstanceGroup\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x30\n\x04kind\x18\x04 \x01(\x0e\x32\".inference.ModelInstanceGroup.Kind\x12\r\n\x05\x63ount\x18\x02 \x01(\x05\x12\x31\n\x0crate_limiter\x18\x06 \x01(\x0b\x32\x1b.inference.ModelRateLimiter\x12\x0c\n\x04gpus\x18\x03 \x03(\x05\x12H\n\x11secondary_devices\x18\x08 \x03(\x0b\x32-.inference.ModelInstanceGroup.SecondaryDevice\x12\x0f\n\x07profile\x18\x05 \x03(\t\x12\x0f\n\x07passive\x18\x07 \x01(\x08\x12\x13\n\x0bhost_policy\x18\t \x01(\t\x1a\x9c\x01\n\x0fSecondaryDevice\x12O\n\x04kind\x18\x01 \x01(\x0e\x32\x41.inference.ModelInstanceGroup.SecondaryDevice.SecondaryDeviceKind\x12\x11\n\tdevice_id\x18\x02 \x01(\x03\"%\n\x13SecondaryDeviceKind\x12\x0e\n\nKIND_NVDLA\x10\x00\"A\n\x04Kind\x12\r\n\tKIND_AUTO\x10\x00\x12\x0c\n\x08KIND_GPU\x10\x01\x12\x0c\n\x08KIND_CPU\x10\x02\x12\x0e\n\nKIND_MODEL\x10\x03\"#\n\x12ModelTensorReshape\x12\r\n\x05shape\x18\x01 \x03(\x03\"\xb2\x02\n\nModelInput\x12\x0c\n\x04name\x18\x01 \x01(\t\x12&\n\tdata_type\x18\x02 \x01(\x0e\x32\x13.inference.DataType\x12,\n\x06\x66ormat\x18\x03 \x01(\x0e\x32\x1c.inference.ModelInput.Format\x12\x0c\n\x04\x64ims\x18\x04 \x03(\x03\x12.\n\x07reshape\x18\x05 \x01(\x0b\x32\x1d.inference.ModelTensorReshape\x12\x17\n\x0fis_shape_tensor\x18\x06 \x01(\x08\x12\x1a\n\x12\x61llow_ragged_batch\x18\x07 \x01(\x08\x12\x10\n\x08optional\x18\x08 \x01(\x08\";\n\x06\x46ormat\x12\x0f\n\x0b\x46ORMAT_NONE\x10\x00\x12\x0f\n\x0b\x46ORMAT_NHWC\x10\x01\x12\x0f\n\x0b\x46ORMAT_NCHW\x10\x02\"\xb2\x01\n\x0bModelOutput\x12\x0c\n\x04name\x18\x01 \x01(\t\x12&\n\tdata_type\x18\x02 \x01(\x0e\x32\x13.inference.DataType\x12\x0c\n\x04\x64ims\x18\x03 \x03(\x03\x12.\n\x07reshape\x18\x05 \x01(\x0b\x32\x1d.inference.ModelTensorReshape\x12\x16\n\x0elabel_filename\x18\x04 \x01(\t\x12\x17\n\x0fis_shape_tensor\x18\x06 \x01(\x08\"\xd9\x02\n\nBatchInput\x12(\n\x04kind\x18\x01 \x01(\x0e\x32\x1a.inference.BatchInput.Kind\x12\x13\n\x0btarget_name\x18\x02 \x03(\t\x12&\n\tdata_type\x18\x03 \x01(\x0e\x32\x13.inference.DataType\x12\x14\n\x0csource_input\x18\x04 \x03(\t\"\xcd\x01\n\x04Kind\x12\x17\n\x13\x42\x41TCH_ELEMENT_COUNT\x10\x00\x12#\n\x1f\x42\x41TCH_ACCUMULATED_ELEMENT_COUNT\x10\x01\x12-\n)BATCH_ACCUMULATED_ELEMENT_COUNT_WITH_ZERO\x10\x02\x12$\n BATCH_MAX_ELEMENT_COUNT_AS_SHAPE\x10\x03\x12\x14\n\x10\x42\x41TCH_ITEM_SHAPE\x10\x04\x12\x1c\n\x18\x42\x41TCH_ITEM_SHAPE_FLATTEN\x10\x05\"\x8f\x01\n\x0b\x42\x61tchOutput\x12\x13\n\x0btarget_name\x18\x01 \x03(\t\x12)\n\x04kind\x18\x02 \x01(\x0e\x32\x1b.inference.BatchOutput.Kind\x12\x14\n\x0csource_input\x18\x03 \x03(\t\"*\n\x04Kind\x12\"\n\x1e\x42\x41TCH_SCATTER_WITH_INPUT_SHAPE\x10\x00\"\x90\x02\n\x12ModelVersionPolicy\x12\x36\n\x06latest\x18\x01 \x01(\x0b\x32$.inference.ModelVersionPolicy.LatestH\x00\x12\x30\n\x03\x61ll\x18\x02 \x01(\x0b\x32!.inference.ModelVersionPolicy.AllH\x00\x12:\n\x08specific\x18\x03 \x01(\x0b\x32&.inference.ModelVersionPolicy.SpecificH\x00\x1a\x1e\n\x06Latest\x12\x14\n\x0cnum_versions\x18\x01 \x01(\r\x1a\x05\n\x03\x41ll\x1a\x1c\n\x08Specific\x12\x10\n\x08versions\x18\x01 \x03(\x03\x42\x0f\n\rpolicy_choice\"\xfd\r\n\x17ModelOptimizationPolicy\x12\x37\n\x05graph\x18\x01 \x01(\x0b\x32(.inference.ModelOptimizationPolicy.Graph\x12\x42\n\x08priority\x18\x02 \x01(\x0e\x32\x30.inference.ModelOptimizationPolicy.ModelPriority\x12\x35\n\x04\x63uda\x18\x03 \x01(\x0b\x32\'.inference.ModelOptimizationPolicy.Cuda\x12X\n\x16\x65xecution_accelerators\x18\x04 \x01(\x0b\x32\x38.inference.ModelOptimizationPolicy.ExecutionAccelerators\x12R\n\x13input_pinned_memory\x18\x05 \x01(\x0b\x32\x35.inference.ModelOptimizationPolicy.PinnedMemoryBuffer\x12S\n\x14output_pinned_memory\x18\x06 \x01(\x0b\x32\x35.inference.ModelOptimizationPolicy.PinnedMemoryBuffer\x12&\n\x1egather_kernel_buffer_threshold\x18\x07 \x01(\r\x12\x16\n\x0e\x65\x61ger_batching\x18\x08 \x01(\x08\x1a\x16\n\x05Graph\x12\r\n\x05level\x18\x01 \x01(\x05\x1a\xba\x05\n\x04\x43uda\x12\x0e\n\x06graphs\x18\x01 \x01(\x08\x12\x18\n\x10\x62usy_wait_events\x18\x02 \x01(\x08\x12\x45\n\ngraph_spec\x18\x03 \x03(\x0b\x32\x31.inference.ModelOptimizationPolicy.Cuda.GraphSpec\x12\x1a\n\x12output_copy_stream\x18\x04 \x01(\x08\x1a\xa4\x04\n\tGraphSpec\x12\x12\n\nbatch_size\x18\x01 \x01(\x05\x12K\n\x05input\x18\x02 \x03(\x0b\x32<.inference.ModelOptimizationPolicy.Cuda.GraphSpec.InputEntry\x12W\n\x11graph_lower_bound\x18\x03 \x01(\x0b\x32<.inference.ModelOptimizationPolicy.Cuda.GraphSpec.LowerBound\x1a\x14\n\x05Shape\x12\x0b\n\x03\x64im\x18\x01 \x03(\x03\x1a\xdf\x01\n\nLowerBound\x12\x12\n\nbatch_size\x18\x01 \x01(\x05\x12V\n\x05input\x18\x02 \x03(\x0b\x32G.inference.ModelOptimizationPolicy.Cuda.GraphSpec.LowerBound.InputEntry\x1a\x65\n\nInputEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x46\n\x05value\x18\x02 \x01(\x0b\x32\x37.inference.ModelOptimizationPolicy.Cuda.GraphSpec.Shape:\x02\x38\x01\x1a\x65\n\nInputEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x46\n\x05value\x18\x02 \x01(\x0b\x32\x37.inference.ModelOptimizationPolicy.Cuda.GraphSpec.Shape:\x02\x38\x01\x1a\xa4\x03\n\x15\x45xecutionAccelerators\x12g\n\x19gpu_execution_accelerator\x18\x01 \x03(\x0b\x32\x44.inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator\x12g\n\x19\x63pu_execution_accelerator\x18\x02 \x03(\x0b\x32\x44.inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator\x1a\xb8\x01\n\x0b\x41\x63\x63\x65lerator\x12\x0c\n\x04name\x18\x01 \x01(\t\x12h\n\nparameters\x18\x02 \x03(\x0b\x32T.inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator.ParametersEntry\x1a\x31\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a$\n\x12PinnedMemoryBuffer\x12\x0e\n\x06\x65nable\x18\x01 \x01(\x08\"I\n\rModelPriority\x12\x14\n\x10PRIORITY_DEFAULT\x10\x00\x12\x10\n\x0cPRIORITY_MAX\x10\x01\x12\x10\n\x0cPRIORITY_MIN\x10\x02\"\xdb\x01\n\x10ModelQueuePolicy\x12\x41\n\x0etimeout_action\x18\x01 \x01(\x0e\x32).inference.ModelQueuePolicy.TimeoutAction\x12$\n\x1c\x64\x65\x66\x61ult_timeout_microseconds\x18\x02 \x01(\x04\x12\x1e\n\x16\x61llow_timeout_override\x18\x03 \x01(\x08\x12\x16\n\x0emax_queue_size\x18\x04 \x01(\r\"&\n\rTimeoutAction\x12\n\n\x06REJECT\x10\x00\x12\t\n\x05\x44\x45LAY\x10\x01\"\x9b\x03\n\x14ModelDynamicBatching\x12\x1c\n\x14preferred_batch_size\x18\x01 \x03(\x05\x12$\n\x1cmax_queue_delay_microseconds\x18\x02 \x01(\x04\x12\x19\n\x11preserve_ordering\x18\x03 \x01(\x08\x12\x17\n\x0fpriority_levels\x18\x04 \x01(\r\x12\x1e\n\x16\x64\x65\x66\x61ult_priority_level\x18\x05 \x01(\r\x12\x39\n\x14\x64\x65\x66\x61ult_queue_policy\x18\x06 \x01(\x0b\x32\x1b.inference.ModelQueuePolicy\x12W\n\x15priority_queue_policy\x18\x07 \x03(\x0b\x32\x38.inference.ModelDynamicBatching.PriorityQueuePolicyEntry\x1aW\n\x18PriorityQueuePolicyEntry\x12\x0b\n\x03key\x18\x01 \x01(\r\x12*\n\x05value\x18\x02 \x01(\x0b\x32\x1b.inference.ModelQueuePolicy:\x02\x38\x01\"\xef\t\n\x15ModelSequenceBatching\x12\x41\n\x06\x64irect\x18\x03 \x01(\x0b\x32/.inference.ModelSequenceBatching.StrategyDirectH\x00\x12\x41\n\x06oldest\x18\x04 \x01(\x0b\x32/.inference.ModelSequenceBatching.StrategyOldestH\x00\x12&\n\x1emax_sequence_idle_microseconds\x18\x01 \x01(\x04\x12\x44\n\rcontrol_input\x18\x02 \x03(\x0b\x32-.inference.ModelSequenceBatching.ControlInput\x12\x35\n\x05state\x18\x05 \x03(\x0b\x32&.inference.ModelSequenceBatching.State\x1a\xb1\x02\n\x07\x43ontrol\x12;\n\x04kind\x18\x01 \x01(\x0e\x32-.inference.ModelSequenceBatching.Control.Kind\x12\x18\n\x10int32_false_true\x18\x02 \x03(\x05\x12\x17\n\x0f\x66p32_false_true\x18\x03 \x03(\x02\x12\x17\n\x0f\x62ool_false_true\x18\x05 \x03(\x08\x12&\n\tdata_type\x18\x04 \x01(\x0e\x32\x13.inference.DataType\"u\n\x04Kind\x12\x1a\n\x16\x43ONTROL_SEQUENCE_START\x10\x00\x12\x1a\n\x16\x43ONTROL_SEQUENCE_READY\x10\x01\x12\x18\n\x14\x43ONTROL_SEQUENCE_END\x10\x02\x12\x1b\n\x17\x43ONTROL_SEQUENCE_CORRID\x10\x03\x1aW\n\x0c\x43ontrolInput\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x39\n\x07\x63ontrol\x18\x02 \x03(\x0b\x32(.inference.ModelSequenceBatching.Control\x1a\x8a\x01\n\x0cInitialState\x12&\n\tdata_type\x18\x01 \x01(\x0e\x32\x13.inference.DataType\x12\x0c\n\x04\x64ims\x18\x02 \x03(\x03\x12\x13\n\tzero_data\x18\x03 \x01(\x08H\x00\x12\x13\n\tdata_file\x18\x04 \x01(\tH\x00\x12\x0c\n\x04name\x18\x05 \x01(\tB\x0c\n\nstate_data\x1a\xac\x01\n\x05State\x12\x12\n\ninput_name\x18\x01 \x01(\t\x12\x13\n\x0boutput_name\x18\x02 \x01(\t\x12&\n\tdata_type\x18\x03 \x01(\x0e\x32\x13.inference.DataType\x12\x0c\n\x04\x64ims\x18\x04 \x03(\x03\x12\x44\n\rinitial_state\x18\x05 \x03(\x0b\x32-.inference.ModelSequenceBatching.InitialState\x1aX\n\x0eStrategyDirect\x12$\n\x1cmax_queue_delay_microseconds\x18\x01 \x01(\x04\x12 \n\x18minimum_slot_utilization\x18\x02 \x01(\x02\x1au\n\x0eStrategyOldest\x12\x1f\n\x17max_candidate_sequences\x18\x01 \x01(\x05\x12\x1c\n\x14preferred_batch_size\x18\x02 \x03(\x05\x12$\n\x1cmax_queue_delay_microseconds\x18\x03 \x01(\x04\x42\x11\n\x0fstrategy_choice\"\xf6\x02\n\x0fModelEnsembling\x12-\n\x04step\x18\x01 \x03(\x0b\x32\x1f.inference.ModelEnsembling.Step\x1a\xb3\x02\n\x04Step\x12\x12\n\nmodel_name\x18\x01 \x01(\t\x12\x15\n\rmodel_version\x18\x02 \x01(\x03\x12@\n\tinput_map\x18\x03 \x03(\x0b\x32-.inference.ModelEnsembling.Step.InputMapEntry\x12\x42\n\noutput_map\x18\x04 \x03(\x0b\x32..inference.ModelEnsembling.Step.OutputMapEntry\x12\x17\n\x0fmodel_namespace\x18\x05 \x01(\t\x1a/\n\rInputMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x30\n\x0eOutputMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"&\n\x0eModelParameter\x12\x14\n\x0cstring_value\x18\x01 \x01(\t\"\xd9\x02\n\x0bModelWarmup\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\nbatch_size\x18\x02 \x01(\r\x12\x32\n\x06inputs\x18\x03 \x03(\x0b\x32\".inference.ModelWarmup.InputsEntry\x12\r\n\x05\x63ount\x18\x04 \x01(\r\x1a\x97\x01\n\x05Input\x12&\n\tdata_type\x18\x01 \x01(\x0e\x32\x13.inference.DataType\x12\x0c\n\x04\x64ims\x18\x02 \x03(\x03\x12\x13\n\tzero_data\x18\x03 \x01(\x08H\x00\x12\x15\n\x0brandom_data\x18\x04 \x01(\x08H\x00\x12\x19\n\x0finput_data_file\x18\x05 \x01(\tH\x00\x42\x11\n\x0finput_data_type\x1aK\n\x0bInputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12+\n\x05value\x18\x02 \x01(\x0b\x32\x1c.inference.ModelWarmup.Input:\x02\x38\x01\".\n\x0fModelOperations\x12\x1b\n\x13op_library_filename\x18\x01 \x03(\t\"+\n\x16ModelTransactionPolicy\x12\x11\n\tdecoupled\x18\x01 \x01(\x08\"\xe6\x01\n\x15ModelRepositoryAgents\x12\x36\n\x06\x61gents\x18\x01 \x03(\x0b\x32&.inference.ModelRepositoryAgents.Agent\x1a\x94\x01\n\x05\x41gent\x12\x0c\n\x04name\x18\x01 \x01(\t\x12J\n\nparameters\x18\x02 \x03(\x0b\x32\x36.inference.ModelRepositoryAgents.Agent.ParametersEntry\x1a\x31\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"$\n\x12ModelResponseCache\x12\x0e\n\x06\x65nable\x18\x01 \x01(\x08\"\xb2\n\n\x0bModelConfig\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08platform\x18\x02 \x01(\t\x12\x0f\n\x07\x62\x61\x63kend\x18\x11 \x01(\t\x12\x35\n\x0eversion_policy\x18\x03 \x01(\x0b\x32\x1d.inference.ModelVersionPolicy\x12\x16\n\x0emax_batch_size\x18\x04 \x01(\x05\x12$\n\x05input\x18\x05 \x03(\x0b\x32\x15.inference.ModelInput\x12&\n\x06output\x18\x06 \x03(\x0b\x32\x16.inference.ModelOutput\x12*\n\x0b\x62\x61tch_input\x18\x14 \x03(\x0b\x32\x15.inference.BatchInput\x12,\n\x0c\x62\x61tch_output\x18\x15 \x03(\x0b\x32\x16.inference.BatchOutput\x12\x38\n\x0coptimization\x18\x0c \x01(\x0b\x32\".inference.ModelOptimizationPolicy\x12;\n\x10\x64ynamic_batching\x18\x0b \x01(\x0b\x32\x1f.inference.ModelDynamicBatchingH\x00\x12=\n\x11sequence_batching\x18\r \x01(\x0b\x32 .inference.ModelSequenceBatchingH\x00\x12\x39\n\x13\x65nsemble_scheduling\x18\x0f \x01(\x0b\x32\x1a.inference.ModelEnsemblingH\x00\x12\x35\n\x0einstance_group\x18\x07 \x03(\x0b\x32\x1d.inference.ModelInstanceGroup\x12\x1e\n\x16\x64\x65\x66\x61ult_model_filename\x18\x08 \x01(\t\x12H\n\x12\x63\x63_model_filenames\x18\t \x03(\x0b\x32,.inference.ModelConfig.CcModelFilenamesEntry\x12;\n\x0bmetric_tags\x18\n \x03(\x0b\x32&.inference.ModelConfig.MetricTagsEntry\x12:\n\nparameters\x18\x0e \x03(\x0b\x32&.inference.ModelConfig.ParametersEntry\x12,\n\x0cmodel_warmup\x18\x10 \x03(\x0b\x32\x16.inference.ModelWarmup\x12\x34\n\x10model_operations\x18\x12 \x01(\x0b\x32\x1a.inference.ModelOperations\x12\x43\n\x18model_transaction_policy\x18\x13 \x01(\x0b\x32!.inference.ModelTransactionPolicy\x12\x41\n\x17model_repository_agents\x18\x17 \x01(\x0b\x32 .inference.ModelRepositoryAgents\x12\x35\n\x0eresponse_cache\x18\x18 \x01(\x0b\x32\x1d.inference.ModelResponseCache\x1a\x37\n\x15\x43\x63ModelFilenamesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x31\n\x0fMetricTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1aL\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.inference.ModelParameter:\x02\x38\x01\x42\x13\n\x11scheduling_choice*\xfa\x01\n\x08\x44\x61taType\x12\x10\n\x0cTYPE_INVALID\x10\x00\x12\r\n\tTYPE_BOOL\x10\x01\x12\x0e\n\nTYPE_UINT8\x10\x02\x12\x0f\n\x0bTYPE_UINT16\x10\x03\x12\x0f\n\x0bTYPE_UINT32\x10\x04\x12\x0f\n\x0bTYPE_UINT64\x10\x05\x12\r\n\tTYPE_INT8\x10\x06\x12\x0e\n\nTYPE_INT16\x10\x07\x12\x0e\n\nTYPE_INT32\x10\x08\x12\x0e\n\nTYPE_INT64\x10\t\x12\r\n\tTYPE_FP16\x10\n\x12\r\n\tTYPE_FP32\x10\x0b\x12\r\n\tTYPE_FP64\x10\x0c\x12\x0f\n\x0bTYPE_STRING\x10\r\x12\r\n\tTYPE_BF16\x10\x0e\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x12model_config.proto\x12\tinference\"\x96\x01\n\x10ModelRateLimiter\x12\x37\n\tresources\x18\x01 \x03(\x0b\x32$.inference.ModelRateLimiter.Resource\x12\x10\n\x08priority\x18\x02 \x01(\r\x1a\x37\n\x08Resource\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06global\x18\x02 \x01(\x08\x12\r\n\x05\x63ount\x18\x03 \x01(\r\"\x87\x04\n\x12ModelInstanceGroup\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x30\n\x04kind\x18\x04 \x01(\x0e\x32\".inference.ModelInstanceGroup.Kind\x12\r\n\x05\x63ount\x18\x02 \x01(\x05\x12\x31\n\x0crate_limiter\x18\x06 \x01(\x0b\x32\x1b.inference.ModelRateLimiter\x12\x0c\n\x04gpus\x18\x03 \x03(\x05\x12H\n\x11secondary_devices\x18\x08 \x03(\x0b\x32-.inference.ModelInstanceGroup.SecondaryDevice\x12\x0f\n\x07profile\x18\x05 \x03(\t\x12\x0f\n\x07passive\x18\x07 \x01(\x08\x12\x13\n\x0bhost_policy\x18\t \x01(\t\x1a\x9c\x01\n\x0fSecondaryDevice\x12O\n\x04kind\x18\x01 \x01(\x0e\x32\x41.inference.ModelInstanceGroup.SecondaryDevice.SecondaryDeviceKind\x12\x11\n\tdevice_id\x18\x02 \x01(\x03\"%\n\x13SecondaryDeviceKind\x12\x0e\n\nKIND_NVDLA\x10\x00\"A\n\x04Kind\x12\r\n\tKIND_AUTO\x10\x00\x12\x0c\n\x08KIND_GPU\x10\x01\x12\x0c\n\x08KIND_CPU\x10\x02\x12\x0e\n\nKIND_MODEL\x10\x03\"#\n\x12ModelTensorReshape\x12\r\n\x05shape\x18\x01 \x03(\x03\"\xb2\x02\n\nModelInput\x12\x0c\n\x04name\x18\x01 \x01(\t\x12&\n\tdata_type\x18\x02 \x01(\x0e\x32\x13.inference.DataType\x12,\n\x06\x66ormat\x18\x03 \x01(\x0e\x32\x1c.inference.ModelInput.Format\x12\x0c\n\x04\x64ims\x18\x04 \x03(\x03\x12.\n\x07reshape\x18\x05 \x01(\x0b\x32\x1d.inference.ModelTensorReshape\x12\x17\n\x0fis_shape_tensor\x18\x06 \x01(\x08\x12\x1a\n\x12\x61llow_ragged_batch\x18\x07 \x01(\x08\x12\x10\n\x08optional\x18\x08 \x01(\x08\";\n\x06\x46ormat\x12\x0f\n\x0b\x46ORMAT_NONE\x10\x00\x12\x0f\n\x0b\x46ORMAT_NHWC\x10\x01\x12\x0f\n\x0b\x46ORMAT_NCHW\x10\x02\"\xb2\x01\n\x0bModelOutput\x12\x0c\n\x04name\x18\x01 \x01(\t\x12&\n\tdata_type\x18\x02 \x01(\x0e\x32\x13.inference.DataType\x12\x0c\n\x04\x64ims\x18\x03 \x03(\x03\x12.\n\x07reshape\x18\x05 \x01(\x0b\x32\x1d.inference.ModelTensorReshape\x12\x16\n\x0elabel_filename\x18\x04 \x01(\t\x12\x17\n\x0fis_shape_tensor\x18\x06 \x01(\x08\"\xd9\x02\n\nBatchInput\x12(\n\x04kind\x18\x01 \x01(\x0e\x32\x1a.inference.BatchInput.Kind\x12\x13\n\x0btarget_name\x18\x02 \x03(\t\x12&\n\tdata_type\x18\x03 \x01(\x0e\x32\x13.inference.DataType\x12\x14\n\x0csource_input\x18\x04 \x03(\t\"\xcd\x01\n\x04Kind\x12\x17\n\x13\x42\x41TCH_ELEMENT_COUNT\x10\x00\x12#\n\x1f\x42\x41TCH_ACCUMULATED_ELEMENT_COUNT\x10\x01\x12-\n)BATCH_ACCUMULATED_ELEMENT_COUNT_WITH_ZERO\x10\x02\x12$\n BATCH_MAX_ELEMENT_COUNT_AS_SHAPE\x10\x03\x12\x14\n\x10\x42\x41TCH_ITEM_SHAPE\x10\x04\x12\x1c\n\x18\x42\x41TCH_ITEM_SHAPE_FLATTEN\x10\x05\"\x8f\x01\n\x0b\x42\x61tchOutput\x12\x13\n\x0btarget_name\x18\x01 \x03(\t\x12)\n\x04kind\x18\x02 \x01(\x0e\x32\x1b.inference.BatchOutput.Kind\x12\x14\n\x0csource_input\x18\x03 \x03(\t\"*\n\x04Kind\x12\"\n\x1e\x42\x41TCH_SCATTER_WITH_INPUT_SHAPE\x10\x00\"\x90\x02\n\x12ModelVersionPolicy\x12\x36\n\x06latest\x18\x01 \x01(\x0b\x32$.inference.ModelVersionPolicy.LatestH\x00\x12\x30\n\x03\x61ll\x18\x02 \x01(\x0b\x32!.inference.ModelVersionPolicy.AllH\x00\x12:\n\x08specific\x18\x03 \x01(\x0b\x32&.inference.ModelVersionPolicy.SpecificH\x00\x1a\x1e\n\x06Latest\x12\x14\n\x0cnum_versions\x18\x01 \x01(\r\x1a\x05\n\x03\x41ll\x1a\x1c\n\x08Specific\x12\x10\n\x08versions\x18\x01 \x03(\x03\x42\x0f\n\rpolicy_choice\"\xfd\r\n\x17ModelOptimizationPolicy\x12\x37\n\x05graph\x18\x01 \x01(\x0b\x32(.inference.ModelOptimizationPolicy.Graph\x12\x42\n\x08priority\x18\x02 \x01(\x0e\x32\x30.inference.ModelOptimizationPolicy.ModelPriority\x12\x35\n\x04\x63uda\x18\x03 \x01(\x0b\x32\'.inference.ModelOptimizationPolicy.Cuda\x12X\n\x16\x65xecution_accelerators\x18\x04 \x01(\x0b\x32\x38.inference.ModelOptimizationPolicy.ExecutionAccelerators\x12R\n\x13input_pinned_memory\x18\x05 \x01(\x0b\x32\x35.inference.ModelOptimizationPolicy.PinnedMemoryBuffer\x12S\n\x14output_pinned_memory\x18\x06 \x01(\x0b\x32\x35.inference.ModelOptimizationPolicy.PinnedMemoryBuffer\x12&\n\x1egather_kernel_buffer_threshold\x18\x07 \x01(\r\x12\x16\n\x0e\x65\x61ger_batching\x18\x08 \x01(\x08\x1a\x16\n\x05Graph\x12\r\n\x05level\x18\x01 \x01(\x05\x1a\xba\x05\n\x04\x43uda\x12\x0e\n\x06graphs\x18\x01 \x01(\x08\x12\x18\n\x10\x62usy_wait_events\x18\x02 \x01(\x08\x12\x45\n\ngraph_spec\x18\x03 \x03(\x0b\x32\x31.inference.ModelOptimizationPolicy.Cuda.GraphSpec\x12\x1a\n\x12output_copy_stream\x18\x04 \x01(\x08\x1a\xa4\x04\n\tGraphSpec\x12\x12\n\nbatch_size\x18\x01 \x01(\x05\x12K\n\x05input\x18\x02 \x03(\x0b\x32<.inference.ModelOptimizationPolicy.Cuda.GraphSpec.InputEntry\x12W\n\x11graph_lower_bound\x18\x03 \x01(\x0b\x32<.inference.ModelOptimizationPolicy.Cuda.GraphSpec.LowerBound\x1a\x14\n\x05Shape\x12\x0b\n\x03\x64im\x18\x01 \x03(\x03\x1a\xdf\x01\n\nLowerBound\x12\x12\n\nbatch_size\x18\x01 \x01(\x05\x12V\n\x05input\x18\x02 \x03(\x0b\x32G.inference.ModelOptimizationPolicy.Cuda.GraphSpec.LowerBound.InputEntry\x1a\x65\n\nInputEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x46\n\x05value\x18\x02 \x01(\x0b\x32\x37.inference.ModelOptimizationPolicy.Cuda.GraphSpec.Shape:\x02\x38\x01\x1a\x65\n\nInputEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x46\n\x05value\x18\x02 \x01(\x0b\x32\x37.inference.ModelOptimizationPolicy.Cuda.GraphSpec.Shape:\x02\x38\x01\x1a\xa4\x03\n\x15\x45xecutionAccelerators\x12g\n\x19gpu_execution_accelerator\x18\x01 \x03(\x0b\x32\x44.inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator\x12g\n\x19\x63pu_execution_accelerator\x18\x02 \x03(\x0b\x32\x44.inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator\x1a\xb8\x01\n\x0b\x41\x63\x63\x65lerator\x12\x0c\n\x04name\x18\x01 \x01(\t\x12h\n\nparameters\x18\x02 \x03(\x0b\x32T.inference.ModelOptimizationPolicy.ExecutionAccelerators.Accelerator.ParametersEntry\x1a\x31\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a$\n\x12PinnedMemoryBuffer\x12\x0e\n\x06\x65nable\x18\x01 \x01(\x08\"I\n\rModelPriority\x12\x14\n\x10PRIORITY_DEFAULT\x10\x00\x12\x10\n\x0cPRIORITY_MAX\x10\x01\x12\x10\n\x0cPRIORITY_MIN\x10\x02\"\xdb\x01\n\x10ModelQueuePolicy\x12\x41\n\x0etimeout_action\x18\x01 \x01(\x0e\x32).inference.ModelQueuePolicy.TimeoutAction\x12$\n\x1c\x64\x65\x66\x61ult_timeout_microseconds\x18\x02 \x01(\x04\x12\x1e\n\x16\x61llow_timeout_override\x18\x03 \x01(\x08\x12\x16\n\x0emax_queue_size\x18\x04 \x01(\r\"&\n\rTimeoutAction\x12\n\n\x06REJECT\x10\x00\x12\t\n\x05\x44\x45LAY\x10\x01\"\x9b\x03\n\x14ModelDynamicBatching\x12\x1c\n\x14preferred_batch_size\x18\x01 \x03(\x05\x12$\n\x1cmax_queue_delay_microseconds\x18\x02 \x01(\x04\x12\x19\n\x11preserve_ordering\x18\x03 \x01(\x08\x12\x17\n\x0fpriority_levels\x18\x04 \x01(\x04\x12\x1e\n\x16\x64\x65\x66\x61ult_priority_level\x18\x05 \x01(\x04\x12\x39\n\x14\x64\x65\x66\x61ult_queue_policy\x18\x06 \x01(\x0b\x32\x1b.inference.ModelQueuePolicy\x12W\n\x15priority_queue_policy\x18\x07 \x03(\x0b\x32\x38.inference.ModelDynamicBatching.PriorityQueuePolicyEntry\x1aW\n\x18PriorityQueuePolicyEntry\x12\x0b\n\x03key\x18\x01 \x01(\x04\x12*\n\x05value\x18\x02 \x01(\x0b\x32\x1b.inference.ModelQueuePolicy:\x02\x38\x01\"\xef\t\n\x15ModelSequenceBatching\x12\x41\n\x06\x64irect\x18\x03 \x01(\x0b\x32/.inference.ModelSequenceBatching.StrategyDirectH\x00\x12\x41\n\x06oldest\x18\x04 \x01(\x0b\x32/.inference.ModelSequenceBatching.StrategyOldestH\x00\x12&\n\x1emax_sequence_idle_microseconds\x18\x01 \x01(\x04\x12\x44\n\rcontrol_input\x18\x02 \x03(\x0b\x32-.inference.ModelSequenceBatching.ControlInput\x12\x35\n\x05state\x18\x05 \x03(\x0b\x32&.inference.ModelSequenceBatching.State\x1a\xb1\x02\n\x07\x43ontrol\x12;\n\x04kind\x18\x01 \x01(\x0e\x32-.inference.ModelSequenceBatching.Control.Kind\x12\x18\n\x10int32_false_true\x18\x02 \x03(\x05\x12\x17\n\x0f\x66p32_false_true\x18\x03 \x03(\x02\x12\x17\n\x0f\x62ool_false_true\x18\x05 \x03(\x08\x12&\n\tdata_type\x18\x04 \x01(\x0e\x32\x13.inference.DataType\"u\n\x04Kind\x12\x1a\n\x16\x43ONTROL_SEQUENCE_START\x10\x00\x12\x1a\n\x16\x43ONTROL_SEQUENCE_READY\x10\x01\x12\x18\n\x14\x43ONTROL_SEQUENCE_END\x10\x02\x12\x1b\n\x17\x43ONTROL_SEQUENCE_CORRID\x10\x03\x1aW\n\x0c\x43ontrolInput\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x39\n\x07\x63ontrol\x18\x02 \x03(\x0b\x32(.inference.ModelSequenceBatching.Control\x1a\x8a\x01\n\x0cInitialState\x12&\n\tdata_type\x18\x01 \x01(\x0e\x32\x13.inference.DataType\x12\x0c\n\x04\x64ims\x18\x02 \x03(\x03\x12\x13\n\tzero_data\x18\x03 \x01(\x08H\x00\x12\x13\n\tdata_file\x18\x04 \x01(\tH\x00\x12\x0c\n\x04name\x18\x05 \x01(\tB\x0c\n\nstate_data\x1a\xac\x01\n\x05State\x12\x12\n\ninput_name\x18\x01 \x01(\t\x12\x13\n\x0boutput_name\x18\x02 \x01(\t\x12&\n\tdata_type\x18\x03 \x01(\x0e\x32\x13.inference.DataType\x12\x0c\n\x04\x64ims\x18\x04 \x03(\x03\x12\x44\n\rinitial_state\x18\x05 \x03(\x0b\x32-.inference.ModelSequenceBatching.InitialState\x1aX\n\x0eStrategyDirect\x12$\n\x1cmax_queue_delay_microseconds\x18\x01 \x01(\x04\x12 \n\x18minimum_slot_utilization\x18\x02 \x01(\x02\x1au\n\x0eStrategyOldest\x12\x1f\n\x17max_candidate_sequences\x18\x01 \x01(\x05\x12\x1c\n\x14preferred_batch_size\x18\x02 \x03(\x05\x12$\n\x1cmax_queue_delay_microseconds\x18\x03 \x01(\x04\x42\x11\n\x0fstrategy_choice\"\xf6\x02\n\x0fModelEnsembling\x12-\n\x04step\x18\x01 \x03(\x0b\x32\x1f.inference.ModelEnsembling.Step\x1a\xb3\x02\n\x04Step\x12\x12\n\nmodel_name\x18\x01 \x01(\t\x12\x15\n\rmodel_version\x18\x02 \x01(\x03\x12@\n\tinput_map\x18\x03 \x03(\x0b\x32-.inference.ModelEnsembling.Step.InputMapEntry\x12\x42\n\noutput_map\x18\x04 \x03(\x0b\x32..inference.ModelEnsembling.Step.OutputMapEntry\x12\x17\n\x0fmodel_namespace\x18\x05 \x01(\t\x1a/\n\rInputMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x30\n\x0eOutputMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"&\n\x0eModelParameter\x12\x14\n\x0cstring_value\x18\x01 \x01(\t\"\xd9\x02\n\x0bModelWarmup\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\nbatch_size\x18\x02 \x01(\r\x12\x32\n\x06inputs\x18\x03 \x03(\x0b\x32\".inference.ModelWarmup.InputsEntry\x12\r\n\x05\x63ount\x18\x04 \x01(\r\x1a\x97\x01\n\x05Input\x12&\n\tdata_type\x18\x01 \x01(\x0e\x32\x13.inference.DataType\x12\x0c\n\x04\x64ims\x18\x02 \x03(\x03\x12\x13\n\tzero_data\x18\x03 \x01(\x08H\x00\x12\x15\n\x0brandom_data\x18\x04 \x01(\x08H\x00\x12\x19\n\x0finput_data_file\x18\x05 \x01(\tH\x00\x42\x11\n\x0finput_data_type\x1aK\n\x0bInputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12+\n\x05value\x18\x02 \x01(\x0b\x32\x1c.inference.ModelWarmup.Input:\x02\x38\x01\".\n\x0fModelOperations\x12\x1b\n\x13op_library_filename\x18\x01 \x03(\t\"+\n\x16ModelTransactionPolicy\x12\x11\n\tdecoupled\x18\x01 \x01(\x08\"\xe6\x01\n\x15ModelRepositoryAgents\x12\x36\n\x06\x61gents\x18\x01 \x03(\x0b\x32&.inference.ModelRepositoryAgents.Agent\x1a\x94\x01\n\x05\x41gent\x12\x0c\n\x04name\x18\x01 \x01(\t\x12J\n\nparameters\x18\x02 \x03(\x0b\x32\x36.inference.ModelRepositoryAgents.Agent.ParametersEntry\x1a\x31\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"$\n\x12ModelResponseCache\x12\x0e\n\x06\x65nable\x18\x01 \x01(\x08\"\xb2\n\n\x0bModelConfig\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08platform\x18\x02 \x01(\t\x12\x0f\n\x07\x62\x61\x63kend\x18\x11 \x01(\t\x12\x35\n\x0eversion_policy\x18\x03 \x01(\x0b\x32\x1d.inference.ModelVersionPolicy\x12\x16\n\x0emax_batch_size\x18\x04 \x01(\x05\x12$\n\x05input\x18\x05 \x03(\x0b\x32\x15.inference.ModelInput\x12&\n\x06output\x18\x06 \x03(\x0b\x32\x16.inference.ModelOutput\x12*\n\x0b\x62\x61tch_input\x18\x14 \x03(\x0b\x32\x15.inference.BatchInput\x12,\n\x0c\x62\x61tch_output\x18\x15 \x03(\x0b\x32\x16.inference.BatchOutput\x12\x38\n\x0coptimization\x18\x0c \x01(\x0b\x32\".inference.ModelOptimizationPolicy\x12;\n\x10\x64ynamic_batching\x18\x0b \x01(\x0b\x32\x1f.inference.ModelDynamicBatchingH\x00\x12=\n\x11sequence_batching\x18\r \x01(\x0b\x32 .inference.ModelSequenceBatchingH\x00\x12\x39\n\x13\x65nsemble_scheduling\x18\x0f \x01(\x0b\x32\x1a.inference.ModelEnsemblingH\x00\x12\x35\n\x0einstance_group\x18\x07 \x03(\x0b\x32\x1d.inference.ModelInstanceGroup\x12\x1e\n\x16\x64\x65\x66\x61ult_model_filename\x18\x08 \x01(\t\x12H\n\x12\x63\x63_model_filenames\x18\t \x03(\x0b\x32,.inference.ModelConfig.CcModelFilenamesEntry\x12;\n\x0bmetric_tags\x18\n \x03(\x0b\x32&.inference.ModelConfig.MetricTagsEntry\x12:\n\nparameters\x18\x0e \x03(\x0b\x32&.inference.ModelConfig.ParametersEntry\x12,\n\x0cmodel_warmup\x18\x10 \x03(\x0b\x32\x16.inference.ModelWarmup\x12\x34\n\x10model_operations\x18\x12 \x01(\x0b\x32\x1a.inference.ModelOperations\x12\x43\n\x18model_transaction_policy\x18\x13 \x01(\x0b\x32!.inference.ModelTransactionPolicy\x12\x41\n\x17model_repository_agents\x18\x17 \x01(\x0b\x32 .inference.ModelRepositoryAgents\x12\x35\n\x0eresponse_cache\x18\x18 \x01(\x0b\x32\x1d.inference.ModelResponseCache\x1a\x37\n\x15\x43\x63ModelFilenamesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x31\n\x0fMetricTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1aL\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.inference.ModelParameter:\x02\x38\x01\x42\x13\n\x11scheduling_choice*\xfa\x01\n\x08\x44\x61taType\x12\x10\n\x0cTYPE_INVALID\x10\x00\x12\r\n\tTYPE_BOOL\x10\x01\x12\x0e\n\nTYPE_UINT8\x10\x02\x12\x0f\n\x0bTYPE_UINT16\x10\x03\x12\x0f\n\x0bTYPE_UINT32\x10\x04\x12\x0f\n\x0bTYPE_UINT64\x10\x05\x12\r\n\tTYPE_INT8\x10\x06\x12\x0e\n\nTYPE_INT16\x10\x07\x12\x0e\n\nTYPE_INT32\x10\x08\x12\x0e\n\nTYPE_INT64\x10\t\x12\r\n\tTYPE_FP16\x10\n\x12\r\n\tTYPE_FP32\x10\x0b\x12\r\n\tTYPE_FP64\x10\x0c\x12\x0f\n\x0bTYPE_STRING\x10\r\x12\r\n\tTYPE_BF16\x10\x0e\x62\x06proto3')
 
 _DATATYPE = DESCRIPTOR.enum_types_by_name['DataType']
 DataType = enum_type_wrapper.EnumTypeWrapper(_DATATYPE)

{tritonclient-2.34.0.data → tritonclient-2.36.0.data}/purelib/tritonclient/grpc/service_pb2.py

@@ -15,7 +15,7 @@ _sym_db = _symbol_database.Default()
 from tritonclient.grpc import model_config_pb2 as model__config__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x12grpc_service.proto\x12\tinference\x1a\x12model_config.proto\"\x13\n\x11ServerLiveRequest\"\"\n\x12ServerLiveResponse\x12\x0c\n\x04live\x18\x01 \x01(\x08\"\x14\n\x12ServerReadyRequest\"$\n\x13ServerReadyResponse\x12\r\n\x05ready\x18\x01 \x01(\x08\"2\n\x11ModelReadyRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\"#\n\x12ModelReadyResponse\x12\r\n\x05ready\x18\x01 \x01(\x08\"\x17\n\x15ServerMetadataRequest\"K\n\x16ServerMetadataResponse\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x12\n\nextensions\x18\x03 \x03(\t\"5\n\x14ModelMetadataRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\"\x8d\x02\n\x15ModelMetadataResponse\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08versions\x18\x02 \x03(\t\x12\x10\n\x08platform\x18\x03 \x01(\t\x12?\n\x06inputs\x18\x04 \x03(\x0b\x32/.inference.ModelMetadataResponse.TensorMetadata\x12@\n\x07outputs\x18\x05 \x03(\x0b\x32/.inference.ModelMetadataResponse.TensorMetadata\x1a?\n\x0eTensorMetadata\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08\x64\x61tatype\x18\x02 \x01(\t\x12\r\n\x05shape\x18\x03 \x03(\x03\"i\n\x0eInferParameter\x12\x14\n\nbool_param\x18\x01 \x01(\x08H\x00\x12\x15\n\x0bint64_param\x18\x02 \x01(\x03H\x00\x12\x16\n\x0cstring_param\x18\x03 \x01(\tH\x00\x42\x12\n\x10parameter_choice\"\xd0\x01\n\x13InferTensorContents\x12\x15\n\rbool_contents\x18\x01 \x03(\x08\x12\x14\n\x0cint_contents\x18\x02 \x03(\x05\x12\x16\n\x0eint64_contents\x18\x03 \x03(\x03\x12\x15\n\ruint_contents\x18\x04 \x03(\r\x12\x17\n\x0fuint64_contents\x18\x05 \x03(\x04\x12\x15\n\rfp32_contents\x18\x06 \x03(\x02\x12\x15\n\rfp64_contents\x18\x07 \x03(\x01\x12\x16\n\x0e\x62ytes_contents\x18\x08 \x03(\x0c\"\xee\x06\n\x11ModelInferRequest\x12\x12\n\nmodel_name\x18\x01 \x01(\t\x12\x15\n\rmodel_version\x18\x02 \x01(\t\x12\n\n\x02id\x18\x03 \x01(\t\x12@\n\nparameters\x18\x04 \x03(\x0b\x32,.inference.ModelInferRequest.ParametersEntry\x12=\n\x06inputs\x18\x05 \x03(\x0b\x32-.inference.ModelInferRequest.InferInputTensor\x12H\n\x07outputs\x18\x06 \x03(\x0b\x32\x37.inference.ModelInferRequest.InferRequestedOutputTensor\x12\x1a\n\x12raw_input_contents\x18\x07 \x03(\x0c\x1a\x94\x02\n\x10InferInputTensor\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08\x64\x61tatype\x18\x02 \x01(\t\x12\r\n\x05shape\x18\x03 \x03(\x03\x12Q\n\nparameters\x18\x04 \x03(\x0b\x32=.inference.ModelInferRequest.InferInputTensor.ParametersEntry\x12\x30\n\x08\x63ontents\x18\x05 \x01(\x0b\x32\x1e.inference.InferTensorContents\x1aL\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.inference.InferParameter:\x02\x38\x01\x1a\xd5\x01\n\x1aInferRequestedOutputTensor\x12\x0c\n\x04name\x18\x01 \x01(\t\x12[\n\nparameters\x18\x02 \x03(\x0b\x32G.inference.ModelInferRequest.InferRequestedOutputTensor.ParametersEntry\x1aL\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.inference.InferParameter:\x02\x38\x01\x1aL\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.inference.InferParameter:\x02\x38\x01\"\xd5\x04\n\x12ModelInferResponse\x12\x12\n\nmodel_name\x18\x01 \x01(\t\x12\x15\n\rmodel_version\x18\x02 \x01(\t\x12\n\n\x02id\x18\x03 \x01(\t\x12\x41\n\nparameters\x18\x04 \x03(\x0b\x32-.inference.ModelInferResponse.ParametersEntry\x12@\n\x07outputs\x18\x05 \x03(\x0b\x32/.inference.ModelInferResponse.InferOutputTensor\x12\x1b\n\x13raw_output_contents\x18\x06 \x03(\x0c\x1a\x97\x02\n\x11InferOutputTensor\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08\x64\x61tatype\x18\x02 \x01(\t\x12\r\n\x05shape\x18\x03 \x03(\x03\x12S\n\nparameters\x18\x04 \x03(\x0b\x32?.inference.ModelInferResponse.InferOutputTensor.ParametersEntry\x12\x30\n\x08\x63ontents\x18\x05 \x01(\x0b\x32\x1e.inference.InferTensorContents\x1aL\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.inference.InferParameter:\x02\x38\x01\x1aL\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.inference.InferParameter:\x02\x38\x01\"h\n\x18ModelStreamInferResponse\x12\x15\n\rerror_message\x18\x01 \x01(\t\x12\x35\n\x0einfer_response\x18\x02 \x01(\x0b\x32\x1d.inference.ModelInferResponse\"3\n\x12ModelConfigRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\"=\n\x13ModelConfigResponse\x12&\n\x06\x63onfig\x18\x01 \x01(\x0b\x32\x16.inference.ModelConfig\"7\n\x16ModelStatisticsRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\".\n\x11StatisticDuration\x12\r\n\x05\x63ount\x18\x01 \x01(\x04\x12\n\n\x02ns\x18\x02 \x01(\x04\"\x9c\x03\n\x0fInferStatistics\x12-\n\x07success\x18\x01 \x01(\x0b\x32\x1c.inference.StatisticDuration\x12*\n\x04\x66\x61il\x18\x02 \x01(\x0b\x32\x1c.inference.StatisticDuration\x12+\n\x05queue\x18\x03 \x01(\x0b\x32\x1c.inference.StatisticDuration\x12\x33\n\rcompute_input\x18\x04 \x01(\x0b\x32\x1c.inference.StatisticDuration\x12\x33\n\rcompute_infer\x18\x05 \x01(\x0b\x32\x1c.inference.StatisticDuration\x12\x34\n\x0e\x63ompute_output\x18\x06 \x01(\x0b\x32\x1c.inference.StatisticDuration\x12/\n\tcache_hit\x18\x07 \x01(\x0b\x32\x1c.inference.StatisticDuration\x12\x30\n\ncache_miss\x18\x08 \x01(\x0b\x32\x1c.inference.StatisticDuration\"\xca\x01\n\x14InferBatchStatistics\x12\x12\n\nbatch_size\x18\x01 \x01(\x04\x12\x33\n\rcompute_input\x18\x02 \x01(\x0b\x32\x1c.inference.StatisticDuration\x12\x33\n\rcompute_infer\x18\x03 \x01(\x0b\x32\x1c.inference.StatisticDuration\x12\x34\n\x0e\x63ompute_output\x18\x04 \x01(\x0b\x32\x1c.inference.StatisticDuration\"\xe5\x01\n\x0fModelStatistics\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x16\n\x0elast_inference\x18\x03 \x01(\x04\x12\x17\n\x0finference_count\x18\x04 \x01(\x04\x12\x17\n\x0f\x65xecution_count\x18\x05 \x01(\x04\x12\x33\n\x0finference_stats\x18\x06 \x01(\x0b\x32\x1a.inference.InferStatistics\x12\x34\n\x0b\x62\x61tch_stats\x18\x07 \x03(\x0b\x32\x1f.inference.InferBatchStatistics\"J\n\x17ModelStatisticsResponse\x12/\n\x0bmodel_stats\x18\x01 \x03(\x0b\x32\x1a.inference.ModelStatistics\"\x8a\x01\n\x18ModelRepositoryParameter\x12\x14\n\nbool_param\x18\x01 \x01(\x08H\x00\x12\x15\n\x0bint64_param\x18\x02 \x01(\x03H\x00\x12\x16\n\x0cstring_param\x18\x03 \x01(\tH\x00\x12\x15\n\x0b\x62ytes_param\x18\x04 \x01(\x0cH\x00\x42\x12\n\x10parameter_choice\"@\n\x16RepositoryIndexRequest\x12\x17\n\x0frepository_name\x18\x01 \x01(\t\x12\r\n\x05ready\x18\x02 \x01(\x08\"\xa4\x01\n\x17RepositoryIndexResponse\x12=\n\x06models\x18\x01 \x03(\x0b\x32-.inference.RepositoryIndexResponse.ModelIndex\x1aJ\n\nModelIndex\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\r\n\x05state\x18\x03 \x01(\t\x12\x0e\n\x06reason\x18\x04 \x01(\t\"\xec\x01\n\x1aRepositoryModelLoadRequest\x12\x17\n\x0frepository_name\x18\x01 \x01(\t\x12\x12\n\nmodel_name\x18\x02 \x01(\t\x12I\n\nparameters\x18\x03 \x03(\x0b\x32\x35.inference.RepositoryModelLoadRequest.ParametersEntry\x1aV\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x32\n\x05value\x18\x02 \x01(\x0b\x32#.inference.ModelRepositoryParameter:\x02\x38\x01\"\x1d\n\x1bRepositoryModelLoadResponse\"\xf0\x01\n\x1cRepositoryModelUnloadRequest\x12\x17\n\x0frepository_name\x18\x01 \x01(\t\x12\x12\n\nmodel_name\x18\x02 \x01(\t\x12K\n\nparameters\x18\x03 \x03(\x0b\x32\x37.inference.RepositoryModelUnloadRequest.ParametersEntry\x1aV\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x32\n\x05value\x18\x02 \x01(\x0b\x32#.inference.ModelRepositoryParameter:\x02\x38\x01\"\x1f\n\x1dRepositoryModelUnloadResponse\"/\n\x1fSystemSharedMemoryStatusRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\xa5\x02\n SystemSharedMemoryStatusResponse\x12I\n\x07regions\x18\x01 \x03(\x0b\x32\x38.inference.SystemSharedMemoryStatusResponse.RegionsEntry\x1aL\n\x0cRegionStatus\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0b\n\x03key\x18\x02 \x01(\t\x12\x0e\n\x06offset\x18\x03 \x01(\x04\x12\x11\n\tbyte_size\x18\x04 \x01(\x04\x1ah\n\x0cRegionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12G\n\x05value\x18\x02 \x01(\x0b\x32\x38.inference.SystemSharedMemoryStatusResponse.RegionStatus:\x02\x38\x01\"a\n!SystemSharedMemoryRegisterRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0b\n\x03key\x18\x02 \x01(\t\x12\x0e\n\x06offset\x18\x03 \x01(\x04\x12\x11\n\tbyte_size\x18\x04 \x01(\x04\"$\n\"SystemSharedMemoryRegisterResponse\"3\n#SystemSharedMemoryUnregisterRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"&\n$SystemSharedMemoryUnregisterResponse\"-\n\x1d\x43udaSharedMemoryStatusRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x95\x02\n\x1e\x43udaSharedMemoryStatusResponse\x12G\n\x07regions\x18\x01 \x03(\x0b\x32\x36.inference.CudaSharedMemoryStatusResponse.RegionsEntry\x1a\x42\n\x0cRegionStatus\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x11\n\tdevice_id\x18\x02 \x01(\x04\x12\x11\n\tbyte_size\x18\x03 \x01(\x04\x1a\x66\n\x0cRegionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x45\n\x05value\x18\x02 \x01(\x0b\x32\x36.inference.CudaSharedMemoryStatusResponse.RegionStatus:\x02\x38\x01\"i\n\x1f\x43udaSharedMemoryRegisterRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\nraw_handle\x18\x02 \x01(\x0c\x12\x11\n\tdevice_id\x18\x03 \x01(\x03\x12\x11\n\tbyte_size\x18\x04 \x01(\x04\"\"\n CudaSharedMemoryRegisterResponse\"1\n!CudaSharedMemoryUnregisterRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"$\n\"CudaSharedMemoryUnregisterResponse\"\xe6\x01\n\x13TraceSettingRequest\x12>\n\x08settings\x18\x01 \x03(\x0b\x32,.inference.TraceSettingRequest.SettingsEntry\x12\x12\n\nmodel_name\x18\x02 \x01(\t\x1a\x1d\n\x0cSettingValue\x12\r\n\x05value\x18\x01 \x03(\t\x1a\\\n\rSettingsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12:\n\x05value\x18\x02 \x01(\x0b\x32+.inference.TraceSettingRequest.SettingValue:\x02\x38\x01\"\xd5\x01\n\x14TraceSettingResponse\x12?\n\x08settings\x18\x01 \x03(\x0b\x32-.inference.TraceSettingResponse.SettingsEntry\x1a\x1d\n\x0cSettingValue\x12\r\n\x05value\x18\x01 \x03(\t\x1a]\n\rSettingsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12;\n\x05value\x18\x02 \x01(\x0b\x32,.inference.TraceSettingResponse.SettingValue:\x02\x38\x01\"\x9a\x02\n\x12LogSettingsRequest\x12=\n\x08settings\x18\x01 \x03(\x0b\x32+.inference.LogSettingsRequest.SettingsEntry\x1ah\n\x0cSettingValue\x12\x14\n\nbool_param\x18\x01 \x01(\x08H\x00\x12\x16\n\x0cuint32_param\x18\x02 \x01(\rH\x00\x12\x16\n\x0cstring_param\x18\x03 \x01(\tH\x00\x42\x12\n\x10parameter_choice\x1a[\n\rSettingsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x39\n\x05value\x18\x02 \x01(\x0b\x32*.inference.LogSettingsRequest.SettingValue:\x02\x38\x01\"\x9d\x02\n\x13LogSettingsResponse\x12>\n\x08settings\x18\x01 \x03(\x0b\x32,.inference.LogSettingsResponse.SettingsEntry\x1ah\n\x0cSettingValue\x12\x14\n\nbool_param\x18\x01 \x01(\x08H\x00\x12\x16\n\x0cuint32_param\x18\x02 \x01(\rH\x00\x12\x16\n\x0cstring_param\x18\x03 \x01(\tH\x00\x42\x12\n\x10parameter_choice\x1a\\\n\rSettingsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12:\n\x05value\x18\x02 \x01(\x0b\x32+.inference.LogSettingsResponse.SettingValue:\x02\x38\x01\x32\xb7\x0f\n\x14GRPCInferenceService\x12K\n\nServerLive\x12\x1c.inference.ServerLiveRequest\x1a\x1d.inference.ServerLiveResponse\"\x00\x12N\n\x0bServerReady\x12\x1d.inference.ServerReadyRequest\x1a\x1e.inference.ServerReadyResponse\"\x00\x12K\n\nModelReady\x12\x1c.inference.ModelReadyRequest\x1a\x1d.inference.ModelReadyResponse\"\x00\x12W\n\x0eServerMetadata\x12 .inference.ServerMetadataRequest\x1a!.inference.ServerMetadataResponse\"\x00\x12T\n\rModelMetadata\x12\x1f.inference.ModelMetadataRequest\x1a .inference.ModelMetadataResponse\"\x00\x12K\n\nModelInfer\x12\x1c.inference.ModelInferRequest\x1a\x1d.inference.ModelInferResponse\"\x00\x12[\n\x10ModelStreamInfer\x12\x1c.inference.ModelInferRequest\x1a#.inference.ModelStreamInferResponse\"\x00(\x01\x30\x01\x12N\n\x0bModelConfig\x12\x1d.inference.ModelConfigRequest\x1a\x1e.inference.ModelConfigResponse\"\x00\x12Z\n\x0fModelStatistics\x12!.inference.ModelStatisticsRequest\x1a\".inference.ModelStatisticsResponse\"\x00\x12Z\n\x0fRepositoryIndex\x12!.inference.RepositoryIndexRequest\x1a\".inference.RepositoryIndexResponse\"\x00\x12\x66\n\x13RepositoryModelLoad\x12%.inference.RepositoryModelLoadRequest\x1a&.inference.RepositoryModelLoadResponse\"\x00\x12l\n\x15RepositoryModelUnload\x12\'.inference.RepositoryModelUnloadRequest\x1a(.inference.RepositoryModelUnloadResponse\"\x00\x12u\n\x18SystemSharedMemoryStatus\x12*.inference.SystemSharedMemoryStatusRequest\x1a+.inference.SystemSharedMemoryStatusResponse\"\x00\x12{\n\x1aSystemSharedMemoryRegister\x12,.inference.SystemSharedMemoryRegisterRequest\x1a-.inference.SystemSharedMemoryRegisterResponse\"\x00\x12\x81\x01\n\x1cSystemSharedMemoryUnregister\x12..inference.SystemSharedMemoryUnregisterRequest\x1a/.inference.SystemSharedMemoryUnregisterResponse\"\x00\x12o\n\x16\x43udaSharedMemoryStatus\x12(.inference.CudaSharedMemoryStatusRequest\x1a).inference.CudaSharedMemoryStatusResponse\"\x00\x12u\n\x18\x43udaSharedMemoryRegister\x12*.inference.CudaSharedMemoryRegisterRequest\x1a+.inference.CudaSharedMemoryRegisterResponse\"\x00\x12{\n\x1a\x43udaSharedMemoryUnregister\x12,.inference.CudaSharedMemoryUnregisterRequest\x1a-.inference.CudaSharedMemoryUnregisterResponse\"\x00\x12Q\n\x0cTraceSetting\x12\x1e.inference.TraceSettingRequest\x1a\x1f.inference.TraceSettingResponse\"\x00\x12N\n\x0bLogSettings\x12\x1d.inference.LogSettingsRequest\x1a\x1e.inference.LogSettingsResponse\"\x00\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x12grpc_service.proto\x12\tinference\x1a\x12model_config.proto\"\x13\n\x11ServerLiveRequest\"\"\n\x12ServerLiveResponse\x12\x0c\n\x04live\x18\x01 \x01(\x08\"\x14\n\x12ServerReadyRequest\"$\n\x13ServerReadyResponse\x12\r\n\x05ready\x18\x01 \x01(\x08\"2\n\x11ModelReadyRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\"#\n\x12ModelReadyResponse\x12\r\n\x05ready\x18\x01 \x01(\x08\"\x17\n\x15ServerMetadataRequest\"K\n\x16ServerMetadataResponse\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x12\n\nextensions\x18\x03 \x03(\t\"5\n\x14ModelMetadataRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\"\x8d\x02\n\x15ModelMetadataResponse\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08versions\x18\x02 \x03(\t\x12\x10\n\x08platform\x18\x03 \x01(\t\x12?\n\x06inputs\x18\x04 \x03(\x0b\x32/.inference.ModelMetadataResponse.TensorMetadata\x12@\n\x07outputs\x18\x05 \x03(\x0b\x32/.inference.ModelMetadataResponse.TensorMetadata\x1a?\n\x0eTensorMetadata\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08\x64\x61tatype\x18\x02 \x01(\t\x12\r\n\x05shape\x18\x03 \x03(\x03\"\x99\x01\n\x0eInferParameter\x12\x14\n\nbool_param\x18\x01 \x01(\x08H\x00\x12\x15\n\x0bint64_param\x18\x02 \x01(\x03H\x00\x12\x16\n\x0cstring_param\x18\x03 \x01(\tH\x00\x12\x16\n\x0c\x64ouble_param\x18\x04 \x01(\x01H\x00\x12\x16\n\x0cuint64_param\x18\x05 \x01(\x04H\x00\x42\x12\n\x10parameter_choice\"\xd0\x01\n\x13InferTensorContents\x12\x15\n\rbool_contents\x18\x01 \x03(\x08\x12\x14\n\x0cint_contents\x18\x02 \x03(\x05\x12\x16\n\x0eint64_contents\x18\x03 \x03(\x03\x12\x15\n\ruint_contents\x18\x04 \x03(\r\x12\x17\n\x0fuint64_contents\x18\x05 \x03(\x04\x12\x15\n\rfp32_contents\x18\x06 \x03(\x02\x12\x15\n\rfp64_contents\x18\x07 \x03(\x01\x12\x16\n\x0e\x62ytes_contents\x18\x08 \x03(\x0c\"\xee\x06\n\x11ModelInferRequest\x12\x12\n\nmodel_name\x18\x01 \x01(\t\x12\x15\n\rmodel_version\x18\x02 \x01(\t\x12\n\n\x02id\x18\x03 \x01(\t\x12@\n\nparameters\x18\x04 \x03(\x0b\x32,.inference.ModelInferRequest.ParametersEntry\x12=\n\x06inputs\x18\x05 \x03(\x0b\x32-.inference.ModelInferRequest.InferInputTensor\x12H\n\x07outputs\x18\x06 \x03(\x0b\x32\x37.inference.ModelInferRequest.InferRequestedOutputTensor\x12\x1a\n\x12raw_input_contents\x18\x07 \x03(\x0c\x1a\x94\x02\n\x10InferInputTensor\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08\x64\x61tatype\x18\x02 \x01(\t\x12\r\n\x05shape\x18\x03 \x03(\x03\x12Q\n\nparameters\x18\x04 \x03(\x0b\x32=.inference.ModelInferRequest.InferInputTensor.ParametersEntry\x12\x30\n\x08\x63ontents\x18\x05 \x01(\x0b\x32\x1e.inference.InferTensorContents\x1aL\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.inference.InferParameter:\x02\x38\x01\x1a\xd5\x01\n\x1aInferRequestedOutputTensor\x12\x0c\n\x04name\x18\x01 \x01(\t\x12[\n\nparameters\x18\x02 \x03(\x0b\x32G.inference.ModelInferRequest.InferRequestedOutputTensor.ParametersEntry\x1aL\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.inference.InferParameter:\x02\x38\x01\x1aL\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.inference.InferParameter:\x02\x38\x01\"\xd5\x04\n\x12ModelInferResponse\x12\x12\n\nmodel_name\x18\x01 \x01(\t\x12\x15\n\rmodel_version\x18\x02 \x01(\t\x12\n\n\x02id\x18\x03 \x01(\t\x12\x41\n\nparameters\x18\x04 \x03(\x0b\x32-.inference.ModelInferResponse.ParametersEntry\x12@\n\x07outputs\x18\x05 \x03(\x0b\x32/.inference.ModelInferResponse.InferOutputTensor\x12\x1b\n\x13raw_output_contents\x18\x06 \x03(\x0c\x1a\x97\x02\n\x11InferOutputTensor\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08\x64\x61tatype\x18\x02 \x01(\t\x12\r\n\x05shape\x18\x03 \x03(\x03\x12S\n\nparameters\x18\x04 \x03(\x0b\x32?.inference.ModelInferResponse.InferOutputTensor.ParametersEntry\x12\x30\n\x08\x63ontents\x18\x05 \x01(\x0b\x32\x1e.inference.InferTensorContents\x1aL\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.inference.InferParameter:\x02\x38\x01\x1aL\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.inference.InferParameter:\x02\x38\x01\"h\n\x18ModelStreamInferResponse\x12\x15\n\rerror_message\x18\x01 \x01(\t\x12\x35\n\x0einfer_response\x18\x02 \x01(\x0b\x32\x1d.inference.ModelInferResponse\"3\n\x12ModelConfigRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\"=\n\x13ModelConfigResponse\x12&\n\x06\x63onfig\x18\x01 \x01(\x0b\x32\x16.inference.ModelConfig\"7\n\x16ModelStatisticsRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\".\n\x11StatisticDuration\x12\r\n\x05\x63ount\x18\x01 \x01(\x04\x12\n\n\x02ns\x18\x02 \x01(\x04\"\x9c\x03\n\x0fInferStatistics\x12-\n\x07success\x18\x01 \x01(\x0b\x32\x1c.inference.StatisticDuration\x12*\n\x04\x66\x61il\x18\x02 \x01(\x0b\x32\x1c.inference.StatisticDuration\x12+\n\x05queue\x18\x03 \x01(\x0b\x32\x1c.inference.StatisticDuration\x12\x33\n\rcompute_input\x18\x04 \x01(\x0b\x32\x1c.inference.StatisticDuration\x12\x33\n\rcompute_infer\x18\x05 \x01(\x0b\x32\x1c.inference.StatisticDuration\x12\x34\n\x0e\x63ompute_output\x18\x06 \x01(\x0b\x32\x1c.inference.StatisticDuration\x12/\n\tcache_hit\x18\x07 \x01(\x0b\x32\x1c.inference.StatisticDuration\x12\x30\n\ncache_miss\x18\x08 \x01(\x0b\x32\x1c.inference.StatisticDuration\"\xca\x01\n\x14InferBatchStatistics\x12\x12\n\nbatch_size\x18\x01 \x01(\x04\x12\x33\n\rcompute_input\x18\x02 \x01(\x0b\x32\x1c.inference.StatisticDuration\x12\x33\n\rcompute_infer\x18\x03 \x01(\x0b\x32\x1c.inference.StatisticDuration\x12\x34\n\x0e\x63ompute_output\x18\x04 \x01(\x0b\x32\x1c.inference.StatisticDuration\":\n\x0bMemoryUsage\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\x03\x12\x11\n\tbyte_size\x18\x03 \x01(\x04\"\x93\x02\n\x0fModelStatistics\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x16\n\x0elast_inference\x18\x03 \x01(\x04\x12\x17\n\x0finference_count\x18\x04 \x01(\x04\x12\x17\n\x0f\x65xecution_count\x18\x05 \x01(\x04\x12\x33\n\x0finference_stats\x18\x06 \x01(\x0b\x32\x1a.inference.InferStatistics\x12\x34\n\x0b\x62\x61tch_stats\x18\x07 \x03(\x0b\x32\x1f.inference.InferBatchStatistics\x12,\n\x0cmemory_usage\x18\x08 \x03(\x0b\x32\x16.inference.MemoryUsage\"J\n\x17ModelStatisticsResponse\x12/\n\x0bmodel_stats\x18\x01 \x03(\x0b\x32\x1a.inference.ModelStatistics\"\x8a\x01\n\x18ModelRepositoryParameter\x12\x14\n\nbool_param\x18\x01 \x01(\x08H\x00\x12\x15\n\x0bint64_param\x18\x02 \x01(\x03H\x00\x12\x16\n\x0cstring_param\x18\x03 \x01(\tH\x00\x12\x15\n\x0b\x62ytes_param\x18\x04 \x01(\x0cH\x00\x42\x12\n\x10parameter_choice\"@\n\x16RepositoryIndexRequest\x12\x17\n\x0frepository_name\x18\x01 \x01(\t\x12\r\n\x05ready\x18\x02 \x01(\x08\"\xa4\x01\n\x17RepositoryIndexResponse\x12=\n\x06models\x18\x01 \x03(\x0b\x32-.inference.RepositoryIndexResponse.ModelIndex\x1aJ\n\nModelIndex\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\r\n\x05state\x18\x03 \x01(\t\x12\x0e\n\x06reason\x18\x04 \x01(\t\"\xec\x01\n\x1aRepositoryModelLoadRequest\x12\x17\n\x0frepository_name\x18\x01 \x01(\t\x12\x12\n\nmodel_name\x18\x02 \x01(\t\x12I\n\nparameters\x18\x03 \x03(\x0b\x32\x35.inference.RepositoryModelLoadRequest.ParametersEntry\x1aV\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x32\n\x05value\x18\x02 \x01(\x0b\x32#.inference.ModelRepositoryParameter:\x02\x38\x01\"\x1d\n\x1bRepositoryModelLoadResponse\"\xf0\x01\n\x1cRepositoryModelUnloadRequest\x12\x17\n\x0frepository_name\x18\x01 \x01(\t\x12\x12\n\nmodel_name\x18\x02 \x01(\t\x12K\n\nparameters\x18\x03 \x03(\x0b\x32\x37.inference.RepositoryModelUnloadRequest.ParametersEntry\x1aV\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x32\n\x05value\x18\x02 \x01(\x0b\x32#.inference.ModelRepositoryParameter:\x02\x38\x01\"\x1f\n\x1dRepositoryModelUnloadResponse\"/\n\x1fSystemSharedMemoryStatusRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\xa5\x02\n SystemSharedMemoryStatusResponse\x12I\n\x07regions\x18\x01 \x03(\x0b\x32\x38.inference.SystemSharedMemoryStatusResponse.RegionsEntry\x1aL\n\x0cRegionStatus\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0b\n\x03key\x18\x02 \x01(\t\x12\x0e\n\x06offset\x18\x03 \x01(\x04\x12\x11\n\tbyte_size\x18\x04 \x01(\x04\x1ah\n\x0cRegionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12G\n\x05value\x18\x02 \x01(\x0b\x32\x38.inference.SystemSharedMemoryStatusResponse.RegionStatus:\x02\x38\x01\"a\n!SystemSharedMemoryRegisterRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0b\n\x03key\x18\x02 \x01(\t\x12\x0e\n\x06offset\x18\x03 \x01(\x04\x12\x11\n\tbyte_size\x18\x04 \x01(\x04\"$\n\"SystemSharedMemoryRegisterResponse\"3\n#SystemSharedMemoryUnregisterRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"&\n$SystemSharedMemoryUnregisterResponse\"-\n\x1d\x43udaSharedMemoryStatusRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x95\x02\n\x1e\x43udaSharedMemoryStatusResponse\x12G\n\x07regions\x18\x01 \x03(\x0b\x32\x36.inference.CudaSharedMemoryStatusResponse.RegionsEntry\x1a\x42\n\x0cRegionStatus\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x11\n\tdevice_id\x18\x02 \x01(\x04\x12\x11\n\tbyte_size\x18\x03 \x01(\x04\x1a\x66\n\x0cRegionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x45\n\x05value\x18\x02 \x01(\x0b\x32\x36.inference.CudaSharedMemoryStatusResponse.RegionStatus:\x02\x38\x01\"i\n\x1f\x43udaSharedMemoryRegisterRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\nraw_handle\x18\x02 \x01(\x0c\x12\x11\n\tdevice_id\x18\x03 \x01(\x03\x12\x11\n\tbyte_size\x18\x04 \x01(\x04\"\"\n CudaSharedMemoryRegisterResponse\"1\n!CudaSharedMemoryUnregisterRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"$\n\"CudaSharedMemoryUnregisterResponse\"\xe6\x01\n\x13TraceSettingRequest\x12>\n\x08settings\x18\x01 \x03(\x0b\x32,.inference.TraceSettingRequest.SettingsEntry\x12\x12\n\nmodel_name\x18\x02 \x01(\t\x1a\x1d\n\x0cSettingValue\x12\r\n\x05value\x18\x01 \x03(\t\x1a\\\n\rSettingsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12:\n\x05value\x18\x02 \x01(\x0b\x32+.inference.TraceSettingRequest.SettingValue:\x02\x38\x01\"\xd5\x01\n\x14TraceSettingResponse\x12?\n\x08settings\x18\x01 \x03(\x0b\x32-.inference.TraceSettingResponse.SettingsEntry\x1a\x1d\n\x0cSettingValue\x12\r\n\x05value\x18\x01 \x03(\t\x1a]\n\rSettingsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12;\n\x05value\x18\x02 \x01(\x0b\x32,.inference.TraceSettingResponse.SettingValue:\x02\x38\x01\"\x9a\x02\n\x12LogSettingsRequest\x12=\n\x08settings\x18\x01 \x03(\x0b\x32+.inference.LogSettingsRequest.SettingsEntry\x1ah\n\x0cSettingValue\x12\x14\n\nbool_param\x18\x01 \x01(\x08H\x00\x12\x16\n\x0cuint32_param\x18\x02 \x01(\rH\x00\x12\x16\n\x0cstring_param\x18\x03 \x01(\tH\x00\x42\x12\n\x10parameter_choice\x1a[\n\rSettingsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x39\n\x05value\x18\x02 \x01(\x0b\x32*.inference.LogSettingsRequest.SettingValue:\x02\x38\x01\"\x9d\x02\n\x13LogSettingsResponse\x12>\n\x08settings\x18\x01 \x03(\x0b\x32,.inference.LogSettingsResponse.SettingsEntry\x1ah\n\x0cSettingValue\x12\x14\n\nbool_param\x18\x01 \x01(\x08H\x00\x12\x16\n\x0cuint32_param\x18\x02 \x01(\rH\x00\x12\x16\n\x0cstring_param\x18\x03 \x01(\tH\x00\x42\x12\n\x10parameter_choice\x1a\\\n\rSettingsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12:\n\x05value\x18\x02 \x01(\x0b\x32+.inference.LogSettingsResponse.SettingValue:\x02\x38\x01\x32\xb7\x0f\n\x14GRPCInferenceService\x12K\n\nServerLive\x12\x1c.inference.ServerLiveRequest\x1a\x1d.inference.ServerLiveResponse\"\x00\x12N\n\x0bServerReady\x12\x1d.inference.ServerReadyRequest\x1a\x1e.inference.ServerReadyResponse\"\x00\x12K\n\nModelReady\x12\x1c.inference.ModelReadyRequest\x1a\x1d.inference.ModelReadyResponse\"\x00\x12W\n\x0eServerMetadata\x12 .inference.ServerMetadataRequest\x1a!.inference.ServerMetadataResponse\"\x00\x12T\n\rModelMetadata\x12\x1f.inference.ModelMetadataRequest\x1a .inference.ModelMetadataResponse\"\x00\x12K\n\nModelInfer\x12\x1c.inference.ModelInferRequest\x1a\x1d.inference.ModelInferResponse\"\x00\x12[\n\x10ModelStreamInfer\x12\x1c.inference.ModelInferRequest\x1a#.inference.ModelStreamInferResponse\"\x00(\x01\x30\x01\x12N\n\x0bModelConfig\x12\x1d.inference.ModelConfigRequest\x1a\x1e.inference.ModelConfigResponse\"\x00\x12Z\n\x0fModelStatistics\x12!.inference.ModelStatisticsRequest\x1a\".inference.ModelStatisticsResponse\"\x00\x12Z\n\x0fRepositoryIndex\x12!.inference.RepositoryIndexRequest\x1a\".inference.RepositoryIndexResponse\"\x00\x12\x66\n\x13RepositoryModelLoad\x12%.inference.RepositoryModelLoadRequest\x1a&.inference.RepositoryModelLoadResponse\"\x00\x12l\n\x15RepositoryModelUnload\x12\'.inference.RepositoryModelUnloadRequest\x1a(.inference.RepositoryModelUnloadResponse\"\x00\x12u\n\x18SystemSharedMemoryStatus\x12*.inference.SystemSharedMemoryStatusRequest\x1a+.inference.SystemSharedMemoryStatusResponse\"\x00\x12{\n\x1aSystemSharedMemoryRegister\x12,.inference.SystemSharedMemoryRegisterRequest\x1a-.inference.SystemSharedMemoryRegisterResponse\"\x00\x12\x81\x01\n\x1cSystemSharedMemoryUnregister\x12..inference.SystemSharedMemoryUnregisterRequest\x1a/.inference.SystemSharedMemoryUnregisterResponse\"\x00\x12o\n\x16\x43udaSharedMemoryStatus\x12(.inference.CudaSharedMemoryStatusRequest\x1a).inference.CudaSharedMemoryStatusResponse\"\x00\x12u\n\x18\x43udaSharedMemoryRegister\x12*.inference.CudaSharedMemoryRegisterRequest\x1a+.inference.CudaSharedMemoryRegisterResponse\"\x00\x12{\n\x1a\x43udaSharedMemoryUnregister\x12,.inference.CudaSharedMemoryUnregisterRequest\x1a-.inference.CudaSharedMemoryUnregisterResponse\"\x00\x12Q\n\x0cTraceSetting\x12\x1e.inference.TraceSettingRequest\x1a\x1f.inference.TraceSettingResponse\"\x00\x12N\n\x0bLogSettings\x12\x1d.inference.LogSettingsRequest\x1a\x1e.inference.LogSettingsResponse\"\x00\x62\x06proto3')
 
 
 
@@ -49,6 +49,7 @@ _MODELSTATISTICSREQUEST = DESCRIPTOR.message_types_by_name['ModelStatisticsReque
 _STATISTICDURATION = DESCRIPTOR.message_types_by_name['StatisticDuration']
 _INFERSTATISTICS = DESCRIPTOR.message_types_by_name['InferStatistics']
 _INFERBATCHSTATISTICS = DESCRIPTOR.message_types_by_name['InferBatchStatistics']
+_MEMORYUSAGE = DESCRIPTOR.message_types_by_name['MemoryUsage']
 _MODELSTATISTICS = DESCRIPTOR.message_types_by_name['ModelStatistics']
 _MODELSTATISTICSRESPONSE = DESCRIPTOR.message_types_by_name['ModelStatisticsResponse']
 _MODELREPOSITORYPARAMETER = DESCRIPTOR.message_types_by_name['ModelRepositoryParameter']
@@ -308,6 +309,13 @@ InferBatchStatistics = _reflection.GeneratedProtocolMessageType('InferBatchStati
   })
 _sym_db.RegisterMessage(InferBatchStatistics)
 
+MemoryUsage = _reflection.GeneratedProtocolMessageType('MemoryUsage', (_message.Message,), {
+  'DESCRIPTOR' : _MEMORYUSAGE,
+  '__module__' : 'grpc_service_pb2'
+  # @@protoc_insertion_point(class_scope:inference.MemoryUsage)
+  })
+_sym_db.RegisterMessage(MemoryUsage)
+
 ModelStatistics = _reflection.GeneratedProtocolMessageType('ModelStatistics', (_message.Message,), {
   'DESCRIPTOR' : _MODELSTATISTICS,
   '__module__' : 'grpc_service_pb2'
@@ -655,124 +663,126 @@ if _descriptor._USE_C_DESCRIPTORS == False:
   _MODELMETADATARESPONSE._serialized_end=686
   _MODELMETADATARESPONSE_TENSORMETADATA._serialized_start=623
   _MODELMETADATARESPONSE_TENSORMETADATA._serialized_end=686
-  _INFERPARAMETER._serialized_start=688
-  _INFERPARAMETER._serialized_end=793
-  _INFERTENSORCONTENTS._serialized_start=796
-  _INFERTENSORCONTENTS._serialized_end=1004
-  _MODELINFERREQUEST._serialized_start=1007
-  _MODELINFERREQUEST._serialized_end=1885
-  _MODELINFERREQUEST_INFERINPUTTENSOR._serialized_start=1315
-  _MODELINFERREQUEST_INFERINPUTTENSOR._serialized_end=1591
-  _MODELINFERREQUEST_INFERINPUTTENSOR_PARAMETERSENTRY._serialized_start=1515
-  _MODELINFERREQUEST_INFERINPUTTENSOR_PARAMETERSENTRY._serialized_end=1591
-  _MODELINFERREQUEST_INFERREQUESTEDOUTPUTTENSOR._serialized_start=1594
-  _MODELINFERREQUEST_INFERREQUESTEDOUTPUTTENSOR._serialized_end=1807
-  _MODELINFERREQUEST_INFERREQUESTEDOUTPUTTENSOR_PARAMETERSENTRY._serialized_start=1515
-  _MODELINFERREQUEST_INFERREQUESTEDOUTPUTTENSOR_PARAMETERSENTRY._serialized_end=1591
-  _MODELINFERREQUEST_PARAMETERSENTRY._serialized_start=1515
-  _MODELINFERREQUEST_PARAMETERSENTRY._serialized_end=1591
-  _MODELINFERRESPONSE._serialized_start=1888
-  _MODELINFERRESPONSE._serialized_end=2485
-  _MODELINFERRESPONSE_INFEROUTPUTTENSOR._serialized_start=2128
-  _MODELINFERRESPONSE_INFEROUTPUTTENSOR._serialized_end=2407
-  _MODELINFERRESPONSE_INFEROUTPUTTENSOR_PARAMETERSENTRY._serialized_start=1515
-  _MODELINFERRESPONSE_INFEROUTPUTTENSOR_PARAMETERSENTRY._serialized_end=1591
-  _MODELINFERRESPONSE_PARAMETERSENTRY._serialized_start=1515
-  _MODELINFERRESPONSE_PARAMETERSENTRY._serialized_end=1591
-  _MODELSTREAMINFERRESPONSE._serialized_start=2487
-  _MODELSTREAMINFERRESPONSE._serialized_end=2591
-  _MODELCONFIGREQUEST._serialized_start=2593
-  _MODELCONFIGREQUEST._serialized_end=2644
-  _MODELCONFIGRESPONSE._serialized_start=2646
-  _MODELCONFIGRESPONSE._serialized_end=2707
-  _MODELSTATISTICSREQUEST._serialized_start=2709
-  _MODELSTATISTICSREQUEST._serialized_end=2764
-  _STATISTICDURATION._serialized_start=2766
-  _STATISTICDURATION._serialized_end=2812
-  _INFERSTATISTICS._serialized_start=2815
-  _INFERSTATISTICS._serialized_end=3227
-  _INFERBATCHSTATISTICS._serialized_start=3230
-  _INFERBATCHSTATISTICS._serialized_end=3432
-  _MODELSTATISTICS._serialized_start=3435
-  _MODELSTATISTICS._serialized_end=3664
-  _MODELSTATISTICSRESPONSE._serialized_start=3666
-  _MODELSTATISTICSRESPONSE._serialized_end=3740
-  _MODELREPOSITORYPARAMETER._serialized_start=3743
-  _MODELREPOSITORYPARAMETER._serialized_end=3881
-  _REPOSITORYINDEXREQUEST._serialized_start=3883
-  _REPOSITORYINDEXREQUEST._serialized_end=3947
-  _REPOSITORYINDEXRESPONSE._serialized_start=3950
-  _REPOSITORYINDEXRESPONSE._serialized_end=4114
-  _REPOSITORYINDEXRESPONSE_MODELINDEX._serialized_start=4040
-  _REPOSITORYINDEXRESPONSE_MODELINDEX._serialized_end=4114
-  _REPOSITORYMODELLOADREQUEST._serialized_start=4117
-  _REPOSITORYMODELLOADREQUEST._serialized_end=4353
-  _REPOSITORYMODELLOADREQUEST_PARAMETERSENTRY._serialized_start=4267
-  _REPOSITORYMODELLOADREQUEST_PARAMETERSENTRY._serialized_end=4353
-  _REPOSITORYMODELLOADRESPONSE._serialized_start=4355
-  _REPOSITORYMODELLOADRESPONSE._serialized_end=4384
-  _REPOSITORYMODELUNLOADREQUEST._serialized_start=4387
-  _REPOSITORYMODELUNLOADREQUEST._serialized_end=4627
-  _REPOSITORYMODELUNLOADREQUEST_PARAMETERSENTRY._serialized_start=4267
-  _REPOSITORYMODELUNLOADREQUEST_PARAMETERSENTRY._serialized_end=4353
-  _REPOSITORYMODELUNLOADRESPONSE._serialized_start=4629
-  _REPOSITORYMODELUNLOADRESPONSE._serialized_end=4660
-  _SYSTEMSHAREDMEMORYSTATUSREQUEST._serialized_start=4662
-  _SYSTEMSHAREDMEMORYSTATUSREQUEST._serialized_end=4709
-  _SYSTEMSHAREDMEMORYSTATUSRESPONSE._serialized_start=4712
-  _SYSTEMSHAREDMEMORYSTATUSRESPONSE._serialized_end=5005
-  _SYSTEMSHAREDMEMORYSTATUSRESPONSE_REGIONSTATUS._serialized_start=4823
-  _SYSTEMSHAREDMEMORYSTATUSRESPONSE_REGIONSTATUS._serialized_end=4899
-  _SYSTEMSHAREDMEMORYSTATUSRESPONSE_REGIONSENTRY._serialized_start=4901
-  _SYSTEMSHAREDMEMORYSTATUSRESPONSE_REGIONSENTRY._serialized_end=5005
-  _SYSTEMSHAREDMEMORYREGISTERREQUEST._serialized_start=5007
-  _SYSTEMSHAREDMEMORYREGISTERREQUEST._serialized_end=5104
-  _SYSTEMSHAREDMEMORYREGISTERRESPONSE._serialized_start=5106
-  _SYSTEMSHAREDMEMORYREGISTERRESPONSE._serialized_end=5142
-  _SYSTEMSHAREDMEMORYUNREGISTERREQUEST._serialized_start=5144
-  _SYSTEMSHAREDMEMORYUNREGISTERREQUEST._serialized_end=5195
-  _SYSTEMSHAREDMEMORYUNREGISTERRESPONSE._serialized_start=5197
-  _SYSTEMSHAREDMEMORYUNREGISTERRESPONSE._serialized_end=5235
-  _CUDASHAREDMEMORYSTATUSREQUEST._serialized_start=5237
-  _CUDASHAREDMEMORYSTATUSREQUEST._serialized_end=5282
-  _CUDASHAREDMEMORYSTATUSRESPONSE._serialized_start=5285
-  _CUDASHAREDMEMORYSTATUSRESPONSE._serialized_end=5562
-  _CUDASHAREDMEMORYSTATUSRESPONSE_REGIONSTATUS._serialized_start=5392
-  _CUDASHAREDMEMORYSTATUSRESPONSE_REGIONSTATUS._serialized_end=5458
-  _CUDASHAREDMEMORYSTATUSRESPONSE_REGIONSENTRY._serialized_start=5460
-  _CUDASHAREDMEMORYSTATUSRESPONSE_REGIONSENTRY._serialized_end=5562
-  _CUDASHAREDMEMORYREGISTERREQUEST._serialized_start=5564
-  _CUDASHAREDMEMORYREGISTERREQUEST._serialized_end=5669
-  _CUDASHAREDMEMORYREGISTERRESPONSE._serialized_start=5671
-  _CUDASHAREDMEMORYREGISTERRESPONSE._serialized_end=5705
-  _CUDASHAREDMEMORYUNREGISTERREQUEST._serialized_start=5707
-  _CUDASHAREDMEMORYUNREGISTERREQUEST._serialized_end=5756
-  _CUDASHAREDMEMORYUNREGISTERRESPONSE._serialized_start=5758
-  _CUDASHAREDMEMORYUNREGISTERRESPONSE._serialized_end=5794
-  _TRACESETTINGREQUEST._serialized_start=5797
-  _TRACESETTINGREQUEST._serialized_end=6027
-  _TRACESETTINGREQUEST_SETTINGVALUE._serialized_start=5904
-  _TRACESETTINGREQUEST_SETTINGVALUE._serialized_end=5933
-  _TRACESETTINGREQUEST_SETTINGSENTRY._serialized_start=5935
-  _TRACESETTINGREQUEST_SETTINGSENTRY._serialized_end=6027
-  _TRACESETTINGRESPONSE._serialized_start=6030
-  _TRACESETTINGRESPONSE._serialized_end=6243
-  _TRACESETTINGRESPONSE_SETTINGVALUE._serialized_start=5904
-  _TRACESETTINGRESPONSE_SETTINGVALUE._serialized_end=5933
-  _TRACESETTINGRESPONSE_SETTINGSENTRY._serialized_start=6150
-  _TRACESETTINGRESPONSE_SETTINGSENTRY._serialized_end=6243
-  _LOGSETTINGSREQUEST._serialized_start=6246
-  _LOGSETTINGSREQUEST._serialized_end=6528
-  _LOGSETTINGSREQUEST_SETTINGVALUE._serialized_start=6331
-  _LOGSETTINGSREQUEST_SETTINGVALUE._serialized_end=6435
-  _LOGSETTINGSREQUEST_SETTINGSENTRY._serialized_start=6437
-  _LOGSETTINGSREQUEST_SETTINGSENTRY._serialized_end=6528
-  _LOGSETTINGSRESPONSE._serialized_start=6531
-  _LOGSETTINGSRESPONSE._serialized_end=6816
-  _LOGSETTINGSRESPONSE_SETTINGVALUE._serialized_start=6331
-  _LOGSETTINGSRESPONSE_SETTINGVALUE._serialized_end=6435
-  _LOGSETTINGSRESPONSE_SETTINGSENTRY._serialized_start=6724
-  _LOGSETTINGSRESPONSE_SETTINGSENTRY._serialized_end=6816
-  _GRPCINFERENCESERVICE._serialized_start=6819
-  _GRPCINFERENCESERVICE._serialized_end=8794
+  _INFERPARAMETER._serialized_start=689
+  _INFERPARAMETER._serialized_end=842
+  _INFERTENSORCONTENTS._serialized_start=845
+  _INFERTENSORCONTENTS._serialized_end=1053
+  _MODELINFERREQUEST._serialized_start=1056
+  _MODELINFERREQUEST._serialized_end=1934
+  _MODELINFERREQUEST_INFERINPUTTENSOR._serialized_start=1364
+  _MODELINFERREQUEST_INFERINPUTTENSOR._serialized_end=1640
+  _MODELINFERREQUEST_INFERINPUTTENSOR_PARAMETERSENTRY._serialized_start=1564
+  _MODELINFERREQUEST_INFERINPUTTENSOR_PARAMETERSENTRY._serialized_end=1640
+  _MODELINFERREQUEST_INFERREQUESTEDOUTPUTTENSOR._serialized_start=1643
+  _MODELINFERREQUEST_INFERREQUESTEDOUTPUTTENSOR._serialized_end=1856
+  _MODELINFERREQUEST_INFERREQUESTEDOUTPUTTENSOR_PARAMETERSENTRY._serialized_start=1564
+  _MODELINFERREQUEST_INFERREQUESTEDOUTPUTTENSOR_PARAMETERSENTRY._serialized_end=1640
+  _MODELINFERREQUEST_PARAMETERSENTRY._serialized_start=1564
+  _MODELINFERREQUEST_PARAMETERSENTRY._serialized_end=1640
+  _MODELINFERRESPONSE._serialized_start=1937
+  _MODELINFERRESPONSE._serialized_end=2534
+  _MODELINFERRESPONSE_INFEROUTPUTTENSOR._serialized_start=2177
+  _MODELINFERRESPONSE_INFEROUTPUTTENSOR._serialized_end=2456
+  _MODELINFERRESPONSE_INFEROUTPUTTENSOR_PARAMETERSENTRY._serialized_start=1564
+  _MODELINFERRESPONSE_INFEROUTPUTTENSOR_PARAMETERSENTRY._serialized_end=1640
+  _MODELINFERRESPONSE_PARAMETERSENTRY._serialized_start=1564
+  _MODELINFERRESPONSE_PARAMETERSENTRY._serialized_end=1640
+  _MODELSTREAMINFERRESPONSE._serialized_start=2536
+  _MODELSTREAMINFERRESPONSE._serialized_end=2640
+  _MODELCONFIGREQUEST._serialized_start=2642
+  _MODELCONFIGREQUEST._serialized_end=2693
+  _MODELCONFIGRESPONSE._serialized_start=2695
+  _MODELCONFIGRESPONSE._serialized_end=2756
+  _MODELSTATISTICSREQUEST._serialized_start=2758
+  _MODELSTATISTICSREQUEST._serialized_end=2813
+  _STATISTICDURATION._serialized_start=2815
+  _STATISTICDURATION._serialized_end=2861
+  _INFERSTATISTICS._serialized_start=2864
+  _INFERSTATISTICS._serialized_end=3276
+  _INFERBATCHSTATISTICS._serialized_start=3279
+  _INFERBATCHSTATISTICS._serialized_end=3481
+  _MEMORYUSAGE._serialized_start=3483
+  _MEMORYUSAGE._serialized_end=3541
+  _MODELSTATISTICS._serialized_start=3544
+  _MODELSTATISTICS._serialized_end=3819
+  _MODELSTATISTICSRESPONSE._serialized_start=3821
+  _MODELSTATISTICSRESPONSE._serialized_end=3895
+  _MODELREPOSITORYPARAMETER._serialized_start=3898
+  _MODELREPOSITORYPARAMETER._serialized_end=4036
+  _REPOSITORYINDEXREQUEST._serialized_start=4038
+  _REPOSITORYINDEXREQUEST._serialized_end=4102
+  _REPOSITORYINDEXRESPONSE._serialized_start=4105
+  _REPOSITORYINDEXRESPONSE._serialized_end=4269
+  _REPOSITORYINDEXRESPONSE_MODELINDEX._serialized_start=4195
+  _REPOSITORYINDEXRESPONSE_MODELINDEX._serialized_end=4269
+  _REPOSITORYMODELLOADREQUEST._serialized_start=4272
+  _REPOSITORYMODELLOADREQUEST._serialized_end=4508
+  _REPOSITORYMODELLOADREQUEST_PARAMETERSENTRY._serialized_start=4422
+  _REPOSITORYMODELLOADREQUEST_PARAMETERSENTRY._serialized_end=4508
+  _REPOSITORYMODELLOADRESPONSE._serialized_start=4510
+  _REPOSITORYMODELLOADRESPONSE._serialized_end=4539
+  _REPOSITORYMODELUNLOADREQUEST._serialized_start=4542
+  _REPOSITORYMODELUNLOADREQUEST._serialized_end=4782
+  _REPOSITORYMODELUNLOADREQUEST_PARAMETERSENTRY._serialized_start=4422
+  _REPOSITORYMODELUNLOADREQUEST_PARAMETERSENTRY._serialized_end=4508
+  _REPOSITORYMODELUNLOADRESPONSE._serialized_start=4784
+  _REPOSITORYMODELUNLOADRESPONSE._serialized_end=4815
+  _SYSTEMSHAREDMEMORYSTATUSREQUEST._serialized_start=4817
+  _SYSTEMSHAREDMEMORYSTATUSREQUEST._serialized_end=4864
+  _SYSTEMSHAREDMEMORYSTATUSRESPONSE._serialized_start=4867
+  _SYSTEMSHAREDMEMORYSTATUSRESPONSE._serialized_end=5160
+  _SYSTEMSHAREDMEMORYSTATUSRESPONSE_REGIONSTATUS._serialized_start=4978
+  _SYSTEMSHAREDMEMORYSTATUSRESPONSE_REGIONSTATUS._serialized_end=5054
+  _SYSTEMSHAREDMEMORYSTATUSRESPONSE_REGIONSENTRY._serialized_start=5056
+  _SYSTEMSHAREDMEMORYSTATUSRESPONSE_REGIONSENTRY._serialized_end=5160
+  _SYSTEMSHAREDMEMORYREGISTERREQUEST._serialized_start=5162
+  _SYSTEMSHAREDMEMORYREGISTERREQUEST._serialized_end=5259
+  _SYSTEMSHAREDMEMORYREGISTERRESPONSE._serialized_start=5261
+  _SYSTEMSHAREDMEMORYREGISTERRESPONSE._serialized_end=5297
+  _SYSTEMSHAREDMEMORYUNREGISTERREQUEST._serialized_start=5299
+  _SYSTEMSHAREDMEMORYUNREGISTERREQUEST._serialized_end=5350
+  _SYSTEMSHAREDMEMORYUNREGISTERRESPONSE._serialized_start=5352
+  _SYSTEMSHAREDMEMORYUNREGISTERRESPONSE._serialized_end=5390
+  _CUDASHAREDMEMORYSTATUSREQUEST._serialized_start=5392
+  _CUDASHAREDMEMORYSTATUSREQUEST._serialized_end=5437
+  _CUDASHAREDMEMORYSTATUSRESPONSE._serialized_start=5440
+  _CUDASHAREDMEMORYSTATUSRESPONSE._serialized_end=5717
+  _CUDASHAREDMEMORYSTATUSRESPONSE_REGIONSTATUS._serialized_start=5547
+  _CUDASHAREDMEMORYSTATUSRESPONSE_REGIONSTATUS._serialized_end=5613
+  _CUDASHAREDMEMORYSTATUSRESPONSE_REGIONSENTRY._serialized_start=5615
+  _CUDASHAREDMEMORYSTATUSRESPONSE_REGIONSENTRY._serialized_end=5717
+  _CUDASHAREDMEMORYREGISTERREQUEST._serialized_start=5719
+  _CUDASHAREDMEMORYREGISTERREQUEST._serialized_end=5824
+  _CUDASHAREDMEMORYREGISTERRESPONSE._serialized_start=5826
+  _CUDASHAREDMEMORYREGISTERRESPONSE._serialized_end=5860
+  _CUDASHAREDMEMORYUNREGISTERREQUEST._serialized_start=5862
+  _CUDASHAREDMEMORYUNREGISTERREQUEST._serialized_end=5911
+  _CUDASHAREDMEMORYUNREGISTERRESPONSE._serialized_start=5913
+  _CUDASHAREDMEMORYUNREGISTERRESPONSE._serialized_end=5949
+  _TRACESETTINGREQUEST._serialized_start=5952
+  _TRACESETTINGREQUEST._serialized_end=6182
+  _TRACESETTINGREQUEST_SETTINGVALUE._serialized_start=6059
+  _TRACESETTINGREQUEST_SETTINGVALUE._serialized_end=6088
+  _TRACESETTINGREQUEST_SETTINGSENTRY._serialized_start=6090
+  _TRACESETTINGREQUEST_SETTINGSENTRY._serialized_end=6182
+  _TRACESETTINGRESPONSE._serialized_start=6185
+  _TRACESETTINGRESPONSE._serialized_end=6398
+  _TRACESETTINGRESPONSE_SETTINGVALUE._serialized_start=6059
+  _TRACESETTINGRESPONSE_SETTINGVALUE._serialized_end=6088
+  _TRACESETTINGRESPONSE_SETTINGSENTRY._serialized_start=6305
+  _TRACESETTINGRESPONSE_SETTINGSENTRY._serialized_end=6398
+  _LOGSETTINGSREQUEST._serialized_start=6401
+  _LOGSETTINGSREQUEST._serialized_end=6683
+  _LOGSETTINGSREQUEST_SETTINGVALUE._serialized_start=6486
+  _LOGSETTINGSREQUEST_SETTINGVALUE._serialized_end=6590
+  _LOGSETTINGSREQUEST_SETTINGSENTRY._serialized_start=6592
+  _LOGSETTINGSREQUEST_SETTINGSENTRY._serialized_end=6683
+  _LOGSETTINGSRESPONSE._serialized_start=6686
+  _LOGSETTINGSRESPONSE._serialized_end=6971
+  _LOGSETTINGSRESPONSE_SETTINGVALUE._serialized_start=6486
+  _LOGSETTINGSRESPONSE_SETTINGVALUE._serialized_end=6590
+  _LOGSETTINGSRESPONSE_SETTINGSENTRY._serialized_start=6879
+  _LOGSETTINGSRESPONSE_SETTINGSENTRY._serialized_end=6971
+  _GRPCINFERENCESERVICE._serialized_start=6974
+  _GRPCINFERENCESERVICE._serialized_end=8949
 # @@protoc_insertion_point(module_scope)

{tritonclient-2.34.0.data → tritonclient-2.36.0.data}/purelib/tritonclient/utils/__init__.py

@@ -26,6 +26,7 @@
 
 import numpy as np
 import struct
+from ._shared_memory_tensor import SharedMemoryTensor
 
 
 def raise_error(msg):

tritonclient-2.36.0.data/purelib/tritonclient/utils/_dlpack.py

@@ -0,0 +1,266 @@
+# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+################################################################################
+# This file contains the DLPack API wrapped in Python style (see
+# 'dlpack.h' for detail) and the utilities for Triton client to interact
+# with DLPack
+#
+# Ref:
+# https://github.com/dmlc/dlpack/blob/main/include/dlpack/dlpack.h
+# https://github.com/dmlc/dlpack/blob/main/apps/numpy_dlpack/dlpack/from_numpy.py
+################################################################################
+
+import ctypes
+
+# Need to explicit set the res / arg types for pythonapi functions to
+# work properly
+ctypes.pythonapi.PyMem_RawMalloc.restype = ctypes.c_void_p
+ctypes.pythonapi.PyMem_RawFree.argtypes = [ctypes.c_void_p]
+
+ctypes.pythonapi.PyCapsule_New.restype = ctypes.py_object
+ctypes.pythonapi.PyCapsule_New.argtypes = [
+    ctypes.c_void_p, ctypes.c_char_p, ctypes.c_void_p
+]
+
+ctypes.pythonapi.PyCapsule_GetPointer.restype = ctypes.c_void_p
+ctypes.pythonapi.PyCapsule_GetPointer.argtypes = [
+    ctypes.py_object, ctypes.c_char_p
+]
+
+c_str_dltensor = b"dltensor"
+
+
+class DLDeviceType(ctypes.c_int):
+    kDLCPU = 1
+    kDLCUDA = 2
+    kDLCUDAHost = 3
+    kDLOpenCL = 4
+    kDLVulkan = 7
+    kDLMetal = 8
+    kDLVPI = 9
+    kDLROCM = 10
+    kDLROCMHost = 11
+    kDLExtDev = 12
+    kDLCUDAManaged = 13
+    kDLOneAPI = 14
+    kDLWebGPU = 15
+    kDLHexagon = 16
+
+
+class DLDevice(ctypes.Structure):
+    _fields_ = [
+        ("device_type", DLDeviceType),
+        ("device_id", ctypes.c_int),
+    ]
+
+
+class DLDataTypeCode(ctypes.c_uint8):
+    kDLInt = 0
+    kDLUInt = 1
+    kDLFloat = 2
+    kDLOpaquePointer = 3
+    kDLBfloat = 4
+    kDLComplex = 5
+    kDLBool = 6
+
+
+class DLDataType(ctypes.Structure):
+    _fields_ = [
+        ("type_code", DLDataTypeCode),
+        ("bits", ctypes.c_uint8),
+        ("lanes", ctypes.c_uint16),
+    ]
+
+
+class DLTensor(ctypes.Structure):
+    _fields_ = [
+        ("data", ctypes.c_void_p),
+        ("device", DLDevice),
+        ("ndim", ctypes.c_int),
+        ("dtype", DLDataType),
+        ("shape", ctypes.POINTER(ctypes.c_int64)),
+        ("strides", ctypes.POINTER(ctypes.c_int64)),
+        ("byte_offset", ctypes.c_uint64),
+    ]
+
+
+class DLManagedTensor(ctypes.Structure):
+    _fields_ = [
+        ("dl_tensor", DLTensor),
+        ("manager_ctx", ctypes.c_void_p),
+        ("deleter", ctypes.CFUNCTYPE(None, ctypes.c_void_p)),
+    ]
+
+
+# Utilities
+
+
+def _raise_error(msg):
+    """
+    Raise error with the provided message
+    """
+    raise Exception(msg=msg) from None
+
+
+# Use as managed context in DLPack that doesn't hold ownership of the
+# data content.
+class DataViewContext:
+
+    def __init__(self, shape) -> None:
+        # Convert the Python object to ctypes objects expected by
+        # DLPack
+        self._shape = (ctypes.c_int64 * len(shape))(*shape)
+        # No strides: compact and row-major
+        self._strides = ctypes.POINTER(ctypes.c_int64)()
+
+    def as_manager_ctx(self) -> ctypes.c_void_p:
+        py_obj = ctypes.py_object(self)
+        py_obj_ptr = ctypes.pointer(py_obj)
+        ctypes.pythonapi.Py_IncRef(py_obj)
+        ctypes.pythonapi.Py_IncRef(ctypes.py_object(py_obj_ptr))
+        return ctypes.cast(py_obj_ptr, ctypes.c_void_p)
+
+
+@ctypes.CFUNCTYPE(None, ctypes.c_void_p)
+def managed_tensor_deleter(handle: ctypes.c_void_p) -> None:
+    dl_managed_tensor = DLManagedTensor.from_address(handle)
+    py_obj_ptr = ctypes.cast(dl_managed_tensor.manager_ctx,
+                             ctypes.POINTER(ctypes.py_object))
+    py_obj = py_obj_ptr.contents
+    ctypes.pythonapi.Py_DecRef(py_obj)
+    ctypes.pythonapi.Py_DecRef(ctypes.py_object(py_obj_ptr))
+    ctypes.pythonapi.PyMem_RawFree(handle)
+
+
+@ctypes.CFUNCTYPE(None, ctypes.c_void_p)
+def pycapsule_deleter(handle: ctypes.c_void_p) -> None:
+    pycapsule: ctypes.py_object = ctypes.cast(handle, ctypes.py_object)
+    if ctypes.pythonapi.PyCapsule_IsValid(pycapsule, c_str_dltensor):
+        dl_managed_tensor = ctypes.pythonapi.PyCapsule_GetPointer(
+            pycapsule, c_str_dltensor)
+        managed_tensor_deleter(dl_managed_tensor)
+        ctypes.pythonapi.PyCapsule_SetDestructor(pycapsule, None)
+
+
+def triton_to_dlpack_dtype(dtype):
+    if dtype == "BOOL":
+        type_code = DLDataTypeCode.kDLBool
+        bits = 1
+    elif dtype == "INT8":
+        type_code = DLDataTypeCode.kDLInt
+        bits = 8
+    elif dtype == "INT16":
+        type_code = DLDataTypeCode.kDLInt
+        bits = 16
+    elif dtype == "INT32":
+        type_code = DLDataTypeCode.kDLInt
+        bits = 32
+    elif dtype == "INT64":
+        type_code = DLDataTypeCode.kDLInt
+        bits = 64
+    elif dtype == "UINT8":
+        type_code = DLDataTypeCode.kDLUInt
+        bits = 8
+    elif dtype == "UINT16":
+        type_code = DLDataTypeCode.kDLUInt
+        bits = 16
+    elif dtype == "UINT32":
+        type_code = DLDataTypeCode.kDLUInt
+        bits = 32
+    elif dtype == "UINT64":
+        type_code = DLDataTypeCode.kDLUInt
+        bits = 64
+    elif dtype == "FP16":
+        type_code = DLDataTypeCode.kDLFloat
+        bits = 16
+    elif dtype == "FP32":
+        type_code = DLDataTypeCode.kDLFloat
+        bits = 32
+    elif dtype == "FP64":
+        type_code = DLDataTypeCode.kDLFloat
+        bits = 64
+    elif dtype == "BF16":
+        type_code = DLDataTypeCode.kDLBfloat
+        bits = 16
+    elif dtype == "BYTES":
+        _raise_error("DLPack currently doesn't suppose BYTES type")
+    else:
+        _raise_error(
+            "Can not covert unknown data type '{}' to DLPack data type".format(
+                dtype))
+    return DLDataType(type_code, bits, 1)
+
+
+def is_device_supported(device: DLDevice):
+    return device[0] in [
+        DLDeviceType.kDLCPU, DLDeviceType.kDLCUDA, DLDeviceType.kDLCUDAHost
+    ]
+
+
+def is_contiguous_data(ndim: ctypes.c_int,
+                       shape: ctypes.POINTER(ctypes.c_int64),
+                       stride: ctypes.POINTER(ctypes.c_int64)):
+    # If 'stride' doesn't capture valid value
+    if (stride is None) or (not bool(stride)):
+        return True
+    calculated_stride = 1
+    # iterate stride in reverse order [ndim-1, -1)
+    for i in reversed(range(ndim)):
+        if stride[i] != calculated_stride:
+            return False
+        calculated_stride *= shape[i]
+    return True
+
+
+def get_byte_size(dtype: DLDataType, ndim: ctypes.c_int,
+                  shape: ctypes.POINTER(ctypes.c_int64)):
+    element_byte_size = dtype.bits * dtype.lanes // 8  # Assume 8 bits in a byte
+    for i in range(ndim):
+        element_byte_size *= shape[i]
+    return element_byte_size
+
+
+def get_dlpack_capsule(dlpack_obj, stream=None):
+    # Extract PyCapsule of the DLPack object
+    if hasattr(dlpack_obj, '__dlpack__'):
+        if not hasattr(dlpack_obj, '__dlpack_device__'):
+            _raise_error(
+                "DLPack expects '__dlpack_device__' if '__dlpack__' has been defined"
+            )
+        device = dlpack_obj.__dlpack_device__()
+        if not is_device_supported(device):
+            _raise_error("DLPack device type {} is not supported".format(
+                device[0]))
+        # Have to condition on the device type as, using numpy as example,
+        # some DLPack implementation doesn't accept 'stream' as arguments
+        if device != DLDeviceType.kDLCUDA:
+            return dlpack_obj.__dlpack__()
+        else:
+            return dlpack_obj.__dlpack__(stream)
+    else:
+        # Old interface where PyCapsule object is passed directly
+        return dlpack_obj

tritonclient-2.36.0.data/purelib/tritonclient/utils/_shared_memory_tensor.py

@@ -0,0 +1,79 @@
+# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import ctypes
+from . import _dlpack
+import collections
+
+
+class SharedMemoryTensor:
+    """An object of SharedMemoryTensor class is a view of the shared memory
+    region that follows DLPack specification. This object should be considered
+    invalidated if there is modification on the corresponding shared memory
+    region.
+
+    https://dmlc.github.io/dlpack/latest/python_spec.html
+
+    """
+
+    def __init__(self, dtype: str, shape: collections.abc.Iterable,
+                 shm_addr: ctypes.c_void_p, offset: ctypes.c_uint64,
+                 byte_size: ctypes.c_uint64, device_id: ctypes.c_int) -> None:
+        self._dtype = dtype
+        self._shape = shape
+        self._shm_addr = shm_addr
+        self._offset = offset
+        self._byte_size = byte_size
+        self._device_id = device_id
+        if device_id.value != -1:
+            self._dl_device = (_dlpack.DLDeviceType.kDLCUDA, device_id.value)
+        else:
+            self._dl_device = (_dlpack.DLDeviceType.kDLCPU, 0)
+
+    def __dlpack__(self, stream=None):
+        context = _dlpack.DataViewContext(self._shape)
+        size = ctypes.c_size_t(ctypes.sizeof(_dlpack.DLManagedTensor))
+        dl_managed_tensor = _dlpack.DLManagedTensor.from_address(
+            ctypes.pythonapi.PyMem_RawMalloc(size))
+        dl_managed_tensor.dl_tensor.data = self._shm_addr
+        dl_managed_tensor.dl_tensor.device = self._dl_device
+        dl_managed_tensor.dl_tensor.dtype = _dlpack.triton_to_dlpack_dtype(
+            self._dtype)
+        dl_managed_tensor.dl_tensor.ndim = len(self._shape)
+        dl_managed_tensor.dl_tensor.shape = context._shape
+        dl_managed_tensor.dl_tensor.strides = context._strides
+        dl_managed_tensor.dl_tensor.byte_offset = self._offset
+        dl_managed_tensor.manager_ctx = context.as_manager_ctx()
+        dl_managed_tensor.deleter = _dlpack.managed_tensor_deleter
+        pycapsule = ctypes.pythonapi.PyCapsule_New(
+            ctypes.byref(dl_managed_tensor),
+            _dlpack.c_str_dltensor,
+            _dlpack.pycapsule_deleter,
+        )
+        return pycapsule
+
+    def __dlpack_device__(self):
+        return self._dl_device

{tritonclient-2.34.0.dist-info → tritonclient-2.36.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tritonclient
-Version: 2.34.0
+Version: 2.36.0
 Summary: Python client library and utilities for communicating with Triton Inference Server
 Home-page: https://developer.nvidia.com/nvidia-triton-inference-server
 Author: NVIDIA Inc.

tritonclient-2.36.0.dist-info/RECORD

@@ -0,0 +1,42 @@
+tritonclient-2.36.0.data/data/LICENSE.txt,sha256=NMqy92YzgvtkIas9IW34i27ARqi_sCunXDu-t-EqxL8,1533
+tritonclient-2.36.0.data/purelib/tritonclient/__init__.py,sha256=erynU-5HRlcIz1tx1Ve0kdCnzAXfNFeUYB5bx2fnL4Y,1542
+tritonclient-2.36.0.data/purelib/tritonclient/_auth.py,sha256=sXf16LlDoQj1qI3GSnFa1mgOv0zhwFmB0OHofkIwVh0,2047
+tritonclient-2.36.0.data/purelib/tritonclient/_client.py,sha256=8-VaArNooS9zKRY0aYTW9xvphmNa5vgWy7PDbBnGVw4,2964
+tritonclient-2.36.0.data/purelib/tritonclient/_plugin.py,sha256=hAaX9FakQBsbh5DZVAlcDujVwrV4EddcmPxMp3puxX0,2069
+tritonclient-2.36.0.data/purelib/tritonclient/_request.py,sha256=RmGnJpiRxTi33mn8Nl7ivGFqeKjicj_HhShtgOTci8I,1789
+tritonclient-2.36.0.data/purelib/tritonclient/grpc/__init__.py,sha256=MDsTzhwtE6603Ixkf6g_N3_5OCqhoDdCZT0KuZFtAmI,2959
+tritonclient-2.36.0.data/purelib/tritonclient/grpc/_client.py,sha256=tGQzAF8ld7JKKgIHYmLvSItGQNHz7CuoTLcW_YyMbuk,69184
+tritonclient-2.36.0.data/purelib/tritonclient/grpc/_infer_input.py,sha256=5cfHBc7lVxWq36HGG12xgnQlx_-3OG8L0fQ18AK3zjA,7213
+tritonclient-2.36.0.data/purelib/tritonclient/grpc/_infer_result.py,sha256=gWA8LO7SiA7BsJnC8TGVh0kfE3hEmRrXNpXRSqMmOwY,6193
+tritonclient-2.36.0.data/purelib/tritonclient/grpc/_infer_stream.py,sha256=AYGC7pD2jXkFQr0wq1moUE1MDgICMMTHVbCFvlujvl8,6618
+tritonclient-2.36.0.data/purelib/tritonclient/grpc/_requested_output.py,sha256=SUHrwwH0RA2Uge6imndW5hNdzhcZtNhrV_0DRwFep8U,4409
+tritonclient-2.36.0.data/purelib/tritonclient/grpc/_utils.py,sha256=aMF8Vi6TtZve7BJ8QlZ6FjQr1MLkPs0PIg7iV5DVxXo,4954
+tritonclient-2.36.0.data/purelib/tritonclient/grpc/model_config_pb2.py,sha256=KL3whNLO69fTXi4RBrdMDQ1OPsOrPPpjYx1xZCsnA78,46321
+tritonclient-2.36.0.data/purelib/tritonclient/grpc/service_pb2.py,sha256=4DTipBPPoTBFW8FwKCD1BMYYHSxhKizuyYW0vYhg1bg,55250
+tritonclient-2.36.0.data/purelib/tritonclient/grpc/service_pb2_grpc.py,sha256=BRXBBgehQgx_LSLqiw_N0zqFl4NYDhstm6AJHP3uxBw,37795
+tritonclient-2.36.0.data/purelib/tritonclient/grpc/aio/__init__.py,sha256=b4AFQnhd_ujj9eGAEGhpqKlfjDtx9AMLJgrdn77OSEI,33059
+tritonclient-2.36.0.data/purelib/tritonclient/grpc/aio/auth/__init__.py,sha256=nOIwuyXvzV1Dha2ZFX5vRjFhgsDa-cBdP2wSWNA6g1w,1573
+tritonclient-2.36.0.data/purelib/tritonclient/grpc/auth/__init__.py,sha256=1JifMIv8ipug7qUP4joArCyRsEXjxV-DIhvrJ2Kj5Rk,1574
+tritonclient-2.36.0.data/purelib/tritonclient/http/__init__.py,sha256=tbJXG_x_NV7x9LbR2h7DAImiwM3a2uA7HBMKfDhpX1A,2128
+tritonclient-2.36.0.data/purelib/tritonclient/http/_client.py,sha256=R8QDM_hFJACsvly0QheM7_ZRLkUrOEp6RMhPOLVUO1g,61716
+tritonclient-2.36.0.data/purelib/tritonclient/http/_infer_input.py,sha256=eEYQEiK8AzAAW5zJq_c_thbxdHq0mAAjdWgeZYv5EoY,9506
+tritonclient-2.36.0.data/purelib/tritonclient/http/_infer_result.py,sha256=vQIcWxxThgvFSnChZwZebKIJW-pSawXG1nVi6ncCYYE,9920
+tritonclient-2.36.0.data/purelib/tritonclient/http/_requested_output.py,sha256=QU-07jLHbKh1S7hCoqvqjPxrHWeb0CsJ8Qd3077AU1U,4695
+tritonclient-2.36.0.data/purelib/tritonclient/http/_utils.py,sha256=rMw1Ojz3UFsbPjrvU4nKVLH65TPJbujMJXhw7klZynk,5118
+tritonclient-2.36.0.data/purelib/tritonclient/http/aio/__init__.py,sha256=edd6GlMqiAb2kJG2jWlBs2P0B6cjAIYig4hEcFs4U68,30390
+tritonclient-2.36.0.data/purelib/tritonclient/http/aio/auth/__init__.py,sha256=2hd0vpFKURRLz9MjB6sf1Tnfq3bzLdhgG_VJBFwj6vA,1575
+tritonclient-2.36.0.data/purelib/tritonclient/http/auth/__init__.py,sha256=1JifMIv8ipug7qUP4joArCyRsEXjxV-DIhvrJ2Kj5Rk,1574
+tritonclient-2.36.0.data/purelib/tritonclient/utils/__init__.py,sha256=JVWOJ5YCglG1gJWgfJGsWrCK4PgzFA_2kz3bag3nZ8E,10557
+tritonclient-2.36.0.data/purelib/tritonclient/utils/_dlpack.py,sha256=eEUc-F0pBO6p8vBtN7KC5bB29DdkYH4wS9YtQPn1oKw,8963
+tritonclient-2.36.0.data/purelib/tritonclient/utils/_shared_memory_tensor.py,sha256=Ei9P4xecCckED_lQ7xXL98NJNNuE_89gIW54B4Sh0MY,3688
+tritonclient-2.36.0.data/purelib/tritonclientutils/__init__.py,sha256=D9hxs7FZaDd50ql_jLvaDFOwH8gCOp2LeFTaW1purzo,1821
+tritonclient-2.36.0.data/purelib/tritongrpcclient/__init__.py,sha256=hRAS-HadBCUVAzEFi2ZNW0Cbtd2-x09mmG0bXu-Q3iI,1818
+tritonclient-2.36.0.data/purelib/tritongrpcclient/grpc_service_pb2.py,sha256=L8UJUpxXzI0AA8YbGActcCUiaYQodaZBRLlOEDcbtEU,1842
+tritonclient-2.36.0.data/purelib/tritongrpcclient/grpc_service_pb2_grpc.py,sha256=YN64p5Y7HZ7eMLnM596As4WAQ2FKh828Rda7jVXF9TM,1852
+tritonclient-2.36.0.data/purelib/tritongrpcclient/model_config_pb2.py,sha256=Mk94wHJMdtG4Ptj0QSTVz7uRDsfX-l4zmN0dPeqiIZE,1852
+tritonclient-2.36.0.data/purelib/tritonhttpclient/__init__.py,sha256=IHIoNJAlBvjTut2ySmh9c0Iw1DyJzc4a5IcYbrzLw3Q,1818
+tritonclient-2.36.0.dist-info/LICENSE.txt,sha256=NMqy92YzgvtkIas9IW34i27ARqi_sCunXDu-t-EqxL8,1533
+tritonclient-2.36.0.dist-info/METADATA,sha256=8H9_Zw6bxz11Sptyuqm93tyy5CApNy8m_Mg4VLivr9k,2716
+tritonclient-2.36.0.dist-info/WHEEL,sha256=HVOIUsWA--OEoo_LBcXUZ_J2MMLFJ69mOq_A9H3ARGY,93
+tritonclient-2.36.0.dist-info/top_level.txt,sha256=J2zLu0JxoFexDB102feHPD6J3JXAUEYUInFJZM9yEkM,65
+tritonclient-2.36.0.dist-info/RECORD,,

tritonclient-2.34.0.dist-info/RECORD

@@ -1,40 +0,0 @@
-tritonclient-2.34.0.data/data/LICENSE.txt,sha256=NMqy92YzgvtkIas9IW34i27ARqi_sCunXDu-t-EqxL8,1533
-tritonclient-2.34.0.data/purelib/tritonclient/__init__.py,sha256=erynU-5HRlcIz1tx1Ve0kdCnzAXfNFeUYB5bx2fnL4Y,1542
-tritonclient-2.34.0.data/purelib/tritonclient/_auth.py,sha256=sXf16LlDoQj1qI3GSnFa1mgOv0zhwFmB0OHofkIwVh0,2047
-tritonclient-2.34.0.data/purelib/tritonclient/_client.py,sha256=8-VaArNooS9zKRY0aYTW9xvphmNa5vgWy7PDbBnGVw4,2964
-tritonclient-2.34.0.data/purelib/tritonclient/_plugin.py,sha256=hAaX9FakQBsbh5DZVAlcDujVwrV4EddcmPxMp3puxX0,2069
-tritonclient-2.34.0.data/purelib/tritonclient/_request.py,sha256=RmGnJpiRxTi33mn8Nl7ivGFqeKjicj_HhShtgOTci8I,1789
-tritonclient-2.34.0.data/purelib/tritonclient/grpc/__init__.py,sha256=MDsTzhwtE6603Ixkf6g_N3_5OCqhoDdCZT0KuZFtAmI,2959
-tritonclient-2.34.0.data/purelib/tritonclient/grpc/_client.py,sha256=HT1E8nMsi5TYjeRRuhLIwT3gD17_223M2nN8spSdzi4,67932
-tritonclient-2.34.0.data/purelib/tritonclient/grpc/_infer_input.py,sha256=5cfHBc7lVxWq36HGG12xgnQlx_-3OG8L0fQ18AK3zjA,7213
-tritonclient-2.34.0.data/purelib/tritonclient/grpc/_infer_result.py,sha256=gWA8LO7SiA7BsJnC8TGVh0kfE3hEmRrXNpXRSqMmOwY,6193
-tritonclient-2.34.0.data/purelib/tritonclient/grpc/_infer_stream.py,sha256=AYGC7pD2jXkFQr0wq1moUE1MDgICMMTHVbCFvlujvl8,6618
-tritonclient-2.34.0.data/purelib/tritonclient/grpc/_requested_output.py,sha256=SUHrwwH0RA2Uge6imndW5hNdzhcZtNhrV_0DRwFep8U,4409
-tritonclient-2.34.0.data/purelib/tritonclient/grpc/_utils.py,sha256=JdmHVv7efZg_Wiuf4tW6Q5hyqK_v9k_9aw7C4Mh3U38,4953
-tritonclient-2.34.0.data/purelib/tritonclient/grpc/model_config_pb2.py,sha256=UI9ApQ47f5WzOtQX0KnNnXzwCA-Ady0VHimxLlKYi4M,46315
-tritonclient-2.34.0.data/purelib/tritonclient/grpc/service_pb2.py,sha256=q3bQJtgKaCFJlWSZxkZjOmCU0BRNG3S5_1QaiV-oG58,54542
-tritonclient-2.34.0.data/purelib/tritonclient/grpc/service_pb2_grpc.py,sha256=BRXBBgehQgx_LSLqiw_N0zqFl4NYDhstm6AJHP3uxBw,37795
-tritonclient-2.34.0.data/purelib/tritonclient/grpc/aio/__init__.py,sha256=8gEdEltJSzd3GF_xSrLG1gh3aGuvX7aQzigi1WPbw1I,32979
-tritonclient-2.34.0.data/purelib/tritonclient/grpc/aio/auth/__init__.py,sha256=nOIwuyXvzV1Dha2ZFX5vRjFhgsDa-cBdP2wSWNA6g1w,1573
-tritonclient-2.34.0.data/purelib/tritonclient/grpc/auth/__init__.py,sha256=1JifMIv8ipug7qUP4joArCyRsEXjxV-DIhvrJ2Kj5Rk,1574
-tritonclient-2.34.0.data/purelib/tritonclient/http/__init__.py,sha256=tbJXG_x_NV7x9LbR2h7DAImiwM3a2uA7HBMKfDhpX1A,2128
-tritonclient-2.34.0.data/purelib/tritonclient/http/_client.py,sha256=R8QDM_hFJACsvly0QheM7_ZRLkUrOEp6RMhPOLVUO1g,61716
-tritonclient-2.34.0.data/purelib/tritonclient/http/_infer_input.py,sha256=eEYQEiK8AzAAW5zJq_c_thbxdHq0mAAjdWgeZYv5EoY,9506
-tritonclient-2.34.0.data/purelib/tritonclient/http/_infer_result.py,sha256=vQIcWxxThgvFSnChZwZebKIJW-pSawXG1nVi6ncCYYE,9920
-tritonclient-2.34.0.data/purelib/tritonclient/http/_requested_output.py,sha256=QU-07jLHbKh1S7hCoqvqjPxrHWeb0CsJ8Qd3077AU1U,4695
-tritonclient-2.34.0.data/purelib/tritonclient/http/_utils.py,sha256=rMw1Ojz3UFsbPjrvU4nKVLH65TPJbujMJXhw7klZynk,5118
-tritonclient-2.34.0.data/purelib/tritonclient/http/aio/__init__.py,sha256=edd6GlMqiAb2kJG2jWlBs2P0B6cjAIYig4hEcFs4U68,30390
-tritonclient-2.34.0.data/purelib/tritonclient/http/aio/auth/__init__.py,sha256=2hd0vpFKURRLz9MjB6sf1Tnfq3bzLdhgG_VJBFwj6vA,1575
-tritonclient-2.34.0.data/purelib/tritonclient/http/auth/__init__.py,sha256=1JifMIv8ipug7qUP4joArCyRsEXjxV-DIhvrJ2Kj5Rk,1574
-tritonclient-2.34.0.data/purelib/tritonclient/utils/__init__.py,sha256=0aKf-KpxgWFqMIXT-5msg_Y4LNCDMe33VyuTcuyRNM4,10503
-tritonclient-2.34.0.data/purelib/tritonclientutils/__init__.py,sha256=D9hxs7FZaDd50ql_jLvaDFOwH8gCOp2LeFTaW1purzo,1821
-tritonclient-2.34.0.data/purelib/tritongrpcclient/__init__.py,sha256=hRAS-HadBCUVAzEFi2ZNW0Cbtd2-x09mmG0bXu-Q3iI,1818
-tritonclient-2.34.0.data/purelib/tritongrpcclient/grpc_service_pb2.py,sha256=L8UJUpxXzI0AA8YbGActcCUiaYQodaZBRLlOEDcbtEU,1842
-tritonclient-2.34.0.data/purelib/tritongrpcclient/grpc_service_pb2_grpc.py,sha256=YN64p5Y7HZ7eMLnM596As4WAQ2FKh828Rda7jVXF9TM,1852
-tritonclient-2.34.0.data/purelib/tritongrpcclient/model_config_pb2.py,sha256=Mk94wHJMdtG4Ptj0QSTVz7uRDsfX-l4zmN0dPeqiIZE,1852
-tritonclient-2.34.0.data/purelib/tritonhttpclient/__init__.py,sha256=IHIoNJAlBvjTut2ySmh9c0Iw1DyJzc4a5IcYbrzLw3Q,1818
-tritonclient-2.34.0.dist-info/LICENSE.txt,sha256=NMqy92YzgvtkIas9IW34i27ARqi_sCunXDu-t-EqxL8,1533
-tritonclient-2.34.0.dist-info/METADATA,sha256=F7faWwgsiTKETF7NL6g7-V_CFRR4JXHye0X3Oy4XyEI,2716
-tritonclient-2.34.0.dist-info/WHEEL,sha256=HVOIUsWA--OEoo_LBcXUZ_J2MMLFJ69mOq_A9H3ARGY,93
-tritonclient-2.34.0.dist-info/top_level.txt,sha256=J2zLu0JxoFexDB102feHPD6J3JXAUEYUInFJZM9yEkM,65
-tritonclient-2.34.0.dist-info/RECORD,,