clarifai 11.1.5rc7__py3-none-any.whl → 11.1.6__py3-none-any.whl

clarifai/runners/__init__.py

@@ -1,9 +1,14 @@
+from .models.base_typed_model import AnyAnyModel, TextInputModel, VisualInputModel
 from .models.model_builder import ModelBuilder
-from .models.model_class import ModelClass
 from .models.model_runner import ModelRunner
+from .utils.data_handler import InputDataHandler, OutputDataHandler
 
 __all__ = [
     "ModelRunner",
     "ModelBuilder",
-    "ModelClass",
+    "InputDataHandler",
+    "OutputDataHandler",
+    "AnyAnyModel",
+    "TextInputModel",
+    "VisualInputModel",
 ]

clarifai/runners/dockerfile_template/Dockerfile.template

@@ -24,7 +24,7 @@ COPY --chown=nonroot:nonroot downloader/unused.yaml /home/nonroot/main/1/checkpo
 #####
 # Download checkpoints if config.yaml has checkpoints.when = "build"
 COPY --link=true config.yaml /home/nonroot/main/
-RUN ["python", "-m", "clarifai.cli", "model", "download-checkpoints", "--model_path", "/home/nonroot/main", "--out_path", "/home/nonroot/main/1/checkpoints", "--stage", "build"]
+RUN ["python", "-m", "clarifai.cli", "model", "download-checkpoints", "/home/nonroot/main", "--out_path", "/home/nonroot/main/1/checkpoints", "--stage", "build"]
 #####
 
 # Copy in the actual files like config.yaml, requirements.txt, and most importantly 1/model.py
@@ -44,9 +44,6 @@ ENV PYTHONPATH=${PYTHONPATH}:/home/nonroot/main \
     CLARIFAI_COMPUTE_CLUSTER_ID=${CLARIFAI_COMPUTE_CLUSTER_ID} \
     CLARIFAI_API_BASE=${CLARIFAI_API_BASE:-https://api.clarifai.com}
 
-# # Write out the model function signatures
-# RUN ["python", "-m", "clarifai.cli", "model", "signatures", "--model_path", "/home/nonroot/main", "--out_path", "/home/nonroot/main/signatures.yaml"]
-
 # Finally run the clarifai entrypoint to start the runner loop and local dev server.
 # Note(zeiler): we may want to make this a clarifai CLI call.
 ENTRYPOINT ["python", "-m", "clarifai.runners.server"]

clarifai/runners/models/base_typed_model.py

@@ -0,0 +1,238 @@
+import itertools
+from typing import Any, Dict, Iterator, List, Tuple
+
+import numpy as np
+from clarifai_grpc.grpc.api import resources_pb2, service_pb2
+from clarifai_grpc.grpc.api.service_pb2 import PostModelOutputsRequest
+from google.protobuf import json_format
+
+from ..utils.data_handler import InputDataHandler, OutputDataHandler
+from .model_class import ModelClass
+
+
+class AnyAnyModel(ModelClass):
+
+  def load_model(self):
+    """
+    Load inference time artifacts that are called frequently .e.g. models, tokenizers, etc.
+    in this method so they are loaded only once for faster inference.
+    """
+    raise NotImplementedError
+
+  def parse_input_request(
+      self, input_request: service_pb2.PostModelOutputsRequest) -> Tuple[List[Dict], Dict]:
+    list_input_dict = [
+        InputDataHandler.from_proto(input).to_python() for input in input_request.inputs
+    ]
+    inference_params = json_format.MessageToDict(
+        input_request.model.model_version.output_info.params)
+
+    return list_input_dict, inference_params
+
+  def convert_output_to_proto(self, outputs: list):
+    assert (isinstance(outputs, Iterator) or isinstance(outputs, list) or
+            isinstance(outputs, tuple)), "outputs must be an Iterator"
+    output_protos = []
+    for output in outputs:
+      if isinstance(output, OutputDataHandler):
+        output = output.proto
+      elif isinstance(output, resources_pb2.Output):
+        pass
+      else:
+        raise NotImplementedError
+      output_protos.append(output)
+
+    return service_pb2.MultiOutputResponse(outputs=output_protos)
+
+  def predict_wrapper(
+      self, request: service_pb2.PostModelOutputsRequest) -> service_pb2.MultiOutputResponse:
+    list_dict_input, inference_params = self.parse_input_request(request)
+    outputs = self.predict(list_dict_input, inference_parameters=inference_params)
+    return self.convert_output_to_proto(outputs)
+
+  def generate_wrapper(
+      self, request: PostModelOutputsRequest) -> Iterator[service_pb2.MultiOutputResponse]:
+    list_dict_input, inference_params = self.parse_input_request(request)
+    outputs = self.generate(list_dict_input, inference_parameters=inference_params)
+    for output in outputs:
+      yield self.convert_output_to_proto(output)
+
+  def _preprocess_stream(
+      self, request: Iterator[PostModelOutputsRequest]) -> Iterator[Tuple[List[Dict], List[Dict]]]:
+    """Return generator of processed data (from proto to python) and inference parameters like predict and generate"""
+    for i, req in enumerate(request):
+      input_data, _ = self.parse_input_request(req)
+      yield input_data
+
+  def stream_wrapper(self, request: Iterator[PostModelOutputsRequest]
+                    ) -> Iterator[service_pb2.MultiOutputResponse]:
+    first_request = next(request)
+    _, inference_params = self.parse_input_request(first_request)
+    request_iterator = itertools.chain([first_request], request)
+    outputs = self.stream(self._preprocess_stream(request_iterator), inference_params)
+    for output in outputs:
+      yield self.convert_output_to_proto(output)
+
+  def predict(self, input_data: List[Dict],
+              inference_parameters: Dict[str, Any] = {}) -> List[OutputDataHandler]:
+    """
+    Prediction method.
+
+    Args:
+    -----
+    - input_data: is list of dict where key is input type name.
+      * image: np.ndarray
+      * text: str
+      * audio: bytes
+
+    - inference_parameters (Dict[str, Union[bool, str, float, int]]): your inference parameters.
+
+    Returns:
+    --------
+      List of OutputDataHandler
+    """
+    raise NotImplementedError
+
+  def generate(self, input_data: List[Dict],
+               inference_parameters: Dict[str, Any] = {}) -> Iterator[List[OutputDataHandler]]:
+    """
+    Generate method.
+
+    Args:
+    -----
+    - input_data: is list of dict where key is input type name.
+      * image: np.ndarray
+      * text: str
+      * audio: bytes
+
+    - inference_parameters (Dict[str, Union[bool, str, float, int]]): your inference parameters.
+
+    Yield:
+    --------
+      List of OutputDataHandler
+    """
+    raise NotImplementedError
+
+  def stream(self, inputs: Iterator[List[Dict[str, Any]]],
+             inference_params: Dict[str, Any]) -> Iterator[List[OutputDataHandler]]:
+    """
+    Stream method.
+
+    Args:
+    -----
+    input_request: is an Iterator of Tuple which
+    - First element (List[Dict[str, Union[np.ndarray, str, bytes]]]) is list of dict input data type which keys and values are:
+        * image: np.ndarray
+        * text: str
+        * audio: bytes
+
+    - Second element (Dict[str, Union[bool, str, float, int]]): is a dict of inference_parameters
+
+    Yield:
+    --------
+      List of OutputDataHandler
+    """
+    raise NotImplementedError
+
+
+class VisualInputModel(AnyAnyModel):
+
+  def parse_input_request(
+      self, input_request: service_pb2.PostModelOutputsRequest) -> Tuple[List[Dict], Dict]:
+    list_input_dict = [
+        InputDataHandler.from_proto(input).image(format="np") for input in input_request.inputs
+    ]
+    inference_params = json_format.MessageToDict(
+        input_request.model.model_version.output_info.params)
+
+    return list_input_dict, inference_params
+
+  def load_model(self):
+    """
+    Load inference time artifacts that are called frequently .e.g. models, tokenizers, etc.
+    in this method so they are loaded only once for faster inference.
+    """
+    raise NotImplementedError
+
+  def predict(self, input_data: List[np.ndarray],
+              inference_parameters: Dict[str, Any] = {}) -> List[OutputDataHandler]:
+    """
+    Prediction method.
+
+    Args:
+    -----
+    - input_data(List[np.ndarray]): is list of image as np.ndarray type
+    - inference_parameters (Dict[str, Union[bool, str, float, int]]): your inference parameters.
+
+    Returns:
+    --------
+      List of OutputDataHandler
+    """
+    raise NotImplementedError
+
+
+class TextInputModel(AnyAnyModel):
+
+  def load_model(self):
+    """
+    Load inference time artifacts that are called frequently .e.g. models, tokenizers, etc.
+    in this method so they are loaded only once for faster inference.
+    """
+    raise NotImplementedError
+
+  def parse_input_request(
+      self, input_request: service_pb2.PostModelOutputsRequest) -> Tuple[List[Dict], Dict]:
+    list_input_text = [InputDataHandler.from_proto(input).text for input in input_request.inputs]
+    inference_params = json_format.MessageToDict(
+        input_request.model.model_version.output_info.params)
+
+    return list_input_text, inference_params
+
+  def predict(self, input_data: List[str],
+              inference_parameters: Dict[str, Any] = {}) -> List[OutputDataHandler]:
+    """
+    Prediction method.
+
+    Args:
+    -----
+    - input_data(List[str]): is list of text as str type
+    - inference_parameters (Dict[str, Union[bool, str, float, int]]): your inference parameters.
+
+    Returns:
+    --------
+      List of OutputDataHandler
+    """
+    raise NotImplementedError
+
+  def generate(self, input_data: List[str],
+               inference_parameters: Dict[str, Any] = {}) -> Iterator[List[OutputDataHandler]]:
+    """
+    Prediction method.
+
+    Args:
+    -----
+    - input_data(List[str]): is list of text as str type
+    - inference_parameters (Dict[str, Union[bool, str, float, int]]): your inference parameters.
+
+    Yield:
+    --------
+      List of OutputDataHandler
+    """
+    raise NotImplementedError
+
+  def stream(self, inputs: Iterator[List[str]],
+             inference_params: Dict[str, Any]) -> Iterator[List[OutputDataHandler]]:
+    """
+    Stream method.
+
+    Args:
+    -----
+    input_request: is an Iterator of Tuple which
+    - First element (List[str]) is list of input text:
+    - Second element (Dict[str, Union[bool, str, float, int]]): is a dict of inference_parameters
+
+    Yield:
+    --------
+      List of OutputDataHandler
+    """
+    raise NotImplementedError

clarifai/runners/models/model_builder.py

@@ -14,14 +14,13 @@ from google.protobuf import json_format
 from rich import print
 from rich.markup import escape
 
-from clarifai.client.base import BaseClient
+from clarifai.client import BaseClient
 from clarifai.runners.models.model_class import ModelClass
 from clarifai.runners.utils.const import (
     AVAILABLE_PYTHON_IMAGES, AVAILABLE_TORCH_IMAGES, CONCEPTS_REQUIRED_MODEL_TYPE,
     DEFAULT_DOWNLOAD_CHECKPOINT_WHEN, DEFAULT_PYTHON_VERSION, DEFAULT_RUNTIME_DOWNLOAD_PATH,
     PYTHON_BASE_IMAGE, TORCH_BASE_IMAGE)
 from clarifai.runners.utils.loader import HuggingFaceLoader
-from clarifai.runners.utils.method_signatures import signatures_to_yaml
 from clarifai.urls.helper import ClarifaiUrlHelper
 from clarifai.utils.logging import logger
 from clarifai.versions import CLIENT_VERSION
@@ -70,18 +69,6 @@ class ModelBuilder:
     """
     Create an instance of the model class, as specified in the config file.
     """
-    model_class = self.load_model_class()
-
-    # initialize the model
-    model = model_class()
-    if load_model:
-      model.load_model()
-    return model
-
-  def load_model_class(self):
-    """
-    Import the model class from the model.py file.
-    """
     # look for default model.py file location
     for loc in ["model.py", "1/model.py"]:
       model_file = os.path.join(self.folder, loc)
@@ -120,7 +107,12 @@ class ModelBuilder:
           "Could not determine model class. There should be exactly one model inheriting from ModelClass defined in the model.py"
       )
     model_class = classes[0]
-    return model_class
+
+    # initialize the model
+    model = model_class()
+    if load_model:
+      model.load_model()
+    return model
 
   def _validate_folder(self, folder):
     if folder == ".":
@@ -261,15 +253,6 @@ class ModelBuilder:
           total_size += member.size
     return total_size
 
-  def method_signatures_yaml(self):
-    """
-    Returns the method signatures for the model class in YAML format.
-    """
-    model_class = self.load_model_class()
-    method_info = model_class._get_method_info()
-    signatures = {name: m.signature for name, m in method_info.items()}
-    return signatures_to_yaml(signatures)
-
   @property
   def client(self):
     if self._client is None:
@@ -639,9 +622,9 @@ class ModelBuilder:
         checkpoint_size = self.DEFAULT_CHECKPOINT_SIZE
       self.storage_request_size += checkpoint_size
 
-    self.maybe_create_model()
+    resp = self.maybe_create_model()
     if not self.check_model_exists():
-      logger.error(f"Failed to create model: {self.model_proto.id}")
+      logger.error(f"Failed to create model: {self.model_proto.id}. Details: {resp}")
       sys.exit(1)
 
     for response in self.client.STUB.PostModelVersionsUpload(

clarifai/runners/models/model_class.py

@@ -1,286 +1,41 @@
-import inspect
-import itertools
-import logging
-import os
-import traceback
-from abc import ABC
-from typing import Any, Dict, Iterator, List
+from abc import ABC, abstractmethod
+from typing import Iterator
 
-from clarifai_grpc.grpc.api import resources_pb2, service_pb2
-from clarifai_grpc.grpc.api.status import status_code_pb2, status_pb2
-
-from clarifai.runners.utils import data_types
-from clarifai.runners.utils.method_signatures import (build_function_signature, deserialize,
-                                                      get_stream_from_signature, serialize,
-                                                      signatures_to_json)
-
-_METHOD_INFO_ATTR = '_cf_method_info'
-
-_RAISE_EXCEPTIONS = os.getenv("RAISE_EXCEPTIONS", "false").lower() == "true"
-
-
-class methods:
-  '''
-  Decorators to mark methods as predict, generate, or stream methods.
-  '''
-
-  @staticmethod
-  def predict(method):
-    setattr(method, _METHOD_INFO_ATTR, _MethodInfo(method, 'predict'))
-    return method
-
-  @staticmethod
-  def generate(method):
-    setattr(method, _METHOD_INFO_ATTR, _MethodInfo(method, 'generate'))
-    return method
-
-  @staticmethod
-  def stream(method):
-    setattr(method, _METHOD_INFO_ATTR, _MethodInfo(method, 'stream'))
-    return method
+from clarifai_grpc.grpc.api import service_pb2
 
 
 class ModelClass(ABC):
-  '''
-  Base class for model classes that can be run as a service.
-
-  Define methods as predict, generate, or stream methods using the @methods decorators.
-
-  Example:
-
-    from clarifai.runners.model_class import ModelClass, methods
-    from clarifai.runners.utils.data_types import Input, Stream
-
-    class MyModel(ModelClass):
-
-      @methods.predict
-      def predict(self, x: str, y: int) -> List[str]:
-        return [x] * y
-
-      @methods.generate
-      def generate(self, x: str, y: int) -> Stream[str]:
-        for i in range(y):
-          yield x + str(i)
-
-      @methods.stream
-      def stream(self, input_stream: Stream[Input(x=str, y=int)]) -> Stream[str]:
-        for item in input_stream:
-          yield item.x + ' ' + str(item.y)
-  '''
-
-  def load_model(self):
-    """Load the model."""
-
-  def _handle_get_signatures_request(self) -> service_pb2.MultiOutputResponse:
-    methods = self._get_method_info()
-    signatures = {method.name: method.signature for method in methods.values()}
-    resp = service_pb2.MultiOutputResponse(status=status_pb2.Status(code=status_code_pb2.SUCCESS))
-    output = resp.outputs.add()
-    output.status.code = status_code_pb2.SUCCESS
-    output.data.text.raw = signatures_to_json(signatures)
-    return resp
-
-  def batch_predict(self, method, inputs: List[Dict[str, Any]]) -> List[Any]:
-    """Batch predict method for multiple inputs."""
-    outputs = []
-    for input in inputs:
-      output = method(**input)
-      outputs.append(output)
-    return outputs
-
-  def batch_generate(self, method, inputs: List[Dict[str, Any]]) -> Iterator[List[Any]]:
-    """Batch generate method for multiple inputs."""
-    generators = [method(**input) for input in inputs]
-    for outputs in itertools.zip_longest(*generators):
-      yield outputs
 
   def predict_wrapper(
       self, request: service_pb2.PostModelOutputsRequest) -> service_pb2.MultiOutputResponse:
-    outputs = []
-    try:
-      # TODO add method name field to proto
-      method_name = None
-      if len(request.inputs) > 0 and '_method_name' in request.inputs[0].data.metadata:
-        method_name = request.inputs[0].data.metadata['_method_name']
-      # call_params = dict(request.model.model_version.output_info.params)
-      # method_name = call_params.get('_method_name', 'predict')
-      if method_name == '_GET_SIGNATURES':  # special case to fetch signatures, TODO add endpoint for this
-        return self._handle_get_signatures_request()
-      if method_name not in self._get_method_info():
-        raise ValueError(f"Method {method_name} not found in model class")
-      method = getattr(self, method_name)
-      method_info = method._cf_method_info
-      signature = method_info.signature
-      python_param_types = method_info.python_param_types
-      inputs = self._convert_input_protos_to_python(request.inputs, signature.inputs,
-                                                    python_param_types)
-      if len(inputs) == 1:
-        inputs = inputs[0]
-        output = method(**inputs)
-        outputs.append(self._convert_output_to_proto(output, signature.outputs))
-      else:
-        outputs = self.batch_predict(method, inputs)
-        outputs = [self._convert_output_to_proto(output, signature.outputs) for output in outputs]
-
-      return service_pb2.MultiOutputResponse(
-          outputs=outputs, status=status_pb2.Status(code=status_code_pb2.SUCCESS))
-    except Exception as e:
-      if _RAISE_EXCEPTIONS:
-        raise
-      logging.exception("Error in predict")
-      return service_pb2.MultiOutputResponse(status=status_pb2.Status(
-          code=status_code_pb2.FAILURE,
-          details=str(e),
-          stack_trace=traceback.format_exc().split('\n')))
+    """This method is used for input/output proto data conversion"""
+    return self.predict(request)
 
   def generate_wrapper(self, request: service_pb2.PostModelOutputsRequest
                       ) -> Iterator[service_pb2.MultiOutputResponse]:
-    try:
-      call_params = dict(request.model.model_version.output_info.params)
-      method_name = call_params.get('_method_name', 'generate')
-      method = getattr(self, method_name)
-      method_info = method._cf_method_info
-      signature = method_info.signature
-      python_param_types = method_info.python_param_types
+    """This method is used for input/output proto data conversion and yield outcome"""
+    return self.generate(request)
 
-      inputs = self._convert_input_protos_to_python(request.inputs, signature.inputs,
-                                                    python_param_types)
-      if len(inputs) == 1:
-        inputs = inputs[0]
-        for output in method(**inputs):
-          resp = service_pb2.MultiOutputResponse()
-          self._convert_output_to_proto(output, signature.outputs, proto=resp.outputs.add())
-          resp.status.code = status_code_pb2.SUCCESS
-          yield resp
-      else:
-        for outputs in self.batch_generate(method, inputs):
-          resp = service_pb2.MultiOutputResponse()
-          for output in outputs:
-            self._convert_output_to_proto(output, signature.outputs, proto=resp.outputs.add())
-          resp.status.code = status_code_pb2.SUCCESS
-          yield resp
-    except Exception as e:
-      if _RAISE_EXCEPTIONS:
-        raise
-      logging.exception("Error in generate")
-      yield service_pb2.MultiOutputResponse(status=status_pb2.Status(
-          code=status_code_pb2.FAILURE,
-          details=str(e),
-          stack_trace=traceback.format_exc().split('\n')))
-
-  def stream_wrapper(self, request_iterator: Iterator[service_pb2.PostModelOutputsRequest]
+  def stream_wrapper(self, request: service_pb2.PostModelOutputsRequest
                     ) -> Iterator[service_pb2.MultiOutputResponse]:
-    try:
-      request = next(request_iterator)  # get first request to determine method
-      assert len(request.inputs) == 1, "Streaming requires exactly one input"
-
-      call_params = dict(request.model.model_version.output_info.params)
-      method_name = call_params.get('_method_name', 'stream')
-      method = getattr(self, method_name)
-      method_info = method._cf_method_info
-      signature = method_info.signature
-      python_param_types = method_info.python_param_types
-
-      # find the streaming vars in the signature
-      stream_argname, streaming_var_signatures = get_stream_from_signature(signature.inputs)
-
-      # convert all inputs for the first request, including the first stream value
-      inputs = self._convert_input_protos_to_python(request.inputs, signature.inputs,
-                                                    python_param_types)
-      kwargs = inputs[0]
-
-      # first streaming item
-      first_item = kwargs.pop(stream_argname)
-
-      # streaming generator
-      def InputStream():
-        yield first_item
-        # subsequent streaming items contain only the streaming input
-        for request in request_iterator:
-          item = self._convert_input_protos_to_python(request.inputs, streaming_var_signatures,
-                                                      python_param_types)
-          item = item[0][stream_argname]
-          yield item
+    """This method is used for input/output proto data conversion and yield outcome"""
+    return self.stream(request)
 
-      # add stream generator back to the input kwargs
-      kwargs[stream_argname] = InputStream()
-
-      for output in method(**kwargs):
-        resp = service_pb2.MultiOutputResponse()
-        self._convert_output_to_proto(output, signature.outputs, proto=resp.outputs.add())
-        resp.status.code = status_code_pb2.SUCCESS
-        yield resp
-    except Exception as e:
-      if _RAISE_EXCEPTIONS:
-        raise
-      logging.exception("Error in stream")
-      yield service_pb2.MultiOutputResponse(status=status_pb2.Status(
-          code=status_code_pb2.FAILURE,
-          details=str(e),
-          stack_trace=traceback.format_exc().split('\n')))
-
-  def _convert_input_protos_to_python(self, inputs: List[resources_pb2.Input], variables_signature,
-                                      python_param_types) -> List[Dict[str, Any]]:
-    result = []
-    for input in inputs:
-      kwargs = deserialize(input.data, variables_signature)
-      # dynamic cast to annotated types
-      for k, v in kwargs.items():
-        if k not in python_param_types:
-          continue
-        kwargs[k] = data_types.cast(v, python_param_types[k])
-      result.append(kwargs)
-    return result
-
-  def _convert_output_to_proto(self, output: Any, variables_signature,
-                               proto=None) -> resources_pb2.Output:
-    if proto is None:
-      proto = resources_pb2.Output()
-    if isinstance(output, tuple):
-      output = {f'return.{i}': item for i, item in enumerate(output)}
-    if not isinstance(output, dict):  # TODO Output type, not just dict
-      output = {'return': output}
-    serialize(output, variables_signature, proto.data, is_output=True)
-    proto.status.code = status_code_pb2.SUCCESS
-    return proto
-
-  @classmethod
-  def _register_model_methods(cls):
-    # go up the class hierarchy to find all decorated methods, and add to registry of current class
-    methods = {}
-    for base in reversed(cls.__mro__):
-      for name, method in base.__dict__.items():
-        method_info = getattr(method, _METHOD_INFO_ATTR, None)
-        if not method_info:  # regular function, not a model method
-          continue
-        methods[name] = method_info
-    # check for generic predict(request) -> response, etc. methods
-    #for name in ('predict', 'generate', 'stream'):
-    #  if hasattr(cls, name):
-    #    method = getattr(cls, name)
-    #    if not hasattr(method, _METHOD_INFO_ATTR):  # not already put in registry
-    #      methods[name] = _MethodInfo(method, method_type=name)
-    # set method table for this class in the registry
-    return methods
-
-  @classmethod
-  def _get_method_info(cls, func_name=None):
-    if not hasattr(cls, _METHOD_INFO_ATTR):
-      setattr(cls, _METHOD_INFO_ATTR, cls._register_model_methods())
-    method_info = getattr(cls, _METHOD_INFO_ATTR)
-    if func_name:
-      return method_info[func_name]
-    return method_info
-
-
-class _MethodInfo:
-
-  def __init__(self, method, method_type):
-    self.name = method.__name__
-    self.signature = build_function_signature(method, method_type)
-    self.python_param_types = {
-        p.name: p.annotation
-        for p in inspect.signature(method).parameters.values()
-        if p.annotation != inspect.Parameter.empty
-    }
-    self.python_param_types.pop('self', None)
+  @abstractmethod
+  def load_model(self):
+    raise NotImplementedError("load_model() not implemented")
+
+  @abstractmethod
+  def predict(self,
+              request: service_pb2.PostModelOutputsRequest) -> service_pb2.MultiOutputResponse:
+    raise NotImplementedError("run_input() not implemented")
+
+  @abstractmethod
+  def generate(self, request: service_pb2.PostModelOutputsRequest
+              ) -> Iterator[service_pb2.MultiOutputResponse]:
+    raise NotImplementedError("generate() not implemented")
+
+  @abstractmethod
+  def stream(self, request_iterator: Iterator[service_pb2.PostModelOutputsRequest]
+            ) -> Iterator[service_pb2.MultiOutputResponse]:
+    raise NotImplementedError("stream() not implemented")