clarifai 11.1.4rc2__py3-none-any.whl → 11.1.5rc1__py3-none-any.whl

clarifai/client/model_client.py

@@ -0,0 +1,400 @@
+import time
+from typing import Any, Dict, Iterator, List
+
+from clarifai_grpc.grpc.api import resources_pb2, service_pb2
+from clarifai_grpc.grpc.api.status import status_code_pb2
+
+from clarifai.constants.model import MAX_MODEL_PREDICT_INPUTS
+from clarifai.errors import UserError
+from clarifai.runners.utils.method_signatures import deserialize, serialize, signatures_from_json
+from clarifai.utils.misc import BackoffIterator, status_is_retryable
+
+
+class ModelClient:
+  '''
+  Client for calling model predict, generate, and stream methods.
+  '''
+
+  def __init__(self, stub, request_template: service_pb2.PostModelOutputsRequest = None):
+    '''
+        Initialize the model client.
+
+        Args:
+            stub: The gRPC stub for the model.
+            request_template: The template for the request to send to the model, including
+            common fields like model_id, model_version, cluster, etc.
+        '''
+    self.STUB = stub
+    self.request_template = request_template or service_pb2.PostModelOutputsRequest()
+    self._fetch_signatures()
+    self._define_functions()
+
+  def _fetch_signatures(self):
+    '''
+      Fetch the method signatures from the model.
+
+      Returns:
+          Dict: The method signatures.
+      '''
+    #request = resources_pb2.GetModelSignaturesRequest()
+    #response = self.stub.GetModelSignatures(request)
+    #self._method_signatures = json.loads(response.signatures)  # or define protos
+    # TODO this could use a new endpoint to get the signatures
+    # for local grpc models, we'll also have to add the endpoint to the model servicer
+    # for now we'll just use the predict endpoint with a special method name
+
+    request = service_pb2.PostModelOutputsRequest()
+    request.CopyFrom(self.request_template)
+    request.model.model_version.output_info.params['_method_name'] = '_GET_SIGNATURES'
+    request.inputs.add()  # empty input for this method
+    start_time = time.time()
+    backoff_iterator = BackoffIterator(10)
+    while True:
+      response = self.STUB.PostModelOutputs(request)
+      if status_is_retryable(
+          response.status.code) and time.time() - start_time < 60 * 10:  # 10 minutes
+        self.logger.info(f"Retrying model info fetch with response {response.status!r}")
+        time.sleep(next(backoff_iterator))
+        continue
+
+      if response.status.code != status_code_pb2.SUCCESS:
+        raise Exception(f"Model failed with response {response.status!r}")
+      break
+    if response.status.code != status_code_pb2.SUCCESS:
+      raise Exception(response.status)
+    self._method_signatures = signatures_from_json(response.outputs[0].data.string_value)
+
+  def _define_functions(self):
+    '''
+    Define the functions based on the method signatures.
+    '''
+    for method_name, method_signature in self._method_signatures.items():
+      # define the function in this client instance
+      if method_signature.method_type == 'predict':
+        call_func = self._predict
+      elif method_signature.method_type == 'generate':
+        call_func = self._generate
+      elif method_signature.method_type == 'stream':
+        call_func = self._stream
+      else:
+        raise ValueError(f"Unknown method type {method_signature.method_type}")
+
+      # method argnames, in order, collapsing nested keys to corresponding user function args
+      method_argnames = []
+      for var in method_signature.inputs:
+        outer = var.name.split('.', 1)[0]
+        if outer in method_argnames:
+          continue
+        method_argnames.append(outer)
+
+      def bind_f(method_name, method_argnames, call_func):
+
+        def f(*args, **kwargs):
+          if len(args) > len(method_argnames):
+            raise TypeError(
+                f"{method_name}() takes {len(method_argnames)} positional arguments but {len(args)} were given"
+            )
+          for name, arg in zip(method_argnames, args):  # handle positional with zip shortest
+            if name in kwargs:
+              raise TypeError(f"Multiple values for argument {name}")
+            kwargs[name] = arg
+          return call_func(kwargs, method_name)
+
+        return f
+
+      # need to bind method_name to the value, not the mutating loop variable
+      f = bind_f(method_name, method_argnames, call_func)
+
+      # set names and docstrings
+      # note we could also have used exec with strings from the signature to define the
+      # function, but this is safer (no xss), and docstrings with the signature is ok enough
+      f.__name__ = method_name
+      f.__qualname__ = f'{self.__class__.__name__}.{method_name}'
+      input_spec = ', '.join(
+          f'{var.name}: {var.data_type}{" = " + str(var.default) if not var.required else ""}'
+          for var in method_signature.inputs)
+      output_vars = method_signature.outputs
+      if len(output_vars) == 1 and output_vars[0].name == 'return':
+        # single output
+        output_spec = output_vars[0].data_type
+      elif output_vars[0].name == 'return.0':
+        # tuple output
+        output_spec = '(' + ', '.join(var.data_type for var in output_vars) + ')'
+      else:
+        # named output
+        output_spec = f'Output({", ".join(f"{var.name}={var.data_type}" for var in output_vars)})'
+      f.__doc__ = f'''{method_name}(self, {input_spec}) -> {output_spec}\n'''
+      #f.__doc__ += method_signature.description  # TODO
+      setattr(self, method_name, f)
+
+  def _predict(
+      self,
+      inputs,  # TODO set up functions according to fetched signatures?
+      method_name: str = 'predict',
+  ) -> Any:
+    input_signature = self._method_signatures[method_name].inputs
+    output_signature = self._method_signatures[method_name].outputs
+
+    batch_input = True
+    if isinstance(inputs, dict):
+      inputs = [inputs]
+      batch_input = False
+
+    proto_inputs = []
+    for input in inputs:
+      proto = resources_pb2.Input()
+      serialize(input, input_signature, proto.data)
+      proto_inputs.append(proto)
+
+    response = self._predict_by_proto(proto_inputs, method_name)
+    #print(response)
+
+    outputs = []
+    for output in response.outputs:
+      outputs.append(deserialize(output.data, output_signature, is_output=True))
+    if batch_input:
+      return outputs
+    return outputs[0]
+
+  def _predict_by_proto(
+      self,
+      inputs: List[resources_pb2.Input],
+      method_name: str = None,
+      inference_params: Dict = None,
+      output_config: Dict = None,
+  ) -> service_pb2.MultiOutputResponse:
+    """Predicts the model based on the given inputs.
+
+      Args:
+          inputs (List[resources_pb2.Input]): The inputs to predict.
+          method_name (str): The remote method name to call.
+          inference_params (Dict): Inference parameters to override.
+          output_config (Dict): Output configuration to override.
+
+      Returns:
+          service_pb2.MultiOutputResponse: The prediction response(s).
+      """
+    if not isinstance(inputs, list):
+      raise UserError('Invalid inputs, inputs must be a list of Input objects.')
+    if len(inputs) > MAX_MODEL_PREDICT_INPUTS:
+      raise UserError(f"Too many inputs. Max is {MAX_MODEL_PREDICT_INPUTS}.")
+
+    request = service_pb2.PostModelOutputsRequest()
+    request.CopyFrom(self.request_template)
+
+    request.inputs.extend(inputs)
+
+    if method_name:
+      # TODO put in new proto field?
+      request.model.model_version.output_info.params['_method_name'] = method_name
+    if inference_params:
+      request.model.model_version.output_info.params.update(inference_params)
+    if output_config:
+      request.model.model_version.output_info.output_config.MergeFromDict(output_config)
+
+    start_time = time.time()
+    backoff_iterator = BackoffIterator(10)
+    while True:
+      response = self.STUB.PostModelOutputs(request)
+      if status_is_retryable(
+          response.status.code) and time.time() - start_time < 60 * 10:  # 10 minutes
+        self.logger.info(f"Model predict failed with response {response.status!r}")
+        time.sleep(next(backoff_iterator))
+        continue
+
+      if response.status.code != status_code_pb2.SUCCESS:
+        raise Exception(f"Model predict failed with response {response.status!r}")
+      break
+
+    return response
+
+  def _generate(
+      self,
+      inputs,  # TODO set up functions according to fetched signatures?
+      method_name: str = 'generate',
+  ) -> Any:
+    input_signature = self._method_signatures[method_name].inputs
+    output_signature = self._method_signatures[method_name].outputs
+
+    batch_input = True
+    if isinstance(inputs, dict):
+      inputs = [inputs]
+      batch_input = False
+
+    proto_inputs = []
+    for input in inputs:
+      proto = resources_pb2.Input()
+      serialize(input, input_signature, proto.data)
+      proto_inputs.append(proto)
+
+    response_stream = self._generate_by_proto(proto_inputs, method_name)
+    #print(response)
+
+    for response in response_stream:
+      outputs = []
+      for output in response.outputs:
+        outputs.append(deserialize(output.data, output_signature, is_output=True))
+      if batch_input:
+        yield outputs
+      yield outputs[0]
+
+  def _generate_by_proto(
+      self,
+      inputs: List[resources_pb2.Input],
+      method_name: str = None,
+      inference_params: Dict = {},
+      output_config: Dict = {},
+  ):
+    """Generate the stream output on model based on the given inputs.
+
+    Args:
+        inputs (list[Input]): The inputs to generate, must be less than 128.
+        method_name (str): The remote method name to call.
+        inference_params (dict): The inference params to override.
+        output_config (dict): The output config to override.
+    """
+    if not isinstance(inputs, list):
+      raise UserError('Invalid inputs, inputs must be a list of Input objects.')
+    if len(inputs) > MAX_MODEL_PREDICT_INPUTS:
+      raise UserError(f"Too many inputs. Max is {MAX_MODEL_PREDICT_INPUTS}."
+                     )  # TODO Use Chunker for inputs len > 128
+
+    request = service_pb2.PostModelOutputsRequest()
+    request.CopyFrom(self.request_template)
+
+    request.inputs.extend(inputs)
+
+    if method_name:
+      # TODO put in new proto field?
+      request.model.model_version.output_info.params['_method_name'] = method_name
+    if inference_params:
+      request.model.model_version.output_info.params.update(inference_params)
+    if output_config:
+      request.model.model_version.output_info.output_config.MergeFromDict(output_config)
+
+    start_time = time.time()
+    backoff_iterator = BackoffIterator(10)
+    started = False
+    while not started:
+      stream_response = self.STUB.GenerateModelOutputs(request)
+      try:
+        response = next(stream_response)  # get the first response
+      except StopIteration:
+        raise Exception("Model Generate failed with no response")
+      if status_is_retryable(response.status.code) and \
+              time.time() - start_time < 60 * 10:
+        self.logger.info("Model is still deploying, please wait...")
+        time.sleep(next(backoff_iterator))
+        continue
+      if response.status.code != status_code_pb2.SUCCESS:
+        raise Exception(f"Model Generate failed with response {response.status!r}")
+      started = True
+
+    yield response  # yield the first response
+
+    for response in stream_response:
+      if response.status.code != status_code_pb2.SUCCESS:
+        raise Exception(f"Model Generate failed with response {response.status!r}")
+      yield response
+
+  def _stream(
+      self,
+      inputs,
+      method_name: str = 'stream',
+  ) -> Any:
+    input_signature = self._method_signatures[method_name].inputs
+    output_signature = self._method_signatures[method_name].outputs
+
+    if isinstance(inputs, list):
+      assert len(inputs) == 1, 'streaming methods do not support batched calls'
+      inputs = inputs[0]
+    assert isinstance(inputs, dict)
+    kwargs = inputs
+
+    # find the streaming vars in the input signature, and the streaming input python param
+    streaming_var_signatures = [var for var in input_signature if var.streaming]
+    stream_argname = set([var.name.split('.', 1)[0] for var in streaming_var_signatures])
+    assert len(
+        stream_argname) == 1, 'streaming methods must have exactly one streaming function arg'
+    stream_argname = stream_argname.pop()
+
+    # get the streaming input generator from the user-provided function arg values
+    user_inputs_generator = kwargs.pop(stream_argname)
+
+    def _input_proto_stream():
+      # first item contains all the inputs and the first stream item
+      proto = resources_pb2.Input()
+      try:
+        item = next(user_inputs_generator)
+      except StopIteration:
+        return  # no items to stream
+      kwargs[stream_argname] = item
+      serialize(kwargs, input_signature, proto.data)
+
+      yield proto
+
+      # subsequent items are just the stream items
+      for item in user_inputs_generator:
+        proto = resources_pb2.Input()
+        serialize({stream_argname: item}, streaming_var_signatures, proto.data)
+        yield proto
+
+    response_stream = self._stream_by_proto(_input_proto_stream(), method_name)
+    #print(response)
+
+    for response in response_stream:
+      assert len(response.outputs) == 1, 'streaming methods must have exactly one output'
+      yield deserialize(response.outputs[0].data, output_signature, is_output=True)
+
+  def _req_iterator(self,
+                    input_iterator: Iterator[List[resources_pb2.Input]],
+                    method_name: str = None,
+                    inference_params: Dict = {},
+                    output_config: Dict = {}):
+    request = service_pb2.PostModelOutputsRequest()
+    request.CopyFrom(self.request_template)
+    request.model.model_version.output_info.params['_method_name'] = method_name
+    if inference_params:
+      request.model.model_version.output_info.params.update(inference_params)
+    if output_config:
+      request.model.model_version.output_info.output_config.MergeFromDict(output_config)
+    for inputs in input_iterator:
+      req = service_pb2.PostModelOutputsRequest()
+      req.CopyFrom(request)
+      if isinstance(inputs, list):
+        req.inputs.extend(inputs)
+      else:
+        req.inputs.append(inputs)
+      yield req
+
+  def _stream_by_proto(self,
+                       inputs: Iterator[List[resources_pb2.Input]],
+                       method_name: str = None,
+                       inference_params: Dict = {},
+                       output_config: Dict = {}):
+    """Generate the stream output on model based on the given stream of inputs.
+    """
+    # if not isinstance(inputs, Iterator[List[Input]]):
+    #   raise UserError('Invalid inputs, inputs must be a iterator of list of Input objects.')
+
+    request = self._req_iterator(inputs, method_name, inference_params, output_config)
+
+    start_time = time.time()
+    backoff_iterator = BackoffIterator(10)
+    generation_started = False
+    while True:
+      if generation_started:
+        break
+      stream_response = self.STUB.StreamModelOutputs(request)
+      for response in stream_response:
+        if status_is_retryable(response.status.code) and \
+                time.time() - start_time < 60 * 10:
+          self.logger.info("Model is still deploying, please wait...")
+          time.sleep(next(backoff_iterator))
+          break
+        if response.status.code != status_code_pb2.SUCCESS:
+          raise Exception(f"Model Predict failed with response {response.status!r}")
+        else:
+          if not generation_started:
+            generation_started = True
+          yield response

clarifai/client/workflow.py

@@ -15,7 +15,7 @@ from clarifai.constants.workflow import MAX_WORKFLOW_PREDICT_INPUTS
 from clarifai.errors import UserError
 from clarifai.urls.helper import ClarifaiUrlHelper
 from clarifai.utils.logging import logger
-from clarifai.utils.misc import BackoffIterator
+from clarifai.utils.misc import BackoffIterator, status_is_retryable
 from clarifai.workflows.export import Exporter
 
 
@@ -99,7 +99,7 @@ class Workflow(Lister, BaseClient):
     while True:
       response = self._grpc_request(self.STUB.PostWorkflowResults, request)
 
-      if response.status.code == status_code_pb2.MODEL_DEPLOYING and \
+      if status_is_retryable(response.status.code) and \
           time.time() - start_time < 60*10:  # 10 minutes
         self.logger.info(f"{self.id} Workflow is still deploying, please wait...")
         time.sleep(next(backoff_iterator))

clarifai/runners/__init__.py

@@ -1,14 +1,9 @@
-from .models.base_typed_model import AnyAnyModel, TextInputModel, VisualInputModel
 from .models.model_builder import ModelBuilder
+from .models.model_class import ModelClass
 from .models.model_runner import ModelRunner
-from .utils.data_handler import InputDataHandler, OutputDataHandler
 
 __all__ = [
     "ModelRunner",
     "ModelBuilder",
-    "InputDataHandler",
-    "OutputDataHandler",
-    "AnyAnyModel",
-    "TextInputModel",
-    "VisualInputModel",
+    "ModelClass",
 ]

clarifai/runners/dockerfile_template/Dockerfile.template

@@ -1,30 +1,11 @@
 # syntax=docker/dockerfile:1.13-labs
-#############################
-# User specific requirements installed in the pip_packages
-#############################
-FROM --platform=$TARGETPLATFORM ${FINAL_IMAGE} as pip_packages
+FROM --platform=$TARGETPLATFORM ${FINAL_IMAGE} as final
 
 COPY --link requirements.txt /home/nonroot/requirements.txt
 
 # Update clarifai package so we always have latest protocol to the API. Everything should land in /venv
 RUN ["pip", "install", "--no-cache-dir", "-r", "/home/nonroot/requirements.txt"]
 RUN ["pip", "show", "clarifai"]
-#############################
-
-#############################
-# Downloader dependencies image
-#############################
-FROM --platform=$TARGETPLATFORM ${DOWNLOADER_IMAGE} as downloader
-
-# make sure we have the latest clarifai package. This version is filled in by SDK.
-RUN ["pip", "install", "clarifai==${CLARIFAI_VERSION}"]
-#####
-
-
-#############################
-# Final runtime image
-#############################
-FROM --platform=$TARGETPLATFORM ${FINAL_IMAGE} as final
 
 # Set the NUMBA cache dir to /tmp
 # Set the TORCHINDUCTOR cache dir to /tmp
@@ -34,12 +15,6 @@ ENV NUMBA_CACHE_DIR=/tmp/numba_cache \
     HOME=/tmp \
     DEBIAN_FRONTEND=noninteractive
 
-#####
-# Copy the python requirements needed to download checkpoints
-#####
-COPY --link=true --from=downloader /venv /venv
-#####
-
 #####
 # Copy the files needed to download
 #####
@@ -52,12 +27,6 @@ COPY --link=true config.yaml /home/nonroot/main/
 RUN ["python", "-m", "clarifai.cli", "model", "download-checkpoints", "--model_path", "/home/nonroot/main", "--out_path", "/home/nonroot/main/1/checkpoints", "--stage", "build"]
 #####
 
-
-#####
-# Copy the python packages from the builder stage.
-COPY --link=true --from=pip_packages /venv /venv
-#####
-
 # Copy in the actual files like config.yaml, requirements.txt, and most importantly 1/model.py
 # for the actual model.
 # If checkpoints aren't downloaded since a checkpoints: block is not provided, then they will
@@ -75,6 +44,9 @@ ENV PYTHONPATH=${PYTHONPATH}:/home/nonroot/main \
     CLARIFAI_COMPUTE_CLUSTER_ID=${CLARIFAI_COMPUTE_CLUSTER_ID} \
     CLARIFAI_API_BASE=${CLARIFAI_API_BASE:-https://api.clarifai.com}
 
+# Write out the model function signatures
+RUN ["python", "-m", "clarifai.cli", "model", "signatures", "--model_path", "/home/nonroot/main", "--out_path", "/home/nonroot/main/signatures.yaml"]
+
 # Finally run the clarifai entrypoint to start the runner loop and local dev server.
 # Note(zeiler): we may want to make this a clarifai CLI call.
 ENTRYPOINT ["python", "-m", "clarifai.runners.server"]