clarifai 11.1.7rc3__py3-none-any.whl → 11.2.1__py3-none-any.whl

clarifai/client/model_client.py

@@ -1,448 +0,0 @@
-import time
-from typing import Any, Dict, Iterator, List
-
-from clarifai_grpc.grpc.api import resources_pb2, service_pb2
-from clarifai_grpc.grpc.api.status import status_code_pb2
-
-from clarifai.constants.model import MAX_MODEL_PREDICT_INPUTS
-from clarifai.errors import UserError
-from clarifai.runners.utils.method_signatures import (CompatibilitySerializer, deserialize,
-                                                      get_stream_from_signature, serialize,
-                                                      signatures_from_json)
-from clarifai.utils.logging import logger
-from clarifai.utils.misc import BackoffIterator, status_is_retryable
-
-
-class ModelClient:
-  '''
-  Client for calling model predict, generate, and stream methods.
-  '''
-
-  def __init__(self, stub, request_template: service_pb2.PostModelOutputsRequest = None):
-    '''
-        Initialize the model client.
-
-        Args:
-            stub: The gRPC stub for the model.
-            request_template: The template for the request to send to the model, including
-            common fields like model_id, model_version, cluster, etc.
-        '''
-    self.STUB = stub
-    self.request_template = request_template or service_pb2.PostModelOutputsRequest()
-    self._method_signatures = None
-    self._defined = False
-
-  def fetch(self):
-    '''
-    Fetch function signature definitions from the model and define the functions in the client
-    '''
-    if self._defined:
-      return
-    try:
-      self._fetch_signatures()
-      self._define_functions()
-    finally:
-      self._defined = True
-
-  def __getattr__(self, name):
-    if not self._defined:
-      self.fetch()
-    return self.__getattribute__(name)
-
-  def _fetch_signatures(self):
-    '''
-      Fetch the method signatures from the model.
-
-      Returns:
-          Dict: The method signatures.
-      '''
-    #request = resources_pb2.GetModelSignaturesRequest()
-    #response = self.stub.GetModelSignatures(request)
-    #self._method_signatures = json.loads(response.signatures)  # or define protos
-    # TODO this could use a new endpoint to get the signatures
-    # for local grpc models, we'll also have to add the endpoint to the model servicer
-    # for now we'll just use the predict endpoint with a special method name
-
-    request = service_pb2.PostModelOutputsRequest()
-    request.CopyFrom(self.request_template)
-    # request.model.model_version.output_info.params['_method_name'] = '_GET_SIGNATURES'
-    inp = request.inputs.add()  # empty input for this method
-    inp.data.parts.add()  # empty part for this input
-    inp.data.metadata['_method_name'] = '_GET_SIGNATURES'
-    start_time = time.time()
-    backoff_iterator = BackoffIterator(10)
-    while True:
-      response = self.STUB.PostModelOutputs(request)
-      if status_is_retryable(
-          response.status.code) and time.time() - start_time < 60 * 10:  # 10 minutes
-        logger.info(f"Retrying model info fetch with response {response.status!r}")
-        time.sleep(next(backoff_iterator))
-        continue
-      break
-    if (response.status.code == status_code_pb2.INPUT_UNSUPPORTED_FORMAT or
-        (response.status.code == status_code_pb2.SUCCESS and
-         response.outputs[0].data.text.raw == '')):
-      # return codes/values from older models that don't support _GET_SIGNATURES
-      self._method_signatures = {}
-      self._define_compatability_functions()
-      return
-    if response.status.code != status_code_pb2.SUCCESS:
-      raise Exception(f"Model failed with response {response!r}")
-    self._method_signatures = signatures_from_json(response.outputs[0].data.text.raw)
-
-  def _define_functions(self):
-    '''
-    Define the functions based on the method signatures.
-    '''
-    for method_name, method_signature in self._method_signatures.items():
-      # define the function in this client instance
-      if resources_pb2.RunnerMethodType.Name(method_signature.method_type) == 'UNARY_UNARY':
-        call_func = self._predict
-      elif resources_pb2.RunnerMethodType.Name(method_signature.method_type) == 'UNARY_STREAMING':
-        call_func = self._generate
-      elif resources_pb2.RunnerMethodType.Name(
-          method_signature.method_type) == 'STREAMING_STREAMING':
-        call_func = self._stream
-      else:
-        raise ValueError(f"Unknown method type {method_signature.method_type}")
-
-      # method argnames, in order, collapsing nested keys to corresponding user function args
-      method_argnames = []
-      for var in method_signature.input_fields:
-        outer = var.name.split('.', 1)[0]
-        if outer in method_argnames:
-          continue
-        method_argnames.append(outer)
-
-      def bind_f(method_name, method_argnames, call_func):
-
-        def f(*args, **kwargs):
-          if len(args) > len(method_argnames):
-            raise TypeError(
-                f"{method_name}() takes {len(method_argnames)} positional arguments but {len(args)} were given"
-            )
-          for name, arg in zip(method_argnames, args):  # handle positional with zip shortest
-            if name in kwargs:
-              raise TypeError(f"Multiple values for argument {name}")
-            kwargs[name] = arg
-          return call_func(kwargs, method_name)
-
-        return f
-
-      # need to bind method_name to the value, not the mutating loop variable
-      f = bind_f(method_name, method_argnames, call_func)
-
-      # set names, annotations and docstrings
-      f.__name__ = method_name
-      f.__qualname__ = f'{self.__class__.__name__}.{method_name}'
-      # TODO: set signature from annotations to the function, currently  MethodSignature don't have `annotations_json` field
-      # input_annotations = json.loads(method_signature.annotations_json)
-      # return_annotation = input_annotations.pop('return', None)
-      # sig = inspect.signature(f).replace(
-      #     parameters=[
-      #         inspect.Parameter(k, inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=v)
-      #         for k, v in input_annotations.items()
-      #     ],
-      #     return_annotation=return_annotation,
-      # )
-      # f.__signature__ = sig
-      f.__doc__ = method_signature.description
-      setattr(self, method_name, f)
-
-  def _define_compatability_functions(self):
-
-    serializer = CompatibilitySerializer()
-
-    def predict(input: Any) -> Any:
-      proto = resources_pb2.Input()
-      serializer.serialize(proto.data, input)
-      # always use text.raw for compat
-      if proto.data.string_value:
-        proto.data.text.raw = proto.data.string_value
-        proto.data.string_value = ''
-      response = self._predict_by_proto([proto])
-      if response.status.code != status_code_pb2.SUCCESS:
-        raise Exception(f"Model predict failed with response {response!r}")
-      response_data = response.outputs[0].data
-      if response_data.text.raw:
-        response_data.string_value = response_data.text.raw
-        response_data.text.raw = ''
-      return serializer.deserialize(response_data)
-
-    self.predict = predict
-
-  def _predict(
-      self,
-      inputs,  # TODO set up functions according to fetched signatures?
-      method_name: str = 'predict',
-  ) -> Any:
-    input_signature = self._method_signatures[method_name].input_fields
-    output_signature = self._method_signatures[method_name].output_fields
-
-    batch_input = True
-    if isinstance(inputs, dict):
-      inputs = [inputs]
-      batch_input = False
-
-    proto_inputs = []
-    for input in inputs:
-      proto = resources_pb2.Input()
-      serialize(input, input_signature, proto.data)
-      proto_inputs.append(proto)
-
-    response = self._predict_by_proto(proto_inputs, method_name)
-    #print(response)
-
-    outputs = []
-    for output in response.outputs:
-      outputs.append(deserialize(output.data, output_signature, is_output=True))
-    if batch_input:
-      return outputs
-    return outputs[0]
-
-  def _predict_by_proto(
-      self,
-      inputs: List[resources_pb2.Input],
-      method_name: str = None,
-      inference_params: Dict = None,
-      output_config: Dict = None,
-  ) -> service_pb2.MultiOutputResponse:
-    """Predicts the model based on the given inputs.
-
-      Args:
-          inputs (List[resources_pb2.Input]): The inputs to predict.
-          method_name (str): The remote method name to call.
-          inference_params (Dict): Inference parameters to override.
-          output_config (Dict): Output configuration to override.
-
-      Returns:
-          service_pb2.MultiOutputResponse: The prediction response(s).
-      """
-    if not isinstance(inputs, list):
-      raise UserError('Invalid inputs, inputs must be a list of Input objects.')
-    if len(inputs) > MAX_MODEL_PREDICT_INPUTS:
-      raise UserError(f"Too many inputs. Max is {MAX_MODEL_PREDICT_INPUTS}.")
-
-    request = service_pb2.PostModelOutputsRequest()
-    request.CopyFrom(self.request_template)
-
-    request.inputs.extend(inputs)
-
-    if method_name:
-      # TODO put in new proto field?
-      for inp in request.inputs:
-        inp.data.metadata['_method_name'] = method_name
-    if inference_params:
-      request.model.model_version.output_info.params.update(inference_params)
-    if output_config:
-      request.model.model_version.output_info.output_config.MergeFrom(
-          resources_pb2.OutputConfig(**output_config))
-
-    start_time = time.time()
-    backoff_iterator = BackoffIterator(10)
-    while True:
-      response = self.STUB.PostModelOutputs(request)
-      if status_is_retryable(
-          response.status.code) and time.time() - start_time < 60 * 10:  # 10 minutes
-        logger.info(f"Model predict failed with response {response!r}")
-        time.sleep(next(backoff_iterator))
-        continue
-
-      if response.status.code != status_code_pb2.SUCCESS:
-        raise Exception(f"Model predict failed with response {response!r}")
-      break
-
-    return response
-
-  def _generate(
-      self,
-      inputs,  # TODO set up functions according to fetched signatures?
-      method_name: str = 'generate',
-  ) -> Any:
-    input_signature = self._method_signatures[method_name].input_fields
-    output_signature = self._method_signatures[method_name].output_fields
-
-    batch_input = True
-    if isinstance(inputs, dict):
-      inputs = [inputs]
-      batch_input = False
-
-    proto_inputs = []
-    for input in inputs:
-      proto = resources_pb2.Input()
-      serialize(input, input_signature, proto.data)
-      proto_inputs.append(proto)
-
-    response_stream = self._generate_by_proto(proto_inputs, method_name)
-    #print(response)
-
-    for response in response_stream:
-      outputs = []
-      for output in response.outputs:
-        outputs.append(deserialize(output.data, output_signature, is_output=True))
-      if batch_input:
-        yield outputs
-      yield outputs[0]
-
-  def _generate_by_proto(
-      self,
-      inputs: List[resources_pb2.Input],
-      method_name: str = None,
-      inference_params: Dict = {},
-      output_config: Dict = {},
-  ):
-    """Generate the stream output on model based on the given inputs.
-
-    Args:
-        inputs (list[Input]): The inputs to generate, must be less than 128.
-        method_name (str): The remote method name to call.
-        inference_params (dict): The inference params to override.
-        output_config (dict): The output config to override.
-    """
-    if not isinstance(inputs, list):
-      raise UserError('Invalid inputs, inputs must be a list of Input objects.')
-    if len(inputs) > MAX_MODEL_PREDICT_INPUTS:
-      raise UserError(f"Too many inputs. Max is {MAX_MODEL_PREDICT_INPUTS}."
-                     )  # TODO Use Chunker for inputs len > 128
-
-    request = service_pb2.PostModelOutputsRequest()
-    request.CopyFrom(self.request_template)
-
-    request.inputs.extend(inputs)
-
-    if method_name:
-      # TODO put in new proto field?
-      for inp in request.inputs:
-        inp.data.metadata['_method_name'] = method_name
-    if inference_params:
-      request.model.model_version.output_info.params.update(inference_params)
-    if output_config:
-      request.model.model_version.output_info.output_config.MergeFromDict(output_config)
-
-    start_time = time.time()
-    backoff_iterator = BackoffIterator(10)
-    started = False
-    while not started:
-      stream_response = self.STUB.GenerateModelOutputs(request)
-      try:
-        response = next(stream_response)  # get the first response
-      except StopIteration:
-        raise Exception("Model Generate failed with no response")
-      if status_is_retryable(response.status.code) and \
-              time.time() - start_time < 60 * 10:
-        logger.info("Model is still deploying, please wait...")
-        time.sleep(next(backoff_iterator))
-        continue
-      if response.status.code != status_code_pb2.SUCCESS:
-        raise Exception(f"Model Generate failed with response {response.status!r}")
-      started = True
-
-    yield response  # yield the first response
-
-    for response in stream_response:
-      if response.status.code != status_code_pb2.SUCCESS:
-        raise Exception(f"Model Generate failed with response {response.status!r}")
-      yield response
-
-  def _stream(
-      self,
-      inputs,
-      method_name: str = 'stream',
-  ) -> Any:
-    input_signature = self._method_signatures[method_name].input_fields
-    output_signature = self._method_signatures[method_name].output_fields
-
-    if isinstance(inputs, list):
-      assert len(inputs) == 1, 'streaming methods do not support batched calls'
-      inputs = inputs[0]
-    assert isinstance(inputs, dict)
-    kwargs = inputs
-
-    # find the streaming vars in the input signature, and the streaming input python param
-    stream_sig = get_stream_from_signature(input_signature)
-    if stream_sig is None:
-      raise ValueError("Streaming method must have a Stream input")
-    stream_argname = stream_sig.name
-
-    # get the streaming input generator from the user-provided function arg values
-    user_inputs_generator = kwargs.pop(stream_argname)
-
-    def _input_proto_stream():
-      # first item contains all the inputs and the first stream item
-      proto = resources_pb2.Input()
-      try:
-        item = next(user_inputs_generator)
-      except StopIteration:
-        return  # no items to stream
-      kwargs[stream_argname] = item
-      serialize(kwargs, input_signature, proto.data)
-
-      yield proto
-
-      # subsequent items are just the stream items
-      for item in user_inputs_generator:
-        proto = resources_pb2.Input()
-        serialize({stream_argname: item}, [stream_sig], proto.data)
-        yield proto
-
-    response_stream = self._stream_by_proto(_input_proto_stream(), method_name)
-    #print(response)
-
-    for response in response_stream:
-      assert len(response.outputs) == 1, 'streaming methods must have exactly one output'
-      yield deserialize(response.outputs[0].data, output_signature, is_output=True)
-
-  def _req_iterator(self,
-                    input_iterator: Iterator[List[resources_pb2.Input]],
-                    method_name: str = None,
-                    inference_params: Dict = {},
-                    output_config: Dict = {}):
-    request = service_pb2.PostModelOutputsRequest()
-    request.CopyFrom(self.request_template)
-    if inference_params:
-      request.model.model_version.output_info.params.update(inference_params)
-    if output_config:
-      request.model.model_version.output_info.output_config.MergeFromDict(output_config)
-    for inputs in input_iterator:
-      req = service_pb2.PostModelOutputsRequest()
-      req.CopyFrom(request)
-      if isinstance(inputs, list):
-        req.inputs.extend(inputs)
-      else:
-        req.inputs.append(inputs)
-      # TODO: put into new proto field?
-      for inp in req.inputs:
-        inp.data.metadata['_method_name'] = method_name
-      yield req
-
-  def _stream_by_proto(self,
-                       inputs: Iterator[List[resources_pb2.Input]],
-                       method_name: str = None,
-                       inference_params: Dict = {},
-                       output_config: Dict = {}):
-    """Generate the stream output on model based on the given stream of inputs.
-    """
-    # if not isinstance(inputs, Iterator[List[Input]]):
-    #   raise UserError('Invalid inputs, inputs must be a iterator of list of Input objects.')
-
-    request = self._req_iterator(inputs, method_name, inference_params, output_config)
-
-    start_time = time.time()
-    backoff_iterator = BackoffIterator(10)
-    generation_started = False
-    while True:
-      if generation_started:
-        break
-      stream_response = self.STUB.StreamModelOutputs(request)
-      for response in stream_response:
-        if status_is_retryable(response.status.code) and \
-                time.time() - start_time < 60 * 10:
-          logger.info("Model is still deploying, please wait...")
-          time.sleep(next(backoff_iterator))
-          break
-        if response.status.code != status_code_pb2.SUCCESS:
-          raise Exception(f"Model Predict failed with response {response.status!r}")
-        else:
-          if not generation_started:
-            generation_started = True
-          yield response

clarifai/runners/dockerfile_template/Dockerfile.cpu.template

@@ -1,31 +0,0 @@
-ARG BASE_IMAGE=${BASE_IMAGE}
-FROM ${BASE_IMAGE} as build
-
-# Set the working directory to /app
-WORKDIR /app
-
-COPY requirements.txt .
-# Install requirements and cleanup before leaving this line.
-# Note(zeiler): this could be in a future template as {{model_python_deps}}
-RUN pip install --no-cache-dir -r requirements.txt
-
-# Install Clarifai SDK
-RUN pip install --no-cache-dir clarifai
-
-# These will be set by the templaing system.
-ENV CLARIFAI_PAT=${CLARIFAI_PAT}
-ENV CLARIFAI_USER_ID=${CLARIFAI_USER_ID}
-ENV CLARIFAI_RUNNER_ID=${CLARIFAI_RUNNER_ID}
-ENV CLARIFAI_NODEPOOL_ID=${CLARIFAI_NODEPOOL_ID}
-ENV CLARIFAI_COMPUTE_CLUSTER_ID=${CLARIFAI_COMPUTE_CLUSTER_ID}
-ENV CLARIFAI_API_BASE=${CLARIFAI_API_BASE}
-
-# Copy the current folder into /app/model_dir that the SDK will expect.
-COPY . /app/model_dir/${name}
-
-# Add the model directory to the python path.
-ENV PYTHONPATH "${PYTHONPATH}:/app/model_dir/${name}"
-
-# Finally run the clarifai entrypoint to start the runner loop and local dev server.
-# Note(zeiler): we may want to make this a clarifai CLI call.
-CMD ["-m", "clarifai.runners.server", "--model_path", "/app/model_dir/${name}"]

clarifai/runners/dockerfile_template/Dockerfile.cuda.template

@@ -1,42 +0,0 @@
-ARG TARGET_PLATFORM=linux/amd64
-FROM --platform=$TARGET_PLATFORM ${BASE_IMAGE} as build
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-#############################
-# User specific requirements
-#############################
-COPY requirements.txt .
-
-# Install requirements and cleanup before leaving this line.
-# Note(zeiler): this could be in a future template as {{model_python_deps}}
-RUN pip install --no-cache-dir -r requirements.txt
-
-# Install Clarifai SDK
-RUN pip install --no-cache-dir clarifai
-
-# These will be set by the templaing system.
-ENV CLARIFAI_PAT=${CLARIFAI_PAT}
-ENV CLARIFAI_USER_ID=${CLARIFAI_USER_ID}
-ENV CLARIFAI_RUNNER_ID=${CLARIFAI_RUNNER_ID}
-ENV CLARIFAI_NODEPOOL_ID=${CLARIFAI_NODEPOOL_ID}
-ENV CLARIFAI_COMPUTE_CLUSTER_ID=${CLARIFAI_COMPUTE_CLUSTER_ID}
-ENV CLARIFAI_API_BASE=${CLARIFAI_API_BASE}
-
-# Set the NUMBA cache dir to /tmp
-ENV NUMBA_CACHE_DIR=/tmp/numba_cache
-ENV HOME=/tmp
-
-# Set the working directory to /app
-WORKDIR /app
-
-# Copy the current folder into /app/model_dir that the SDK will expect.
-# Note(zeiler): would be nice to exclude checkpoints in case they were pre-downloaded.
-COPY . /app/model_dir/${name}
-
-# Add the model directory to the python path.
-ENV PYTHONPATH=${PYTHONPATH}:/app/model_dir/${name}
-
-# Finally run the clarifai entrypoint to start the runner loop and local dev server.
-# Note(zeiler): we may want to make this a clarifai CLI call.
-CMD ["-m", "clarifai.runners.server", "--model_path", "/app/model_dir/${name}"]

clarifai/runners/dockerfile_template/Dockerfile.nim

@@ -1,71 +0,0 @@
-FROM nvcr.io/nim/meta/llama-3.1-8b-instruct:1.1.2 as build
-
-FROM gcr.io/distroless/python3-debian12:debug
-
-
-COPY --from=build /bin/bash /bin/rbash
-COPY --from=build /bin/sh /bin/sh
-COPY --from=build /bin/rsh /bin/rsh
-
-# we have to overwrite the python3 binary that the distroless image uses
-COPY --from=build /opt/nim/llm/.venv/bin/python3.10 /usr/bin/python3
-COPY --from=build /opt/nim/llm/.venv/bin/python3.10 /usr/local/bin/python3.10
-
-# also copy in all the lib files for it.
-COPY --from=build /lib /lib
-COPY --from=build /lib64 /lib64
-COPY --from=build /usr/lib/ /usr/lib/
-COPY --from=build /usr/local/lib/ /usr/local/lib/
-# ldconfig is needed to update the shared library cache so system libraries (like CUDA) can be found
-COPY --from=build /usr/sbin/ldconfig /sbin/ldconfig
-COPY --from=build /usr/sbin/ldconfig.real /sbin/ldconfig.real
-COPY --from=build /etc/ld.so.conf /etc/ld.so.conf
-COPY --from=build /etc/ld.so.cache /etc/ld.so.cache
-COPY --from=build /etc/ld.so.conf.d/ /etc/ld.so.conf.d/
-
-# COPY NIM files
-COPY  --from=build /opt /opt
-COPY  --from=build /etc/nim /etc/nim
-
-# Set environment variables to use the nim libraries and python
-ENV PYTHONPATH=${PYTHONPATH}:/opt/nim/llm/.venv/lib/python3.10/site-packages:/opt/nim/llm
-ENV PATH="/opt/nim/llm/.venv/bin:/opt/hpcx/ucc/bin:/opt/hpcx/ucx/bin:/opt/hpcx/ompi/bin:$PATH"
-
-ENV LD_LIBRARY_PATH="/opt/hpcx/ucc/lib/ucc:/opt/hpcx/ucc/lib:/opt/hpcx/ucx/lib/ucx:/opt/hpcx/ucx/lib:/opt/hpcx/ompi/lib:/opt/hpcx/ompi/lib/openmpi:/opt/nim/llm/.venv/lib/python3.10/site-packages/tensorrt_llm/libs:/opt/nim/llm/.venv/lib/python3.10/site-packages/nvidia/cublas/lib:/opt/nim/llm/.venv/lib/python3.10/site-packages/tensorrt_libs:/opt/nim/llm/.venv/lib/python3.10/site-packages/nvidia/nccl/lib:$LD_LIBRARY_PATH"
-
-ENV LIBRARY_PATH=/opt/hpcx/ucc/lib:/opt/hpcx/ucx/lib:/opt/hpcx/ompi/lib:$LIBRARY_PATH
-
-ENV CPATH=/opt/hpcx/ompi/include:/opt/hpcx/ucc/include:/opt/hpcx/ucx/include:$CPATH
-ENV LLM_PROJECT_DIR=/opt/nim/llm
-
-# Set environment variables for MPI
-ENV OMPI_HOME=/opt/hpcx/ompi
-ENV HPCX_MPI_DIR=/opt/hpcx/ompi
-ENV MPIf_HOME=/opt/hpcx/ompi
-ENV OPAL_PREFIX=/opt/hpcx/ompi
-
-# Set environment variables for UCC
-ENV UCC_DIR=/opt/hpcx/ucc/lib/cmake/ucc
-ENV UCC_HOME=/opt/hpcx/ucc
-ENV HPCX_UCC_DIR=/opt/hpcx/ucc
-ENV USE_UCC=1
-ENV USE_SYSTEM_UCC=1
-
-# Set environment variables for HPC-X
-ENV HPCX_DIR=/opt/hpcx
-ENV HPCX_UCX_DIR=/opt/hpcx/ucx
-ENV HPCX_MPI_DIR=/opt/hpcx/ompi
-
-# Set environment variables for UCX
-ENV UCX_DIR=/opt/hpcx/ucx/lib/cmake/ucx
-ENV UCX_HOME=/opt/hpcx/ucx
-
-ENV HOME=/opt/nim/llm
-
-# ln is needed to create symbolic links (needed by nvidia-container-runtime)
-COPY --from=build /usr/bin/ln /usr/bin/ln
-
-# Run ldconfig in the build stage to update the library cache else CUDA libraries won't be found
-RUN ldconfig -v
-
-SHELL ["/bin/rbash", "-c"]