clarifai 11.2.4rc2__py3-none-any.whl → 11.3.0__py3-none-any.whl

clarifai/runners/dockerfile_template/Dockerfile.cuda.template

@@ -1,42 +0,0 @@
-ARG TARGET_PLATFORM=linux/amd64
-FROM --platform=$TARGET_PLATFORM ${BASE_IMAGE} as build
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-#############################
-# User specific requirements
-#############################
-COPY requirements.txt .
-
-# Install requirements and cleanup before leaving this line.
-# Note(zeiler): this could be in a future template as {{model_python_deps}}
-RUN pip install --no-cache-dir -r requirements.txt
-
-# Install Clarifai SDK
-RUN pip install --no-cache-dir clarifai
-
-# These will be set by the templaing system.
-ENV CLARIFAI_PAT=${CLARIFAI_PAT}
-ENV CLARIFAI_USER_ID=${CLARIFAI_USER_ID}
-ENV CLARIFAI_RUNNER_ID=${CLARIFAI_RUNNER_ID}
-ENV CLARIFAI_NODEPOOL_ID=${CLARIFAI_NODEPOOL_ID}
-ENV CLARIFAI_COMPUTE_CLUSTER_ID=${CLARIFAI_COMPUTE_CLUSTER_ID}
-ENV CLARIFAI_API_BASE=${CLARIFAI_API_BASE}
-
-# Set the NUMBA cache dir to /tmp
-ENV NUMBA_CACHE_DIR=/tmp/numba_cache
-ENV HOME=/tmp
-
-# Set the working directory to /app
-WORKDIR /app
-
-# Copy the current folder into /app/model_dir that the SDK will expect.
-# Note(zeiler): would be nice to exclude checkpoints in case they were pre-downloaded.
-COPY . /app/model_dir/${name}
-
-# Add the model directory to the python path.
-ENV PYTHONPATH=${PYTHONPATH}:/app/model_dir/${name}
-
-# Finally run the clarifai entrypoint to start the runner loop and local dev server.
-# Note(zeiler): we may want to make this a clarifai CLI call.
-CMD ["-m", "clarifai.runners.server", "--model_path", "/app/model_dir/${name}"]

clarifai/runners/dockerfile_template/Dockerfile.nim

@@ -1,71 +0,0 @@
-FROM nvcr.io/nim/meta/llama-3.1-8b-instruct:1.1.2 as build
-
-FROM gcr.io/distroless/python3-debian12:debug
-
-
-COPY --from=build /bin/bash /bin/rbash
-COPY --from=build /bin/sh /bin/sh
-COPY --from=build /bin/rsh /bin/rsh
-
-# we have to overwrite the python3 binary that the distroless image uses
-COPY --from=build /opt/nim/llm/.venv/bin/python3.10 /usr/bin/python3
-COPY --from=build /opt/nim/llm/.venv/bin/python3.10 /usr/local/bin/python3.10
-
-# also copy in all the lib files for it.
-COPY --from=build /lib /lib
-COPY --from=build /lib64 /lib64
-COPY --from=build /usr/lib/ /usr/lib/
-COPY --from=build /usr/local/lib/ /usr/local/lib/
-# ldconfig is needed to update the shared library cache so system libraries (like CUDA) can be found
-COPY --from=build /usr/sbin/ldconfig /sbin/ldconfig
-COPY --from=build /usr/sbin/ldconfig.real /sbin/ldconfig.real
-COPY --from=build /etc/ld.so.conf /etc/ld.so.conf
-COPY --from=build /etc/ld.so.cache /etc/ld.so.cache
-COPY --from=build /etc/ld.so.conf.d/ /etc/ld.so.conf.d/
-
-# COPY NIM files
-COPY  --from=build /opt /opt
-COPY  --from=build /etc/nim /etc/nim
-
-# Set environment variables to use the nim libraries and python
-ENV PYTHONPATH=${PYTHONPATH}:/opt/nim/llm/.venv/lib/python3.10/site-packages:/opt/nim/llm
-ENV PATH="/opt/nim/llm/.venv/bin:/opt/hpcx/ucc/bin:/opt/hpcx/ucx/bin:/opt/hpcx/ompi/bin:$PATH"
-
-ENV LD_LIBRARY_PATH="/opt/hpcx/ucc/lib/ucc:/opt/hpcx/ucc/lib:/opt/hpcx/ucx/lib/ucx:/opt/hpcx/ucx/lib:/opt/hpcx/ompi/lib:/opt/hpcx/ompi/lib/openmpi:/opt/nim/llm/.venv/lib/python3.10/site-packages/tensorrt_llm/libs:/opt/nim/llm/.venv/lib/python3.10/site-packages/nvidia/cublas/lib:/opt/nim/llm/.venv/lib/python3.10/site-packages/tensorrt_libs:/opt/nim/llm/.venv/lib/python3.10/site-packages/nvidia/nccl/lib:$LD_LIBRARY_PATH"
-
-ENV LIBRARY_PATH=/opt/hpcx/ucc/lib:/opt/hpcx/ucx/lib:/opt/hpcx/ompi/lib:$LIBRARY_PATH
-
-ENV CPATH=/opt/hpcx/ompi/include:/opt/hpcx/ucc/include:/opt/hpcx/ucx/include:$CPATH
-ENV LLM_PROJECT_DIR=/opt/nim/llm
-
-# Set environment variables for MPI
-ENV OMPI_HOME=/opt/hpcx/ompi
-ENV HPCX_MPI_DIR=/opt/hpcx/ompi
-ENV MPIf_HOME=/opt/hpcx/ompi
-ENV OPAL_PREFIX=/opt/hpcx/ompi
-
-# Set environment variables for UCC
-ENV UCC_DIR=/opt/hpcx/ucc/lib/cmake/ucc
-ENV UCC_HOME=/opt/hpcx/ucc
-ENV HPCX_UCC_DIR=/opt/hpcx/ucc
-ENV USE_UCC=1
-ENV USE_SYSTEM_UCC=1
-
-# Set environment variables for HPC-X
-ENV HPCX_DIR=/opt/hpcx
-ENV HPCX_UCX_DIR=/opt/hpcx/ucx
-ENV HPCX_MPI_DIR=/opt/hpcx/ompi
-
-# Set environment variables for UCX
-ENV UCX_DIR=/opt/hpcx/ucx/lib/cmake/ucx
-ENV UCX_HOME=/opt/hpcx/ucx
-
-ENV HOME=/opt/nim/llm
-
-# ln is needed to create symbolic links (needed by nvidia-container-runtime)
-COPY --from=build /usr/bin/ln /usr/bin/ln
-
-# Run ldconfig in the build stage to update the library cache else CUDA libraries won't be found
-RUN ldconfig -v
-
-SHELL ["/bin/rbash", "-c"]

clarifai/runners/models/base_typed_model.py

@@ -1,238 +0,0 @@
-import itertools
-from typing import Any, Dict, Iterator, List, Tuple
-
-import numpy as np
-from clarifai_grpc.grpc.api import resources_pb2, service_pb2
-from clarifai_grpc.grpc.api.service_pb2 import PostModelOutputsRequest
-from google.protobuf import json_format
-
-from ..utils.data_handler import InputDataHandler, OutputDataHandler
-from .model_class import ModelClass
-
-
-class AnyAnyModel(ModelClass):
-
-  def load_model(self):
-    """
-    Load inference time artifacts that are called frequently .e.g. models, tokenizers, etc.
-    in this method so they are loaded only once for faster inference.
-    """
-    raise NotImplementedError
-
-  def parse_input_request(
-      self, input_request: service_pb2.PostModelOutputsRequest) -> Tuple[List[Dict], Dict]:
-    list_input_dict = [
-        InputDataHandler.from_proto(input).to_python() for input in input_request.inputs
-    ]
-    inference_params = json_format.MessageToDict(
-        input_request.model.model_version.output_info.params)
-
-    return list_input_dict, inference_params
-
-  def convert_output_to_proto(self, outputs: list):
-    assert (isinstance(outputs, Iterator) or isinstance(outputs, list) or
-            isinstance(outputs, tuple)), "outputs must be an Iterator"
-    output_protos = []
-    for output in outputs:
-      if isinstance(output, OutputDataHandler):
-        output = output.proto
-      elif isinstance(output, resources_pb2.Output):
-        pass
-      else:
-        raise NotImplementedError
-      output_protos.append(output)
-
-    return service_pb2.MultiOutputResponse(outputs=output_protos)
-
-  def predict_wrapper(
-      self, request: service_pb2.PostModelOutputsRequest) -> service_pb2.MultiOutputResponse:
-    list_dict_input, inference_params = self.parse_input_request(request)
-    outputs = self.predict(list_dict_input, inference_parameters=inference_params)
-    return self.convert_output_to_proto(outputs)
-
-  def generate_wrapper(
-      self, request: PostModelOutputsRequest) -> Iterator[service_pb2.MultiOutputResponse]:
-    list_dict_input, inference_params = self.parse_input_request(request)
-    outputs = self.generate(list_dict_input, inference_parameters=inference_params)
-    for output in outputs:
-      yield self.convert_output_to_proto(output)
-
-  def _preprocess_stream(
-      self, request: Iterator[PostModelOutputsRequest]) -> Iterator[Tuple[List[Dict], List[Dict]]]:
-    """Return generator of processed data (from proto to python) and inference parameters like predict and generate"""
-    for i, req in enumerate(request):
-      input_data, _ = self.parse_input_request(req)
-      yield input_data
-
-  def stream_wrapper(self, request: Iterator[PostModelOutputsRequest]
-                    ) -> Iterator[service_pb2.MultiOutputResponse]:
-    first_request = next(request)
-    _, inference_params = self.parse_input_request(first_request)
-    request_iterator = itertools.chain([first_request], request)
-    outputs = self.stream(self._preprocess_stream(request_iterator), inference_params)
-    for output in outputs:
-      yield self.convert_output_to_proto(output)
-
-  def predict(self, input_data: List[Dict],
-              inference_parameters: Dict[str, Any] = {}) -> List[OutputDataHandler]:
-    """
-    Prediction method.
-
-    Args:
-    -----
-    - input_data: is list of dict where key is input type name.
-      * image: np.ndarray
-      * text: str
-      * audio: bytes
-
-    - inference_parameters (Dict[str, Union[bool, str, float, int]]): your inference parameters.
-
-    Returns:
-    --------
-      List of OutputDataHandler
-    """
-    raise NotImplementedError
-
-  def generate(self, input_data: List[Dict],
-               inference_parameters: Dict[str, Any] = {}) -> Iterator[List[OutputDataHandler]]:
-    """
-    Generate method.
-
-    Args:
-    -----
-    - input_data: is list of dict where key is input type name.
-      * image: np.ndarray
-      * text: str
-      * audio: bytes
-
-    - inference_parameters (Dict[str, Union[bool, str, float, int]]): your inference parameters.
-
-    Yield:
-    --------
-      List of OutputDataHandler
-    """
-    raise NotImplementedError
-
-  def stream(self, inputs: Iterator[List[Dict[str, Any]]],
-             inference_params: Dict[str, Any]) -> Iterator[List[OutputDataHandler]]:
-    """
-    Stream method.
-
-    Args:
-    -----
-    input_request: is an Iterator of Tuple which
-    - First element (List[Dict[str, Union[np.ndarray, str, bytes]]]) is list of dict input data type which keys and values are:
-        * image: np.ndarray
-        * text: str
-        * audio: bytes
-
-    - Second element (Dict[str, Union[bool, str, float, int]]): is a dict of inference_parameters
-
-    Yield:
-    --------
-      List of OutputDataHandler
-    """
-    raise NotImplementedError
-
-
-class VisualInputModel(AnyAnyModel):
-
-  def parse_input_request(
-      self, input_request: service_pb2.PostModelOutputsRequest) -> Tuple[List[Dict], Dict]:
-    list_input_dict = [
-        InputDataHandler.from_proto(input).image(format="np") for input in input_request.inputs
-    ]
-    inference_params = json_format.MessageToDict(
-        input_request.model.model_version.output_info.params)
-
-    return list_input_dict, inference_params
-
-  def load_model(self):
-    """
-    Load inference time artifacts that are called frequently .e.g. models, tokenizers, etc.
-    in this method so they are loaded only once for faster inference.
-    """
-    raise NotImplementedError
-
-  def predict(self, input_data: List[np.ndarray],
-              inference_parameters: Dict[str, Any] = {}) -> List[OutputDataHandler]:
-    """
-    Prediction method.
-
-    Args:
-    -----
-    - input_data(List[np.ndarray]): is list of image as np.ndarray type
-    - inference_parameters (Dict[str, Union[bool, str, float, int]]): your inference parameters.
-
-    Returns:
-    --------
-      List of OutputDataHandler
-    """
-    raise NotImplementedError
-
-
-class TextInputModel(AnyAnyModel):
-
-  def load_model(self):
-    """
-    Load inference time artifacts that are called frequently .e.g. models, tokenizers, etc.
-    in this method so they are loaded only once for faster inference.
-    """
-    raise NotImplementedError
-
-  def parse_input_request(
-      self, input_request: service_pb2.PostModelOutputsRequest) -> Tuple[List[Dict], Dict]:
-    list_input_text = [InputDataHandler.from_proto(input).text for input in input_request.inputs]
-    inference_params = json_format.MessageToDict(
-        input_request.model.model_version.output_info.params)
-
-    return list_input_text, inference_params
-
-  def predict(self, input_data: List[str],
-              inference_parameters: Dict[str, Any] = {}) -> List[OutputDataHandler]:
-    """
-    Prediction method.
-
-    Args:
-    -----
-    - input_data(List[str]): is list of text as str type
-    - inference_parameters (Dict[str, Union[bool, str, float, int]]): your inference parameters.
-
-    Returns:
-    --------
-      List of OutputDataHandler
-    """
-    raise NotImplementedError
-
-  def generate(self, input_data: List[str],
-               inference_parameters: Dict[str, Any] = {}) -> Iterator[List[OutputDataHandler]]:
-    """
-    Prediction method.
-
-    Args:
-    -----
-    - input_data(List[str]): is list of text as str type
-    - inference_parameters (Dict[str, Union[bool, str, float, int]]): your inference parameters.
-
-    Yield:
-    --------
-      List of OutputDataHandler
-    """
-    raise NotImplementedError
-
-  def stream(self, inputs: Iterator[List[str]],
-             inference_params: Dict[str, Any]) -> Iterator[List[OutputDataHandler]]:
-    """
-    Stream method.
-
-    Args:
-    -----
-    input_request: is an Iterator of Tuple which
-    - First element (List[str]) is list of input text:
-    - Second element (Dict[str, Union[bool, str, float, int]]): is a dict of inference_parameters
-
-    Yield:
-    --------
-      List of OutputDataHandler
-    """
-    raise NotImplementedError

clarifai/runners/models/model_class_refract.py

@@ -1,80 +0,0 @@
-import inspect
-from abc import ABC, abstractmethod
-from concurrent.futures import ThreadPoolExecutor
-from typing import Any, Dict, Iterator, List, get_type_hints
-
-from clarifai_grpc.grpc.api import resources_pb2, service_pb2
-
-from clarifai.runners.utils.data_handler import Output, kwargs_to_proto, proto_to_kwargs
-
-
-class ModelClass(ABC):
-
-  @abstractmethod
-  def load_model(self):
-    raise NotImplementedError("load_model() not implemented")
-
-  @abstractmethod
-  def predict(self, *args, **kwargs) -> Output:
-    raise NotImplementedError("predict() not implemented")
-
-  @abstractmethod
-  def generate(self, *args, **kwargs) -> Iterator[Output]:
-    raise NotImplementedError("generate() not implemented")
-
-  @abstractmethod
-  def stream(self, *args, **kwargs) -> Iterator[Output]:
-    raise NotImplementedError("stream() not implemented")
-
-  def batch_predict(self, inputs: List[Dict[str, Any]]) -> List[Output]:
-    with ThreadPoolExecutor() as executor:
-      return list(executor.map(lambda x: self.predict(**x), inputs))
-
-  def _process_request(self, request, process_func, is_stream=False):
-    inputs = self._convert_proto_to_python(request.inputs)
-    if len(inputs) == 1:
-      result = process_func(**inputs[0])
-      if is_stream:
-        return (self._convert_output_to_proto(output) for output in result)
-      else:
-        return [self._convert_output_to_proto(result)]
-    else:
-      results = self.batch_predict(inputs) if not is_stream else []
-      return [self._convert_output_to_proto(output) for output in results]
-
-  def predict_wrapper(
-      self, request: service_pb2.PostModelOutputsRequest) -> service_pb2.MultiOutputResponse:
-    outputs = self._process_request(request, self.predict)
-    return service_pb2.MultiOutputResponse(outputs=outputs)
-
-  def generate_wrapper(self, request: service_pb2.PostModelOutputsRequest
-                      ) -> Iterator[service_pb2.MultiOutputResponse]:
-    outputs = self._process_request(request, self.generate, is_stream=True)
-    for output in outputs:
-      yield service_pb2.MultiOutputResponse(outputs=[output])
-
-  def stream_wrapper(self, requests: Iterator[service_pb2.PostModelOutputsRequest]
-                    ) -> Iterator[service_pb2.MultiOutputResponse]:
-    for request in requests:
-      outputs = self._process_request(request, self.stream, is_stream=True)
-      yield service_pb2.MultiOutputResponse(outputs=outputs)
-
-  def _convert_proto_to_python(self, inputs: List[resources_pb2.Input]) -> List[Dict[str, Any]]:
-    get_type_hints(self.predict)
-    required_params = [
-        name for name, param in inspect.signature(self.predict).parameters.items()
-        if param.default == inspect.Parameter.empty
-    ]
-    kwargs_list = []
-    for input_proto in inputs:
-      kwargs = proto_to_kwargs(input_proto.data)
-      missing = [name for name in required_params if name not in kwargs]
-      if missing:
-        raise ValueError(f"Missing required parameters: {missing}")
-      kwargs_list.append(kwargs)
-    return kwargs_list
-
-  def _convert_output_to_proto(self, output: Any) -> resources_pb2.Output:
-    if isinstance(output, Output):
-      return output.to_proto()
-    return kwargs_to_proto(**output).outputs.add()