clarifai 11.2.3rc1__py3-none-any.whl → 11.2.3rc3__py3-none-any.whl

clarifai/runners/dockerfile_template/Dockerfile.nim

@@ -0,0 +1,71 @@
+FROM nvcr.io/nim/meta/llama-3.1-8b-instruct:1.1.2 as build
+
+FROM gcr.io/distroless/python3-debian12:debug
+
+
+COPY --from=build /bin/bash /bin/rbash
+COPY --from=build /bin/sh /bin/sh
+COPY --from=build /bin/rsh /bin/rsh
+
+# we have to overwrite the python3 binary that the distroless image uses
+COPY --from=build /opt/nim/llm/.venv/bin/python3.10 /usr/bin/python3
+COPY --from=build /opt/nim/llm/.venv/bin/python3.10 /usr/local/bin/python3.10
+
+# also copy in all the lib files for it.
+COPY --from=build /lib /lib
+COPY --from=build /lib64 /lib64
+COPY --from=build /usr/lib/ /usr/lib/
+COPY --from=build /usr/local/lib/ /usr/local/lib/
+# ldconfig is needed to update the shared library cache so system libraries (like CUDA) can be found
+COPY --from=build /usr/sbin/ldconfig /sbin/ldconfig
+COPY --from=build /usr/sbin/ldconfig.real /sbin/ldconfig.real
+COPY --from=build /etc/ld.so.conf /etc/ld.so.conf
+COPY --from=build /etc/ld.so.cache /etc/ld.so.cache
+COPY --from=build /etc/ld.so.conf.d/ /etc/ld.so.conf.d/
+
+# COPY NIM files
+COPY  --from=build /opt /opt
+COPY  --from=build /etc/nim /etc/nim
+
+# Set environment variables to use the nim libraries and python
+ENV PYTHONPATH=${PYTHONPATH}:/opt/nim/llm/.venv/lib/python3.10/site-packages:/opt/nim/llm
+ENV PATH="/opt/nim/llm/.venv/bin:/opt/hpcx/ucc/bin:/opt/hpcx/ucx/bin:/opt/hpcx/ompi/bin:$PATH"
+
+ENV LD_LIBRARY_PATH="/opt/hpcx/ucc/lib/ucc:/opt/hpcx/ucc/lib:/opt/hpcx/ucx/lib/ucx:/opt/hpcx/ucx/lib:/opt/hpcx/ompi/lib:/opt/hpcx/ompi/lib/openmpi:/opt/nim/llm/.venv/lib/python3.10/site-packages/tensorrt_llm/libs:/opt/nim/llm/.venv/lib/python3.10/site-packages/nvidia/cublas/lib:/opt/nim/llm/.venv/lib/python3.10/site-packages/tensorrt_libs:/opt/nim/llm/.venv/lib/python3.10/site-packages/nvidia/nccl/lib:$LD_LIBRARY_PATH"
+
+ENV LIBRARY_PATH=/opt/hpcx/ucc/lib:/opt/hpcx/ucx/lib:/opt/hpcx/ompi/lib:$LIBRARY_PATH
+
+ENV CPATH=/opt/hpcx/ompi/include:/opt/hpcx/ucc/include:/opt/hpcx/ucx/include:$CPATH
+ENV LLM_PROJECT_DIR=/opt/nim/llm
+
+# Set environment variables for MPI
+ENV OMPI_HOME=/opt/hpcx/ompi
+ENV HPCX_MPI_DIR=/opt/hpcx/ompi
+ENV MPIf_HOME=/opt/hpcx/ompi
+ENV OPAL_PREFIX=/opt/hpcx/ompi
+
+# Set environment variables for UCC
+ENV UCC_DIR=/opt/hpcx/ucc/lib/cmake/ucc
+ENV UCC_HOME=/opt/hpcx/ucc
+ENV HPCX_UCC_DIR=/opt/hpcx/ucc
+ENV USE_UCC=1
+ENV USE_SYSTEM_UCC=1
+
+# Set environment variables for HPC-X
+ENV HPCX_DIR=/opt/hpcx
+ENV HPCX_UCX_DIR=/opt/hpcx/ucx
+ENV HPCX_MPI_DIR=/opt/hpcx/ompi
+
+# Set environment variables for UCX
+ENV UCX_DIR=/opt/hpcx/ucx/lib/cmake/ucx
+ENV UCX_HOME=/opt/hpcx/ucx
+
+ENV HOME=/opt/nim/llm
+
+# ln is needed to create symbolic links (needed by nvidia-container-runtime)
+COPY --from=build /usr/bin/ln /usr/bin/ln
+
+# Run ldconfig in the build stage to update the library cache else CUDA libraries won't be found
+RUN ldconfig -v
+
+SHELL ["/bin/rbash", "-c"]

clarifai/runners/models/model_builder.py

@@ -1,3 +1,4 @@
+import builtins
 import importlib
 import inspect
 import os
@@ -6,6 +7,7 @@ import sys
 import tarfile
 import time
 from string import Template
+from unittest.mock import MagicMock
 
 import yaml
 from clarifai_grpc.grpc.api import resources_pb2, service_pb2
@@ -14,13 +16,14 @@ from google.protobuf import json_format
 from rich import print
 from rich.markup import escape
 
-from clarifai.client import BaseClient
+from clarifai.client.base import BaseClient
 from clarifai.runners.models.model_class import ModelClass
 from clarifai.runners.utils.const import (
     AVAILABLE_PYTHON_IMAGES, AVAILABLE_TORCH_IMAGES, CONCEPTS_REQUIRED_MODEL_TYPE,
     DEFAULT_DOWNLOAD_CHECKPOINT_WHEN, DEFAULT_PYTHON_VERSION, DEFAULT_RUNTIME_DOWNLOAD_PATH,
     PYTHON_BASE_IMAGE, TORCH_BASE_IMAGE)
 from clarifai.runners.utils.loader import HuggingFaceLoader
+from clarifai.runners.utils.method_signatures import signatures_to_yaml
 from clarifai.urls.helper import ClarifaiUrlHelper
 from clarifai.utils.logging import logger
 from clarifai.versions import CLIENT_VERSION
@@ -65,10 +68,22 @@ class ModelBuilder:
     self.inference_compute_info = self._get_inference_compute_info()
     self.is_v3 = True  # Do model build for v3
 
-  def create_model_instance(self, load_model=True):
+  def create_model_instance(self, load_model=True, mocking=False):
     """
     Create an instance of the model class, as specified in the config file.
     """
+    model_class = self.load_model_class(mocking=mocking)
+
+    # initialize the model
+    model = model_class()
+    if load_model:
+      model.load_model()
+    return model
+
+  def load_model_class(self, mocking=False):
+    """
+    Import the model class from the model.py file, dynamically handling missing dependencies
+    """
     # look for default model.py file location
     for loc in ["model.py", "1/model.py"]:
       model_file = os.path.join(self.folder, loc)
@@ -82,7 +97,30 @@ class ModelBuilder:
     spec = importlib.util.spec_from_file_location(module_name, model_file)
     module = importlib.util.module_from_spec(spec)
     sys.modules[module_name] = module
-    spec.loader.exec_module(module)
+
+    original_import = builtins.__import__
+
+    def custom_import(name, globals=None, locals=None, fromlist=(), level=0):
+
+      # Allow standard libraries and clarifai
+      if self._is_standard_or_clarifai(name):
+        return original_import(name, globals, locals, fromlist, level)
+
+      # Mock all third-party imports to avoid ImportErrors or other issues
+      return MagicMock()
+
+    if mocking:
+      # Replace the built-in __import__ function with our custom one
+      builtins.__import__ = custom_import
+
+    try:
+      spec.loader.exec_module(module)
+    except Exception as e:
+      logger.error(f"Error loading model.py: {e}")
+      raise
+    finally:
+      # Restore the original __import__ function
+      builtins.__import__ = original_import
 
     # Find all classes in the model.py file that are subclasses of ModelClass
     classes = [
@@ -107,12 +145,24 @@ class ModelBuilder:
           "Could not determine model class. There should be exactly one model inheriting from ModelClass defined in the model.py"
       )
     model_class = classes[0]
+    return model_class
 
-    # initialize the model
-    model = model_class()
-    if load_model:
-      model.load_model()
-    return model
+  def _is_standard_or_clarifai(self, name):
+    """Check if import is from standard library or clarifai"""
+    if name.startswith("clarifai"):
+      return True
+
+    # Handle Python <3.10 compatibility
+    stdlib_names = getattr(sys, "stdlib_module_names", sys.builtin_module_names)
+    if name in stdlib_names:
+      return True
+
+    # Handle submodules (e.g., os.path)
+    parts = name.split(".")
+    for i in range(1, len(parts)):
+      if ".".join(parts[:i]) in stdlib_names:
+        return True
+    return False
 
   def _validate_folder(self, folder):
     if folder == ".":
@@ -142,21 +192,17 @@ class ModelBuilder:
   def _validate_config_checkpoints(self):
     """
     Validates the checkpoints section in the config file.
-      return loader_type, repo_id, hf_token, when, allowed_file_patterns, ignore_file_patterns
     :return: loader_type the type of loader or None if no checkpoints.
     :return: repo_id location of checkpoint.
     :return: hf_token token to access checkpoint.
-    :return: when one of ['upload', 'build', 'runtime'] to download checkpoint
-    :return: allowed_file_patterns patterns to allow in downloaded checkpoint
-    :return: ignore_file_patterns patterns to ignore in downloaded checkpoint
     """
     if "checkpoints" not in self.config:
-      return None, None, None, DEFAULT_DOWNLOAD_CHECKPOINT_WHEN, None, None
+      return None, None, None, DEFAULT_DOWNLOAD_CHECKPOINT_WHEN
     assert "type" in self.config.get("checkpoints"), "No loader type specified in the config file"
     loader_type = self.config.get("checkpoints").get("type")
     if not loader_type:
       logger.info("No loader type specified in the config file for checkpoints")
-      return None, None, None, DEFAULT_DOWNLOAD_CHECKPOINT_WHEN, None, None
+      return None, None, None
     checkpoints = self.config.get("checkpoints")
     if 'when' not in checkpoints:
       logger.warn(
@@ -175,30 +221,15 @@ class ModelBuilder:
 
       # get from config.yaml otherwise fall back to HF_TOKEN env var.
       hf_token = self.config.get("checkpoints").get("hf_token", os.environ.get("HF_TOKEN", None))
-
-      allowed_file_patterns = self.config.get("checkpoints").get('allowed_file_patterns', None)
-      if isinstance(allowed_file_patterns, str):
-        allowed_file_patterns = [allowed_file_patterns]
-      ignore_file_patterns = self.config.get("checkpoints").get('ignore_file_patterns', None)
-      if isinstance(ignore_file_patterns, str):
-        ignore_file_patterns = [ignore_file_patterns]
-      return loader_type, repo_id, hf_token, when, allowed_file_patterns, ignore_file_patterns
+      return loader_type, repo_id, hf_token, when
 
   def _check_app_exists(self):
     resp = self.client.STUB.GetApp(service_pb2.GetAppRequest(user_app_id=self.client.user_app_id))
     if resp.status.code == status_code_pb2.SUCCESS:
       return True
-    if resp.status.code == status_code_pb2.CONN_KEY_INVALID:
-      logger.error(
-          f"Invalid PAT provided for user {self.client.user_app_id.user_id}. Please check your PAT and try again."
-      )
-      return False
     logger.error(
         f"Error checking API {self._base_api} for user app {self.client.user_app_id.user_id}/{self.client.user_app_id.app_id}. Error code: {resp.status.code}"
     )
-    logger.error(
-        f"App {self.client.user_app_id.app_id} not found for user {self.client.user_app_id.user_id}. Please create the app first and try again."
-    )
     return False
 
   def _validate_config_model(self):
@@ -219,6 +250,9 @@ class ModelBuilder:
     assert model.get('id') != "", "model_id cannot be empty in the config file"
 
     if not self._check_app_exists():
+      logger.error(
+          f"App {self.client.user_app_id.app_id} not found for user {self.client.user_app_id.user_id}"
+      )
       sys.exit(1)
 
   def _validate_config(self):
@@ -232,7 +266,7 @@ class ModelBuilder:
         assert model_type_id in CONCEPTS_REQUIRED_MODEL_TYPE, f"Model type {model_type_id} not supported for concepts"
 
     if self.config.get("checkpoints"):
-      loader_type, _, hf_token, _, _, _ = self._validate_config_checkpoints()
+      loader_type, _, hf_token, _ = self._validate_config_checkpoints()
 
       if loader_type == "huggingface" and hf_token:
         is_valid_token = HuggingFaceLoader.validate_hftoken(hf_token)
@@ -248,7 +282,7 @@ class ModelBuilder:
           f"`num_threads` must be an integer greater than or equal to 1. Received type {type(num_threads)} with value {num_threads}."
       )
     else:
-      num_threads = int(os.environ.get("CLARIFAI_NUM_THREADS", 16))
+      num_threads = int(os.environ.get("CLARIFAI_NUM_THREADS", 1))
       self.config["num_threads"] = num_threads
 
   @staticmethod
@@ -269,6 +303,24 @@ class ModelBuilder:
           total_size += member.size
     return total_size
 
+  def method_signatures_yaml(self):
+    """
+    Returns the method signatures for the model class in YAML format.
+    """
+    model_class = self.load_model_class(mocking=True)
+    method_info = model_class._get_method_info()
+    signatures = {method.name: method.signature for method in method_info.values()}
+    return signatures_to_yaml(signatures)
+
+  def get_method_signatures(self):
+    """
+    Returns the method signatures for the model class.
+    """
+    model_class = self.load_model_class(mocking=True)
+    method_info = model_class._get_method_info()
+    signatures = [method.signature for method in method_info.values()]
+    return signatures
+
   @property
   def client(self):
     if self._client is None:
@@ -302,9 +354,8 @@ class ModelBuilder:
 
     assert "model_type_id" in model, "model_type_id not found in the config file"
     assert "id" in model, "model_id not found in the config file"
-    if not self.download_validation_only:
-      assert "user_id" in model, "user_id not found in the config file"
-      assert "app_id" in model, "app_id not found in the config file"
+    assert "user_id" in model, "user_id not found in the config file"
+    assert "app_id" in model, "app_id not found in the config file"
 
     model_proto = json_format.ParseDict(model, resources_pb2.Model())
 
@@ -415,12 +466,11 @@ class ModelBuilder:
       # Sort in reverse so that newer cuda versions come first and are preferred.
       for image in sorted(AVAILABLE_TORCH_IMAGES, reverse=True):
         if torch_version in image and f'py{python_version}' in image:
-          # like cu124, rocm6.3, etc.
-          gpu_version = image.split('-')[-1]
+          cuda_version = image.split('-')[-1].replace('cuda', '')
           final_image = TORCH_BASE_IMAGE.format(
               torch_version=torch_version,
               python_version=python_version,
-              gpu_version=gpu_version,
+              cuda_version=cuda_version,
           )
           logger.info(f"Using Torch version {torch_version} base image to build the Docker image")
           break
@@ -497,10 +547,8 @@ class ModelBuilder:
     if not self.config.get("checkpoints"):
       logger.info("No checkpoints specified in the config file")
       return path
-    clarifai_model_type_id = self.config.get('model').get('model_type_id')
 
-    loader_type, repo_id, hf_token, when, allowed_file_patterns, ignore_file_patterns = self._validate_config_checkpoints(
-    )
+    loader_type, repo_id, hf_token, when = self._validate_config_checkpoints()
     if stage not in ["build", "upload", "runtime"]:
       raise Exception("Invalid stage provided, must be one of ['build', 'upload', 'runtime']")
     if when != stage:
@@ -509,18 +557,14 @@ class ModelBuilder:
       )
       return path
 
-    success = False
+    success = True
     if loader_type == "huggingface":
-      loader = HuggingFaceLoader(
-          repo_id=repo_id, token=hf_token, model_type_id=clarifai_model_type_id)
+      loader = HuggingFaceLoader(repo_id=repo_id, token=hf_token)
       # for runtime default to /tmp path
       if stage == "runtime" and checkpoint_path_override is None:
         checkpoint_path_override = self.default_runtime_checkpoint_path()
       path = checkpoint_path_override if checkpoint_path_override else self.checkpoint_path
-      success = loader.download_checkpoints(
-          path,
-          allowed_file_patterns=allowed_file_patterns,
-          ignore_file_patterns=ignore_file_patterns)
+      success = loader.download_checkpoints(path)
 
     if loader_type:
       if not success:
@@ -554,11 +598,53 @@ class ModelBuilder:
     concepts = config.get('concepts')
     logger.info(f"Updated config.yaml with {len(concepts)} concepts.")
 
-  def get_model_version_proto(self):
+  def filled_params_specs_with_inference_params(self, method_signatures: list[resources_pb2.MethodSignature]) -> list[resources_pb2.ModelTypeField]:
+    """
+    Fills the params_specs with the inference params.
+    """
+    inference_params = set()
+    for i, signature in enumerate(method_signatures):
+      for field in signature.input_fields:
+        if field.is_param:
+          if i==0:
+            inference_params.add(field.name)
+          else:
+            # if field.name not in inference_params then remove from inference_params
+            if field.name not in inference_params:
+              inference_params.remove(field.name)
+    output=[]
+    for signature in method_signatures:
+      for field in signature.input_fields:
+        if field.is_param and field.name in inference_params:
+          field.path = field.name
+          if field.type == resources_pb2.ModelTypeField.DataType.STR:
+            field.default_value= str(field.default)
+            field.field_type = resources_pb2.ModelTypeField.ModelTypeFieldType.STRING
+          elif field.type == resources_pb2.ModelTypeField.DataType.INT:
+            field.default_value= int(field.default)
+            field.field_type = resources_pb2.ModelTypeField.ModelTypeFieldType.NUMBER
+          elif field.type == resources_pb2.ModelTypeField.DataType.FLOAT:
+            field.default_value= float(field.default)
+            field.field_type = resources_pb2.ModelTypeField.ModelTypeFieldType.NUMBER
+          elif field.type == resources_pb2.ModelTypeField.DataType.BOOL:
+            field.default_value= bool(field.default)
+            field.field_type = resources_pb2.ModelTypeField.ModelTypeFieldType.BOOLEAN
+          else:
+            field.default_value= field.default
+            field.field_type = resources_pb2.ModelTypeField.ModelTypeFieldType.STRING
+          output.append(field)
+    return output
 
+
+  def get_model_version_proto(self):
+    signatures = self.get_method_signatures()
     model_version_proto = resources_pb2.ModelVersion(
         pretrained_model_config=resources_pb2.PretrainedModelConfig(),
         inference_compute_info=self.inference_compute_info,
+        method_signatures=signatures,
+        # output_info= resources_pb2.OutputInfo(
+        #   params_specs=self.filled_params_specs_with_inference_params(signatures),
+        # )
     )
 
     model_type_id = self.config.get('model').get('model_type_id')
@@ -592,7 +678,7 @@ class ModelBuilder:
     logger.debug(f"Will tar it into file: {file_path}")
 
     model_type_id = self.config.get('model').get('model_type_id')
-    loader_type, repo_id, hf_token, when, _, _ = self._validate_config_checkpoints()
+    loader_type, repo_id, hf_token, when = self._validate_config_checkpoints()
 
     if (model_type_id in CONCEPTS_REQUIRED_MODEL_TYPE) and 'concepts' not in self.config:
       logger.info(
@@ -640,7 +726,7 @@ class ModelBuilder:
       # First check for the env variable, then try querying huggingface. If all else fails, use the default.
       checkpoint_size = os.environ.get('CHECKPOINT_SIZE_BYTES', 0)
       if not checkpoint_size:
-        _, repo_id, _, _, _, _ = self._validate_config_checkpoints()
+        _, repo_id, _, _ = self._validate_config_checkpoints()
         checkpoint_size = HuggingFaceLoader.get_huggingface_checkpoint_total_size(repo_id)
       if not checkpoint_size:
         checkpoint_size = self.DEFAULT_CHECKPOINT_SIZE
@@ -746,6 +832,7 @@ class ModelBuilder:
               model_id=self.model_proto.id,
               version_id=self.model_version_id,
           ))
+
       status_code = resp.model_version.status.code
       logs = self.get_model_build_logs()
       for log_entry in logs.log_entries: