clarifai 11.2.3rc1__py3-none-any.whl → 11.2.3rc2__py3-none-any.whl

clarifai/runners/utils/data_utils.py

@@ -1,7 +1,12 @@
 from io import BytesIO
 
+from clarifai_grpc.grpc.api.resources_pb2 import ModelTypeEnumOption
+from clarifai_grpc.grpc.api.resources_pb2 import ModelTypeField as InputFieldProto
+from clarifai_grpc.grpc.api.resources_pb2 import ModelTypeRangeInfo
 from PIL import Image
 
+from clarifai.runners.utils.data_types import MessageData
+
 
 def image_to_bytes(img: Image.Image, format="JPEG") -> bytes:
   buffered = BytesIO()
@@ -13,3 +18,163 @@ def image_to_bytes(img: Image.Image, format="JPEG") -> bytes:
 def bytes_to_image(bytes_img) -> Image.Image:
   img = Image.open(BytesIO(bytes_img))
   return img
+
+
+def is_openai_chat_format(messages):
+  """
+    Verify if the given argument follows the OpenAI chat messages format.
+
+    Args:
+        messages (list): A list of dictionaries representing chat messages.
+
+    Returns:
+        bool: True if valid, False otherwise.
+    """
+  if not isinstance(messages, list):
+    return False
+
+  valid_roles = {"system", "user", "assistant", "function"}
+
+  for msg in messages:
+    if not isinstance(msg, dict):
+      return False
+    if "role" not in msg or "content" not in msg:
+      return False
+    if msg["role"] not in valid_roles:
+      return False
+
+    content = msg["content"]
+
+    # Content should be either a string (text message) or a multimodal list
+    if isinstance(content, str):
+      continue  # Valid text message
+
+    elif isinstance(content, list):
+      for item in content:
+        if not isinstance(item, dict):
+          return False
+  return True
+
+
+class InputField(MessageData):
+  """A field that can be used to store input data."""
+
+  def __init__(
+      self,
+      default=None,
+      description=None,
+      min_value=None,
+      max_value=None,
+      choices=None,
+      #  is_param=True
+  ):
+    self.default = default
+    self.description = description
+    self.min_value = min_value
+    self.max_value = max_value
+    self.choices = choices
+    # self.is_param = is_param
+
+  def __repr__(self) -> str:
+    attrs = []
+    if self.default is not None:
+      attrs.append(f"default={self.default!r}")
+    if self.description is not None:
+      attrs.append(f"description={self.description!r}")
+    if self.min_value is not None:
+      attrs.append(f"min_value={self.min_value!r}")
+    if self.max_value is not None:
+      attrs.append(f"max_value={self.max_value!r}")
+    if self.choices is not None:
+      attrs.append(f"choices={self.choices!r}")
+    # attrs.append(f"is_param={self.is_param!r}")
+    return f"InputField({', '.join(attrs)})"
+
+  def to_proto(self, proto=None) -> InputFieldProto:
+    if proto is None:
+      proto = InputFieldProto()
+    if self.description is not None:
+      proto.description = self.description
+
+    if self.choices is not None:
+      for choice in self.choices:
+        option = ModelTypeEnumOption(id=str(choice))
+        proto.model_type_enum_options.append(option)
+
+    proto.required = self.default is None
+
+    if self.min_value is not None or self.max_value is not None:
+      range_info = ModelTypeRangeInfo()
+      if self.min_value is not None:
+        range_info.min = float(self.min_value)
+      if self.max_value is not None:
+        range_info.max = float(self.max_value)
+      proto.model_type_range_info.CopyFrom(range_info)
+    # proto.is_param = self.is_param
+
+    if self.default is not None:
+      if isinstance(self.default, str) or isinstance(self.default, bool) or isinstance(
+          self.default, (int, float)):
+        proto.default = str(self.default)
+      else:
+        import json
+        proto.default = json.dumps(self.default)
+
+    return proto
+
+  @classmethod
+  def from_proto(cls, proto):
+    default = None
+    if proto.HasField('default'):
+      pb_value = proto.default
+      if pb_value.HasField('string_value'):
+        default = pb_value.string_value
+        try:
+          import json
+          default = json.loads(default)
+        except json.JSONDecodeError:
+          pass
+      elif pb_value.HasField('number_value'):
+        default = pb_value.number_value
+        if default.is_integer():
+          default = int(default)
+        else:
+          default = float(default)
+      elif pb_value.HasField('bool_value'):
+        default = pb_value.bool_value
+
+    choices = [option.id for option in proto.model_type_enum_options
+              ] if proto.model_type_enum_options else None
+
+    min_value = None
+    max_value = None
+    if proto.HasField('model_type_range_info'):
+      min_value = proto.model_type_range_info.min
+      max_value = proto.model_type_range_info.max
+      if min_value.is_integer():
+        min_value = int(min_value)
+      if max_value.is_integer():
+        max_value = int(max_value)
+
+    return cls(
+        default=default,
+        description=proto.description if proto.description else None,
+        min_value=min_value,
+        max_value=max_value,
+        choices=choices,
+        # is_param=proto.is_param
+    )
+
+  @classmethod
+  def set_default(cls, proto=None, default=None):
+
+    if proto is None:
+      proto = InputFieldProto()
+    if default is not None:
+      if isinstance(default, str) or isinstance(default, bool) or isinstance(
+          default, (int, float)):
+        proto.default = str(default)
+      else:
+        import json
+        proto.default = json.dumps(default)
+    return proto

clarifai/runners/utils/loader.py

@@ -6,7 +6,6 @@ import shutil
 
 import requests
 
-from clarifai.runners.utils.const import CONCEPTS_REQUIRED_MODEL_TYPE
 from clarifai.utils.logging import logger
 
 
@@ -14,10 +13,9 @@ class HuggingFaceLoader:
 
   HF_DOWNLOAD_TEXT = "The 'huggingface_hub' package is not installed. Please install it using 'pip install huggingface_hub'."
 
-  def __init__(self, repo_id=None, token=None, model_type_id=None):
+  def __init__(self, repo_id=None, token=None):
     self.repo_id = repo_id
     self.token = token
-    self.clarifai_model_type_id = model_type_id
     if token:
       if self.validate_hftoken(token):
         try:
@@ -45,17 +43,13 @@ class HuggingFaceLoader:
           f"Error setting up Hugging Face token, please make sure you have the correct token: {e}")
       return False
 
-  def download_checkpoints(self,
-                           checkpoint_path: str,
-                           allowed_file_patterns=None,
-                           ignore_file_patterns=None):
+  def download_checkpoints(self, checkpoint_path: str):
     # throw error if huggingface_hub wasn't installed
     try:
       from huggingface_hub import snapshot_download
     except ImportError:
       raise ImportError(self.HF_DOWNLOAD_TEXT)
-    if os.path.exists(checkpoint_path) and self.validate_download(
-        checkpoint_path, allowed_file_patterns, ignore_file_patterns):
+    if os.path.exists(checkpoint_path) and self.validate_download(checkpoint_path):
       logger.info("Checkpoints already exist")
       return True
     else:
@@ -67,16 +61,10 @@ class HuggingFaceLoader:
           return False
 
         self.ignore_patterns = self._get_ignore_patterns()
-        if ignore_file_patterns:
-          if self.ignore_patterns:
-            self.ignore_patterns.extend(ignore_file_patterns)
-          else:
-            self.ignore_patterns = ignore_file_patterns
         snapshot_download(
             repo_id=self.repo_id,
             local_dir=checkpoint_path,
             local_dir_use_symlinks=False,
-            allow_patterns=allowed_file_patterns,
             ignore_patterns=self.ignore_patterns)
         # Remove the `.cache` folder if it exists
         cache_path = os.path.join(checkpoint_path, ".cache")
@@ -87,8 +75,7 @@ class HuggingFaceLoader:
         logger.error(f"Error downloading model checkpoints {e}")
         return False
       finally:
-        is_downloaded = self.validate_download(checkpoint_path, allowed_file_patterns,
-                                               ignore_file_patterns)
+        is_downloaded = self.validate_download(checkpoint_path)
         if not is_downloaded:
           logger.error("Error validating downloaded model checkpoints")
           return False
@@ -122,13 +109,9 @@ class HuggingFaceLoader:
       from huggingface_hub import file_exists, repo_exists
     except ImportError:
       raise ImportError(self.HF_DOWNLOAD_TEXT)
-    if self.clarifai_model_type_id in CONCEPTS_REQUIRED_MODEL_TYPE:
-      return repo_exists(self.repo_id) and file_exists(self.repo_id, 'config.json')
-    else:
-      return repo_exists(self.repo_id)
+    return repo_exists(self.repo_id) and file_exists(self.repo_id, 'config.json')
 
-  def validate_download(self, checkpoint_path: str, allowed_file_patterns: list,
-                        ignore_file_patterns: list):
+  def validate_download(self, checkpoint_path: str):
     # check if model exists on HF
     try:
       from huggingface_hub import list_repo_files
@@ -137,20 +120,7 @@ class HuggingFaceLoader:
     # Get the list of files on the repo
     repo_files = list_repo_files(self.repo_id, token=self.token)
 
-    # Get the list of files on the repo that are allowed
-    if allowed_file_patterns:
-
-      def should_allow(file_path):
-        return any(fnmatch.fnmatch(file_path, pattern) for pattern in allowed_file_patterns)
-
-      repo_files = [f for f in repo_files if should_allow(f)]
-
     self.ignore_patterns = self._get_ignore_patterns()
-    if ignore_file_patterns:
-      if self.ignore_patterns:
-        self.ignore_patterns.extend(ignore_file_patterns)
-      else:
-        self.ignore_patterns = ignore_file_patterns
     # Get the list of files on the repo that are not ignored
     if getattr(self, "ignore_patterns", None):
       patterns = self.ignore_patterns