clarifai 11.2.1__tar.gz → 11.2.2__tar.gz

{clarifai-11.2.1/clarifai.egg-info → clarifai-11.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: clarifai
-Version: 11.2.1
+Version: 11.2.2
 Summary: Clarifai Python SDK
 Home-page: https://github.com/Clarifai/clarifai-python
 Author: Clarifai

clarifai-11.2.2/clarifai/__init__.py

@@ -0,0 +1 @@
+__version__ = "11.2.2"

{clarifai-11.2.1 → clarifai-11.2.2}/clarifai/client/app.py

@@ -629,7 +629,7 @@ class App(Lister, BaseClient):
 
     Args:
         model_id (str): The model ID for the model to interact with.
-        model_version_id (str): The model version ID for the model version to interact with.
+        model_version (Dict): The model version ID for the model version to interact with.
 
     Returns:
         Model: A Model object for the existing model ID.

{clarifai-11.2.1 → clarifai-11.2.2}/clarifai/runners/models/model_builder.py

@@ -171,7 +171,14 @@ class ModelBuilder:
 
       # get from config.yaml otherwise fall back to HF_TOKEN env var.
       hf_token = self.config.get("checkpoints").get("hf_token", os.environ.get("HF_TOKEN", None))
-      return loader_type, repo_id, hf_token, when
+
+      allowed_file_patterns = self.config.get("checkpoints").get('allowed_file_patterns', None)
+      if isinstance(allowed_file_patterns, str):
+        allowed_file_patterns = [allowed_file_patterns]
+      ignore_file_patterns = self.config.get("checkpoints").get('ignore_file_patterns', None)
+      if isinstance(ignore_file_patterns, str):
+        ignore_file_patterns = [ignore_file_patterns]
+      return loader_type, repo_id, hf_token, when, allowed_file_patterns, ignore_file_patterns
 
   def _check_app_exists(self):
     resp = self.client.STUB.GetApp(service_pb2.GetAppRequest(user_app_id=self.client.user_app_id))
@@ -216,7 +223,7 @@ class ModelBuilder:
         assert model_type_id in CONCEPTS_REQUIRED_MODEL_TYPE, f"Model type {model_type_id} not supported for concepts"
 
     if self.config.get("checkpoints"):
-      loader_type, _, hf_token, _ = self._validate_config_checkpoints()
+      loader_type, _, hf_token, _, _, _ = self._validate_config_checkpoints()
 
       if loader_type == "huggingface" and hf_token:
         is_valid_token = HuggingFaceLoader.validate_hftoken(hf_token)
@@ -480,8 +487,10 @@ class ModelBuilder:
     if not self.config.get("checkpoints"):
       logger.info("No checkpoints specified in the config file")
       return path
+    clarifai_model_type_id = self.config.get('model').get('model_type_id')
 
-    loader_type, repo_id, hf_token, when = self._validate_config_checkpoints()
+    loader_type, repo_id, hf_token, when, allowed_file_patterns, ignore_file_patterns = self._validate_config_checkpoints(
+    )
     if stage not in ["build", "upload", "runtime"]:
       raise Exception("Invalid stage provided, must be one of ['build', 'upload', 'runtime']")
     if when != stage:
@@ -490,14 +499,18 @@ class ModelBuilder:
       )
       return path
 
-    success = True
+    success = False
     if loader_type == "huggingface":
-      loader = HuggingFaceLoader(repo_id=repo_id, token=hf_token)
+      loader = HuggingFaceLoader(
+          repo_id=repo_id, token=hf_token, model_type_id=clarifai_model_type_id)
       # for runtime default to /tmp path
       if stage == "runtime" and checkpoint_path_override is None:
         checkpoint_path_override = self.default_runtime_checkpoint_path()
       path = checkpoint_path_override if checkpoint_path_override else self.checkpoint_path
-      success = loader.download_checkpoints(path)
+      success = loader.download_checkpoints(
+          path,
+          allowed_file_patterns=allowed_file_patterns,
+          ignore_file_patterns=ignore_file_patterns)
 
     if loader_type:
       if not success:
@@ -569,7 +582,7 @@ class ModelBuilder:
     logger.debug(f"Will tar it into file: {file_path}")
 
     model_type_id = self.config.get('model').get('model_type_id')
-    loader_type, repo_id, hf_token, when = self._validate_config_checkpoints()
+    loader_type, repo_id, hf_token, when, _, _ = self._validate_config_checkpoints()
 
     if (model_type_id in CONCEPTS_REQUIRED_MODEL_TYPE) and 'concepts' not in self.config:
       logger.info(
@@ -617,7 +630,7 @@ class ModelBuilder:
       # First check for the env variable, then try querying huggingface. If all else fails, use the default.
       checkpoint_size = os.environ.get('CHECKPOINT_SIZE_BYTES', 0)
       if not checkpoint_size:
-        _, repo_id, _, _ = self._validate_config_checkpoints()
+        _, repo_id, _, _, _, _ = self._validate_config_checkpoints()
         checkpoint_size = HuggingFaceLoader.get_huggingface_checkpoint_total_size(repo_id)
       if not checkpoint_size:
         checkpoint_size = self.DEFAULT_CHECKPOINT_SIZE

{clarifai-11.2.1 → clarifai-11.2.2}/clarifai/runners/models/model_run_locally.py

@@ -479,7 +479,7 @@ def main(model_path,
   manager = ModelRunLocally(model_path)
   # get whatever stage is in config.yaml to force download now
   # also always write to where upload/build wants to, not the /tmp folder that runtime stage uses
-  _, _, _, when = manager.builder._validate_config_checkpoints()
+  _, _, _, when, _, _ = manager.builder._validate_config_checkpoints()
   manager.builder.download_checkpoints(
       stage=when, checkpoint_path_override=manager.builder.checkpoint_path)
   if inside_container:

{clarifai-11.2.1 → clarifai-11.2.2}/clarifai/runners/utils/loader.py

@@ -6,6 +6,7 @@ import shutil
 
 import requests
 
+from clarifai.runners.utils.const import CONCEPTS_REQUIRED_MODEL_TYPE
 from clarifai.utils.logging import logger
 
 
@@ -13,9 +14,10 @@ class HuggingFaceLoader:
 
   HF_DOWNLOAD_TEXT = "The 'huggingface_hub' package is not installed. Please install it using 'pip install huggingface_hub'."
 
-  def __init__(self, repo_id=None, token=None):
+  def __init__(self, repo_id=None, token=None, model_type_id=None):
     self.repo_id = repo_id
     self.token = token
+    self.clarifai_model_type_id = model_type_id
     if token:
       if self.validate_hftoken(token):
         try:
@@ -43,13 +45,17 @@ class HuggingFaceLoader:
           f"Error setting up Hugging Face token, please make sure you have the correct token: {e}")
       return False
 
-  def download_checkpoints(self, checkpoint_path: str):
+  def download_checkpoints(self,
+                           checkpoint_path: str,
+                           allowed_file_patterns=None,
+                           ignore_file_patterns=None):
     # throw error if huggingface_hub wasn't installed
     try:
       from huggingface_hub import snapshot_download
     except ImportError:
       raise ImportError(self.HF_DOWNLOAD_TEXT)
-    if os.path.exists(checkpoint_path) and self.validate_download(checkpoint_path):
+    if os.path.exists(checkpoint_path) and self.validate_download(
+        checkpoint_path, allowed_file_patterns, ignore_file_patterns):
       logger.info("Checkpoints already exist")
       return True
     else:
@@ -61,10 +67,16 @@ class HuggingFaceLoader:
           return False
 
         self.ignore_patterns = self._get_ignore_patterns()
+        if ignore_file_patterns:
+          if self.ignore_patterns:
+            self.ignore_patterns.extend(ignore_file_patterns)
+          else:
+            self.ignore_patterns = ignore_file_patterns
         snapshot_download(
             repo_id=self.repo_id,
             local_dir=checkpoint_path,
             local_dir_use_symlinks=False,
+            allow_patterns=allowed_file_patterns,
             ignore_patterns=self.ignore_patterns)
         # Remove the `.cache` folder if it exists
         cache_path = os.path.join(checkpoint_path, ".cache")
@@ -75,7 +87,8 @@ class HuggingFaceLoader:
         logger.error(f"Error downloading model checkpoints {e}")
         return False
       finally:
-        is_downloaded = self.validate_download(checkpoint_path)
+        is_downloaded = self.validate_download(checkpoint_path, allowed_file_patterns,
+                                               ignore_file_patterns)
         if not is_downloaded:
           logger.error("Error validating downloaded model checkpoints")
           return False
@@ -109,9 +122,13 @@ class HuggingFaceLoader:
       from huggingface_hub import file_exists, repo_exists
     except ImportError:
       raise ImportError(self.HF_DOWNLOAD_TEXT)
-    return repo_exists(self.repo_id) and file_exists(self.repo_id, 'config.json')
+    if self.clarifai_model_type_id in CONCEPTS_REQUIRED_MODEL_TYPE:
+      return repo_exists(self.repo_id) and file_exists(self.repo_id, 'config.json')
+    else:
+      return repo_exists(self.repo_id)
 
-  def validate_download(self, checkpoint_path: str):
+  def validate_download(self, checkpoint_path: str, allowed_file_patterns: list,
+                        ignore_file_patterns: list):
     # check if model exists on HF
     try:
       from huggingface_hub import list_repo_files
@@ -120,7 +137,20 @@ class HuggingFaceLoader:
     # Get the list of files on the repo
     repo_files = list_repo_files(self.repo_id, token=self.token)
 
+    # Get the list of files on the repo that are allowed
+    if allowed_file_patterns:
+
+      def should_allow(file_path):
+        return any(fnmatch.fnmatch(file_path, pattern) for pattern in allowed_file_patterns)
+
+      repo_files = [f for f in repo_files if should_allow(f)]
+
     self.ignore_patterns = self._get_ignore_patterns()
+    if ignore_file_patterns:
+      if self.ignore_patterns:
+        self.ignore_patterns.extend(ignore_file_patterns)
+      else:
+        self.ignore_patterns = ignore_file_patterns
     # Get the list of files on the repo that are not ignored
     if getattr(self, "ignore_patterns", None):
       patterns = self.ignore_patterns

{clarifai-11.2.1 → clarifai-11.2.2/clarifai.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: clarifai
-Version: 11.2.1
+Version: 11.2.2
 Summary: Clarifai Python SDK
 Home-page: https://github.com/Clarifai/clarifai-python
 Author: Clarifai

clarifai-11.2.1/clarifai/__init__.py

@@ -1 +0,0 @@
-__version__ = "11.2.1"