clarifai 11.2.1__py3-none-any.whl → 11.2.2__py3-none-any.whl

clarifai/__init__.py

@@ -1 +1 @@
-__version__ = "11.2.1"
+__version__ = "11.2.2"

clarifai/client/app.py

@@ -629,7 +629,7 @@ class App(Lister, BaseClient):
 
     Args:
         model_id (str): The model ID for the model to interact with.
-        model_version_id (str): The model version ID for the model version to interact with.
+        model_version (Dict): The model version ID for the model version to interact with.
 
     Returns:
         Model: A Model object for the existing model ID.

clarifai/runners/models/model_builder.py

@@ -171,7 +171,14 @@ class ModelBuilder:
 
       # get from config.yaml otherwise fall back to HF_TOKEN env var.
       hf_token = self.config.get("checkpoints").get("hf_token", os.environ.get("HF_TOKEN", None))
-      return loader_type, repo_id, hf_token, when
+
+      allowed_file_patterns = self.config.get("checkpoints").get('allowed_file_patterns', None)
+      if isinstance(allowed_file_patterns, str):
+        allowed_file_patterns = [allowed_file_patterns]
+      ignore_file_patterns = self.config.get("checkpoints").get('ignore_file_patterns', None)
+      if isinstance(ignore_file_patterns, str):
+        ignore_file_patterns = [ignore_file_patterns]
+      return loader_type, repo_id, hf_token, when, allowed_file_patterns, ignore_file_patterns
 
   def _check_app_exists(self):
     resp = self.client.STUB.GetApp(service_pb2.GetAppRequest(user_app_id=self.client.user_app_id))
@@ -216,7 +223,7 @@ class ModelBuilder:
         assert model_type_id in CONCEPTS_REQUIRED_MODEL_TYPE, f"Model type {model_type_id} not supported for concepts"
 
     if self.config.get("checkpoints"):
-      loader_type, _, hf_token, _ = self._validate_config_checkpoints()
+      loader_type, _, hf_token, _, _, _ = self._validate_config_checkpoints()
 
       if loader_type == "huggingface" and hf_token:
         is_valid_token = HuggingFaceLoader.validate_hftoken(hf_token)
@@ -480,8 +487,10 @@ class ModelBuilder:
     if not self.config.get("checkpoints"):
       logger.info("No checkpoints specified in the config file")
       return path
+    clarifai_model_type_id = self.config.get('model').get('model_type_id')
 
-    loader_type, repo_id, hf_token, when = self._validate_config_checkpoints()
+    loader_type, repo_id, hf_token, when, allowed_file_patterns, ignore_file_patterns = self._validate_config_checkpoints(
+    )
     if stage not in ["build", "upload", "runtime"]:
       raise Exception("Invalid stage provided, must be one of ['build', 'upload', 'runtime']")
     if when != stage:
@@ -490,14 +499,18 @@ class ModelBuilder:
       )
       return path
 
-    success = True
+    success = False
     if loader_type == "huggingface":
-      loader = HuggingFaceLoader(repo_id=repo_id, token=hf_token)
+      loader = HuggingFaceLoader(
+          repo_id=repo_id, token=hf_token, model_type_id=clarifai_model_type_id)
       # for runtime default to /tmp path
       if stage == "runtime" and checkpoint_path_override is None:
         checkpoint_path_override = self.default_runtime_checkpoint_path()
       path = checkpoint_path_override if checkpoint_path_override else self.checkpoint_path
-      success = loader.download_checkpoints(path)
+      success = loader.download_checkpoints(
+          path,
+          allowed_file_patterns=allowed_file_patterns,
+          ignore_file_patterns=ignore_file_patterns)
 
     if loader_type:
       if not success:
@@ -569,7 +582,7 @@ class ModelBuilder:
     logger.debug(f"Will tar it into file: {file_path}")
 
     model_type_id = self.config.get('model').get('model_type_id')
-    loader_type, repo_id, hf_token, when = self._validate_config_checkpoints()
+    loader_type, repo_id, hf_token, when, _, _ = self._validate_config_checkpoints()
 
     if (model_type_id in CONCEPTS_REQUIRED_MODEL_TYPE) and 'concepts' not in self.config:
       logger.info(
@@ -617,7 +630,7 @@ class ModelBuilder:
       # First check for the env variable, then try querying huggingface. If all else fails, use the default.
       checkpoint_size = os.environ.get('CHECKPOINT_SIZE_BYTES', 0)
       if not checkpoint_size:
-        _, repo_id, _, _ = self._validate_config_checkpoints()
+        _, repo_id, _, _, _, _ = self._validate_config_checkpoints()
         checkpoint_size = HuggingFaceLoader.get_huggingface_checkpoint_total_size(repo_id)
       if not checkpoint_size:
         checkpoint_size = self.DEFAULT_CHECKPOINT_SIZE

clarifai/runners/models/model_run_locally.py

@@ -479,7 +479,7 @@ def main(model_path,
   manager = ModelRunLocally(model_path)
   # get whatever stage is in config.yaml to force download now
   # also always write to where upload/build wants to, not the /tmp folder that runtime stage uses
-  _, _, _, when = manager.builder._validate_config_checkpoints()
+  _, _, _, when, _, _ = manager.builder._validate_config_checkpoints()
   manager.builder.download_checkpoints(
       stage=when, checkpoint_path_override=manager.builder.checkpoint_path)
   if inside_container:

clarifai/runners/utils/loader.py

@@ -6,6 +6,7 @@ import shutil
 
 import requests
 
+from clarifai.runners.utils.const import CONCEPTS_REQUIRED_MODEL_TYPE
 from clarifai.utils.logging import logger
 
 
@@ -13,9 +14,10 @@ class HuggingFaceLoader:
 
   HF_DOWNLOAD_TEXT = "The 'huggingface_hub' package is not installed. Please install it using 'pip install huggingface_hub'."
 
-  def __init__(self, repo_id=None, token=None):
+  def __init__(self, repo_id=None, token=None, model_type_id=None):
     self.repo_id = repo_id
     self.token = token
+    self.clarifai_model_type_id = model_type_id
     if token:
       if self.validate_hftoken(token):
         try:
@@ -43,13 +45,17 @@ class HuggingFaceLoader:
           f"Error setting up Hugging Face token, please make sure you have the correct token: {e}")
       return False
 
-  def download_checkpoints(self, checkpoint_path: str):
+  def download_checkpoints(self,
+                           checkpoint_path: str,
+                           allowed_file_patterns=None,
+                           ignore_file_patterns=None):
     # throw error if huggingface_hub wasn't installed
     try:
       from huggingface_hub import snapshot_download
     except ImportError:
       raise ImportError(self.HF_DOWNLOAD_TEXT)
-    if os.path.exists(checkpoint_path) and self.validate_download(checkpoint_path):
+    if os.path.exists(checkpoint_path) and self.validate_download(
+        checkpoint_path, allowed_file_patterns, ignore_file_patterns):
       logger.info("Checkpoints already exist")
       return True
     else:
@@ -61,10 +67,16 @@ class HuggingFaceLoader:
           return False
 
         self.ignore_patterns = self._get_ignore_patterns()
+        if ignore_file_patterns:
+          if self.ignore_patterns:
+            self.ignore_patterns.extend(ignore_file_patterns)
+          else:
+            self.ignore_patterns = ignore_file_patterns
         snapshot_download(
             repo_id=self.repo_id,
             local_dir=checkpoint_path,
             local_dir_use_symlinks=False,
+            allow_patterns=allowed_file_patterns,
             ignore_patterns=self.ignore_patterns)
         # Remove the `.cache` folder if it exists
         cache_path = os.path.join(checkpoint_path, ".cache")
@@ -75,7 +87,8 @@ class HuggingFaceLoader:
         logger.error(f"Error downloading model checkpoints {e}")
         return False
       finally:
-        is_downloaded = self.validate_download(checkpoint_path)
+        is_downloaded = self.validate_download(checkpoint_path, allowed_file_patterns,
+                                               ignore_file_patterns)
         if not is_downloaded:
           logger.error("Error validating downloaded model checkpoints")
           return False
@@ -109,9 +122,13 @@ class HuggingFaceLoader:
       from huggingface_hub import file_exists, repo_exists
     except ImportError:
       raise ImportError(self.HF_DOWNLOAD_TEXT)
-    return repo_exists(self.repo_id) and file_exists(self.repo_id, 'config.json')
+    if self.clarifai_model_type_id in CONCEPTS_REQUIRED_MODEL_TYPE:
+      return repo_exists(self.repo_id) and file_exists(self.repo_id, 'config.json')
+    else:
+      return repo_exists(self.repo_id)
 
-  def validate_download(self, checkpoint_path: str):
+  def validate_download(self, checkpoint_path: str, allowed_file_patterns: list,
+                        ignore_file_patterns: list):
     # check if model exists on HF
     try:
       from huggingface_hub import list_repo_files
@@ -120,7 +137,20 @@ class HuggingFaceLoader:
     # Get the list of files on the repo
     repo_files = list_repo_files(self.repo_id, token=self.token)
 
+    # Get the list of files on the repo that are allowed
+    if allowed_file_patterns:
+
+      def should_allow(file_path):
+        return any(fnmatch.fnmatch(file_path, pattern) for pattern in allowed_file_patterns)
+
+      repo_files = [f for f in repo_files if should_allow(f)]
+
     self.ignore_patterns = self._get_ignore_patterns()
+    if ignore_file_patterns:
+      if self.ignore_patterns:
+        self.ignore_patterns.extend(ignore_file_patterns)
+      else:
+        self.ignore_patterns = ignore_file_patterns
     # Get the list of files on the repo that are not ignored
     if getattr(self, "ignore_patterns", None):
       patterns = self.ignore_patterns

{clarifai-11.2.1.dist-info → clarifai-11.2.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: clarifai
-Version: 11.2.1
+Version: 11.2.2
 Summary: Clarifai Python SDK
 Home-page: https://github.com/Clarifai/clarifai-python
 Author: Clarifai

{clarifai-11.2.1.dist-info → clarifai-11.2.2.dist-info}/RECORD

@@ -1,4 +1,4 @@
-clarifai/__init__.py,sha256=R3NVFNSpKk1o0N9GhBBuvfPhVFPrY5oLJJk5xhzNpGY,23
+clarifai/__init__.py,sha256=_cDM_DxpY5KiS6333RgrXKdnvdwjkQID_vppBsYvGPM,23
 clarifai/cli.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 clarifai/errors.py,sha256=RwzTajwds51wLD0MVlMC5kcpBnzRpreDLlazPSBZxrg,2605
 clarifai/versions.py,sha256=jctnczzfGk_S3EnVqb2FjRKfSREkNmvNEwAAa_VoKiQ,222
@@ -11,7 +11,7 @@ clarifai/cli/deployment.py,sha256=Vv92Qo7Q3UmMiCdrGQ4qNVNVzfcDyl-GB6B7kx8gyFw,25
 clarifai/cli/model.py,sha256=PR-A7V3D78iAjNx2hNeputiWuIyINrx2SJmjKtfMoQ4,12079
 clarifai/cli/nodepool.py,sha256=eYo3l9NMSwjeFzsuwKSYVkR1FdZ7_NcCdNvzGsBBMzY,3139
 clarifai/client/__init__.py,sha256=xI1U0l5AZdRThvQAXCLsd9axxyFzXXJ22m8LHqVjQRU,662
-clarifai/client/app.py,sha256=6pckYme1urV2YJjLIYfeZ-vH0Z5YSQa51jzIMcEfwug,38342
+clarifai/client/app.py,sha256=FnKvKksYZwdry0e4Obh-trdSO1mv6QcVAa3kzKiQMpU,38340
 clarifai/client/base.py,sha256=hSHOqkXbSKyaRDeylMMnkhUHCAHhEqno4KI0CXGziBA,7536
 clarifai/client/compute_cluster.py,sha256=EvW9TJjPvInUlggfg1A98sxoWH8_PY5rCVXZhsj6ac0,8705
 clarifai/client/dataset.py,sha256=y3zKT_VhP1gyN3OO-b3cPeW21ZXyKbQ7ZJkEG06bsTU,32096
@@ -66,16 +66,16 @@ clarifai/runners/server.py,sha256=xHDLdhQApCgYG19QOKXqJNCGNyw1Vsvobq3UdryDrVc,41
 clarifai/runners/dockerfile_template/Dockerfile.template,sha256=5cjv7U8PmWa3DB_5B1CqSYh_6GE0E0np52TIAa7EIDE,2312
 clarifai/runners/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 clarifai/runners/models/base_typed_model.py,sha256=0QCWxch8CcyJSKvE1D4PILd2RSnQZHTmx4DXlQQ6dpo,7856
-clarifai/runners/models/model_builder.py,sha256=YCnZsAIE0CB-M0c2cvdEzY4O11I80STshd36dqDAN2Q,32565
+clarifai/runners/models/model_builder.py,sha256=NAZiOm2ok-ThQ-jcAyBTRRCFufmyxGY76vlkLMpYHq4,33320
 clarifai/runners/models/model_class.py,sha256=9JSPAr4U4K7xI0kSl-q0mHB06zknm2OR-8XIgBCto94,1611
-clarifai/runners/models/model_run_locally.py,sha256=hSkXrln4q1_EEtRyGH5g0nMg-ieGWmamD7cI5YV_qIk,20550
+clarifai/runners/models/model_run_locally.py,sha256=jz4ZvRj0eMCEziV3X37wH2AEe3Gl9Icx7Quy1ymaDe0,20556
 clarifai/runners/models/model_runner.py,sha256=PyxwK-33hLlhkD07tTXkjWZ_iNlZHl9_8AZ2W7WfExI,6097
 clarifai/runners/models/model_servicer.py,sha256=jtQmtGeQlvQ5ttMvVw7CMnNzq-rLkTaxR2IWF9SnHwk,2808
 clarifai/runners/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 clarifai/runners/utils/const.py,sha256=bwj-Pcw558-pasdIFbNhnkn-9oiCdojYH1fNTTUG2gU,1048
 clarifai/runners/utils/data_handler.py,sha256=sxy9zlAgI6ETuxCQhUgEXAn2GCsaW1GxpK6GTaMne0g,6966
 clarifai/runners/utils/data_utils.py,sha256=R1iQ82TuQ9JwxCJk8yEB1Lyb0BYVhVbWJI9YDi1zGOs,318
-clarifai/runners/utils/loader.py,sha256=SgNHMwRmCCymFQm8aDp73NmIUHhM-N60CBlTKbPzmVc,7470
+clarifai/runners/utils/loader.py,sha256=Sl0m29RDtMPx2cIiSbbDFtKHQj2ktXQ5CnkvaHi-zDc,8804
 clarifai/runners/utils/url_fetcher.py,sha256=v_8JOWmkyFAzsBulsieKX7Nfjy1Yg7wGSZeqfEvw2cg,1640
 clarifai/schema/search.py,sha256=JjTi8ammJgZZ2OGl4K6tIA4zEJ1Fr2ASZARXavI1j5c,2448
 clarifai/urls/helper.py,sha256=tjoMGGHuWX68DUB0pk4MEjrmFsClUAQj2jmVEM_Sy78,4751
@@ -93,9 +93,9 @@ clarifai/workflows/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 clarifai/workflows/export.py,sha256=vICRhIreqDSShxLKjHNM2JwzKsf1B4fdXB0ciMcA70k,1945
 clarifai/workflows/utils.py,sha256=nGeB_yjVgUO9kOeKTg4OBBaBz-AwXI3m-huSVj-9W18,1924
 clarifai/workflows/validate.py,sha256=yJq03MaJqi5AK3alKGJJBR89xmmjAQ31sVufJUiOqY8,2556
-clarifai-11.2.1.dist-info/licenses/LICENSE,sha256=mUqF_d12-qE2n41g7C5_sq-BMLOcj6CNN-jevr15YHU,555
-clarifai-11.2.1.dist-info/METADATA,sha256=te276wArqCAEYdsF-LYLQSTxMZiPtAVO_a2lEQngFNU,22472
-clarifai-11.2.1.dist-info/WHEEL,sha256=DK49LOLCYiurdXXOXwGJm6U4DkHkg4lcxjhqwRa0CP4,91
-clarifai-11.2.1.dist-info/entry_points.txt,sha256=X9FZ4Z-i_r2Ud1RpZ9sNIFYuu_-9fogzCMCRUD9hyX0,51
-clarifai-11.2.1.dist-info/top_level.txt,sha256=wUMdCQGjkxaynZ6nZ9FAnvBUCgp5RJUVFSy2j-KYo0s,9
-clarifai-11.2.1.dist-info/RECORD,,
+clarifai-11.2.2.dist-info/licenses/LICENSE,sha256=mUqF_d12-qE2n41g7C5_sq-BMLOcj6CNN-jevr15YHU,555
+clarifai-11.2.2.dist-info/METADATA,sha256=mWqPwNuC1LdGkPWUu5ugnqADyHzcwR18PA66roRCPj4,22472
+clarifai-11.2.2.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+clarifai-11.2.2.dist-info/entry_points.txt,sha256=X9FZ4Z-i_r2Ud1RpZ9sNIFYuu_-9fogzCMCRUD9hyX0,51
+clarifai-11.2.2.dist-info/top_level.txt,sha256=wUMdCQGjkxaynZ6nZ9FAnvBUCgp5RJUVFSy2j-KYo0s,9
+clarifai-11.2.2.dist-info/RECORD,,

{clarifai-11.2.1.dist-info → clarifai-11.2.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.0.2)
+Generator: setuptools (78.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any