clarifai 10.11.2rc1__py3-none-any.whl → 10.11.2rc3__py3-none-any.whl

clarifai/__init__.py

@@ -1 +1 @@
-__version__ = "10.11.2rc1"
+__version__ = "10.11.2rc3"

clarifai/runners/models/model_upload.py

@@ -29,16 +29,19 @@ def _clear_line(n: int = 1) -> None:
 
 class ModelUploader:
 
-  def __init__(self, folder: str, validate_api_ids: bool = True):
+  def __init__(self, folder: str, validate_api_ids: bool = True, download_validation_only=False):
     """
     :param folder: The folder containing the model.py, config.yaml, requirements.txt and
     checkpoints.
-    :param validate_api_ids: Whether to validate the user_id and app_id in the config file.
+    :param validate_api_ids: Whether to validate the user_id and app_id in the config file. TODO:
+    deprecate in favor of download_validation_only.
+    :param download_validation_only: Whether to skip the API config validation. Set to True if
+    just downloading a checkpoint.
     """
     self._client = None
+    self.download_validation_only = download_validation_only
     self.folder = self._validate_folder(folder)
     self.config = self._load_config(os.path.join(self.folder, 'config.yaml'))
-    self.validate_api_ids = validate_api_ids
     self._validate_config()
     self.model_proto = self._get_model_proto()
     self.model_id = self.model_proto.id
@@ -46,19 +49,21 @@ class ModelUploader:
     self.inference_compute_info = self._get_inference_compute_info()
     self.is_v3 = True  # Do model build for v3
 
-  @staticmethod
-  def _validate_folder(folder):
+  def _validate_folder(self, folder):
     if not folder.startswith("/"):
       folder = os.path.join(os.getcwd(), folder)
     logger.info(f"Validating folder: {folder}")
     if not os.path.exists(folder):
       raise FileNotFoundError(f"Folder {folder} not found, please provide a valid folder path")
     files = os.listdir(folder)
-    assert "requirements.txt" in files, "requirements.txt not found in the folder"
     assert "config.yaml" in files, "config.yaml not found in the folder"
+    # If just downloading we don't need requirements.txt or the python code, we do need the
+    # 1/ folder to put 1/checkpoints into.
     assert "1" in files, "Subfolder '1' not found in the folder"
-    subfolder_files = os.listdir(os.path.join(folder, '1'))
-    assert 'model.py' in subfolder_files, "model.py not found in the folder"
+    if not self.download_validation_only:
+      assert "requirements.txt" in files, "requirements.txt not found in the folder"
+      subfolder_files = os.listdir(os.path.join(folder, '1'))
+      assert 'model.py' in subfolder_files, "model.py not found in the folder"
     return folder
 
   @staticmethod
@@ -83,8 +88,6 @@ class ModelUploader:
       return repo_id, hf_token
 
   def _check_app_exists(self):
-    if not self.validate_api_ids:
-      return True
     resp = self.client.STUB.GetApp(service_pb2.GetAppRequest(user_app_id=self.client.user_app_id))
     if resp.status.code == status_code_pb2.SUCCESS:
       return True
@@ -114,16 +117,14 @@ class ModelUploader:
       sys.exit(1)
 
   def _validate_config(self):
-    self._validate_config_model()
-
-    if self.config.get("checkpoints"):
-      self._validate_config_checkpoints()
+    if not self.download_validation_only:
+      self._validate_config_model()
 
-    assert "inference_compute_info" in self.config, "inference_compute_info not found in the config file"
+      assert "inference_compute_info" in self.config, "inference_compute_info not found in the config file"
 
-    if self.config.get("concepts"):
-      model_type_id = self.config.get('model').get('model_type_id')
-      assert model_type_id in CONCEPTS_REQUIRED_MODEL_TYPE, f"Model type {model_type_id} not supported for concepts"
+      if self.config.get("concepts"):
+        model_type_id = self.config.get('model').get('model_type_id')
+        assert model_type_id in CONCEPTS_REQUIRED_MODEL_TYPE, f"Model type {model_type_id} not supported for concepts"
 
     if self.config.get("checkpoints"):
       _, hf_token = self._validate_config_checkpoints()

clarifai/runners/utils/loader.py

@@ -1,6 +1,8 @@
+import fnmatch
 import importlib.util
 import json
 import os
+import shutil
 import subprocess
 
 from clarifai.utils.logging import logger
@@ -39,7 +41,7 @@ class HuggingFaceLoader:
   def download_checkpoints(self, checkpoint_path: str):
     # throw error if huggingface_hub wasn't installed
     try:
-      from huggingface_hub import list_repo_files, snapshot_download
+      from huggingface_hub import snapshot_download
     except ImportError:
       raise ImportError(self.HF_DOWNLOAD_TEXT)
     if os.path.exists(checkpoint_path) and self.validate_download(checkpoint_path):
@@ -53,16 +55,17 @@ class HuggingFaceLoader:
           logger.error("Model %s not found on Hugging Face" % (self.repo_id))
           return False
 
-        ignore_patterns = None  # Download everything.
-        repo_files = list_repo_files(repo_id=self.repo_id, token=self.token)
-        if any(f.endswith(".safetensors") for f in repo_files):
-          logger.info(f"SafeTensors found in {self.repo_id}, downloading only .safetensors files.")
-          ignore_patterns = ["**/original/*", "**/*.pth", "**/*.bin"]
+        self.ignore_patterns = self._get_ignore_patterns()
         snapshot_download(
             repo_id=self.repo_id,
             local_dir=checkpoint_path,
             local_dir_use_symlinks=False,
-            ignore_patterns=ignore_patterns)
+            ignore_patterns=self.ignore_patterns)
+        # Remove the `.cache` folder if it exists
+        cache_path = os.path.join(checkpoint_path, ".cache")
+        if os.path.exists(cache_path) and os.path.isdir(cache_path):
+          shutil.rmtree(cache_path)
+
       except Exception as e:
         logger.error(f"Error downloading model checkpoints {e}")
         return False
@@ -109,11 +112,41 @@ class HuggingFaceLoader:
       from huggingface_hub import list_repo_files
     except ImportError:
       raise ImportError(self.HF_DOWNLOAD_TEXT)
+    # Get the list of files on the repo
+    repo_files = list_repo_files(self.repo_id, token=self.token)
+
+    self.ignore_patterns = self._get_ignore_patterns()
+    # Get the list of files on the repo that are not ignored
+    if getattr(self, "ignore_patterns", None):
+      patterns = self.ignore_patterns
+
+      def should_ignore(file_path):
+        return any(fnmatch.fnmatch(file_path, pattern) for pattern in patterns)
+
+      repo_files = [f for f in repo_files if not should_ignore(f)]
+
+    # Check if downloaded files match the files we expect (ignoring ignored patterns)
     checkpoint_dir_files = [
         f for dp, dn, fn in os.walk(os.path.expanduser(checkpoint_path)) for f in fn
     ]
-    return (len(checkpoint_dir_files) >= len(list_repo_files(self.repo_id))) and len(
-        list_repo_files(self.repo_id)) > 0
+
+    # Validate by comparing file lists
+    return len(checkpoint_dir_files) >= len(repo_files) and not (
+        len(set(repo_files) - set(checkpoint_dir_files)) > 0) and len(repo_files) > 0
+
+  def _get_ignore_patterns(self):
+    # check if model exists on HF
+    try:
+      from huggingface_hub import list_repo_files
+    except ImportError:
+      raise ImportError(self.HF_DOWNLOAD_TEXT)
+
+    # Get the list of files on the repo that are not ignored
+    repo_files = list_repo_files(self.repo_id, token=self.token)
+    self.ignore_patterns = None
+    if any(f.endswith(".safetensors") for f in repo_files):
+      self.ignore_patterns = ["**/original/*", "**/*.pth", "**/*.bin", "*.pth", "*.bin"]
+    return self.ignore_patterns
 
   @staticmethod
   def validate_config(checkpoint_path: str):

{clarifai-10.11.2rc1.dist-info → clarifai-10.11.2rc3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: clarifai
-Version: 10.11.2rc1
+Version: 10.11.2rc3
 Summary: Clarifai Python SDK
 Home-page: https://github.com/Clarifai/clarifai-python
 Author: Clarifai

{clarifai-10.11.2rc1.dist-info → clarifai-10.11.2rc3.dist-info}/RECORD

@@ -1,4 +1,4 @@
-clarifai/__init__.py,sha256=TUFYMVETe0TPo-c1k5cztKVyUKS4v93hkc5aMO20brk,27
+clarifai/__init__.py,sha256=CeWkfe22NXjzgsoSQ84l50ZL9fR1MatZEVOZm2BBGZ4,27
 clarifai/cli.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 clarifai/errors.py,sha256=RwzTajwds51wLD0MVlMC5kcpBnzRpreDLlazPSBZxrg,2605
 clarifai/versions.py,sha256=jctnczzfGk_S3EnVqb2FjRKfSREkNmvNEwAAa_VoKiQ,222
@@ -186,26 +186,26 @@ clarifai/runners/models/model_class.py,sha256=9JSPAr4U4K7xI0kSl-q0mHB06zknm2OR-8
 clarifai/runners/models/model_run_locally.py,sha256=OhzQbmaV8Wwgs2H0KhdDF6Z7bYSaIh4RRA0QwSiv5vY,20644
 clarifai/runners/models/model_runner.py,sha256=3vzoastQxkGRDK8T9aojDsLNBb9A3IiKm6YmbFrE9S0,6241
 clarifai/runners/models/model_servicer.py,sha256=X4715PVA5PBurRTYcwSEudg8fShGV6InAF4mmRlRcHg,2826
-clarifai/runners/models/model_upload.py,sha256=4_tyT18unNV7QpFBfFCanKny4wMHDulfULZT4Son2yo,22923
+clarifai/runners/models/model_upload.py,sha256=L5BbODARRDBXIV8DBe9_6jT37oNQw4PuzRrwKfy37mg,23250
 clarifai/runners/models/__pycache__/__init__.cpython-310.pyc,sha256=GTFjzypyx5wnDGqxYeY01iya1CELKb5fOFBFLV031yU,171
 clarifai/runners/models/__pycache__/base_typed_model.cpython-310.pyc,sha256=xni4ij1KhISxAia1j-Q1PDSX65tOYYjyfPlIxOxtlFs,8273
 clarifai/runners/models/__pycache__/model_class.cpython-310.pyc,sha256=OxOQDH4G_XXvXCOChWLjONFVcDWdr6T00JjQvKx1AgA,1889
 clarifai/runners/models/__pycache__/model_run_locally.cpython-310.pyc,sha256=_vm1H_HBCwWRpl-3mVjRIVbvHDYM6-sz9OheR1BmzcE,17040
 clarifai/runners/models/__pycache__/model_runner.cpython-310.pyc,sha256=lMrT9MMO3sLTo11zSv5XYTBQte3oY23esUFV4Wdnquw,5108
 clarifai/runners/models/__pycache__/model_servicer.cpython-310.pyc,sha256=w5kENnKCiiZSpZF6g7N1wljHbnan3pi_mhZBm_VvD4Q,2489
-clarifai/runners/models/__pycache__/model_upload.cpython-310.pyc,sha256=vU27DTu3KJKQjRJOLi8GdC1s4HMJrdb9yzRm6tMwS8M,19274
+clarifai/runners/models/__pycache__/model_upload.cpython-310.pyc,sha256=3iBqzh3IXS42yi4fYLJTZ594G9UGYX9xXw4NItrA5sI,19274
 clarifai/runners/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 clarifai/runners/utils/const.py,sha256=eyBrj5ywuGKPF-IFipm7yjiYyLhnsKhMNZ6xF-OvykQ,1250
 clarifai/runners/utils/data_handler.py,sha256=sxy9zlAgI6ETuxCQhUgEXAn2GCsaW1GxpK6GTaMne0g,6966
 clarifai/runners/utils/data_utils.py,sha256=R1iQ82TuQ9JwxCJk8yEB1Lyb0BYVhVbWJI9YDi1zGOs,318
-clarifai/runners/utils/loader.py,sha256=_bbFWaq5B41rKUqD3i7L3DjxExmCzGqWfIMeJ7wj0P0,4867
+clarifai/runners/utils/loader.py,sha256=u4dSnOf8JcAlR2wKaWxvz4KzMfJ9oL-9orhgwtVGhJU,5975
 clarifai/runners/utils/logging.py,sha256=xan5ZQH5XHVyNIIjUAQoc_bS3MrjQjEvBYuo0CQKdho,153
 clarifai/runners/utils/url_fetcher.py,sha256=v_8JOWmkyFAzsBulsieKX7Nfjy1Yg7wGSZeqfEvw2cg,1640
 clarifai/runners/utils/__pycache__/__init__.cpython-310.pyc,sha256=PRPZOzUV5Z8grWizu5RKOkki0iLYxZDJBgsLfmCcieE,170
 clarifai/runners/utils/__pycache__/const.cpython-310.pyc,sha256=oEBBn71Dyzuo25Rrt-UwPSUT7EAOliC1QZUYFVhW4vI,1028
 clarifai/runners/utils/__pycache__/data_handler.cpython-310.pyc,sha256=YVncnM0NaHeMaZAyALxaHCdtUT6n5E3BI99-54Bs6HM,7961
 clarifai/runners/utils/__pycache__/data_utils.cpython-310.pyc,sha256=1e6NiK6bnJiiAo2KPsDmm91BSlbI3mVkQZKbDfh5hBI,642
-clarifai/runners/utils/__pycache__/loader.cpython-310.pyc,sha256=CU9LwYthYQSOEsb2C9bamdN7rKLIwxDlaIrI79Klvuc,5049
+clarifai/runners/utils/__pycache__/loader.cpython-310.pyc,sha256=smsd1eEst3buel_C8OcOhnyGBzvtBe_wrCEaBqhGTVo,6002
 clarifai/runners/utils/__pycache__/logging.cpython-310.pyc,sha256=VV0KFcnuYpFFtaG4EeDIgg7c4QEsBLo-eX_NnsyFEhA,331
 clarifai/runners/utils/__pycache__/url_fetcher.cpython-310.pyc,sha256=jFxVdOmm7DCkgatv1GwIXeefHthpvlkg4ybBlMnmxss,1739
 clarifai/schema/search.py,sha256=JjTi8ammJgZZ2OGl4K6tIA4zEJ1Fr2ASZARXavI1j5c,2448
@@ -234,9 +234,9 @@ clarifai/workflows/__pycache__/__init__.cpython-310.pyc,sha256=oRKg6B7Z-wWQy0EW2
 clarifai/workflows/__pycache__/export.cpython-310.pyc,sha256=cNmGLnww7xVpm4htd1vRhQJoEZ1dhpN1oD8iLLAtVzM,2418
 clarifai/workflows/__pycache__/utils.cpython-310.pyc,sha256=rm2kWk4a3GOKWoerXpEAEeRvGhEe7wPd0ZZ6jHtEGqY,1925
 clarifai/workflows/__pycache__/validate.cpython-310.pyc,sha256=QA1i6YdDpY824cqtQvkEaFPpaCa2iqfOwFouqwZfAKY,2139
-clarifai-10.11.2rc1.dist-info/LICENSE,sha256=mUqF_d12-qE2n41g7C5_sq-BMLOcj6CNN-jevr15YHU,555
-clarifai-10.11.2rc1.dist-info/METADATA,sha256=lCXDIqQH3Ipj4nLIGgYpF3lSJMMxiljkCJxN0TN_Bxw,22237
-clarifai-10.11.2rc1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-clarifai-10.11.2rc1.dist-info/entry_points.txt,sha256=X9FZ4Z-i_r2Ud1RpZ9sNIFYuu_-9fogzCMCRUD9hyX0,51
-clarifai-10.11.2rc1.dist-info/top_level.txt,sha256=wUMdCQGjkxaynZ6nZ9FAnvBUCgp5RJUVFSy2j-KYo0s,9
-clarifai-10.11.2rc1.dist-info/RECORD,,
+clarifai-10.11.2rc3.dist-info/LICENSE,sha256=mUqF_d12-qE2n41g7C5_sq-BMLOcj6CNN-jevr15YHU,555
+clarifai-10.11.2rc3.dist-info/METADATA,sha256=asimq_wgoMJRN83rnydqa8kUCc-AQBkwExjt_A01W34,22237
+clarifai-10.11.2rc3.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+clarifai-10.11.2rc3.dist-info/entry_points.txt,sha256=X9FZ4Z-i_r2Ud1RpZ9sNIFYuu_-9fogzCMCRUD9hyX0,51
+clarifai-10.11.2rc3.dist-info/top_level.txt,sha256=wUMdCQGjkxaynZ6nZ9FAnvBUCgp5RJUVFSy2j-KYo0s,9
+clarifai-10.11.2rc3.dist-info/RECORD,,