clarifai 11.0.2__py3-none-any.whl → 11.0.4__py3-none-any.whl

clarifai/__init__.py

@@ -1 +1 @@
-__version__ = "11.0.2"
+__version__ = "11.0.4"

clarifai/runners/models/model_upload.py

@@ -29,8 +29,17 @@ def _clear_line(n: int = 1) -> None:
 
 class ModelUploader:
 
-  def __init__(self, folder: str):
+  def __init__(self, folder: str, validate_api_ids: bool = True, download_validation_only=False):
+    """
+    :param folder: The folder containing the model.py, config.yaml, requirements.txt and
+    checkpoints.
+    :param validate_api_ids: Whether to validate the user_id and app_id in the config file. TODO(zeiler):
+    deprecate in favor of download_validation_only.
+    :param download_validation_only: Whether to skip the API config validation. Set to True if
+    just downloading a checkpoint.
+    """
     self._client = None
+    self.download_validation_only = download_validation_only
     self.folder = self._validate_folder(folder)
     self.config = self._load_config(os.path.join(self.folder, 'config.yaml'))
     self._validate_config()
@@ -40,19 +49,23 @@ class ModelUploader:
     self.inference_compute_info = self._get_inference_compute_info()
     self.is_v3 = True  # Do model build for v3
 
-  @staticmethod
-  def _validate_folder(folder):
+  def _validate_folder(self, folder):
+    if folder == ".":
+      folder = ""  # will getcwd() next which ends with /
     if not folder.startswith("/"):
       folder = os.path.join(os.getcwd(), folder)
     logger.info(f"Validating folder: {folder}")
     if not os.path.exists(folder):
       raise FileNotFoundError(f"Folder {folder} not found, please provide a valid folder path")
     files = os.listdir(folder)
-    assert "requirements.txt" in files, "requirements.txt not found in the folder"
     assert "config.yaml" in files, "config.yaml not found in the folder"
+    # If just downloading we don't need requirements.txt or the python code, we do need the
+    # 1/ folder to put 1/checkpoints into.
     assert "1" in files, "Subfolder '1' not found in the folder"
-    subfolder_files = os.listdir(os.path.join(folder, '1'))
-    assert 'model.py' in subfolder_files, "model.py not found in the folder"
+    if not self.download_validation_only:
+      assert "requirements.txt" in files, "requirements.txt not found in the folder"
+      subfolder_files = os.listdir(os.path.join(folder, '1'))
+      assert 'model.py' in subfolder_files, "model.py not found in the folder"
     return folder
 
   @staticmethod
@@ -62,18 +75,25 @@ class ModelUploader:
     return config
 
   def _validate_config_checkpoints(self):
-
+    """
+    Validates the checkpoints section in the config file.
+    :return: loader_type the type of loader or None if no checkpoints.
+    :return: repo_id location of checkpoint.
+    :return: hf_token token to access checkpoint.
+    """
     assert "type" in self.config.get("checkpoints"), "No loader type specified in the config file"
     loader_type = self.config.get("checkpoints").get("type")
     if not loader_type:
       logger.info("No loader type specified in the config file for checkpoints")
+      return None, None, None
     assert loader_type == "huggingface", "Only huggingface loader supported for now"
     if loader_type == "huggingface":
       assert "repo_id" in self.config.get("checkpoints"), "No repo_id specified in the config file"
       repo_id = self.config.get("checkpoints").get("repo_id")
 
-      hf_token = self.config.get("checkpoints").get("hf_token", None)
-      return repo_id, hf_token
+      # get from config.yaml otherwise fall back to HF_TOKEN env var.
+      hf_token = self.config.get("checkpoints").get("hf_token", os.environ.get("HF_TOKEN", None))
+      return loader_type, repo_id, hf_token
 
   def _check_app_exists(self):
     resp = self.client.STUB.GetApp(service_pb2.GetAppRequest(user_app_id=self.client.user_app_id))
@@ -105,21 +125,19 @@ class ModelUploader:
       sys.exit(1)
 
   def _validate_config(self):
-    self._validate_config_model()
-
-    if self.config.get("checkpoints"):
-      self._validate_config_checkpoints()
+    if not self.download_validation_only:
+      self._validate_config_model()
 
-    assert "inference_compute_info" in self.config, "inference_compute_info not found in the config file"
+      assert "inference_compute_info" in self.config, "inference_compute_info not found in the config file"
 
-    if self.config.get("concepts"):
-      model_type_id = self.config.get('model').get('model_type_id')
-      assert model_type_id in CONCEPTS_REQUIRED_MODEL_TYPE, f"Model type {model_type_id} not supported for concepts"
+      if self.config.get("concepts"):
+        model_type_id = self.config.get('model').get('model_type_id')
+        assert model_type_id in CONCEPTS_REQUIRED_MODEL_TYPE, f"Model type {model_type_id} not supported for concepts"
 
     if self.config.get("checkpoints"):
-      _, hf_token = self._validate_config_checkpoints()
+      loader_type, _, hf_token = self._validate_config_checkpoints()
 
-      if hf_token:
+      if loader_type == "huggingface" and hf_token:
         is_valid_token = HuggingFaceLoader.validate_hftoken(hf_token)
         if not is_valid_token:
           logger.error(
@@ -303,16 +321,19 @@ class ModelUploader:
       logger.info("No checkpoints specified in the config file")
       return True
 
-    repo_id, hf_token = self._validate_config_checkpoints()
+    loader_type, repo_id, hf_token = self._validate_config_checkpoints()
 
-    loader = HuggingFaceLoader(repo_id=repo_id, token=hf_token)
-    success = loader.download_checkpoints(self.checkpoint_path)
+    success = True
+    if loader_type == "huggingface":
+      loader = HuggingFaceLoader(repo_id=repo_id, token=hf_token)
+      success = loader.download_checkpoints(self.checkpoint_path)
 
-    if not success:
-      logger.error(f"Failed to download checkpoints for model {repo_id}")
-      sys.exit(1)
-    else:
-      logger.info(f"Downloaded checkpoints for model {repo_id}")
+    if loader_type:
+      if not success:
+        logger.error(f"Failed to download checkpoints for model {repo_id}")
+        sys.exit(1)
+      else:
+        logger.info(f"Downloaded checkpoints for model {repo_id}")
     return success
 
   def _concepts_protos_from_concepts(self, concepts):
@@ -392,9 +413,10 @@ class ModelUploader:
           input(
               "Press Enter to download the HuggingFace model's config.json file to infer the concepts and continue..."
           )
-          repo_id, hf_token = self._validate_config_checkpoints()
-          loader = HuggingFaceLoader(repo_id=repo_id, token=hf_token)
-          loader.download_config(self.checkpoint_path)
+          loader_type, repo_id, hf_token = self._validate_config_checkpoints()
+          if loader_type == "huggingface":
+            loader = HuggingFaceLoader(repo_id=repo_id, token=hf_token)
+            loader.download_config(self.checkpoint_path)
 
       else:
         logger.error(
@@ -405,10 +427,10 @@ class ModelUploader:
     model_version_proto = self.get_model_version_proto()
 
     if download_checkpoints:
-      tar_cmd = f"tar --exclude=*~ -czvf {self.tar_file} -C {self.folder} ."
+      tar_cmd = f"tar --exclude=*~ --exclude={self.tar_file} -czvf {self.tar_file} -C {self.folder} ."
     else:  # we don't want to send the checkpoints up even if they are in the folder.
       logger.info(f"Skipping {self.checkpoint_path} in the tar file that is uploaded.")
-      tar_cmd = f"tar --exclude={self.checkpoint_suffix} --exclude=*~ -czvf {self.tar_file} -C {self.folder} ."
+      tar_cmd = f"tar --exclude={self.checkpoint_suffix} --exclude=*~ --exclude={self.tar_file} -czvf {self.tar_file} -C {self.folder} ."
     # Tar the folder
     logger.debug(tar_cmd)
     os.system(tar_cmd)
@@ -485,7 +507,7 @@ class ModelUploader:
     file_size = os.path.getsize(file_path)
     logger.info(f"Uploading model version of model {self.model_proto.id}")
     logger.info(f"Using file '{os.path.basename(file_path)}' of size: {file_size} bytes")
-    return service_pb2.PostModelVersionsUploadRequest(
+    result = service_pb2.PostModelVersionsUploadRequest(
         upload_config=service_pb2.PostModelVersionsUploadConfig(
             user_app_id=self.client.user_app_id,
             model_id=self.model_proto.id,
@@ -493,6 +515,7 @@ class ModelUploader:
             total_size=file_size,
             is_v3=self.is_v3,
         ))
+    return result
 
   def get_model_build_logs(self):
     logs_request = service_pb2.ListLogEntriesRequest(

clarifai/runners/utils/loader.py

@@ -1,6 +1,8 @@
+import fnmatch
 import importlib.util
 import json
 import os
+import shutil
 import subprocess
 
 from clarifai.utils.logging import logger
@@ -39,7 +41,7 @@ class HuggingFaceLoader:
   def download_checkpoints(self, checkpoint_path: str):
     # throw error if huggingface_hub wasn't installed
     try:
-      from huggingface_hub import list_repo_files, snapshot_download
+      from huggingface_hub import snapshot_download
     except ImportError:
       raise ImportError(self.HF_DOWNLOAD_TEXT)
     if os.path.exists(checkpoint_path) and self.validate_download(checkpoint_path):
@@ -53,16 +55,17 @@ class HuggingFaceLoader:
           logger.error("Model %s not found on Hugging Face" % (self.repo_id))
           return False
 
-        ignore_patterns = None  # Download everything.
-        repo_files = list_repo_files(repo_id=self.repo_id, token=self.token)
-        if any(f.endswith(".safetensors") for f in repo_files):
-          logger.info(f"SafeTensors found in {self.repo_id}, downloading only .safetensors files.")
-          ignore_patterns = ["original/*", "*.pth", "*.bin"]
+        self.ignore_patterns = self._get_ignore_patterns()
         snapshot_download(
             repo_id=self.repo_id,
             local_dir=checkpoint_path,
             local_dir_use_symlinks=False,
-            ignore_patterns=ignore_patterns)
+            ignore_patterns=self.ignore_patterns)
+        # Remove the `.cache` folder if it exists
+        cache_path = os.path.join(checkpoint_path, ".cache")
+        if os.path.exists(cache_path) and os.path.isdir(cache_path):
+          shutil.rmtree(cache_path)
+
       except Exception as e:
         logger.error(f"Error downloading model checkpoints {e}")
         return False
@@ -109,11 +112,44 @@ class HuggingFaceLoader:
       from huggingface_hub import list_repo_files
     except ImportError:
       raise ImportError(self.HF_DOWNLOAD_TEXT)
-    checkpoint_dir_files = [
-        f for dp, dn, fn in os.walk(os.path.expanduser(checkpoint_path)) for f in fn
-    ]
-    return (len(checkpoint_dir_files) >= len(list_repo_files(self.repo_id))) and len(
-        list_repo_files(self.repo_id)) > 0
+    # Get the list of files on the repo
+    repo_files = list_repo_files(self.repo_id, token=self.token)
+
+    self.ignore_patterns = self._get_ignore_patterns()
+    # Get the list of files on the repo that are not ignored
+    if getattr(self, "ignore_patterns", None):
+      patterns = self.ignore_patterns
+
+      def should_ignore(file_path):
+        return any(fnmatch.fnmatch(file_path, pattern) for pattern in patterns)
+
+      repo_files = [f for f in repo_files if not should_ignore(f)]
+
+    # Check if downloaded files match the files we expect (ignoring ignored patterns)
+    checkpoint_dir_files = []
+    for dp, dn, fn in os.walk(os.path.expanduser(checkpoint_path)):
+      checkpoint_dir_files.extend(
+          [os.path.relpath(os.path.join(dp, f), checkpoint_path) for f in fn])
+
+    # Validate by comparing file lists
+    return len(checkpoint_dir_files) >= len(repo_files) and not (
+        len(set(repo_files) - set(checkpoint_dir_files)) > 0) and len(repo_files) > 0
+
+  def _get_ignore_patterns(self):
+    # check if model exists on HF
+    try:
+      from huggingface_hub import list_repo_files
+    except ImportError:
+      raise ImportError(self.HF_DOWNLOAD_TEXT)
+
+    # Get the list of files on the repo that are not ignored
+    repo_files = list_repo_files(self.repo_id, token=self.token)
+    self.ignore_patterns = None
+    if any(f.endswith(".safetensors") for f in repo_files):
+      self.ignore_patterns = [
+          "**/original/*", "**/*.pth", "**/*.bin", "*.pth", "*.bin", "**/.cache/*"
+      ]
+    return self.ignore_patterns
 
   @staticmethod
   def validate_config(checkpoint_path: str):

clarifai/utils/logging.py

@@ -282,6 +282,11 @@ class JsonFormatter(logging.Formatter):
       except Exception:
         self.source_host = ""
 
+    self.extra_blacklist_fields = []
+    extra_blacklist_fields = os.getenv('EXTRA_JSON_LOGGER_BLACKLIST_FIELDS', None)
+    if extra_blacklist_fields:
+      self.extra_blacklist_fields = extra_blacklist_fields.split(",")
+
   def _build_fields(self, defaults, fields):
     """Return provided fields including any in defaults
     """
@@ -302,6 +307,8 @@ class JsonFormatter(logging.Formatter):
       msg = record.getMessage()
     for k in FIELD_BLACKLIST:
       fields.pop(k, None)
+    for k in self.extra_blacklist_fields:
+      fields.pop(k, None)
     # Rename 'levelname' to 'level' and make the value lowercase to match Go logs
     level = fields.pop('levelname', None)
     if level:

{clarifai-11.0.2.dist-info → clarifai-11.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: clarifai
-Version: 11.0.2
+Version: 11.0.4
 Summary: Clarifai Python SDK
 Home-page: https://github.com/Clarifai/clarifai-python
 Author: Clarifai

{clarifai-11.0.2.dist-info → clarifai-11.0.4.dist-info}/RECORD

@@ -1,4 +1,4 @@
-clarifai/__init__.py,sha256=RDbyFSHLm1s06WmNHjem0y67LhbGXNidjKUmieqBJ5c,23
+clarifai/__init__.py,sha256=vo9MZbRrD_gywPOV0Tq9RYPDXhSDrU97AuJT6cCihkg,23
 clarifai/cli.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 clarifai/errors.py,sha256=RwzTajwds51wLD0MVlMC5kcpBnzRpreDLlazPSBZxrg,2605
 clarifai/versions.py,sha256=jctnczzfGk_S3EnVqb2FjRKfSREkNmvNEwAAa_VoKiQ,222
@@ -69,19 +69,19 @@ clarifai/runners/models/model_class.py,sha256=9JSPAr4U4K7xI0kSl-q0mHB06zknm2OR-8
 clarifai/runners/models/model_run_locally.py,sha256=OhzQbmaV8Wwgs2H0KhdDF6Z7bYSaIh4RRA0QwSiv5vY,20644
 clarifai/runners/models/model_runner.py,sha256=3vzoastQxkGRDK8T9aojDsLNBb9A3IiKm6YmbFrE9S0,6241
 clarifai/runners/models/model_servicer.py,sha256=X4715PVA5PBurRTYcwSEudg8fShGV6InAF4mmRlRcHg,2826
-clarifai/runners/models/model_upload.py,sha256=ggUa1OwqZg57C-Dagf6U22fSC4SHKZ_mB4xMSynCtPg,22411
+clarifai/runners/models/model_upload.py,sha256=pao_genthc8pW9ENFrJjd4TI3TRsWKgnceZtgD5M8sY,23829
 clarifai/runners/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 clarifai/runners/utils/const.py,sha256=eyBrj5ywuGKPF-IFipm7yjiYyLhnsKhMNZ6xF-OvykQ,1250
 clarifai/runners/utils/data_handler.py,sha256=sxy9zlAgI6ETuxCQhUgEXAn2GCsaW1GxpK6GTaMne0g,6966
 clarifai/runners/utils/data_utils.py,sha256=R1iQ82TuQ9JwxCJk8yEB1Lyb0BYVhVbWJI9YDi1zGOs,318
-clarifai/runners/utils/loader.py,sha256=O7L8Foc98CALA4FLUeQaRNw0d-y3K46lCJPXigsJd18,4858
+clarifai/runners/utils/loader.py,sha256=Q4psyvHjGPDHA6GMFLEhSSVqPRVq4vJyYyDItKn1WwU,6100
 clarifai/runners/utils/url_fetcher.py,sha256=v_8JOWmkyFAzsBulsieKX7Nfjy1Yg7wGSZeqfEvw2cg,1640
 clarifai/schema/search.py,sha256=JjTi8ammJgZZ2OGl4K6tIA4zEJ1Fr2ASZARXavI1j5c,2448
 clarifai/urls/helper.py,sha256=tjoMGGHuWX68DUB0pk4MEjrmFsClUAQj2jmVEM_Sy78,4751
 clarifai/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 clarifai/utils/cli.py,sha256=O6ukcQb05pFIgdQKWn0tL0AALAjT3U3DFRjd_GgPCvk,1918
 clarifai/utils/constants.py,sha256=MG_iHnSwNEyUZOpvsrTicNwaT4CIjmlK_Ixk_qqEX8g,142
-clarifai/utils/logging.py,sha256=rhutBRQJLtkNRz8IErNCgbIpvtl2fQ3D2otYcGqd3-Q,11565
+clarifai/utils/logging.py,sha256=Co2J4JzltrVEf-Fbll3X1n-iVxYOGXbfLVwx-X49bMg,11855
 clarifai/utils/misc.py,sha256=ptjt1NtteDT0EhrPoyQ7mgWtvoAQ-XNncQaZvNHb0KI,2253
 clarifai/utils/model_train.py,sha256=Mndqy5GNu7kjQHjDyNVyamL0hQFLGSHcWhOuPyOvr1w,8005
 clarifai/utils/evaluation/__init__.py,sha256=PYkurUrXrGevByj7RFb6CoU1iC7fllyQSfnnlo9WnY8,69
@@ -92,9 +92,9 @@ clarifai/workflows/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 clarifai/workflows/export.py,sha256=vICRhIreqDSShxLKjHNM2JwzKsf1B4fdXB0ciMcA70k,1945
 clarifai/workflows/utils.py,sha256=nGeB_yjVgUO9kOeKTg4OBBaBz-AwXI3m-huSVj-9W18,1924
 clarifai/workflows/validate.py,sha256=yJq03MaJqi5AK3alKGJJBR89xmmjAQ31sVufJUiOqY8,2556
-clarifai-11.0.2.dist-info/LICENSE,sha256=mUqF_d12-qE2n41g7C5_sq-BMLOcj6CNN-jevr15YHU,555
-clarifai-11.0.2.dist-info/METADATA,sha256=UR2QCaE64X7v_Lc3VfVyclCDTp1LUfRTScyTY4wOxrU,22456
-clarifai-11.0.2.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-clarifai-11.0.2.dist-info/entry_points.txt,sha256=X9FZ4Z-i_r2Ud1RpZ9sNIFYuu_-9fogzCMCRUD9hyX0,51
-clarifai-11.0.2.dist-info/top_level.txt,sha256=wUMdCQGjkxaynZ6nZ9FAnvBUCgp5RJUVFSy2j-KYo0s,9
-clarifai-11.0.2.dist-info/RECORD,,
+clarifai-11.0.4.dist-info/LICENSE,sha256=mUqF_d12-qE2n41g7C5_sq-BMLOcj6CNN-jevr15YHU,555
+clarifai-11.0.4.dist-info/METADATA,sha256=pBd3sHEtOUF5sOlRGEixOA3cdZ39PhqWxEiLW-e6sSs,22456
+clarifai-11.0.4.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+clarifai-11.0.4.dist-info/entry_points.txt,sha256=X9FZ4Z-i_r2Ud1RpZ9sNIFYuu_-9fogzCMCRUD9hyX0,51
+clarifai-11.0.4.dist-info/top_level.txt,sha256=wUMdCQGjkxaynZ6nZ9FAnvBUCgp5RJUVFSy2j-KYo0s,9
+clarifai-11.0.4.dist-info/RECORD,,