clarifai 10.9.1__py3-none-any.whl → 10.9.2__py3-none-any.whl

clarifai/__init__.py

@@ -1 +1 @@
-__version__ = "10.9.1"
+__version__ = "10.9.2"

clarifai/client/dataset.py

@@ -628,6 +628,28 @@ class Dataset(Lister, BaseClient):
     if delete_version:
       self.delete_version(dataset_version_id)
 
+  def merge_dataset(self, merge_dataset_id: str) -> None:
+    """Merges the another dataset into current dataset.
+
+    Args:
+        merge_dataset_id (str): The dataset ID of the dataset to merge.
+
+    Example:
+        >>> from clarifai.client.dataset import Dataset
+        >>> dataset = Dataset(dataset_id='dataset_id', user_id='user_id', app_id='app_id')
+        >>> dataset.merge_dataset(merge_dataset_id='merge_dataset_id')
+    """
+    dataset_filter = resources_pb2.Filter(
+        input=resources_pb2.Input(dataset_ids=[merge_dataset_id]))
+    query = resources_pb2.Search(query=resources_pb2.Query(filters=[dataset_filter]))
+    request = service_pb2.PostDatasetInputsRequest(
+        user_app_id=self.user_app_id, dataset_id=self.id, search=query)
+
+    response = self._grpc_request(self.STUB.PostDatasetInputs, request)
+    if response.status.code != status_code_pb2.SUCCESS:
+      raise Exception(response.status)
+    self.logger.info("\nDataset Merged\n%s", response.status)
+
   def archive_zip(self, wait: bool = True) -> str:
     """Exports the dataset to a zip file URL."""
     request = service_pb2.PutDatasetVersionExportsRequest(

clarifai/runners/dockerfile_template/Dockerfile.cuda.template

@@ -64,6 +64,10 @@ ENV CLARIFAI_NODEPOOL_ID=${CLARIFAI_NODEPOOL_ID}
 ENV CLARIFAI_COMPUTE_CLUSTER_ID=${CLARIFAI_COMPUTE_CLUSTER_ID}
 ENV CLARIFAI_API_BASE=${CLARIFAI_API_BASE}
 
+# Set the NUMBA cache dir to /tmp
+ENV NUMBA_CACHE_DIR=/tmp/numba_cache
+ENV HOME=/tmp
+
 # Set the working directory to /app
 WORKDIR /app
 

clarifai/runners/models/model_run_locally.py

@@ -22,8 +22,6 @@ class ModelRunLocally:
   def __init__(self, model_path):
     self.model_path = model_path
     self.requirements_file = os.path.join(self.model_path, "requirements.txt")
-    self.venv_dir, self.temp_dir = self.create_temp_venv()
-    self.python_executable = os.path.join(self.venv_dir, "bin", "python")
 
   def create_temp_venv(self):
     """Create a temporary virtual environment."""
@@ -32,6 +30,10 @@ class ModelRunLocally:
     venv_dir = os.path.join(temp_dir, "venv")
     venv.create(venv_dir, with_pip=True)
 
+    self.venv_dir = venv_dir
+    self.temp_dir = temp_dir
+    self.python_executable = os.path.join(venv_dir, "bin", "python")
+
     logger.info(f"Created temporary virtual environment at {venv_dir}")
     return venv_dir, temp_dir
 
@@ -125,7 +127,6 @@ class ModelRunLocally:
         nodepool_id="n/a",
         compute_cluster_id="n/a",
     )
-    runner.load_model()
 
     # send an inference.
     response = self._run_model_inference(runner)
@@ -182,6 +183,7 @@ def main():
 
   model_path = args.model_path
   manager = ModelRunLocally(model_path)
+  manager.create_temp_venv()
 
   try:
     manager.install_requirements()

clarifai/runners/models/model_upload.py

@@ -60,9 +60,6 @@ class ModelUploader:
     return config
 
   def _validate_config_checkpoints(self):
-    if not self.config.get("checkpoints"):
-      logger.info("No checkpoints specified in the config file")
-      return None
 
     assert "type" in self.config.get("checkpoints"), "No loader type specified in the config file"
     loader_type = self.config.get("checkpoints").get("type")
@@ -201,15 +198,20 @@ class ModelUploader:
     return f"{self.folder}.tar.gz"
 
   def download_checkpoints(self):
+    if not self.config.get("checkpoints"):
+      logger.info("No checkpoints specified in the config file")
+      return True
+
     repo_id, hf_token = self._validate_config_checkpoints()
-    if repo_id and hf_token:
-      loader = HuggingFaceLoader(repo_id=repo_id, token=hf_token)
-      success = loader.download_checkpoints(self.checkpoint_path)
 
-      if not success:
-        logger.error(f"Failed to download checkpoints for model {repo_id}")
-        return
+    loader = HuggingFaceLoader(repo_id=repo_id, token=hf_token)
+    success = loader.download_checkpoints(self.checkpoint_path)
+
+    if not success:
+      logger.error(f"Failed to download checkpoints for model {repo_id}")
+    else:
       logger.info(f"Downloaded checkpoints for model {repo_id}")
+    return success
 
   def _concepts_protos_from_concepts(self, concepts):
     concept_protos = []
@@ -245,15 +247,23 @@ class ModelUploader:
     model_type_id = self.config.get('model').get('model_type_id')
     if model_type_id in self.CONCEPTS_REQUIRED_MODEL_TYPE:
 
-      labels = HuggingFaceLoader.fetch_labels(self.checkpoint_path)
-      # sort the concepts by id and then update the config file
-      labels = sorted(labels.items(), key=lambda x: int(x[0]))
+      if 'concepts' in self.config:
+        labels = self.config.get('concepts')
+        logger.info(f"Found {len(labels)} concepts in the config file.")
+        for concept in labels:
+          concept_proto = json_format.ParseDict(concept, resources_pb2.Concept())
+          model_version_proto.output_info.data.concepts.append(concept_proto)
+      else:
+        labels = HuggingFaceLoader.fetch_labels(self.checkpoint_path)
+        logger.info(f"Found {len(labels)} concepts from the model checkpoints.")
+        # sort the concepts by id and then update the config file
+        labels = sorted(labels.items(), key=lambda x: int(x[0]))
 
-      config_file = os.path.join(self.folder, 'config.yaml')
-      self.hf_labels_to_config(labels, config_file)
+        config_file = os.path.join(self.folder, 'config.yaml')
+        self.hf_labels_to_config(labels, config_file)
 
-      model_version_proto.output_info.data.concepts.extend(
-          self._concepts_protos_from_concepts(labels))
+        model_version_proto.output_info.data.concepts.extend(
+            self._concepts_protos_from_concepts(labels))
     return model_version_proto
 
   def upload_model_version(self, download_checkpoints):
@@ -261,17 +271,31 @@ class ModelUploader:
     logger.info(f"Will tar it into file: {file_path}")
 
     model_type_id = self.config.get('model').get('model_type_id')
-    repo_id, hf_token = self._validate_config_checkpoints()
-
-    loader = HuggingFaceLoader(repo_id=repo_id, token=hf_token)
 
-    if not download_checkpoints and not loader.validate_download(self.checkpoint_path) and (
-        model_type_id in self.CONCEPTS_REQUIRED_MODEL_TYPE) and 'concepts' not in self.config:
-      logger.error(
-          f"Model type {model_type_id} requires concepts to be specified in the config file or download the model checkpoints to infer the concepts."
+    if (model_type_id in self.CONCEPTS_REQUIRED_MODEL_TYPE) and 'concepts' not in self.config:
+      logger.info(
+          f"Model type {model_type_id} requires concepts to be specified in the config.yaml file.."
       )
-      input("Press Enter to download the checkpoints to infer the concepts and continue...")
-      self.download_checkpoints()
+      if self.config.get("checkpoints"):
+        logger.info(
+            "Checkpoints specified in the config.yaml file, will download the HF model's config.json file to infer the concepts."
+        )
+
+        if not download_checkpoints and not HuggingFaceLoader.validate_config(
+            self.checkpoint_path):
+
+          input(
+              "Press Enter to download the HuggingFace model's config.json file to infer the concepts and continue..."
+          )
+          repo_id, hf_token = self._validate_config_checkpoints()
+          loader = HuggingFaceLoader(repo_id=repo_id, token=hf_token)
+          loader.download_config(self.checkpoint_path)
+
+      else:
+        logger.error(
+            "No checkpoints specified in the config.yaml file to infer the concepts. Please either specify the concepts directly in the config.yaml file or include a checkpoints section to download the HF model's config.json file to infer the concepts."
+        )
+        return
 
     model_version_proto = self.get_model_version_proto()
 

clarifai/runners/utils/loader.py

@@ -18,9 +18,17 @@ class HuggingFaceLoader:
         if importlib.util.find_spec("huggingface_hub") is None:
           raise ImportError(self.HF_DOWNLOAD_TEXT)
         os.environ['HF_TOKEN'] = token
+        from huggingface_hub import HfApi
+
+        api = HfApi()
+        api.whoami(token=token)
+
         subprocess.run(f'huggingface-cli login --token={os.environ["HF_TOKEN"]}', shell=True)
       except Exception as e:
-        Exception("Error setting up Hugging Face token ", e)
+        logger.error(
+            f"Error setting up Hugging Face token, please make sure you have the correct token: {e}"
+        )
+        logger.info("Continuing without Hugging Face token")
 
   def download_checkpoints(self, checkpoint_path: str):
     # throw error if huggingface_hub wasn't installed
@@ -50,6 +58,28 @@ class HuggingFaceLoader:
           return False
       return True
 
+  def download_config(self, checkpoint_path: str):
+    # throw error if huggingface_hub wasn't installed
+    try:
+      from huggingface_hub import hf_hub_download
+    except ImportError:
+      raise ImportError(self.HF_DOWNLOAD_TEXT)
+    if os.path.exists(checkpoint_path) and os.path.exists(
+        os.path.join(checkpoint_path, 'config.json')):
+      logger.info("HF model's config.json already exists")
+      return True
+    os.makedirs(checkpoint_path, exist_ok=True)
+    try:
+      is_hf_model_exists = self.validate_hf_model()
+      if not is_hf_model_exists:
+        logger.error("Model %s not found on Hugging Face" % (self.repo_id))
+        return False
+      hf_hub_download(repo_id=self.repo_id, filename='config.json', local_dir=checkpoint_path)
+    except Exception as e:
+      logger.error(f"Error downloading model's config.json {e}")
+      return False
+    return True
+
   def validate_hf_model(self,):
     # check if model exists on HF
     try:
@@ -70,6 +100,12 @@ class HuggingFaceLoader:
     return (len(checkpoint_dir_files) >= len(list_repo_files(self.repo_id))) and len(
         list_repo_files(self.repo_id)) > 0
 
+  @staticmethod
+  def validate_config(checkpoint_path: str):
+    # check if downloaded config.json exists
+    return os.path.exists(checkpoint_path) and os.path.exists(
+        os.path.join(checkpoint_path, 'config.json'))
+
   @staticmethod
   def fetch_labels(checkpoint_path: str):
     # Fetch labels for classification, detection and segmentation models

{clarifai-10.9.1.dist-info → clarifai-10.9.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: clarifai
-Version: 10.9.1
+Version: 10.9.2
 Summary: Clarifai Python SDK
 Home-page: https://github.com/Clarifai/clarifai-python
 Author: Clarifai
@@ -20,7 +20,7 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: clarifai-grpc >=10.8.8
+Requires-Dist: clarifai-grpc >=10.9.4
 Requires-Dist: clarifai-protocol >=0.0.6
 Requires-Dist: numpy >=1.22.0
 Requires-Dist: tqdm >=4.65.0

{clarifai-10.9.1.dist-info → clarifai-10.9.2.dist-info}/RECORD

@@ -1,4 +1,4 @@
-clarifai/__init__.py,sha256=1-Xk7E7-IsivFNQIvqGyYjTSA2tEoxgp679FIfjZd-w,23
+clarifai/__init__.py,sha256=pIn_ah9SsqsljuRvOukQQ4KCQQ4aaDew3A8OtaeuEqU,23
 clarifai/cli.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 clarifai/errors.py,sha256=RwzTajwds51wLD0MVlMC5kcpBnzRpreDLlazPSBZxrg,2605
 clarifai/versions.py,sha256=jctnczzfGk_S3EnVqb2FjRKfSREkNmvNEwAAa_VoKiQ,222
@@ -6,7 +6,7 @@ clarifai/client/__init__.py,sha256=xI1U0l5AZdRThvQAXCLsd9axxyFzXXJ22m8LHqVjQRU,6
 clarifai/client/app.py,sha256=6pckYme1urV2YJjLIYfeZ-vH0Z5YSQa51jzIMcEfwug,38342
 clarifai/client/base.py,sha256=hSHOqkXbSKyaRDeylMMnkhUHCAHhEqno4KI0CXGziBA,7536
 clarifai/client/compute_cluster.py,sha256=lntZDLVDhS71Yj7mZrgq5uhnAuNPUnj48i3zMSuoUpk,8693
-clarifai/client/dataset.py,sha256=oqp6ryg7IyxCZcItzownadYJKK0s1DtghHwITN71_6E,30160
+clarifai/client/dataset.py,sha256=Xt8M9sP7RLeAqqTu17-8cpupThhkU9n_bl6jXtG_25A,31148
 clarifai/client/deployment.py,sha256=4gfvUvQY9adFS98B0vP9C5fR9OnDRV2JbUIdAkMymT8,2551
 clarifai/client/input.py,sha256=cEVRytrMF1gCgwHLbXlSbPSEQN8uHpUAoKcCdyHO1pc,44406
 clarifai/client/lister.py,sha256=03KGMvs5RVyYqxLsSrWhNc34I8kiF1Ph0NeyEwu7nMU,2082
@@ -56,18 +56,18 @@ clarifai/rag/utils.py,sha256=yr1jAcbpws4vFGBqlAwPPE7v1DRba48g8gixLFw8OhQ,4070
 clarifai/runners/__init__.py,sha256=3vr4RVvN1IRy2SxJpyycAAvrUBbH-mXR7pqUmu4w36A,412
 clarifai/runners/server.py,sha256=CVLrv2DjzCvKVXcJ4SWvcFWUZq0bdlBmyEpfVlfgT2A,4902
 clarifai/runners/dockerfile_template/Dockerfile.cpu.template,sha256=B35jcpqWBP3ALa2WRtbtBg8uvDyqP_PWZnJtIeAnjT0,1222
-clarifai/runners/dockerfile_template/Dockerfile.cuda.template,sha256=TMqTZBN1exMYzjLotn17DO4Je0rg9pBapIuwdohwht8,3228
+clarifai/runners/dockerfile_template/Dockerfile.cuda.template,sha256=8uQp2sX_bIzgQk84FNlS19PwKH_l0Qi54xE7_NVxUTE,3314
 clarifai/runners/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 clarifai/runners/models/base_typed_model.py,sha256=OnAk08Lo2Y1fGiBc6JJ6UvJ8P435cTsikTNYDkStDpI,7790
 clarifai/runners/models/model_class.py,sha256=9JSPAr4U4K7xI0kSl-q0mHB06zknm2OR-8XIgBCto94,1611
-clarifai/runners/models/model_run_locally.py,sha256=35K37fq9Zkd0tautnRf9y1fvfbp_57e05-6wREBxdUI,6720
+clarifai/runners/models/model_run_locally.py,sha256=eP_QFx3fuoKbf5VyRm-rfD9k6fLrXG7u1EaWz-Qjr68,6720
 clarifai/runners/models/model_runner.py,sha256=3vzoastQxkGRDK8T9aojDsLNBb9A3IiKm6YmbFrE9S0,6241
 clarifai/runners/models/model_servicer.py,sha256=L5AuqKDZrsKOnv_Fz1Ld4-nzqehltLTsYAS7NIclm1g,2880
-clarifai/runners/models/model_upload.py,sha256=QxuRtr5dFc5_6cEPQBklACGgHVdxeIVIMYujs_ZKO28,16648
+clarifai/runners/models/model_upload.py,sha256=7Oi09Y93WUH7EKTzrKf6JQawZzLj3EnmWUDUgHOrFq8,17654
 clarifai/runners/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 clarifai/runners/utils/data_handler.py,sha256=sxy9zlAgI6ETuxCQhUgEXAn2GCsaW1GxpK6GTaMne0g,6966
 clarifai/runners/utils/data_utils.py,sha256=R1iQ82TuQ9JwxCJk8yEB1Lyb0BYVhVbWJI9YDi1zGOs,318
-clarifai/runners/utils/loader.py,sha256=uSjn36-Henlht-mOuQN3p5XPUvsFlYPBs_J2Gu3YAOA,2900
+clarifai/runners/utils/loader.py,sha256=1oktDUQA1Lpv0NiCXFwoxpp0jqqbvB7sWvpymwyWY2E,4243
 clarifai/runners/utils/url_fetcher.py,sha256=-Hwjb1SURszn7zUVwi4Of0-nrksfZy-uqT4SvPGCgSU,1446
 clarifai/schema/search.py,sha256=JjTi8ammJgZZ2OGl4K6tIA4zEJ1Fr2ASZARXavI1j5c,2448
 clarifai/urls/helper.py,sha256=tjoMGGHuWX68DUB0pk4MEjrmFsClUAQj2jmVEM_Sy78,4751
@@ -84,9 +84,9 @@ clarifai/workflows/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 clarifai/workflows/export.py,sha256=vICRhIreqDSShxLKjHNM2JwzKsf1B4fdXB0ciMcA70k,1945
 clarifai/workflows/utils.py,sha256=nGeB_yjVgUO9kOeKTg4OBBaBz-AwXI3m-huSVj-9W18,1924
 clarifai/workflows/validate.py,sha256=yJq03MaJqi5AK3alKGJJBR89xmmjAQ31sVufJUiOqY8,2556
-clarifai-10.9.1.dist-info/LICENSE,sha256=mUqF_d12-qE2n41g7C5_sq-BMLOcj6CNN-jevr15YHU,555
-clarifai-10.9.1.dist-info/METADATA,sha256=TRfuw_q3tLo09odVd5cB2JTVyCunUt1DstFmvmo7CJE,19479
-clarifai-10.9.1.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-clarifai-10.9.1.dist-info/entry_points.txt,sha256=qZOr_MIPG0dBBE1zringDJS_wXNGTAA_SQ-zcbmDHOw,82
-clarifai-10.9.1.dist-info/top_level.txt,sha256=wUMdCQGjkxaynZ6nZ9FAnvBUCgp5RJUVFSy2j-KYo0s,9
-clarifai-10.9.1.dist-info/RECORD,,
+clarifai-10.9.2.dist-info/LICENSE,sha256=mUqF_d12-qE2n41g7C5_sq-BMLOcj6CNN-jevr15YHU,555
+clarifai-10.9.2.dist-info/METADATA,sha256=BcdNjVzD08g4xW4RgxN-zkLGLsG2P590d6tAzY4f5ME,19479
+clarifai-10.9.2.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+clarifai-10.9.2.dist-info/entry_points.txt,sha256=qZOr_MIPG0dBBE1zringDJS_wXNGTAA_SQ-zcbmDHOw,82
+clarifai-10.9.2.dist-info/top_level.txt,sha256=wUMdCQGjkxaynZ6nZ9FAnvBUCgp5RJUVFSy2j-KYo0s,9
+clarifai-10.9.2.dist-info/RECORD,,