clarifai 10.9.1__tar.gz → 10.9.2__tar.gz

{clarifai-10.9.1/clarifai.egg-info → clarifai-10.9.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: clarifai
-Version: 10.9.1
+Version: 10.9.2
 Summary: Clarifai Python SDK
 Home-page: https://github.com/Clarifai/clarifai-python
 Author: Clarifai
@@ -20,7 +20,7 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: clarifai-grpc>=10.8.8
+Requires-Dist: clarifai-grpc>=10.9.4
 Requires-Dist: clarifai-protocol>=0.0.6
 Requires-Dist: numpy>=1.22.0
 Requires-Dist: tqdm>=4.65.0

clarifai-10.9.2/clarifai/__init__.py

@@ -0,0 +1 @@
+__version__ = "10.9.2"

{clarifai-10.9.1 → clarifai-10.9.2}/clarifai/client/dataset.py

@@ -628,6 +628,28 @@ class Dataset(Lister, BaseClient):
     if delete_version:
       self.delete_version(dataset_version_id)
 
+  def merge_dataset(self, merge_dataset_id: str) -> None:
+    """Merges the another dataset into current dataset.
+
+    Args:
+        merge_dataset_id (str): The dataset ID of the dataset to merge.
+
+    Example:
+        >>> from clarifai.client.dataset import Dataset
+        >>> dataset = Dataset(dataset_id='dataset_id', user_id='user_id', app_id='app_id')
+        >>> dataset.merge_dataset(merge_dataset_id='merge_dataset_id')
+    """
+    dataset_filter = resources_pb2.Filter(
+        input=resources_pb2.Input(dataset_ids=[merge_dataset_id]))
+    query = resources_pb2.Search(query=resources_pb2.Query(filters=[dataset_filter]))
+    request = service_pb2.PostDatasetInputsRequest(
+        user_app_id=self.user_app_id, dataset_id=self.id, search=query)
+
+    response = self._grpc_request(self.STUB.PostDatasetInputs, request)
+    if response.status.code != status_code_pb2.SUCCESS:
+      raise Exception(response.status)
+    self.logger.info("\nDataset Merged\n%s", response.status)
+
   def archive_zip(self, wait: bool = True) -> str:
     """Exports the dataset to a zip file URL."""
     request = service_pb2.PutDatasetVersionExportsRequest(

{clarifai-10.9.1 → clarifai-10.9.2}/clarifai/runners/dockerfile_template/Dockerfile.cuda.template

@@ -64,6 +64,10 @@ ENV CLARIFAI_NODEPOOL_ID=${CLARIFAI_NODEPOOL_ID}
 ENV CLARIFAI_COMPUTE_CLUSTER_ID=${CLARIFAI_COMPUTE_CLUSTER_ID}
 ENV CLARIFAI_API_BASE=${CLARIFAI_API_BASE}
 
+# Set the NUMBA cache dir to /tmp
+ENV NUMBA_CACHE_DIR=/tmp/numba_cache
+ENV HOME=/tmp
+
 # Set the working directory to /app
 WORKDIR /app
 

{clarifai-10.9.1 → clarifai-10.9.2}/clarifai/runners/models/model_run_locally.py

@@ -22,8 +22,6 @@ class ModelRunLocally:
   def __init__(self, model_path):
     self.model_path = model_path
     self.requirements_file = os.path.join(self.model_path, "requirements.txt")
-    self.venv_dir, self.temp_dir = self.create_temp_venv()
-    self.python_executable = os.path.join(self.venv_dir, "bin", "python")
 
   def create_temp_venv(self):
     """Create a temporary virtual environment."""
@@ -32,6 +30,10 @@ class ModelRunLocally:
     venv_dir = os.path.join(temp_dir, "venv")
     venv.create(venv_dir, with_pip=True)
 
+    self.venv_dir = venv_dir
+    self.temp_dir = temp_dir
+    self.python_executable = os.path.join(venv_dir, "bin", "python")
+
     logger.info(f"Created temporary virtual environment at {venv_dir}")
     return venv_dir, temp_dir
 
@@ -125,7 +127,6 @@ class ModelRunLocally:
         nodepool_id="n/a",
         compute_cluster_id="n/a",
     )
-    runner.load_model()
 
     # send an inference.
     response = self._run_model_inference(runner)
@@ -182,6 +183,7 @@ def main():
 
   model_path = args.model_path
   manager = ModelRunLocally(model_path)
+  manager.create_temp_venv()
 
   try:
     manager.install_requirements()

{clarifai-10.9.1 → clarifai-10.9.2}/clarifai/runners/models/model_upload.py

@@ -60,9 +60,6 @@ class ModelUploader:
     return config
 
   def _validate_config_checkpoints(self):
-    if not self.config.get("checkpoints"):
-      logger.info("No checkpoints specified in the config file")
-      return None
 
     assert "type" in self.config.get("checkpoints"), "No loader type specified in the config file"
     loader_type = self.config.get("checkpoints").get("type")
@@ -201,15 +198,20 @@ class ModelUploader:
     return f"{self.folder}.tar.gz"
 
   def download_checkpoints(self):
+    if not self.config.get("checkpoints"):
+      logger.info("No checkpoints specified in the config file")
+      return True
+
     repo_id, hf_token = self._validate_config_checkpoints()
-    if repo_id and hf_token:
-      loader = HuggingFaceLoader(repo_id=repo_id, token=hf_token)
-      success = loader.download_checkpoints(self.checkpoint_path)
 
-      if not success:
-        logger.error(f"Failed to download checkpoints for model {repo_id}")
-        return
+    loader = HuggingFaceLoader(repo_id=repo_id, token=hf_token)
+    success = loader.download_checkpoints(self.checkpoint_path)
+
+    if not success:
+      logger.error(f"Failed to download checkpoints for model {repo_id}")
+    else:
       logger.info(f"Downloaded checkpoints for model {repo_id}")
+    return success
 
   def _concepts_protos_from_concepts(self, concepts):
     concept_protos = []
@@ -245,15 +247,23 @@ class ModelUploader:
     model_type_id = self.config.get('model').get('model_type_id')
     if model_type_id in self.CONCEPTS_REQUIRED_MODEL_TYPE:
 
-      labels = HuggingFaceLoader.fetch_labels(self.checkpoint_path)
-      # sort the concepts by id and then update the config file
-      labels = sorted(labels.items(), key=lambda x: int(x[0]))
+      if 'concepts' in self.config:
+        labels = self.config.get('concepts')
+        logger.info(f"Found {len(labels)} concepts in the config file.")
+        for concept in labels:
+          concept_proto = json_format.ParseDict(concept, resources_pb2.Concept())
+          model_version_proto.output_info.data.concepts.append(concept_proto)
+      else:
+        labels = HuggingFaceLoader.fetch_labels(self.checkpoint_path)
+        logger.info(f"Found {len(labels)} concepts from the model checkpoints.")
+        # sort the concepts by id and then update the config file
+        labels = sorted(labels.items(), key=lambda x: int(x[0]))
 
-      config_file = os.path.join(self.folder, 'config.yaml')
-      self.hf_labels_to_config(labels, config_file)
+        config_file = os.path.join(self.folder, 'config.yaml')
+        self.hf_labels_to_config(labels, config_file)
 
-      model_version_proto.output_info.data.concepts.extend(
-          self._concepts_protos_from_concepts(labels))
+        model_version_proto.output_info.data.concepts.extend(
+            self._concepts_protos_from_concepts(labels))
     return model_version_proto
 
   def upload_model_version(self, download_checkpoints):
@@ -261,17 +271,31 @@ class ModelUploader:
     logger.info(f"Will tar it into file: {file_path}")
 
     model_type_id = self.config.get('model').get('model_type_id')
-    repo_id, hf_token = self._validate_config_checkpoints()
-
-    loader = HuggingFaceLoader(repo_id=repo_id, token=hf_token)
 
-    if not download_checkpoints and not loader.validate_download(self.checkpoint_path) and (
-        model_type_id in self.CONCEPTS_REQUIRED_MODEL_TYPE) and 'concepts' not in self.config:
-      logger.error(
-          f"Model type {model_type_id} requires concepts to be specified in the config file or download the model checkpoints to infer the concepts."
+    if (model_type_id in self.CONCEPTS_REQUIRED_MODEL_TYPE) and 'concepts' not in self.config:
+      logger.info(
+          f"Model type {model_type_id} requires concepts to be specified in the config.yaml file.."
       )
-      input("Press Enter to download the checkpoints to infer the concepts and continue...")
-      self.download_checkpoints()
+      if self.config.get("checkpoints"):
+        logger.info(
+            "Checkpoints specified in the config.yaml file, will download the HF model's config.json file to infer the concepts."
+        )
+
+        if not download_checkpoints and not HuggingFaceLoader.validate_config(
+            self.checkpoint_path):
+
+          input(
+              "Press Enter to download the HuggingFace model's config.json file to infer the concepts and continue..."
+          )
+          repo_id, hf_token = self._validate_config_checkpoints()
+          loader = HuggingFaceLoader(repo_id=repo_id, token=hf_token)
+          loader.download_config(self.checkpoint_path)
+
+      else:
+        logger.error(
+            "No checkpoints specified in the config.yaml file to infer the concepts. Please either specify the concepts directly in the config.yaml file or include a checkpoints section to download the HF model's config.json file to infer the concepts."
+        )
+        return
 
     model_version_proto = self.get_model_version_proto()
 

{clarifai-10.9.1 → clarifai-10.9.2}/clarifai/runners/utils/loader.py

@@ -18,9 +18,17 @@ class HuggingFaceLoader:
         if importlib.util.find_spec("huggingface_hub") is None:
           raise ImportError(self.HF_DOWNLOAD_TEXT)
         os.environ['HF_TOKEN'] = token
+        from huggingface_hub import HfApi
+
+        api = HfApi()
+        api.whoami(token=token)
+
         subprocess.run(f'huggingface-cli login --token={os.environ["HF_TOKEN"]}', shell=True)
       except Exception as e:
-        Exception("Error setting up Hugging Face token ", e)
+        logger.error(
+            f"Error setting up Hugging Face token, please make sure you have the correct token: {e}"
+        )
+        logger.info("Continuing without Hugging Face token")
 
   def download_checkpoints(self, checkpoint_path: str):
     # throw error if huggingface_hub wasn't installed
@@ -50,6 +58,28 @@ class HuggingFaceLoader:
           return False
       return True
 
+  def download_config(self, checkpoint_path: str):
+    # throw error if huggingface_hub wasn't installed
+    try:
+      from huggingface_hub import hf_hub_download
+    except ImportError:
+      raise ImportError(self.HF_DOWNLOAD_TEXT)
+    if os.path.exists(checkpoint_path) and os.path.exists(
+        os.path.join(checkpoint_path, 'config.json')):
+      logger.info("HF model's config.json already exists")
+      return True
+    os.makedirs(checkpoint_path, exist_ok=True)
+    try:
+      is_hf_model_exists = self.validate_hf_model()
+      if not is_hf_model_exists:
+        logger.error("Model %s not found on Hugging Face" % (self.repo_id))
+        return False
+      hf_hub_download(repo_id=self.repo_id, filename='config.json', local_dir=checkpoint_path)
+    except Exception as e:
+      logger.error(f"Error downloading model's config.json {e}")
+      return False
+    return True
+
   def validate_hf_model(self,):
     # check if model exists on HF
     try:
@@ -70,6 +100,12 @@ class HuggingFaceLoader:
     return (len(checkpoint_dir_files) >= len(list_repo_files(self.repo_id))) and len(
         list_repo_files(self.repo_id)) > 0
 
+  @staticmethod
+  def validate_config(checkpoint_path: str):
+    # check if downloaded config.json exists
+    return os.path.exists(checkpoint_path) and os.path.exists(
+        os.path.join(checkpoint_path, 'config.json'))
+
   @staticmethod
   def fetch_labels(checkpoint_path: str):
     # Fetch labels for classification, detection and segmentation models

{clarifai-10.9.1 → clarifai-10.9.2/clarifai.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: clarifai
-Version: 10.9.1
+Version: 10.9.2
 Summary: Clarifai Python SDK
 Home-page: https://github.com/Clarifai/clarifai-python
 Author: Clarifai
@@ -20,7 +20,7 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: clarifai-grpc>=10.8.8
+Requires-Dist: clarifai-grpc>=10.9.4
 Requires-Dist: clarifai-protocol>=0.0.6
 Requires-Dist: numpy>=1.22.0
 Requires-Dist: tqdm>=4.65.0

{clarifai-10.9.1 → clarifai-10.9.2}/clarifai.egg-info/requires.txt

@@ -1,4 +1,4 @@
-clarifai-grpc>=10.8.8
+clarifai-grpc>=10.9.4
 clarifai-protocol>=0.0.6
 numpy>=1.22.0
 tqdm>=4.65.0

{clarifai-10.9.1 → clarifai-10.9.2}/requirements.txt

@@ -1,4 +1,4 @@
-clarifai-grpc>=10.8.8
+clarifai-grpc>=10.9.4
 clarifai-protocol>=0.0.6
 numpy>=1.22.0
 tqdm>=4.65.0

{clarifai-10.9.1 → clarifai-10.9.2}/tests/test_data_upload.py

@@ -15,6 +15,7 @@ CREATE_APP_USER_ID = os.environ["CLARIFAI_USER_ID"]
 NOW = uuid.uuid4().hex[:10]
 CREATE_APP_ID = f"ci_input_app_{NOW}"
 CREATE_DATASET_ID = "ci_input_test_dataset"
+CREATE_DATASET_NEW_ID = "ci_input_test_dataset_new"
 
 #assets
 IMAGE_URL = "https://samples.clarifai.com/metro-north.jpg"
@@ -60,6 +61,7 @@ class Testdataupload:
     self.app = create_app()
     self.input_object = self.app.inputs()
     self.dataset = self.app.create_dataset(dataset_id=CREATE_DATASET_ID)
+    self.dataset_new = self.app.create_dataset(dataset_id=CREATE_DATASET_NEW_ID)
 
   def test_upload_image_url(self, caplog):
     with caplog.at_level(logging.INFO):
@@ -265,6 +267,11 @@ class Testdataupload:
     assert len(uploaded_inputs) == 10  # 10 inputs are uploaded from the folder
     assert len(annotations) == 10  # Test for list annotations
 
+  def test_merge_datasets(self, caplog):
+    with caplog.at_level(logging.INFO):
+      self.dataset_new.merge_dataset(merge_dataset_id=CREATE_DATASET_ID)
+      assert "SUCCESS" in caplog.text
+
   def test_export_dataset(self):
     dataset_demo_version = self.dataset.create_version()
     time.sleep(5)

clarifai-10.9.1/clarifai/__init__.py

@@ -1 +0,0 @@
-__version__ = "10.9.1"