clarifai 10.0.0__py3-none-any.whl → 10.1.0__py3-none-any.whl

clarifai/client/base.py

@@ -118,8 +118,15 @@ class BaseClient:
             value_s = struct_pb2.Struct()
             value_s.update(value)
             value = value_s
+          elif key == 'metrics':
+            continue
           elif key in ['metadata']:
-            continue  # TODO Fix "app_duplication"
+            if isinstance(value, dict) and value != {}:
+              value_s = struct_pb2.Struct()
+              value_s.update(value)
+              value = value_s
+            else:
+              continue
           new_item[key] = convert_recursive(value)
         return new_item
       elif isinstance(item, list):

clarifai/client/dataset.py

@@ -2,11 +2,13 @@ import os
 import time
 import uuid
 from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime
 from multiprocessing import cpu_count
-from typing import Generator, List, Tuple, Type, TypeVar, Union
+from typing import Dict, Generator, List, Optional, Tuple, Type, TypeVar, Union
 
 import requests
 from clarifai_grpc.grpc.api import resources_pb2, service_pb2
+from clarifai_grpc.grpc.api.service_pb2 import MultiInputResponse
 from clarifai_grpc.grpc.api.status import status_code_pb2, status_pb2
 from google.protobuf.json_format import MessageToDict
 from requests.adapters import HTTPAdapter, Retry
@@ -25,7 +27,7 @@ from clarifai.datasets.upload.text import TextClassificationDataset
 from clarifai.datasets.upload.utils import DisplayUploadStatus
 from clarifai.errors import UserError
 from clarifai.urls.helper import ClarifaiUrlHelper
-from clarifai.utils.logging import get_logger
+from clarifai.utils.logging import add_file_handler, get_logger
 from clarifai.utils.misc import BackoffIterator, Chunker
 
 ClarifaiDatasetType = TypeVar('ClarifaiDatasetType', VisualClassificationDataset,
@@ -65,7 +67,7 @@ class Dataset(Lister, BaseClient):
     self.batch_size = 128  # limit max protos in a req
     self.task = None  # Upload dataset type
     self.input_object = Inputs(user_id=self.user_id, app_id=self.app_id, pat=pat)
-    self.logger = get_logger(logger_level="INFO")
+    self.logger = get_logger(logger_level="INFO", name=__name__)
     BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id, base=base_url, pat=pat)
     Lister.__init__(self)
 
@@ -194,13 +196,17 @@ class Dataset(Lister, BaseClient):
 
     return retry_annot_upload
 
-  def _delete_failed_inputs(self, batch_input_ids: List[int],
-                            dataset_obj: ClarifaiDatasetType) -> Tuple[List[int], List[int]]:
+  def _delete_failed_inputs(
+      self,
+      batch_input_ids: List[int],
+      dataset_obj: ClarifaiDatasetType,
+      upload_response: MultiInputResponse = None) -> Tuple[List[int], List[int]]:
     """Delete failed input ids from clarifai platform dataset.
 
     Args:
       batch_input_ids: batch input ids
       dataset_obj: ClarifaiDataset object
+      upload_response: upload response proto
 
     Returns:
       success_inputs: upload success input ids
@@ -220,7 +226,19 @@ class Dataset(Lister, BaseClient):
     success_inputs = response_dict.get('inputs', [])
 
     success_input_ids = [input.get('id') for input in success_inputs]
-    failed_input_ids = list(set(input_ids) - set(success_input_ids))
+    failed_input_ids = list(set(input_ids) - set(success_input_ids.copy()))
+    #check duplicate input ids
+    duplicate_input_ids = [
+        input.id for input in upload_response.inputs
+        if input.status.details == 'Input has a duplicate ID.'
+    ]  #handling duplicte ID failures.
+    if duplicate_input_ids:
+      success_input_ids = list(set(success_input_ids.copy()) - set(duplicate_input_ids.copy()))
+      failed_input_ids = list(set(failed_input_ids) - set(duplicate_input_ids))
+      self.logger.warning(
+          f"Upload Failed for {len(duplicate_input_ids)} inputs in current batch: Duplicate input ids: {duplicate_input_ids}"
+      )
+
     #delete failed inputs
     self._grpc_request(
         self.STUB.DeleteInputs,
@@ -228,8 +246,9 @@ class Dataset(Lister, BaseClient):
     )
     return [input_ids[id] for id in success_input_ids], [input_ids[id] for id in failed_input_ids]
 
-  def _upload_inputs_annotations(self, batch_input_ids: List[int], dataset_obj: ClarifaiDatasetType
-                                ) -> Tuple[List[int], List[resources_pb2.Annotation]]:
+  def _upload_inputs_annotations(
+      self, batch_input_ids: List[int], dataset_obj: ClarifaiDatasetType
+  ) -> Tuple[List[int], List[resources_pb2.Annotation], MultiInputResponse]:
     """Uploads batch of inputs and annotations concurrently to clarifai platform dataset.
 
     Args:
@@ -239,20 +258,22 @@ class Dataset(Lister, BaseClient):
     Returns:
       failed_input_ids: failed input ids
       retry_annot_protos: failed annot protos
+      response: upload response proto
     """
     input_protos, _ = dataset_obj.get_protos(batch_input_ids)
-    input_job_id = self.input_object.upload_inputs(inputs=input_protos, show_log=False)
+    input_job_id, _response = self.input_object.upload_inputs(inputs=input_protos, show_log=False)
     retry_annot_protos = []
 
     self.input_object._wait_for_inputs(input_job_id)
-    success_input_ids, failed_input_ids = self._delete_failed_inputs(batch_input_ids, dataset_obj)
+    success_input_ids, failed_input_ids = self._delete_failed_inputs(batch_input_ids, dataset_obj,
+                                                                     _response)
 
-    if self.task in ["visual_detection", "visual_segmentation"]:
+    if self.task in ["visual_detection", "visual_segmentation"] and success_input_ids:
       _, annotation_protos = dataset_obj.get_protos(success_input_ids)
       chunked_annotation_protos = Chunker(annotation_protos, self.batch_size).chunk()
       retry_annot_protos.extend(self._concurrent_annot_upload(chunked_annotation_protos))
 
-    return failed_input_ids, retry_annot_protos
+    return failed_input_ids, retry_annot_protos, _response
 
   def _retry_uploads(self, failed_input_ids: List[int],
                      retry_annot_protos: List[resources_pb2.Annotation],
@@ -265,7 +286,25 @@ class Dataset(Lister, BaseClient):
       dataset_obj: ClarifaiDataset object
     """
     if failed_input_ids:
-      self._upload_inputs_annotations(failed_input_ids, dataset_obj)
+      retry_input_ids = [dataset_obj.all_input_ids[id] for id in failed_input_ids]
+      #Log Retrying inputs
+      self.logger.warning(
+          f"Retrying upload for {len(failed_input_ids)} inputs in current batch: {retry_input_ids}"
+      )
+      failed_retrying_inputs, _, retry_response = self._upload_inputs_annotations(
+          failed_input_ids, dataset_obj)
+      #Log failed inputs
+      if failed_retrying_inputs:
+        failed_retrying_input_ids = [
+            dataset_obj.all_input_ids[id] for id in failed_retrying_inputs
+        ]
+        failed_inputs_logs = {
+            input.id: input.status.details
+            for input in retry_response.inputs if input.id in failed_retrying_input_ids
+        }
+        self.logger.warning(
+            f"Failed to upload {len(failed_retrying_inputs)} inputs in current batch: {failed_inputs_logs}"
+        )
     if retry_annot_protos:
       chunked_annotation_protos = Chunker(retry_annot_protos, self.batch_size).chunk()
       _ = self._concurrent_annot_upload(chunked_annotation_protos)
@@ -287,21 +326,27 @@ class Dataset(Lister, BaseClient):
         ]
 
         for job in as_completed(futures):
-          retry_input_ids, retry_annot_protos = job.result()
+          retry_input_ids, retry_annot_protos, _ = job.result()
           self._retry_uploads(retry_input_ids, retry_annot_protos, dataset_obj)
           progress.update()
 
   def upload_dataset(self,
                      dataloader: Type[ClarifaiDataLoader],
                      batch_size: int = 32,
-                     get_upload_status: bool = False) -> None:
+                     get_upload_status: bool = False,
+                     log_warnings: bool = False) -> None:
     """Uploads a dataset to the app.
 
     Args:
       dataloader (Type[ClarifaiDataLoader]): ClarifaiDataLoader object
       batch_size (int): batch size for concurrent upload of inputs and annotations (max: 128)
       get_upload_status (bool): True if you want to get the upload status of the dataset
+      log_warnings (bool): True if you want to save log warnings in a file
     """
+    #add file handler to log warnings
+    if log_warnings:
+      add_file_handler(self.logger, f"Dataset_Upload{str(int(datetime.now().timestamp()))}.log")
+    #set batch size and task
     self.batch_size = min(self.batch_size, batch_size)
     self.task = dataloader.task
     if self.task not in DATASET_UPLOAD_TASKS:
@@ -321,10 +366,13 @@ class Dataset(Lister, BaseClient):
     else:  # visual_classification & visual_captioning
       dataset_obj = VisualClassificationDataset(dataloader, self.id)
 
+    if get_upload_status:
+      pre_upload_stats = self.get_upload_status(pre_upload=True)
+
     self._data_upload(dataset_obj)
 
     if get_upload_status:
-      self.get_upload_status(dataloader)
+      self.get_upload_status(dataloader=dataloader, pre_upload_stats=pre_upload_stats)
 
   def upload_from_csv(self,
                       csv_path: str,
@@ -398,16 +446,21 @@ class Dataset(Lister, BaseClient):
           folder_path=folder_path, dataset_id=self.id, labels=labels)
     self.input_object._bulk_upload(inputs=input_protos, batch_size=batch_size)
 
-  def get_upload_status(self,
-                        dataloader: Type[ClarifaiDataLoader],
-                        delete_version: bool = False,
-                        timeout: int = 600) -> None:
+  def get_upload_status(
+      self,
+      dataloader: Type[ClarifaiDataLoader] = None,
+      delete_version: bool = False,
+      timeout: int = 600,
+      pre_upload_stats: Tuple[Dict[str, int], Dict[str, int]] = None,
+      pre_upload: bool = False) -> Optional[Tuple[Dict[str, int], Dict[str, int]]]:
     """Creates a new dataset version and displays the upload status of the dataset.
 
     Args:
         dataloader (Type[ClarifaiDataLoader]): ClarifaiDataLoader object
         delete_version (bool): True if you want to delete the version after getting the upload status
         timeout (int): Timeout in seconds for getting the upload status. Default is 600 seconds.
+        pre_upload_stats (Tuple[Dict[str, int], Dict[str, int]]): The pre upload stats for the dataset.
+        pre_upload (bool): True if you want to get the pre upload stats for the dataset.
 
     Example:
         >>> from clarifai.client.dataset import Dataset
@@ -450,9 +503,12 @@ class Dataset(Lister, BaseClient):
           raise UserError(
               "Dataset metrics are taking too long to process. Please try again later.")
         break
+    #get pre upload stats
+    if pre_upload:
+      return DisplayUploadStatus.get_dataset_version_stats(dataset_metrics_response)
 
     dataset_info_dict = dict(user_id=self.user_id, app_id=self.app_id, dataset_id=self.id)
-    DisplayUploadStatus(dataloader, dataset_metrics_response, dataset_info_dict)
+    DisplayUploadStatus(dataloader, dataset_metrics_response, dataset_info_dict, pre_upload_stats)
 
     if delete_version:
       self.delete_version(dataset_version_id)

clarifai/client/input.py

@@ -660,15 +660,15 @@ class Inputs(Lister, BaseClient):
         user_app_id=self.user_app_id, inputs=inputs, inputs_add_job_id=input_job_id)
     response = self._grpc_request(self.STUB.PostInputs, request)
     if response.status.code != status_code_pb2.SUCCESS:
-      try:
-        self.logger.warning(response.inputs[0].status)
-      except IndexError:
-        self.logger.warning(response.status)
+      if show_log:
+        self.logger.warning(response)
+      else:
+        return input_job_id, response
     else:
       if show_log:
         self.logger.info("\nInputs Uploaded\n%s", response.status)
 
-    return input_job_id
+    return input_job_id, response
 
   def upload_annotations(self, batch_annot: List[resources_pb2.Annotation], show_log: bool = True
                         ) -> Union[List[resources_pb2.Annotation], List[None]]:
@@ -705,7 +705,7 @@ class Inputs(Lister, BaseClient):
     Returns:
         input_job_id: job id for the upload request.
     """
-    input_job_id = self.upload_inputs(inputs, False)
+    input_job_id, _ = self.upload_inputs(inputs, False)
     self._wait_for_inputs(input_job_id)
     failed_inputs = self._delete_failed_inputs(inputs)
 

clarifai/client/model.py

@@ -53,7 +53,7 @@ class Model(Lister, BaseClient):
       kwargs = {'user_id': user_id, 'app_id': app_id}
     self.kwargs = {**kwargs, 'id': model_id, 'model_version': model_version,}
     self.model_info = resources_pb2.Model(**self.kwargs)
-    self.logger = get_logger(logger_level="INFO")
+    self.logger = get_logger(logger_level="INFO", name=__name__)
     self.training_params = {}
     BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id, base=base_url, pat=pat)
     Lister.__init__(self)

clarifai/client/module.py

@@ -40,7 +40,7 @@ class Module(Lister, BaseClient):
 
     self.kwargs = {**kwargs, 'id': module_id, 'module_version': module_version}
     self.module_info = resources_pb2.Module(**self.kwargs)
-    self.logger = get_logger(logger_level="INFO")
+    self.logger = get_logger(logger_level="INFO", name=__name__)
     BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id, base=base_url, pat=pat)
     Lister.__init__(self)
 

clarifai/client/workflow.py

@@ -54,7 +54,7 @@ class Workflow(Lister, BaseClient):
     self.kwargs = {**kwargs, 'id': workflow_id, 'version': workflow_version}
     self.output_config = output_config
     self.workflow_info = resources_pb2.Workflow(**self.kwargs)
-    self.logger = get_logger(logger_level="INFO")
+    self.logger = get_logger(logger_level="INFO", name=__name__)
     BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id, base=base_url, pat=pat)
     Lister.__init__(self)
 

clarifai/datasets/upload/features.py

@@ -20,6 +20,7 @@ class VisualClassificationFeatures:
   geo_info: Optional[List[float]] = None  #[Longitude, Latitude]
   id: Optional[int] = None  # image_id
   metadata: Optional[dict] = None
+  image_bytes: Optional[bytes] = None
 
 
 @dataclass
@@ -31,6 +32,7 @@ class VisualDetectionFeatures:
   geo_info: Optional[List[float]] = None  #[Longitude, Latitude]
   id: Optional[int] = None  # image_id
   metadata: Optional[dict] = None
+  image_bytes: Optional[bytes] = None
 
 
 @dataclass
@@ -42,3 +44,4 @@ class VisualSegmentationFeatures:
   geo_info: Optional[List[float]] = None  #[Longitude, Latitude]
   id: Optional[int] = None  # image_id
   metadata: Optional[dict] = None
+  image_bytes: Optional[bytes] = None

clarifai/datasets/upload/image.py

@@ -1,4 +1,5 @@
 import os
+import uuid
 from concurrent.futures import ThreadPoolExecutor
 from typing import List, Tuple, Type
 
@@ -31,22 +32,34 @@ class VisualClassificationDataset(ClarifaiDataset):
       image_path = data_item.image_path
       labels = data_item.labels if isinstance(data_item.labels,
                                               list) else [data_item.labels]  # clarifai concept
-      input_id = f"{self.dataset_id}-{id}" if data_item.id is None else f"{self.dataset_id}-{str(data_item.id)}"
+      input_id = f"{self.dataset_id}-{uuid.uuid4().hex[:8]}" if data_item.id is None else f"{self.dataset_id}-{str(data_item.id)}"
       geo_info = data_item.geo_info
       if data_item.metadata is not None:
         metadata.update(data_item.metadata)
-      else:
+      elif image_path is not None:
         metadata.update({"filename": os.path.basename(image_path)})
+      else:
+        metadata = None
 
       self.all_input_ids[id] = input_id
-      input_protos.append(
-          Inputs.get_input_from_file(
-              input_id=input_id,
-              image_file=image_path,
-              dataset_id=self.dataset_id,
-              labels=labels,
-              geo_info=geo_info,
-              metadata=metadata))
+      if data_item.image_bytes is not None:
+        input_protos.append(
+            Inputs.get_input_from_bytes(
+                input_id=input_id,
+                image_bytes=data_item.image_bytes,
+                dataset_id=self.dataset_id,
+                labels=labels,
+                geo_info=geo_info,
+                metadata=metadata))
+      else:
+        input_protos.append(
+            Inputs.get_input_from_file(
+                input_id=input_id,
+                image_file=image_path,
+                dataset_id=self.dataset_id,
+                labels=labels,
+                geo_info=geo_info,
+                metadata=metadata))
 
     with ThreadPoolExecutor(max_workers=4) as executor:
       futures = [executor.submit(process_data_item, id) for id in batch_input_ids]
@@ -79,7 +92,7 @@ class VisualDetectionDataset(ClarifaiDataset):
       image = data_item.image_path
       labels = data_item.labels  # list:[l1,...,ln]
       bboxes = data_item.bboxes  # [[xmin,ymin,xmax,ymax],...,[xmin,ymin,xmax,ymax]]
-      input_id = f"{self.dataset_id}-{id}" if data_item.id is None else f"{self.dataset_id}-{str(data_item.id)}"
+      input_id = f"{self.dataset_id}-{uuid.uuid4().hex[:8]}" if data_item.id is None else f"{self.dataset_id}-{str(data_item.id)}"
       if data_item.metadata is not None:
         metadata.update(data_item.metadata)
       else:
@@ -87,13 +100,22 @@ class VisualDetectionDataset(ClarifaiDataset):
       geo_info = data_item.geo_info
 
       self.all_input_ids[id] = input_id
-      input_protos.append(
-          Inputs.get_input_from_file(
-              input_id=input_id,
-              image_file=image,
-              dataset_id=self.dataset_id,
-              geo_info=geo_info,
-              metadata=metadata))
+      if data_item.image_bytes is not None:
+        input_protos.append(
+            Inputs.get_input_from_bytes(
+                input_id=input_id,
+                image_bytes=data_item.image_bytes,
+                dataset_id=self.dataset_id,
+                geo_info=geo_info,
+                metadata=metadata))
+      else:
+        input_protos.append(
+            Inputs.get_input_from_file(
+                input_id=input_id,
+                image_file=image,
+                dataset_id=self.dataset_id,
+                geo_info=geo_info,
+                metadata=metadata))
       # iter over bboxes and labels
       # one id could have more than one bbox and label
       for i in range(len(bboxes)):
@@ -131,7 +153,7 @@ class VisualSegmentationDataset(ClarifaiDataset):
       image = data_item.image_path
       labels = data_item.labels
       _polygons = data_item.polygons  # list of polygons: [[[x,y],...,[x,y]],...]
-      input_id = f"{self.dataset_id}-{id}" if data_item.id is None else f"{self.dataset_id}-{str(data_item.id)}"
+      input_id = f"{self.dataset_id}-{uuid.uuid4().hex[:8]}" if data_item.id is None else f"{self.dataset_id}-{str(data_item.id)}"
       if data_item.metadata is not None:
         metadata.update(data_item.metadata)
       else:
@@ -139,13 +161,22 @@ class VisualSegmentationDataset(ClarifaiDataset):
       geo_info = data_item.geo_info
 
       self.all_input_ids[id] = input_id
-      input_protos.append(
-          Inputs.get_input_from_file(
-              input_id=input_id,
-              image_file=image,
-              dataset_id=self.dataset_id,
-              geo_info=geo_info,
-              metadata=metadata))
+      if data_item.image_bytes is not None:
+        input_protos.append(
+            Inputs.get_input_from_bytes(
+                input_id=input_id,
+                image_bytes=data_item.image_bytes,
+                dataset_id=self.dataset_id,
+                geo_info=geo_info,
+                metadata=metadata))
+      else:
+        input_protos.append(
+            Inputs.get_input_from_file(
+                input_id=input_id,
+                image_file=image,
+                dataset_id=self.dataset_id,
+                geo_info=geo_info,
+                metadata=metadata))
 
       ## Iterate over each masked image and create a proto for upload to clarifai
       ## The length of masks/polygons-list and labels must be equal

clarifai/datasets/upload/loaders/xview_detection.py

@@ -47,6 +47,10 @@ class xviewDetectionDataLoader(ClarifaiDataLoader):
 
     self.load_data()
 
+  @property
+  def task(self):
+    return "visual_detection"
+
   def compress_tiff(self, img_path: str) -> None:
     """Compress tiff image"""
     img_comp_path = os.path.join(self.img_comp_dir, os.path.basename(img_path))

clarifai/datasets/upload/utils.py

@@ -53,16 +53,19 @@ class DisplayUploadStatus:
 
   def __init__(self, dataloader: ClarifaiDataLoader,
                dataset_metrics_response: Type[MultiDatasetVersionMetricsGroupResponse],
-               dataset_info_dict: Dict[str, str]) -> None:
+               dataset_info_dict: Dict[str, str],
+               pre_upload_stats: Tuple[Dict[str, int], Dict[str, int]]) -> None:
     """Initialize the class.
     Args:
       dataloader: ClarifaiDataLoader object
       dataset_metrics_response: The dataset version metrics response from the server.
       dataset_info_dict: The dataset info dictionary.
+      pre_upload_stats: The pre upload stats for the dataset.
     """
     self.dataloader = dataloader
     self.dataset_metrics_response = dataset_metrics_response
     self.dataset_info_dict = dataset_info_dict
+    self.pre_upload_stats = pre_upload_stats
 
     self.display()
 
@@ -71,7 +74,18 @@ class DisplayUploadStatus:
     from rich.console import Console
 
     local_inputs_count, local_annotations_dict = self.get_dataloader_stats()
-    uploaded_inputs_dict, uploaded_annotations_dict = self.get_uploaded_dataset_stats()
+    uploaded_inputs_dict, uploaded_annotations_dict = self.get_dataset_version_stats(
+        self.dataset_metrics_response)
+
+    # Subtract the pre upload stats from the uploaded stats
+    uploaded_inputs_dict = {
+        key: int(uploaded_inputs_dict[key]) - int(self.pre_upload_stats[0].get(key, 0))
+        for key in uploaded_inputs_dict
+    }
+    uploaded_annotations_dict = {
+        key: uploaded_annotations_dict[key] - self.pre_upload_stats[1].get(key, 0)
+        for key in uploaded_annotations_dict
+    }
 
     self.local_annotations_count = sum(local_annotations_dict.values())
     self.uploaded_annotations_count = sum(uploaded_annotations_dict.values())
@@ -99,9 +113,6 @@ class DisplayUploadStatus:
     """
     from clarifai.constants.dataset import DATASET_UPLOAD_TASKS
 
-    if not isinstance(self.dataloader, ClarifaiDataLoader):
-      raise UserError("Dataloader is not an instance of ClarifaiDataLoader")
-
     task = self.dataloader.task
     if task not in DATASET_UPLOAD_TASKS:
       raise UserError(
@@ -113,8 +124,13 @@ class DisplayUploadStatus:
       local_annotations_dict[key] += len(getattr(self.dataloader[i], attr))
     return local_inputs_count, local_annotations_dict
 
-  def get_uploaded_dataset_stats(self) -> Tuple[Dict[str, int], Dict[str, int]]:
+  @staticmethod
+  def get_dataset_version_stats(
+      dataset_metrics_response: Type[MultiDatasetVersionMetricsGroupResponse]
+  ) -> Tuple[Dict[str, int], Dict[str, int]]:
     """Parse the response from the server for the dataset version metrics groups.
+    Args:
+      dataset_metrics_response: The dataset version metrics response from the server.
 
     Returns:
       uploaded_inputs_dict (Dict[str, int]): The input statistics for the dataset.
@@ -123,7 +139,7 @@ class DisplayUploadStatus:
     dataset_statistics = []
     uploaded_inputs_dict = {}
     uploaded_annotations_dict = dict(concepts=0, bboxes=0, polygons=0)
-    dict_response = MessageToDict(self.dataset_metrics_response)
+    dict_response = MessageToDict(dataset_metrics_response)
 
     for data in dict_response["datasetVersionMetricsGroups"]:
       if isinstance(data["value"], str):