clarifai 9.10.1__py3-none-any.whl → 9.10.3__py3-none-any.whl

clarifai/client/__init__.py

@@ -1,16 +1,17 @@
-# Note: V2Stub, create_stub is just added for backwards compatibility. It will be removed in the future.
 from clarifai.client.app import App
 from clarifai.client.auth.register import V2Stub
 from clarifai.client.auth.stub import create_stub
 from clarifai.client.base import BaseClient
 from clarifai.client.dataset import Dataset
+from clarifai.client.input import Inputs
 from clarifai.client.lister import Lister
 from clarifai.client.model import Model
 from clarifai.client.module import Module
+from clarifai.client.search import Search
 from clarifai.client.user import User
 from clarifai.client.workflow import Workflow
 
 __all__ = [
     'V2Stub', 'create_stub', 'User', 'App', 'Model', 'Workflow', 'Module', 'Lister', 'Dataset',
-    'BaseClient'
+    'Inputs', 'BaseClient', 'Search'
 ]

clarifai/client/app.py

@@ -28,29 +28,31 @@ class App(Lister, BaseClient):
   """App is a class that provides access to Clarifai API endpoints related to App information."""
 
   def __init__(self,
-               url_init: str = "",
-               app_id: str = "",
+               url: str = None,
+               app_id: str = None,
                base_url: str = "https://api.clarifai.com",
+               pat: str = None,
                **kwargs):
     """Initializes an App object.
 
     Args:
-        url_init (str): The URL to initialize the app object.
+        url (str): The URL to initialize the app object.
         app_id (str): The App ID for the App to interact with.
         base_url (str): Base API url. Default "https://api.clarifai.com"
+        pat (str): A personal access token for authentication. Can be set as env var CLARIFAI_PAT
         **kwargs: Additional keyword arguments to be passed to the App.
             - name (str): The name of the app.
             - description (str): The description of the app.
     """
-    if url_init != "" and app_id != "":
-      raise UserError("You can only specify one of url_init or app_id.")
-    if url_init != "":
-      user_id, app_id, _, _, _ = ClarifaiUrlHelper.split_clarifai_url(url_init)
+    if url and app_id:
+      raise UserError("You can only specify one of url or app_id.")
+    if url:
+      user_id, app_id, _, _, _ = ClarifaiUrlHelper.split_clarifai_url(url)
       kwargs = {'user_id': user_id}
     self.kwargs = {**kwargs, 'id': app_id}
     self.app_info = resources_pb2.App(**self.kwargs)
     self.logger = get_logger(logger_level="INFO", name=__name__)
-    BaseClient.__init__(self, user_id=self.user_id, app_id=self.id, base=base_url)
+    BaseClient.__init__(self, user_id=self.user_id, app_id=self.id, base=base_url, pat=pat)
     Lister.__init__(self)
 
   def list_datasets(self, page_no: int = None,
@@ -83,7 +85,7 @@ class App(Lister, BaseClient):
     for dataset_info in all_datasets_info:
       if 'version' in list(dataset_info.keys()):
         del dataset_info['version']['metrics']
-      yield Dataset(base_url=self.base, **dataset_info)
+      yield Dataset(base_url=self.base, pat=self.pat, **dataset_info)
 
   def list_models(self,
                   filter_by: Dict[str, Any] = {},
@@ -124,7 +126,7 @@ class App(Lister, BaseClient):
       if only_in_app:
         if model_info['app_id'] != self.id:
           continue
-      yield Model(base_url=self.base, **model_info)
+      yield Model(base_url=self.base, pat=self.pat, **model_info)
 
   def list_workflows(self,
                      filter_by: Dict[str, Any] = {},
@@ -163,7 +165,7 @@ class App(Lister, BaseClient):
       if only_in_app:
         if workflow_info['app_id'] != self.id:
           continue
-      yield Workflow(base_url=self.base, **workflow_info)
+      yield Workflow(base_url=self.base, pat=self.pat, **workflow_info)
 
   def list_modules(self,
                    filter_by: Dict[str, Any] = {},
@@ -202,7 +204,7 @@ class App(Lister, BaseClient):
       if only_in_app:
         if module_info['app_id'] != self.id:
           continue
-      yield Module(base_url=self.base, **module_info)
+      yield Module(base_url=self.base, pat=self.pat, **module_info)
 
   def list_installed_module_versions(self,
                                      filter_by: Dict[str, Any] = {},
@@ -238,7 +240,10 @@ class App(Lister, BaseClient):
       del imv_info['deploy_url']
       del imv_info['installed_module_version_id']  # TODO: remove this after the backend fix
       yield Module(
-          module_id=imv_info['module_version']['module_id'], base_url=self.base, **imv_info)
+          module_id=imv_info['module_version']['module_id'],
+          base_url=self.base,
+          pat=self.pat,
+          **imv_info)
 
   def list_concepts(self, page_no: int = None,
                     per_page: int = None) -> Generator[Concept, None, None]:
@@ -303,7 +308,12 @@ class App(Lister, BaseClient):
     if response.status.code != status_code_pb2.SUCCESS:
       raise Exception(response.status)
     self.logger.info("\nDataset created\n%s", response.status)
-    kwargs.update({'app_id': self.id, 'user_id': self.user_id, 'base_url': self.base})
+    kwargs.update({
+        'app_id': self.id,
+        'user_id': self.user_id,
+        'base_url': self.base,
+        'pat': self.pat
+    })
 
     return Dataset(dataset_id=dataset_id, **kwargs)
 
@@ -332,7 +342,8 @@ class App(Lister, BaseClient):
         'app_id': self.id,
         'user_id': self.user_id,
         'model_type_id': response.model.model_type_id,
-        'base_url': self.base
+        'base_url': self.base,
+        'pat': self.pat
     })
 
     return Model(model_id=model_id, **kwargs)
@@ -425,7 +436,7 @@ class App(Lister, BaseClient):
       display_workflow_tree(dict_response["workflows"][0]["nodes"])
     kwargs = self.process_response_keys(dict_response[list(dict_response.keys())[1]][0],
                                         "workflow")
-    kwargs.update({'base_url': self.base})
+    kwargs.update({'base_url': self.base, 'pat': self.pat})
 
     return Workflow(**kwargs)
 
@@ -453,7 +464,12 @@ class App(Lister, BaseClient):
     if response.status.code != status_code_pb2.SUCCESS:
       raise Exception(response.status)
     self.logger.info("\nModule created\n%s", response.status)
-    kwargs.update({'app_id': self.id, 'user_id': self.user_id, 'base_url': self.base})
+    kwargs.update({
+        'app_id': self.id,
+        'user_id': self.user_id,
+        'base_url': self.base,
+        'pat': self.pat
+    })
 
     return Module(module_id=module_id, **kwargs)
 
@@ -480,7 +496,7 @@ class App(Lister, BaseClient):
     kwargs = self.process_response_keys(dict_response[list(dict_response.keys())[1]],
                                         list(dict_response.keys())[1])
     kwargs['version'] = response.dataset.version if response.dataset.version else None
-    kwargs.update({'base_url': self.base})
+    kwargs.update({'base_url': self.base, 'pat': self.pat})
     return Dataset(**kwargs)
 
   def model(self, model_id: str, model_version_id: str = "", **kwargs) -> Model:
@@ -516,7 +532,7 @@ class App(Lister, BaseClient):
     kwargs = self.process_response_keys(dict_response['model'], 'model')
     kwargs[
         'model_version'] = response.model.model_version if response.model.model_version else None
-    kwargs.update({'base_url': self.base})
+    kwargs.update({'base_url': self.base, 'pat': self.pat})
 
     return Model(**kwargs)
 
@@ -542,7 +558,7 @@ class App(Lister, BaseClient):
     dict_response = MessageToDict(response, preserving_proto_field_name=True)
     kwargs = self.process_response_keys(dict_response[list(dict_response.keys())[1]],
                                         list(dict_response.keys())[1])
-    kwargs.update({'base_url': self.base})
+    kwargs.update({'base_url': self.base, 'pat': self.pat})
 
     return Workflow(**kwargs)
 
@@ -569,7 +585,7 @@ class App(Lister, BaseClient):
       raise Exception(response.status)
     dict_response = MessageToDict(response, preserving_proto_field_name=True)
     kwargs = self.process_response_keys(dict_response['module'], 'module')
-    kwargs.update({'base_url': self.base})
+    kwargs.update({'base_url': self.base, 'pat': self.pat})
 
     return Module(**kwargs)
 
@@ -579,7 +595,7 @@ class App(Lister, BaseClient):
     Returns:
         Inputs: An input object.
     """
-    return Inputs(self.user_id, self.id, base_url=self.base)
+    return Inputs(self.user_id, self.id, base_url=self.base, pat=self.pat)
 
   def delete_dataset(self, dataset_id: str) -> None:
     """Deletes an dataset for the user.
@@ -670,7 +686,7 @@ class App(Lister, BaseClient):
     """
     user_id = kwargs.get("user_id", self.user_app_id.user_id)
     app_id = kwargs.get("app_id", self.user_app_id.app_id)
-    return Search(user_id=user_id, app_id=app_id, **kwargs)
+    return Search(user_id=user_id, app_id=app_id, base_url=self.base, pat=self.pat, **kwargs)
 
   def __getattr__(self, name):
     return getattr(self.app_info, name)

clarifai/client/base.py

@@ -1,4 +1,3 @@
-import os
 from datetime import datetime
 from typing import Any, Callable
 
@@ -8,7 +7,8 @@ from google.protobuf.wrappers_pb2 import BoolValue
 
 from clarifai.client.auth import create_stub
 from clarifai.client.auth.helper import ClarifaiAuthHelper
-from clarifai.errors import ApiError, UserError
+from clarifai.errors import ApiError
+from clarifai.utils.misc import get_from_dict_or_env
 
 
 class BaseClient:
@@ -31,12 +31,12 @@ class BaseClient:
   """
 
   def __init__(self, **kwargs):
-    pat = os.environ.get('CLARIFAI_PAT', "")
-    if pat == "":
-      raise UserError("CLARIFAI_PAT must be set as env vars")
-    self.auth_helper = ClarifaiAuthHelper(**kwargs, pat=pat, validate=False)
+    pat = get_from_dict_or_env(key="pat", env_key="CLARIFAI_PAT", **kwargs)
+    kwargs.update({'pat': pat})
+    self.auth_helper = ClarifaiAuthHelper(**kwargs, validate=False)
     self.STUB = create_stub(self.auth_helper)
     self.metadata = self.auth_helper.metadata
+    self.pat = self.auth_helper.pat
     self.user_app_id = self.auth_helper.get_user_app_id_proto()
     self.base = self.auth_helper.base
 

clarifai/client/dataset.py

@@ -1,7 +1,9 @@
 import os
+import time
+import uuid
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from multiprocessing import cpu_count
-from typing import Generator, List, Tuple, TypeVar, Union
+from typing import Generator, List, Tuple, Type, TypeVar, Union
 
 import requests
 from clarifai_grpc.grpc.api import resources_pb2, service_pb2
@@ -13,16 +15,18 @@ from tqdm import tqdm
 from clarifai.client.base import BaseClient
 from clarifai.client.input import Inputs
 from clarifai.client.lister import Lister
+from clarifai.constants.dataset import DATASET_UPLOAD_TASKS
 from clarifai.datasets.export.inputs_annotations import (DatasetExportReader,
                                                          InputAnnotationDownloader)
+from clarifai.datasets.upload.base import ClarifaiDataLoader
 from clarifai.datasets.upload.image import (VisualClassificationDataset, VisualDetectionDataset,
                                             VisualSegmentationDataset)
 from clarifai.datasets.upload.text import TextClassificationDataset
-from clarifai.datasets.upload.utils import load_dataloader, load_module_dataloader
+from clarifai.datasets.upload.utils import DisplayUploadStatus
 from clarifai.errors import UserError
 from clarifai.urls.helper import ClarifaiUrlHelper
 from clarifai.utils.logging import get_logger
-from clarifai.utils.misc import Chunker
+from clarifai.utils.misc import BackoffIterator, Chunker
 
 ClarifaiDatasetType = TypeVar('ClarifaiDatasetType', VisualClassificationDataset,
                               VisualDetectionDataset, VisualSegmentationDataset,
@@ -33,22 +37,24 @@ class Dataset(Lister, BaseClient):
   """Dataset is a class that provides access to Clarifai API endpoints related to Dataset information."""
 
   def __init__(self,
-               url_init: str = "",
-               dataset_id: str = "",
+               url: str = None,
+               dataset_id: str = None,
                base_url: str = "https://api.clarifai.com",
+               pat: str = None,
                **kwargs):
     """Initializes a Dataset object.
 
     Args:
-        url_init (str): The URL to initialize the dataset object.
+        url (str): The URL to initialize the dataset object.
         dataset_id (str): The Dataset ID within the App to interact with.
         base_url (str): Base API url. Default "https://api.clarifai.com"
+        pat (str): A personal access token for authentication. Can be set as env var CLARIFAI_PAT
         **kwargs: Additional keyword arguments to be passed to the Dataset.
     """
-    if url_init != "" and dataset_id != "":
-      raise UserError("You can only specify one of url_init or dataset_id.")
-    if url_init != "":
-      user_id, app_id, _, dataset_id, _ = ClarifaiUrlHelper.split_clarifai_url(url_init)
+    if url and dataset_id:
+      raise UserError("You can only specify one of url or dataset_id.")
+    if url:
+      user_id, app_id, _, dataset_id, _ = ClarifaiUrlHelper.split_clarifai_url(url)
       kwargs = {'user_id': user_id, 'app_id': app_id}
     self.kwargs = {**kwargs, 'id': dataset_id}
     self.dataset_info = resources_pb2.Dataset(**self.kwargs)
@@ -56,11 +62,11 @@ class Dataset(Lister, BaseClient):
     self.num_workers: int = min(10, cpu_count())  #15 req/sec rate limit
     self.annot_num_workers = 4
     self.max_retires = 10
-    self.chunk_size = 128  # limit max protos in a req
+    self.batch_size = 128  # limit max protos in a req
     self.task = None  # Upload dataset type
-    self.input_object = Inputs(user_id=self.user_id, app_id=self.app_id)
+    self.input_object = Inputs(user_id=self.user_id, app_id=self.app_id, pat=pat)
     self.logger = get_logger(logger_level="INFO")
-    BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id, base=base_url)
+    BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id, base=base_url, pat=pat)
     Lister.__init__(self)
 
   def create_version(self, **kwargs) -> 'Dataset':
@@ -93,7 +99,8 @@ class Dataset(Lister, BaseClient):
         'app_id': self.app_id,
         'user_id': self.user_id,
         'version': response.dataset_versions[0],
-        'base_url': self.base
+        'base_url': self.base,
+        'pat': self.pat
     })
     return Dataset(**kwargs)
 
@@ -156,7 +163,8 @@ class Dataset(Lister, BaseClient):
           'app_id': self.app_id,
           'user_id': self.user_id,
           'version': resources_pb2.DatasetVersion(**dataset_version_info),
-          'base_url': self.base
+          'base_url': self.base,
+          'pat': self.pat
       }
       yield Dataset(**kwargs)
 
@@ -241,7 +249,7 @@ class Dataset(Lister, BaseClient):
 
     if self.task in ["visual_detection", "visual_segmentation"]:
       _, annotation_protos = dataset_obj.get_protos(success_input_ids)
-      chunked_annotation_protos = Chunker(annotation_protos, self.chunk_size).chunk()
+      chunked_annotation_protos = Chunker(annotation_protos, self.batch_size).chunk()
       retry_annot_protos.extend(self._concurrent_annot_upload(chunked_annotation_protos))
 
     return failed_input_ids, retry_annot_protos
@@ -259,7 +267,7 @@ class Dataset(Lister, BaseClient):
     if failed_input_ids:
       self._upload_inputs_annotations(failed_input_ids, dataset_obj)
     if retry_annot_protos:
-      chunked_annotation_protos = Chunker(retry_annot_protos, self.chunk_size).chunk()
+      chunked_annotation_protos = Chunker(retry_annot_protos, self.batch_size).chunk()
       _ = self._concurrent_annot_upload(chunked_annotation_protos)
 
   def _data_upload(self, dataset_obj: ClarifaiDatasetType) -> None:
@@ -269,7 +277,7 @@ class Dataset(Lister, BaseClient):
       dataset_obj: ClarifaiDataset object
     """
     input_ids = list(range(len(dataset_obj)))
-    chunk_input_ids = Chunker(input_ids, self.chunk_size).chunk()
+    chunk_input_ids = Chunker(input_ids, self.batch_size).chunk()
     with ThreadPoolExecutor(max_workers=self.num_workers) as executor:
       with tqdm(total=len(chunk_input_ids), desc='Uploading Dataset') as progress:
         # Submit all jobs to the executor and store the returned futures
@@ -284,55 +292,46 @@ class Dataset(Lister, BaseClient):
           progress.update()
 
   def upload_dataset(self,
-                     task: str,
-                     split: str,
-                     module_dir: str = None,
-                     dataset_loader: str = None,
-                     chunk_size: int = 128) -> None:
+                     dataloader: Type[ClarifaiDataLoader],
+                     batch_size: int = 32,
+                     get_upload_status: bool = False) -> None:
     """Uploads a dataset to the app.
 
     Args:
-      task (str): task type(text_clf, visual-classification, visual_detection, visual_segmentation, visual-captioning)
-      split (str): split type(train, test, val)
-      module_dir (str): path to the module directory
-      dataset_loader (str): name of the dataset loader
-      chunk_size (int): chunk size for concurrent upload of inputs and annotations
+      dataloader (Type[ClarifaiDataLoader]): ClarifaiDataLoader object
+      batch_size (int): batch size for concurrent upload of inputs and annotations (max: 128)
+      get_upload_status (bool): True if you want to get the upload status of the dataset
     """
-    self.chunk_size = min(self.chunk_size, chunk_size)
-    self.task = task
-    datagen_object = None
-
-    if module_dir is None and dataset_loader is None:
-      raise UserError("One of `from_module` and `dataset_loader` must be \
-      specified. Both can't be None or defined at the same time.")
-    elif module_dir is not None and dataset_loader is not None:
-      raise UserError("Use either of `from_module` or `dataset_loader` \
-      but NOT both.")
-    elif module_dir is not None:
-      datagen_object = load_module_dataloader(module_dir, split)
-    else:
-      datagen_object = load_dataloader(dataset_loader, split)
-
-    if self.task == "text_clf":
-      dataset_obj = TextClassificationDataset(datagen_object, self.id, split)
+    self.batch_size = min(self.batch_size, batch_size)
+    self.task = dataloader.task
+    if self.task not in DATASET_UPLOAD_TASKS:
+      raise UserError("Task should be one of \
+                      'text_classification', 'visual_classification', \
+                      'visual_detection', 'visual_segmentation', 'visual_captioning'")
+
+    if self.task == "text_classification":
+      dataset_obj = TextClassificationDataset(dataloader, self.id)
 
     elif self.task == "visual_detection":
-      dataset_obj = VisualDetectionDataset(datagen_object, self.id, split)
+      dataset_obj = VisualDetectionDataset(dataloader, self.id)
 
     elif self.task == "visual_segmentation":
-      dataset_obj = VisualSegmentationDataset(datagen_object, self.id, split)
+      dataset_obj = VisualSegmentationDataset(dataloader, self.id)
 
     else:  # visual_classification & visual_captioning
-      dataset_obj = VisualClassificationDataset(datagen_object, self.id, split)
+      dataset_obj = VisualClassificationDataset(dataloader, self.id)
 
     self._data_upload(dataset_obj)
 
+    if get_upload_status:
+      self.get_upload_status(dataloader)
+
   def upload_from_csv(self,
                       csv_path: str,
                       input_type: str = 'text',
                       csv_type: str = None,
                       labels: bool = True,
-                      chunk_size: int = 128) -> None:
+                      batch_size: int = 128) -> None:
     """Uploads dataset from a csv file.
 
     Args:
@@ -340,7 +339,7 @@ class Dataset(Lister, BaseClient):
         input_type (str): type of the dataset(text, image)
         csv_type (str): type of the csv file(raw, url, file_path)
         labels (bool): True if csv file has labels column
-        chunk_size (int): chunk size for concurrent upload of inputs and annotations
+        batch_size (int): batch size for concurrent upload of inputs and annotations
 
     Example:
         >>> from clarifai.client.dataset import Dataset
@@ -360,27 +359,27 @@ class Dataset(Lister, BaseClient):
     assert csv_path.endswith('.csv'), 'csv_path should be a csv file'
     if csv_type == 'raw' and input_type != 'text':
       raise UserError('Only text input type is supported for raw csv type')
-    chunk_size = min(128, chunk_size)
+    batch_size = min(128, batch_size)
     input_protos = self.input_object.get_inputs_from_csv(
         csv_path=csv_path,
         input_type=input_type,
         csv_type=csv_type,
         dataset_id=self.id,
         labels=labels)
-    self.input_object._bulk_upload(inputs=input_protos, chunk_size=chunk_size)
+    self.input_object._bulk_upload(inputs=input_protos, batch_size=batch_size)
 
   def upload_from_folder(self,
                          folder_path: str,
                          input_type: str,
                          labels: bool = False,
-                         chunk_size: int = 128) -> None:
+                         batch_size: int = 128) -> None:
     """Upload dataset from folder.
 
     Args:
         folder_path (str): Path to the folder containing images.
         input_type (str): type of the dataset(text, image)
         labels (bool): True if folder name is the label for the inputs
-        chunk_size (int): chunk size for concurrent upload of inputs and annotations
+        batch_size (int): batch size for concurrent upload of inputs and annotations
 
     Example:
         >>> from clarifai.client.dataset import Dataset
@@ -397,7 +396,66 @@ class Dataset(Lister, BaseClient):
     if input_type == 'text':
       input_protos = self.input_object.get_text_inputs_from_folder(
           folder_path=folder_path, dataset_id=self.id, labels=labels)
-    self.input_object._bulk_upload(inputs=input_protos, chunk_size=chunk_size)
+    self.input_object._bulk_upload(inputs=input_protos, batch_size=batch_size)
+
+  def get_upload_status(self,
+                        dataloader: Type[ClarifaiDataLoader],
+                        delete_version: bool = False,
+                        timeout: int = 600) -> None:
+    """Creates a new dataset version and displays the upload status of the dataset.
+
+    Args:
+        dataloader (Type[ClarifaiDataLoader]): ClarifaiDataLoader object
+        delete_version (bool): True if you want to delete the version after getting the upload status
+        timeout (int): Timeout in seconds for getting the upload status. Default is 600 seconds.
+
+    Example:
+        >>> from clarifai.client.dataset import Dataset
+        >>> dataset = Dataset(dataset_id='dataset_id', user_id='user_id', app_id='app_id')
+        >>> dataset.get_upload_status(dataloader)
+
+    Note:
+        This is a beta feature and is subject to change.
+    """
+    self.logger.info("Getting dataset upload status...")
+    dataset_version_id = uuid.uuid4().hex
+    _ = self.create_version(id=dataset_version_id, description="SDK Upload Status")
+
+    request_data = dict(
+        user_app_id=self.user_app_id,
+        dataset_id=self.id,
+        dataset_version_id=dataset_version_id,
+    )
+
+    start_time = time.time()
+    backoff_iterator = BackoffIterator()
+    while (True):
+      dataset_metrics_response = self._grpc_request(
+          self.STUB.ListDatasetVersionMetricsGroups,
+          service_pb2.ListDatasetVersionMetricsGroupsRequest(**request_data),
+      )
+
+      if dataset_metrics_response.status.code != status_code_pb2.SUCCESS:
+        self.delete_version(dataset_version_id)
+        raise Exception("Failed to get dataset metrics {}".format(dataset_metrics_response.status))
+
+      dict_response = MessageToDict(dataset_metrics_response)
+      if len(dict_response.keys()) == 1 and time.time() - start_time < timeout:
+        self.logger.info("Crunching the dataset metrics. Please wait...")
+        time.sleep(next(backoff_iterator))
+        continue
+      else:
+        if time.time() - start_time > timeout:
+          self.delete_version(dataset_version_id)
+          raise UserError(
+              "Dataset metrics are taking too long to process. Please try again later.")
+        break
+
+    dataset_info_dict = dict(user_id=self.user_id, app_id=self.app_id, dataset_id=self.id)
+    DisplayUploadStatus(dataloader, dataset_metrics_response, dataset_info_dict)
+
+    if delete_version:
+      self.delete_version(dataset_version_id)
 
   def export(self,
              save_path: str,