clarifai 10.2.0__py3-none-any.whl → 10.3.0__py3-none-any.whl

clarifai/client/app.py

@@ -33,6 +33,7 @@ class App(Lister, BaseClient):
                base_url: str = "https://api.clarifai.com",
                pat: str = None,
                token: str = None,
+               root_certificates_path: str = None,
                **kwargs):
     """Initializes an App object.
 
@@ -42,6 +43,7 @@ class App(Lister, BaseClient):
         base_url (str): Base API url. Default "https://api.clarifai.com"
         pat (str): A personal access token for authentication. Can be set as env var CLARIFAI_PAT
         token (str): A session token for authentication. Accepts either a session token or a pat. Can be set as env var CLARIFAI_SESSION_TOKEN
+        root_certificates_path (str): Path to the SSL root certificates file, used to establish secure gRPC connections.
         **kwargs: Additional keyword arguments to be passed to the App.
             - name (str): The name of the app.
             - description (str): The description of the app.
@@ -55,7 +57,13 @@ class App(Lister, BaseClient):
     self.app_info = resources_pb2.App(**self.kwargs)
     self.logger = get_logger(logger_level="INFO", name=__name__)
     BaseClient.__init__(
-        self, user_id=self.user_id, app_id=self.id, base=base_url, pat=pat, token=token)
+        self,
+        user_id=self.user_id,
+        app_id=self.id,
+        base=base_url,
+        pat=pat,
+        token=token,
+        root_certificates_path=root_certificates_path)
     Lister.__init__(self)
 
   def list_datasets(self, page_no: int = None,
@@ -86,8 +94,9 @@ class App(Lister, BaseClient):
         per_page=per_page,
         page_no=page_no)
     for dataset_info in all_datasets_info:
-      if 'version' in list(dataset_info.keys()):
-        del dataset_info['version']['metrics']
+      if 'version' in dataset_info:
+        dataset_info['version'].pop('metrics', None)
+        dataset_info['version'].pop('export_info', None)
       yield Dataset.from_auth_helper(auth=self.auth_helper, **dataset_info)
 
   def list_models(self,
@@ -373,8 +382,8 @@ class App(Lister, BaseClient):
       output_info = get_yaml_output_info_proto(node['model'].get('output_info', None))
       try:
         model = self.model(
-            node['model']['model_id'],
-            node['model'].get('model_version_id', ""),
+            model_id=node['model']['model_id'],
+            model_version={"id": node['model'].get('model_version_id', "")},
             user_id=node['model'].get('user_id', ""),
             app_id=node['model'].get('app_id', ""))
       except Exception as e:
@@ -457,11 +466,12 @@ class App(Lister, BaseClient):
 
     return Module.from_auth_helper(auth=self.auth_helper, module_id=module_id, **kwargs)
 
-  def dataset(self, dataset_id: str, **kwargs) -> Dataset:
+  def dataset(self, dataset_id: str, dataset_version_id: str = None, **kwargs) -> Dataset:
     """Returns a Dataset object for the existing dataset ID.
 
     Args:
         dataset_id (str): The dataset ID for the dataset to interact with.
+        dataset_version_id (str): The version ID for the dataset version to interact with.
 
     Returns:
         Dataset: A Dataset object for the existing dataset ID.
@@ -480,9 +490,10 @@ class App(Lister, BaseClient):
     kwargs = self.process_response_keys(dict_response[list(dict_response.keys())[1]],
                                         list(dict_response.keys())[1])
     kwargs['version'] = response.dataset.version if response.dataset.version else None
+    kwargs['dataset_version_id'] = dataset_version_id
     return Dataset.from_auth_helper(auth=self.auth_helper, **kwargs)
 
-  def model(self, model_id: str, model_version_id: str = "", **kwargs) -> Model:
+  def model(self, model_id: str, model_version: Dict = {'id': ""}, **kwargs) -> Model:
     """Returns a Model object for the existing model ID.
 
     Args:
@@ -495,7 +506,7 @@ class App(Lister, BaseClient):
     Example:
         >>> from clarifai.client.app import App
         >>> app = App(app_id="app_id", user_id="user_id")
-        >>> model_v1 = app.model(model_id="model_id", model_version_id="model_version_id")
+        >>> model_v1 = app.model(model_id="model_id", model_version={"id":"model_version_id")
     """
     # Change user_app_id based on whether user_id or app_id is specified.
     if kwargs.get("user_id") or kwargs.get("app_id"):
@@ -503,10 +514,10 @@ class App(Lister, BaseClient):
           user_app_id=self.auth_helper.get_user_app_id_proto(
               kwargs.get("user_id"), kwargs.get("app_id")),
           model_id=model_id,
-          version_id=model_version_id)
+          version_id=model_version["id"])
     else:
       request = service_pb2.GetModelRequest(
-          user_app_id=self.user_app_id, model_id=model_id, version_id=model_version_id)
+          user_app_id=self.user_app_id, model_id=model_id, version_id=model_version["id"])
     response = self._grpc_request(self.STUB.GetModel, request)
 
     if response.status.code != status_code_pb2.SUCCESS:

clarifai/client/auth/helper.py

@@ -65,6 +65,7 @@ class ClarifaiAuthHelper:
       token: str = "",
       base: str = DEFAULT_BASE,
       ui: str = DEFAULT_UI,
+      root_certificates_path: str = None,
       validate: bool = True,
   ):
     """
@@ -85,6 +86,7 @@ class ClarifaiAuthHelper:
     https://api.clarifai.com (default), https://host:port, http://host:port, host:port (will be treated as http, not https). It's highly recommended to include the http:// or https:// otherwise we need to check the endpoint to determine if it has SSL during this __init__
       ui: a url to the UI. Examples include clarifai.com,
     https://clarifai.com (default), https://host:port, http://host:port, host:port (will be treated as http, not https). It's highly recommended to include the http:// or https:// otherwise we need to check the endpoint to determine if it has SSL during this __init__
+      root_certificates_path: path to the root certificates file. This is only used for grpc secure channels.
       validate: whether to validate the inputs. This is useful for overriding vars then validating
     """
 
@@ -92,6 +94,7 @@ class ClarifaiAuthHelper:
     self.app_id = app_id
     self._pat = pat
     self._token = token
+    self._root_certificates_path = root_certificates_path
 
     self.set_base(base)
     self.set_ui(ui)
@@ -113,6 +116,8 @@ class ClarifaiAuthHelper:
       raise Exception(
           "Need 'pat' or 'token' in the query params or use one of the CLARIFAI_PAT or CLARIFAI_SESSION_TOKEN env vars"
       )
+    if (self._root_certificates_path) and (not os.path.exists(self._root_certificates_path)):
+      raise Exception("Root certificates path %s does not exist" % self._root_certificates_path)
 
   @classmethod
   def from_streamlit(cls, st: Any) -> "ClarifaiAuthHelper":
@@ -219,6 +224,8 @@ Additionally, these optional params are supported:
       self.set_base(query_params["base"][0])
     if "ui" in query_params:
       self.set_ui(query_params["ui"][0])
+    if "root_certificates_path" in query_params:
+      self._root_certificates_path = query_params["root_certificates_path"][0]
 
   @classmethod
   def from_env(cls, validate: bool = True) -> "ClarifaiAuthHelper":
@@ -229,6 +236,7 @@ Additionally, these optional params are supported:
           token: CLARIFAI_SESSION_TOKEN env var.
           pat: CLARIFAI_PAT env var.
         base: CLARIFAI_API_BASE env var.
+        root_certificates_path: CLARIFAI_ROOT_CERTIFICATES_PATH env var.
     """
     user_id = os.environ.get("CLARIFAI_USER_ID", "")
     app_id = os.environ.get("CLARIFAI_APP_ID", "")
@@ -236,7 +244,8 @@ Additionally, these optional params are supported:
     pat = os.environ.get("CLARIFAI_PAT", "")
     base = os.environ.get("CLARIFAI_API_BASE", DEFAULT_BASE)
     ui = os.environ.get("CLARIFAI_UI", DEFAULT_UI)
-    return cls(user_id, app_id, pat, token, base, ui, validate)
+    root_certificates_path = os.environ.get("CLARIFAI_ROOT_CERTIFICATES_PATH", None)
+    return cls(user_id, app_id, pat, token, base, ui, root_certificates_path, validate)
 
   def get_user_app_id_proto(
       self,
@@ -281,7 +290,8 @@ Additionally, these optional params are supported:
 
     https = base_https_cache[self._base]
     if https:
-      channel = ClarifaiChannel.get_grpc_channel(base=self._base)
+      channel = ClarifaiChannel.get_grpc_channel(
+          base=self._base, root_certificates_path=self._root_certificates_path)
     else:
       if self._base.find(":") >= 0:
         host, port = self._base.split(":")

clarifai/client/base.py

@@ -22,6 +22,7 @@ class BaseClient:
           - token (str): A session token for authentication. Accepts either a session token or a pat.
           - base (str): The base URL for the API endpoint. Defaults to 'https://api.clarifai.com'.
           - ui (str): The URL for the UI. Defaults to 'https://clarifai.com'.
+          - root_certificates_path (str): Path to the SSL root certificates file, used to establish secure gRPC connections.
 
 
   Attributes:
@@ -51,14 +52,21 @@ class BaseClient:
     self.token = self.auth_helper._token
     self.user_app_id = self.auth_helper.get_user_app_id_proto()
     self.base = self.auth_helper.base
+    self.root_certificates_path = self.auth_helper._root_certificates_path
 
   @classmethod
   def from_auth_helper(cls, auth: ClarifaiAuthHelper, **kwargs):
     default_kwargs = {
-        "user_id": kwargs.get("user_id", None) or auth.user_id,
-        "app_id": kwargs.get("app_id", None) or auth.app_id,
-        "pat": kwargs.get("pat", None) or auth.pat,
-        "token": kwargs.get("token", None) or auth._token,
+        "user_id":
+            kwargs.get("user_id", None) or auth.user_id,
+        "app_id":
+            kwargs.get("app_id", None) or auth.app_id,
+        "pat":
+            kwargs.get("pat", None) or auth.pat,
+        "token":
+            kwargs.get("token", None) or auth._token,
+        "root_certificates_path":
+            kwargs.get("root_certificates_path", None) or auth._root_certificates_path
     }
     _base = kwargs.get("base", None) or auth.base
     _clss = cls.__mro__[0]
@@ -160,6 +168,8 @@ class BaseClient:
             value = value_s
           elif key == 'metrics':
             continue
+          elif key == 'size':
+            value = int(value)
           elif key in ['metadata']:
             if isinstance(value, dict) and value != {}:
               value_s = struct_pb2.Struct()

clarifai/client/dataset.py

@@ -43,26 +43,34 @@ class Dataset(Lister, BaseClient):
   def __init__(self,
                url: str = None,
                dataset_id: str = None,
+               dataset_version_id: str = None,
                base_url: str = "https://api.clarifai.com",
                pat: str = None,
                token: str = None,
+               root_certificates_path: str = None,
                **kwargs):
     """Initializes a Dataset object.
 
     Args:
         url (str): The URL to initialize the dataset object.
         dataset_id (str): The Dataset ID within the App to interact with.
+        dataset_version_id (str): The Dataset Version ID within the Dataset to interact with.
         base_url (str): Base API url. Default "https://api.clarifai.com"
         pat (str): A personal access token for authentication. Can be set as env var CLARIFAI_PAT
         token (str): A session token for authentication. Accepts either a session token or a pat. Can be set as env var CLARIFAI_SESSION_TOKEN
+        root_certificates_path (str): Path to the SSL root certificates file, used to establish secure gRPC connections.
         **kwargs: Additional keyword arguments to be passed to the Dataset.
     """
     if url and dataset_id:
       raise UserError("You can only specify one of url or dataset_id.")
     if url:
-      user_id, app_id, _, dataset_id, _ = ClarifaiUrlHelper.split_clarifai_url(url)
+      user_id, app_id, _, dataset_id, dataset_version_id = ClarifaiUrlHelper.split_clarifai_url(
+          url)
       kwargs = {'user_id': user_id, 'app_id': app_id}
-    self.kwargs = {**kwargs, 'id': dataset_id}
+    dataset_version = {
+        'id': dataset_version_id
+    } if dataset_version_id else kwargs['version'] if 'version' in kwargs else None
+    self.kwargs = {**kwargs, 'id': dataset_id, 'version': dataset_version}
     self.dataset_info = resources_pb2.Dataset(**self.kwargs)
     # Related to Dataset Upload
     self.num_workers: int = min(10, cpu_count())  #15 req/sec rate limit
@@ -71,10 +79,21 @@ class Dataset(Lister, BaseClient):
     self.batch_size = 128  # limit max protos in a req
     self.task = None  # Upload dataset type
     self.input_object = Inputs(
-        user_id=self.user_id, app_id=self.app_id, pat=pat, token=token, base_url=base_url)
+        user_id=self.user_id,
+        app_id=self.app_id,
+        pat=pat,
+        token=token,
+        base_url=base_url,
+        root_certificates_path=root_certificates_path)
     self.logger = get_logger(logger_level="INFO", name=__name__)
     BaseClient.__init__(
-        self, user_id=self.user_id, app_id=self.app_id, base=base_url, pat=pat, token=token)
+        self,
+        user_id=self.user_id,
+        app_id=self.app_id,
+        base=base_url,
+        pat=pat,
+        token=token,
+        root_certificates_path=root_certificates_path)
     Lister.__init__(self)
 
   def create_version(self, **kwargs) -> 'Dataset':
@@ -162,13 +181,17 @@ class Dataset(Lister, BaseClient):
     for dataset_version_info in all_dataset_versions_info:
       dataset_version_info['id'] = dataset_version_info['dataset_version_id']
       del dataset_version_info['dataset_version_id']
-      del dataset_version_info['metrics']
+      dataset_version_info.pop('metrics', None)
+      dataset_version_info.pop('export_info', None)
       kwargs = {
           'dataset_id': self.id,
           'version': resources_pb2.DatasetVersion(**dataset_version_info),
       }
       yield Dataset.from_auth_helper(self.auth_helper, **kwargs)
 
+  def __iter__(self):
+    return iter(DatasetExportReader(archive_url=self.archive_zip()))
+
   def _concurrent_annot_upload(self, annots: List[List[resources_pb2.Annotation]]
                               ) -> Union[List[resources_pb2.Annotation], List[None]]:
     """Uploads annotations concurrently.
@@ -567,7 +590,7 @@ class Dataset(Lister, BaseClient):
     )
 
     start_time = time.time()
-    backoff_iterator = BackoffIterator()
+    backoff_iterator = BackoffIterator(10)
     while (True):
       dataset_metrics_response = self._grpc_request(
           self.STUB.ListDatasetVersionMetricsGroups,
@@ -599,11 +622,37 @@ class Dataset(Lister, BaseClient):
     if delete_version:
       self.delete_version(dataset_version_id)
 
+  def archive_zip(self, wait: bool = True) -> str:
+    """Exports the dataset to a zip file URL."""
+    request = service_pb2.PutDatasetVersionExportsRequest(
+        user_app_id=self.user_app_id,
+        dataset_id=self.id,
+        dataset_version_id=self.version.id,
+        exports=[
+            resources_pb2.DatasetVersionExport(
+                format=resources_pb2.DatasetVersionExportFormat.CLARIFAI_DATA_PROTOBUF)
+        ])
+
+    response = self._grpc_request(self.STUB.PutDatasetVersionExports, request)
+    if response.status.code != status_code_pb2.SUCCESS:
+      raise Exception(response.status)
+    if wait:
+      while response.exports[0].status.code in (
+          status_code_pb2.DATASET_VERSION_EXPORT_PENDING,
+          status_code_pb2.DATASET_VERSION_EXPORT_IN_PROGRESS):
+        time.sleep(1)
+        response = self._grpc_request(self.STUB.PutDatasetVersionExports, request)
+        if response.status.code != status_code_pb2.SUCCESS:
+          raise Exception(response.status)
+    if response.exports[0].status.code != status_code_pb2.DATASET_VERSION_EXPORT_SUCCESS:
+      raise Exception(response.exports[0].status)
+    return response.exports[0].url
+
   def export(self,
              save_path: str,
              archive_url: str = None,
              local_archive_path: str = None,
-             split: str = None,
+             split: str = 'all',
              num_workers: int = 4) -> None:
     """Exports the Clarifai protobuf dataset to a local archive.
 
@@ -616,10 +665,12 @@ class Dataset(Lister, BaseClient):
 
     Example:
         >>> from clarifai.client.dataset import Dataset
-        >>> Dataset().export(save_path='output.zip', local_archive_path='clarifai-data-protobuf.zip')
+        >>> Dataset().export(save_path='output.zip')
     """
     if local_archive_path and not os.path.exists(local_archive_path):
       raise UserError(f"Archive {local_archive_path} does not exist.")
+    if not archive_url and not local_archive_path:
+      archive_url = self.archive_zip()
     # Create a session object and set auth header
     session = requests.Session()
     retries = Retry(total=5, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504])

clarifai/client/input.py

@@ -19,6 +19,7 @@ from tqdm import tqdm
 from clarifai.client.base import BaseClient
 from clarifai.client.lister import Lister
 from clarifai.constants.dataset import MAX_RETRIES
+from clarifai.constants.input import MAX_UPLOAD_BATCH_SIZE
 from clarifai.errors import UserError
 from clarifai.utils.logging import get_logger
 from clarifai.utils.misc import BackoffIterator, Chunker
@@ -34,6 +35,7 @@ class Inputs(Lister, BaseClient):
                base_url: str = "https://api.clarifai.com",
                pat: str = None,
                token: str = None,
+               root_certificates_path: str = None,
                **kwargs):
     """Initializes an Input object.
 
@@ -43,6 +45,7 @@ class Inputs(Lister, BaseClient):
         base_url (str): Base API url. Default "https://api.clarifai.com"
         pat (str): A personal access token for authentication. Can be set as env var CLARIFAI_PAT
         token (str): A session token for authentication. Accepts either a session token or a pat. Can be set as env var CLARIFAI_SESSION_TOKEN
+        root_certificates_path (str): Path to the SSL root certificates file, used to establish secure gRPC connections.
         **kwargs: Additional keyword arguments to be passed to the Input
     """
     self.user_id = user_id
@@ -51,7 +54,13 @@ class Inputs(Lister, BaseClient):
     self.input_info = resources_pb2.Input(**self.kwargs)
     self.logger = get_logger(logger_level=logger_level, name=__name__)
     BaseClient.__init__(
-        self, user_id=self.user_id, app_id=self.app_id, base=base_url, pat=pat, token=token)
+        self,
+        user_id=self.user_id,
+        app_id=self.app_id,
+        base=base_url,
+        pat=pat,
+        token=token,
+        root_certificates_path=root_certificates_path)
     Lister.__init__(self)
 
   @staticmethod
@@ -660,6 +669,10 @@ class Inputs(Lister, BaseClient):
     """
     if not isinstance(inputs, list):
       raise UserError("inputs must be a list of Input objects")
+    if len(inputs) > MAX_UPLOAD_BATCH_SIZE:
+      raise UserError(
+          f"Number of inputs to upload exceeds the maximum batch size of {MAX_UPLOAD_BATCH_SIZE}. Please reduce batch size."
+      )
     input_job_id = uuid.uuid4().hex  # generate a unique id for this job
     request = service_pb2.PostInputsRequest(
         user_app_id=self.user_app_id, inputs=inputs, inputs_add_job_id=input_job_id)
@@ -912,7 +925,7 @@ class Inputs(Lister, BaseClient):
     Returns:
         True if inputs are processed, False otherwise
     """
-    backoff_iterator = BackoffIterator()
+    backoff_iterator = BackoffIterator(10)
     max_retries = 10
     start_time = time.time()
     while True: