clarifai 10.2.1__py3-none-any.whl → 10.3.0__py3-none-any.whl

clarifai/client/app.py

@@ -33,6 +33,7 @@ class App(Lister, BaseClient):
                base_url: str = "https://api.clarifai.com",
                pat: str = None,
                token: str = None,
+               root_certificates_path: str = None,
                **kwargs):
     """Initializes an App object.
 
@@ -42,6 +43,7 @@ class App(Lister, BaseClient):
         base_url (str): Base API url. Default "https://api.clarifai.com"
         pat (str): A personal access token for authentication. Can be set as env var CLARIFAI_PAT
         token (str): A session token for authentication. Accepts either a session token or a pat. Can be set as env var CLARIFAI_SESSION_TOKEN
+        root_certificates_path (str): Path to the SSL root certificates file, used to establish secure gRPC connections.
         **kwargs: Additional keyword arguments to be passed to the App.
             - name (str): The name of the app.
             - description (str): The description of the app.
@@ -55,7 +57,13 @@ class App(Lister, BaseClient):
     self.app_info = resources_pb2.App(**self.kwargs)
     self.logger = get_logger(logger_level="INFO", name=__name__)
     BaseClient.__init__(
-        self, user_id=self.user_id, app_id=self.id, base=base_url, pat=pat, token=token)
+        self,
+        user_id=self.user_id,
+        app_id=self.id,
+        base=base_url,
+        pat=pat,
+        token=token,
+        root_certificates_path=root_certificates_path)
     Lister.__init__(self)
 
   def list_datasets(self, page_no: int = None,
@@ -374,8 +382,8 @@ class App(Lister, BaseClient):
       output_info = get_yaml_output_info_proto(node['model'].get('output_info', None))
       try:
         model = self.model(
-            node['model']['model_id'],
-            node['model'].get('model_version_id', ""),
+            model_id=node['model']['model_id'],
+            model_version={"id": node['model'].get('model_version_id', "")},
             user_id=node['model'].get('user_id', ""),
             app_id=node['model'].get('app_id', ""))
       except Exception as e:
@@ -485,7 +493,7 @@ class App(Lister, BaseClient):
     kwargs['dataset_version_id'] = dataset_version_id
     return Dataset.from_auth_helper(auth=self.auth_helper, **kwargs)
 
-  def model(self, model_id: str, model_version_id: str = "", **kwargs) -> Model:
+  def model(self, model_id: str, model_version: Dict = {'id': ""}, **kwargs) -> Model:
     """Returns a Model object for the existing model ID.
 
     Args:
@@ -498,7 +506,7 @@ class App(Lister, BaseClient):
     Example:
         >>> from clarifai.client.app import App
         >>> app = App(app_id="app_id", user_id="user_id")
-        >>> model_v1 = app.model(model_id="model_id", model_version_id="model_version_id")
+        >>> model_v1 = app.model(model_id="model_id", model_version={"id":"model_version_id")
     """
     # Change user_app_id based on whether user_id or app_id is specified.
     if kwargs.get("user_id") or kwargs.get("app_id"):
@@ -506,10 +514,10 @@ class App(Lister, BaseClient):
           user_app_id=self.auth_helper.get_user_app_id_proto(
               kwargs.get("user_id"), kwargs.get("app_id")),
           model_id=model_id,
-          version_id=model_version_id)
+          version_id=model_version["id"])
     else:
       request = service_pb2.GetModelRequest(
-          user_app_id=self.user_app_id, model_id=model_id, version_id=model_version_id)
+          user_app_id=self.user_app_id, model_id=model_id, version_id=model_version["id"])
     response = self._grpc_request(self.STUB.GetModel, request)
 
     if response.status.code != status_code_pb2.SUCCESS:

clarifai/client/auth/helper.py

@@ -65,6 +65,7 @@ class ClarifaiAuthHelper:
       token: str = "",
       base: str = DEFAULT_BASE,
       ui: str = DEFAULT_UI,
+      root_certificates_path: str = None,
       validate: bool = True,
   ):
     """
@@ -85,6 +86,7 @@ class ClarifaiAuthHelper:
     https://api.clarifai.com (default), https://host:port, http://host:port, host:port (will be treated as http, not https). It's highly recommended to include the http:// or https:// otherwise we need to check the endpoint to determine if it has SSL during this __init__
       ui: a url to the UI. Examples include clarifai.com,
     https://clarifai.com (default), https://host:port, http://host:port, host:port (will be treated as http, not https). It's highly recommended to include the http:// or https:// otherwise we need to check the endpoint to determine if it has SSL during this __init__
+      root_certificates_path: path to the root certificates file. This is only used for grpc secure channels.
       validate: whether to validate the inputs. This is useful for overriding vars then validating
     """
 
@@ -92,6 +94,7 @@ class ClarifaiAuthHelper:
     self.app_id = app_id
     self._pat = pat
     self._token = token
+    self._root_certificates_path = root_certificates_path
 
     self.set_base(base)
     self.set_ui(ui)
@@ -113,6 +116,8 @@ class ClarifaiAuthHelper:
       raise Exception(
           "Need 'pat' or 'token' in the query params or use one of the CLARIFAI_PAT or CLARIFAI_SESSION_TOKEN env vars"
       )
+    if (self._root_certificates_path) and (not os.path.exists(self._root_certificates_path)):
+      raise Exception("Root certificates path %s does not exist" % self._root_certificates_path)
 
   @classmethod
   def from_streamlit(cls, st: Any) -> "ClarifaiAuthHelper":
@@ -219,6 +224,8 @@ Additionally, these optional params are supported:
       self.set_base(query_params["base"][0])
     if "ui" in query_params:
       self.set_ui(query_params["ui"][0])
+    if "root_certificates_path" in query_params:
+      self._root_certificates_path = query_params["root_certificates_path"][0]
 
   @classmethod
   def from_env(cls, validate: bool = True) -> "ClarifaiAuthHelper":
@@ -229,6 +236,7 @@ Additionally, these optional params are supported:
           token: CLARIFAI_SESSION_TOKEN env var.
           pat: CLARIFAI_PAT env var.
         base: CLARIFAI_API_BASE env var.
+        root_certificates_path: CLARIFAI_ROOT_CERTIFICATES_PATH env var.
     """
     user_id = os.environ.get("CLARIFAI_USER_ID", "")
     app_id = os.environ.get("CLARIFAI_APP_ID", "")
@@ -236,7 +244,8 @@ Additionally, these optional params are supported:
     pat = os.environ.get("CLARIFAI_PAT", "")
     base = os.environ.get("CLARIFAI_API_BASE", DEFAULT_BASE)
     ui = os.environ.get("CLARIFAI_UI", DEFAULT_UI)
-    return cls(user_id, app_id, pat, token, base, ui, validate)
+    root_certificates_path = os.environ.get("CLARIFAI_ROOT_CERTIFICATES_PATH", None)
+    return cls(user_id, app_id, pat, token, base, ui, root_certificates_path, validate)
 
   def get_user_app_id_proto(
       self,
@@ -281,7 +290,8 @@ Additionally, these optional params are supported:
 
     https = base_https_cache[self._base]
     if https:
-      channel = ClarifaiChannel.get_grpc_channel(base=self._base)
+      channel = ClarifaiChannel.get_grpc_channel(
+          base=self._base, root_certificates_path=self._root_certificates_path)
     else:
       if self._base.find(":") >= 0:
         host, port = self._base.split(":")

clarifai/client/base.py

@@ -22,6 +22,7 @@ class BaseClient:
           - token (str): A session token for authentication. Accepts either a session token or a pat.
           - base (str): The base URL for the API endpoint. Defaults to 'https://api.clarifai.com'.
           - ui (str): The URL for the UI. Defaults to 'https://clarifai.com'.
+          - root_certificates_path (str): Path to the SSL root certificates file, used to establish secure gRPC connections.
 
 
   Attributes:
@@ -51,14 +52,21 @@ class BaseClient:
     self.token = self.auth_helper._token
     self.user_app_id = self.auth_helper.get_user_app_id_proto()
     self.base = self.auth_helper.base
+    self.root_certificates_path = self.auth_helper._root_certificates_path
 
   @classmethod
   def from_auth_helper(cls, auth: ClarifaiAuthHelper, **kwargs):
     default_kwargs = {
-        "user_id": kwargs.get("user_id", None) or auth.user_id,
-        "app_id": kwargs.get("app_id", None) or auth.app_id,
-        "pat": kwargs.get("pat", None) or auth.pat,
-        "token": kwargs.get("token", None) or auth._token,
+        "user_id":
+            kwargs.get("user_id", None) or auth.user_id,
+        "app_id":
+            kwargs.get("app_id", None) or auth.app_id,
+        "pat":
+            kwargs.get("pat", None) or auth.pat,
+        "token":
+            kwargs.get("token", None) or auth._token,
+        "root_certificates_path":
+            kwargs.get("root_certificates_path", None) or auth._root_certificates_path
     }
     _base = kwargs.get("base", None) or auth.base
     _clss = cls.__mro__[0]
@@ -160,6 +168,8 @@ class BaseClient:
             value = value_s
           elif key == 'metrics':
             continue
+          elif key == 'size':
+            value = int(value)
           elif key in ['metadata']:
             if isinstance(value, dict) and value != {}:
               value_s = struct_pb2.Struct()

clarifai/client/dataset.py

@@ -47,6 +47,7 @@ class Dataset(Lister, BaseClient):
                base_url: str = "https://api.clarifai.com",
                pat: str = None,
                token: str = None,
+               root_certificates_path: str = None,
                **kwargs):
     """Initializes a Dataset object.
 
@@ -57,6 +58,7 @@ class Dataset(Lister, BaseClient):
         base_url (str): Base API url. Default "https://api.clarifai.com"
         pat (str): A personal access token for authentication. Can be set as env var CLARIFAI_PAT
         token (str): A session token for authentication. Accepts either a session token or a pat. Can be set as env var CLARIFAI_SESSION_TOKEN
+        root_certificates_path (str): Path to the SSL root certificates file, used to establish secure gRPC connections.
         **kwargs: Additional keyword arguments to be passed to the Dataset.
     """
     if url and dataset_id:
@@ -77,10 +79,21 @@ class Dataset(Lister, BaseClient):
     self.batch_size = 128  # limit max protos in a req
     self.task = None  # Upload dataset type
     self.input_object = Inputs(
-        user_id=self.user_id, app_id=self.app_id, pat=pat, token=token, base_url=base_url)
+        user_id=self.user_id,
+        app_id=self.app_id,
+        pat=pat,
+        token=token,
+        base_url=base_url,
+        root_certificates_path=root_certificates_path)
     self.logger = get_logger(logger_level="INFO", name=__name__)
     BaseClient.__init__(
-        self, user_id=self.user_id, app_id=self.app_id, base=base_url, pat=pat, token=token)
+        self,
+        user_id=self.user_id,
+        app_id=self.app_id,
+        base=base_url,
+        pat=pat,
+        token=token,
+        root_certificates_path=root_certificates_path)
     Lister.__init__(self)
 
   def create_version(self, **kwargs) -> 'Dataset':

clarifai/client/input.py

@@ -19,6 +19,7 @@ from tqdm import tqdm
 from clarifai.client.base import BaseClient
 from clarifai.client.lister import Lister
 from clarifai.constants.dataset import MAX_RETRIES
+from clarifai.constants.input import MAX_UPLOAD_BATCH_SIZE
 from clarifai.errors import UserError
 from clarifai.utils.logging import get_logger
 from clarifai.utils.misc import BackoffIterator, Chunker
@@ -34,6 +35,7 @@ class Inputs(Lister, BaseClient):
                base_url: str = "https://api.clarifai.com",
                pat: str = None,
                token: str = None,
+               root_certificates_path: str = None,
                **kwargs):
     """Initializes an Input object.
 
@@ -43,6 +45,7 @@ class Inputs(Lister, BaseClient):
         base_url (str): Base API url. Default "https://api.clarifai.com"
         pat (str): A personal access token for authentication. Can be set as env var CLARIFAI_PAT
         token (str): A session token for authentication. Accepts either a session token or a pat. Can be set as env var CLARIFAI_SESSION_TOKEN
+        root_certificates_path (str): Path to the SSL root certificates file, used to establish secure gRPC connections.
         **kwargs: Additional keyword arguments to be passed to the Input
     """
     self.user_id = user_id
@@ -51,7 +54,13 @@ class Inputs(Lister, BaseClient):
     self.input_info = resources_pb2.Input(**self.kwargs)
     self.logger = get_logger(logger_level=logger_level, name=__name__)
     BaseClient.__init__(
-        self, user_id=self.user_id, app_id=self.app_id, base=base_url, pat=pat, token=token)
+        self,
+        user_id=self.user_id,
+        app_id=self.app_id,
+        base=base_url,
+        pat=pat,
+        token=token,
+        root_certificates_path=root_certificates_path)
     Lister.__init__(self)
 
   @staticmethod
@@ -660,6 +669,10 @@ class Inputs(Lister, BaseClient):
     """
     if not isinstance(inputs, list):
       raise UserError("inputs must be a list of Input objects")
+    if len(inputs) > MAX_UPLOAD_BATCH_SIZE:
+      raise UserError(
+          f"Number of inputs to upload exceeds the maximum batch size of {MAX_UPLOAD_BATCH_SIZE}. Please reduce batch size."
+      )
     input_job_id = uuid.uuid4().hex  # generate a unique id for this job
     request = service_pb2.PostInputsRequest(
         user_app_id=self.user_app_id, inputs=inputs, inputs_add_job_id=input_job_id)

clarifai/client/model.py

@@ -1,7 +1,8 @@
 import os
 import time
-from typing import Any, Dict, Generator, List, Union
+from typing import Any, Dict, Generator, List, Tuple, Union
 
+import numpy as np
 import requests
 import yaml
 from clarifai_grpc.grpc.api import resources_pb2, service_pb2
@@ -12,6 +13,7 @@ from google.protobuf.struct_pb2 import Struct
 from tqdm import tqdm
 
 from clarifai.client.base import BaseClient
+from clarifai.client.dataset import Dataset
 from clarifai.client.input import Inputs
 from clarifai.client.lister import Lister
 from clarifai.constants.model import MAX_MODEL_PREDICT_INPUTS, TRAINABLE_MODEL_TYPES
@@ -34,6 +36,7 @@ class Model(Lister, BaseClient):
                base_url: str = "https://api.clarifai.com",
                pat: str = None,
                token: str = None,
+               root_certificates_path: str = None,
                **kwargs):
     """Initializes a Model object.
 
@@ -44,6 +47,7 @@ class Model(Lister, BaseClient):
         base_url (str): Base API url. Default "https://api.clarifai.com"
         pat (str): A personal access token for authentication. Can be set as env var CLARIFAI_PAT
         token (str): A session token for authentication. Accepts either a session token or a pat. Can be set as env var CLARIFAI_SESSION_TOKEN
+        root_certificates_path (str): Path to the SSL root certificates file, used to establish secure gRPC connections.
         **kwargs: Additional keyword arguments to be passed to the Model.
     """
     if url and model_id:
@@ -59,7 +63,13 @@ class Model(Lister, BaseClient):
     self.logger = get_logger(logger_level="INFO", name=__name__)
     self.training_params = {}
     BaseClient.__init__(
-        self, user_id=self.user_id, app_id=self.app_id, base=base_url, pat=pat, token=token)
+        self,
+        user_id=self.user_id,
+        app_id=self.app_id,
+        base=base_url,
+        pat=pat,
+        token=token,
+        root_certificates_path=root_certificates_path)
     Lister.__init__(self)
 
   def list_training_templates(self) -> List[str]:
@@ -243,7 +253,7 @@ class Model(Lister, BaseClient):
 
     return response.model.model_version.id
 
-  def training_status(self, version_id: str, training_logs: bool = False) -> Dict[str, str]:
+  def training_status(self, version_id: str = None, training_logs: bool = False) -> Dict[str, str]:
     """Get the training status for the model version. Also stores training logs
 
     Args:
@@ -258,19 +268,20 @@ class Model(Lister, BaseClient):
         >>> model = Model(model_id='model_id', user_id='user_id', app_id='app_id')
         >>> model.training_status(version_id='version_id',training_logs=True)
     """
+    if not version_id and not self.model_info.model_version.id:
+      raise UserError(
+          "Model version ID is missing. Please provide a `model_version` with a valid `id` as an argument or as a URL in the following format: '{user_id}/{app_id}/models/{your_model_id}/model_version_id/{your_version_model_id}' when initializing."
+      )
+
+    if not self.model_info.model_type_id or not self.model_info.model_version.train_log:
+      self.load_info()
     if self.model_info.model_type_id not in TRAINABLE_MODEL_TYPES:
       raise UserError(f"Model type {self.model_info.model_type_id} is not trainable")
 
-    request = service_pb2.GetModelVersionRequest(
-        user_app_id=self.user_app_id, model_id=self.id, version_id=version_id)
-    response = self._grpc_request(self.STUB.GetModelVersion, request)
-    if response.status.code != status_code_pb2.SUCCESS:
-      raise Exception(response.status)
-
     if training_logs:
       try:
-        if response.model_version.train_log:
-          log_response = requests.get(response.model_version.train_log)
+        if self.model_info.model_version.train_log:
+          log_response = requests.get(self.model_info.model_version.train_log)
           log_response.raise_for_status()  # Check for any HTTP errors
           with open(version_id + '.log', 'wb') as file:
             for chunk in log_response.iter_content(chunk_size=4096):  # 4KB
@@ -280,7 +291,7 @@ class Model(Lister, BaseClient):
       except requests.exceptions.RequestException as e:
         raise Exception(f"An error occurred while getting training logs: {e}")
 
-    return response.model_version.status
+    return self.model_info.model_version.status
 
   def delete_version(self, version_id: str) -> None:
     """Deletes a model version for the Model.
@@ -617,18 +628,22 @@ class Model(Lister, BaseClient):
     return response.eval_metrics
 
   def evaluate(self,
-               dataset_id: str,
+               dataset: Dataset = None,
+               dataset_id: str = None,
                dataset_app_id: str = None,
                dataset_user_id: str = None,
+               dataset_version_id: str = None,
                eval_id: str = None,
                extended_metrics: dict = None,
                eval_info: dict = None) -> resources_pb2.EvalMetrics:
     """ Run evaluation
 
     Args:
-      dataset_id (str): Dataset Id.
+      dataset (Dataset): If Clarifai Dataset is set, it will ignore other arguments prefixed with 'dataset_'.
+      dataset_id (str): Dataset Id. Default is None.
       dataset_app_id (str): App ID for cross app evaluation, leave it as None to use Model App ID. Default is None.
       dataset_user_id (str): User ID for cross app evaluation, leave it as None to use Model User ID. Default is None.
+      dataset_version_id (str): Dataset version Id. Default is None.
       eval_id (str): Specific ID for the evaluation. You must specify this parameter to either overwrite the result with the dataset ID or format your evaluation in an informative manner. If you don't, it will use random ID from system. Default is None.
       extended_metrics (dict): user custom metrics result. Default is None.
       eval_info (dict): custom eval info. Default is empty dict.
@@ -638,6 +653,23 @@ class Model(Lister, BaseClient):
 
     """
     assert self.model_info.model_version.id, "Model version is empty. Please provide `model_version` as arguments or with a URL as the format '{user_id}/{app_id}/models/{your_model_id}/model_version_id/{your_version_model_id}' when initializing."
+
+    if dataset:
+      self.logger.info("Using dataset, ignore other arguments prefixed with 'dataset_'")
+      dataset_id = dataset.id
+      dataset_app_id = dataset.app_id
+      dataset_user_id = dataset.user_id
+      dataset_version_id = dataset.version.id
+    else:
+      self.logger.warning(
+          "Arguments prefixed with `dataset_` will be removed soon, please use dataset")
+
+    gt_dataset = resources_pb2.Dataset(
+        id=dataset_id,
+        app_id=dataset_app_id or self.auth_helper.app_id,
+        user_id=dataset_user_id or self.auth_helper.user_id,
+        version=resources_pb2.DatasetVersion(id=dataset_version_id))
+
     metrics = None
     if isinstance(extended_metrics, dict):
       metrics = Struct()
@@ -659,11 +691,7 @@ class Model(Lister, BaseClient):
             model_version=resources_pb2.ModelVersion(id=self.model_info.model_version.id),
         ),
         extended_metrics=metrics,
-        ground_truth_dataset=resources_pb2.Dataset(
-            id=dataset_id,
-            app_id=dataset_app_id or self.auth_helper.app_id,
-            user_id=dataset_user_id or self.auth_helper.user_id,
-        ),
+        ground_truth_dataset=gt_dataset,
         eval_info=eval_info_params,
     )
     request = service_pb2.PostEvaluationsRequest(
@@ -761,6 +789,157 @@ class Model(Lister, BaseClient):
 
     return result
 
+  def get_eval_by_dataset(self, dataset: Dataset) -> List[resources_pb2.EvalMetrics]:
+    """Get all eval data of dataset
+
+    Args:
+        dataset (Dataset): Clarifai dataset
+
+    Returns:
+        List[resources_pb2.EvalMetrics]
+    """
+    _id = dataset.id
+    app = dataset.app_id or self.app_id
+    user_id = dataset.user_id or self.user_id
+    version = dataset.version.id
+
+    list_eval: resources_pb2.EvalMetrics = self.list_evaluations()
+    outputs = []
+    for _eval in list_eval:
+      if _eval.status.code == status_code_pb2.MODEL_EVALUATED:
+        gt_ds = _eval.ground_truth_dataset
+        if (_id == gt_ds.id and user_id == gt_ds.user_id and app == gt_ds.app_id):
+          if not version or version == gt_ds.version.id:
+            outputs.append(_eval)
+
+    return outputs
+
+  def get_raw_eval(self,
+                   dataset: Dataset = None,
+                   eval_id: str = None,
+                   return_format: str = 'array') -> Union[resources_pb2.EvalTestSetEntry, Tuple[
+                       np.array, np.array, list, List[Input]], Tuple[List[dict], List[dict]]]:
+    """Get ground truths, predictions and input information. Do not pass dataset and eval_id at same time
+
+    Args:
+        dataset (Dataset): Clarifai dataset, get eval data of latest eval result of dataset.
+        eval_id (str): Evaluation ID, get eval data of specific eval id.
+        return_format (str, optional): Choice {proto, array, coco}. !Note that `coco` is only applicable for 'visual-detector'. Defaults to 'array'.
+
+    Returns:
+
+        Depends on `return_format`.
+
+        * if return_format == proto
+          `resources_pb2.EvalTestSetEntry`
+
+        * if return_format == array
+          `Tuple(np.array, np.array, List[str], List[Input])`: Tuple has 4 elements (y, y_pred, concept_ids, inputs).
+            y, y_pred, concept_ids can be used to compute metrics. 'inputs' can be use to download
+            - if model is 'classifier': 'y' and 'y_pred' are both arrays with a shape of (num_inputs,)
+            - if model is 'visual-detector': 'y' and 'y_pred' are arrays with a shape of (num_inputs,), where each element is array has shape (num_annotation, 6) consists of [x_min, y_min, x_max, y_max, concept_index, score]. The score is always 1 for 'y'
+
+        * if return_format == coco: Applicable only for 'visual-detector'
+          `Tuple[List[Dict], List[Dict]]`: Tuple has 2 elemnts where first element is COCO Ground Truth and last one is COCO Prediction Annotation
+
+    Example Usages:
+    ------
+    * Evaluate `visual-classifier` using sklearn
+
+    ```python
+    import os
+    from sklearn.metrics import accuracy_score
+    from sklearn.metrics import classification_report
+    import numpy as np
+    from clarifai.client.model import Model
+    from clarifai.client.dataset import Dataset
+    os.environ["CLARIFAI_PAT"] = "???"
+    model = Model(url="url/of/model/includes/version-id")
+    dataset = Dataset(dataset_id="dataset-id")
+    y, y_pred, clss, input_protos = model.get_raw_eval(dataset, return_format="array")
+    y = np.argmax(y, axis=1)
+    y_pred = np.argmax(y_pred, axis=1)
+    report = classification_report(y, y_pred, target_names=clss)
+    print(report)
+    acc = accuracy_score(y, y_pred)
+    print("acc ", acc)
+    ```
+
+    * Evaluate `visual-detector` using COCOeval
+
+    ```python
+    import os
+    import json
+    from pycocotools.coco import COCO
+    from pycocotools.cocoeval import COCOeval
+    from clarifai.client.model import Model
+    from clarifai.client.dataset import Dataset
+    os.environ["CLARIFAI_PAT"] = "???" # Insert your PAT
+    model = Model(url=model_url)
+    dataset = Dataset(url=dataset_url)
+    y, y_pred = model.get_raw_eval(dataset, return_format="coco")
+    # save as files to load in COCO API
+    def save_annot(d, path):
+      with open(path, "w") as fp:
+        json.dump(d, fp, indent=2)
+    gt_path = os.path.join("gt.json")
+    pred_path = os.path.join("pred.json")
+    save_annot(y, gt_path)
+    save_annot(y_pred, pred_path)
+
+    cocoGt = COCO(gt_path)
+    cocoPred = COCO(pred_path)
+    cocoEval = COCOeval(cocoGt, cocoPred, "bbox")
+    cocoEval.evaluate()
+    cocoEval.accumulate()
+    cocoEval.summarize() # Print out result of all classes with all area type
+    # Example:
+    # Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.863
+    # Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.973
+    # Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.939
+    # ...
+    ```
+
+    """
+    from clarifai.utils.evaluation.testset_annotation_parser import (
+        parse_eval_annotation_classifier, parse_eval_annotation_detector,
+        parse_eval_annotation_detector_coco)
+
+    valid_model_types = ["visual-classifier", "text-classifier", "visual-detector"]
+    supported_format = ['proto', 'array', 'coco']
+    assert return_format in supported_format, ValueError(
+        f"Expected return_format in {supported_format}, got {return_format}")
+    self.load_info()
+    model_type_id = self.model_info.model_type_id
+    assert model_type_id in valid_model_types, \
+      f"This method only supports model types {valid_model_types}, but your model type is {self.model_info.model_type_id}."
+    assert not (dataset and
+                eval_id), "Using both `dataset` and `eval_id`, but only one should be passed."
+    assert not dataset or not eval_id, "Please provide either `dataset` or `eval_id`, but nothing was passed."
+    if model_type_id.endswith("-classifier") and return_format == "coco":
+      raise ValueError(
+          f"return_format coco only applies for `visual-detector`, however your model is `{model_type_id}`"
+      )
+
+    if dataset:
+      eval_by_ds = self.get_eval_by_dataset(dataset)
+      if len(eval_by_ds) == 0:
+        raise Exception(f"Model is not valuated with dataset: {dataset}")
+      eval_id = eval_by_ds[0].id
+
+    detail_eval_data = self.get_eval_by_id(eval_id=eval_id, test_set=True, metrics_by_class=True)
+
+    if return_format == "proto":
+      return detail_eval_data.test_set
+    else:
+      if model_type_id.endswith("-classifier"):
+        return parse_eval_annotation_classifier(detail_eval_data)
+      elif model_type_id == "visual-detector":
+        if return_format == "array":
+          return parse_eval_annotation_detector(detail_eval_data)
+        elif return_format == "coco":
+          return parse_eval_annotation_detector_coco(detail_eval_data)
+
   def export(self, export_dir: str = None) -> None:
     """Export the model, stores the exported model as model.tar file
 

clarifai/client/module.py

@@ -19,6 +19,7 @@ class Module(Lister, BaseClient):
                base_url: str = "https://api.clarifai.com",
                pat: str = None,
                token: str = None,
+               root_certificates_path: str = None,
                **kwargs):
     """Initializes a Module object.
 
@@ -29,6 +30,7 @@ class Module(Lister, BaseClient):
             base_url (str): Base API url. Default "https://api.clarifai.com"
             pat (str): A personal access token for authentication. Can be set as env var CLARIFAI_PAT.
             token (str): A session token for authentication. Accepts either a session token or a pat. Can be set as env var CLARIFAI_SESSION_TOKEN.
+            root_certificates_path (str): Path to the SSL root certificates file, used to establish secure gRPC connections.
             **kwargs: Additional keyword arguments to be passed to the Module.
         """
     if url and module_id:
@@ -44,7 +46,13 @@ class Module(Lister, BaseClient):
     self.module_info = resources_pb2.Module(**self.kwargs)
     self.logger = get_logger(logger_level="INFO", name=__name__)
     BaseClient.__init__(
-        self, user_id=self.user_id, app_id=self.app_id, base=base_url, pat=pat, token=token)
+        self,
+        user_id=self.user_id,
+        app_id=self.app_id,
+        base=base_url,
+        pat=pat,
+        token=token,
+        root_certificates_path=root_certificates_path)
     Lister.__init__(self)
 
   def list_versions(self, page_no: int = None,

clarifai/client/search.py

@@ -24,7 +24,8 @@ class Search(Lister, BaseClient):
                metric: str = DEFAULT_SEARCH_METRIC,
                base_url: str = "https://api.clarifai.com",
                pat: str = None,
-               token: str = None):
+               token: str = None,
+               root_certificates_path: str = None):
     """Initialize the Search object.
 
     Args:
@@ -35,6 +36,7 @@ class Search(Lister, BaseClient):
         base_url (str, optional): Base API url. Defaults to "https://api.clarifai.com".
         pat (str, optional): A personal access token for authentication. Can be set as env var CLARIFAI_PAT
         token (str): A session token for authentication. Accepts either a session token or a pat. Can be set as env var CLARIFAI_SESSION_TOKEN
+        root_certificates_path (str): Path to the SSL root certificates file, used to establish secure gRPC connections.
 
     Raises:
         UserError: If the metric is not 'cosine' or 'euclidean'.
@@ -52,7 +54,13 @@ class Search(Lister, BaseClient):
         user_id=self.user_id, app_id=self.app_id, pat=pat, token=token, base_url=base_url)
     self.rank_filter_schema = get_schema()
     BaseClient.__init__(
-        self, user_id=self.user_id, app_id=self.app_id, base=base_url, pat=pat, token=token)
+        self,
+        user_id=self.user_id,
+        app_id=self.app_id,
+        base=base_url,
+        pat=pat,
+        token=token,
+        root_certificates_path=root_certificates_path)
     Lister.__init__(self, page_size=1000)
 
   def _get_annot_proto(self, **kwargs):