clarifai 10.2.1__py3-none-any.whl → 10.3.1__py3-none-any.whl

clarifai/client/model.py

@@ -1,17 +1,20 @@
+import json
 import os
 import time
-from typing import Any, Dict, Generator, List, Union
+from typing import Any, Dict, Generator, List, Tuple, Union
 
+import numpy as np
 import requests
 import yaml
 from clarifai_grpc.grpc.api import resources_pb2, service_pb2
 from clarifai_grpc.grpc.api.resources_pb2 import Input
 from clarifai_grpc.grpc.api.status import status_code_pb2
 from google.protobuf.json_format import MessageToDict
-from google.protobuf.struct_pb2 import Struct
+from google.protobuf.struct_pb2 import Struct, Value
 from tqdm import tqdm
 
 from clarifai.client.base import BaseClient
+from clarifai.client.dataset import Dataset
 from clarifai.client.input import Inputs
 from clarifai.client.lister import Lister
 from clarifai.constants.model import MAX_MODEL_PREDICT_INPUTS, TRAINABLE_MODEL_TYPES
@@ -23,6 +26,10 @@ from clarifai.utils.model_train import (find_and_replace_key, params_parser,
                                         response_to_model_params, response_to_param_info,
                                         response_to_templates)
 
+MAX_SIZE_PER_STREAM = int(89_128_960)  # 85GiB
+MIN_CHUNK_FOR_UPLOAD_FILE = int(5_242_880)  # 5MiB
+MAX_CHUNK_FOR_UPLOAD_FILE = int(5_242_880_000)  # 5GiB
+
 
 class Model(Lister, BaseClient):
   """Model is a class that provides access to Clarifai API endpoints related to Model information."""
@@ -34,6 +41,7 @@ class Model(Lister, BaseClient):
                base_url: str = "https://api.clarifai.com",
                pat: str = None,
                token: str = None,
+               root_certificates_path: str = None,
                **kwargs):
     """Initializes a Model object.
 
@@ -44,6 +52,7 @@ class Model(Lister, BaseClient):
         base_url (str): Base API url. Default "https://api.clarifai.com"
         pat (str): A personal access token for authentication. Can be set as env var CLARIFAI_PAT
         token (str): A session token for authentication. Accepts either a session token or a pat. Can be set as env var CLARIFAI_SESSION_TOKEN
+        root_certificates_path (str): Path to the SSL root certificates file, used to establish secure gRPC connections.
         **kwargs: Additional keyword arguments to be passed to the Model.
     """
     if url and model_id:
@@ -54,12 +63,18 @@ class Model(Lister, BaseClient):
       user_id, app_id, _, model_id, model_version_id = ClarifaiUrlHelper.split_clarifai_url(url)
       model_version = {'id': model_version_id}
       kwargs = {'user_id': user_id, 'app_id': app_id}
-    self.kwargs = {**kwargs, 'id': model_id, 'model_version': model_version,}
+    self.kwargs = {**kwargs, 'id': model_id, 'model_version': model_version, }
     self.model_info = resources_pb2.Model(**self.kwargs)
     self.logger = get_logger(logger_level="INFO", name=__name__)
     self.training_params = {}
     BaseClient.__init__(
-        self, user_id=self.user_id, app_id=self.app_id, base=base_url, pat=pat, token=token)
+        self,
+        user_id=self.user_id,
+        app_id=self.app_id,
+        base=base_url,
+        pat=pat,
+        token=token,
+        root_certificates_path=root_certificates_path)
     Lister.__init__(self)
 
   def list_training_templates(self) -> List[str]:
@@ -122,11 +137,11 @@ class Model(Lister, BaseClient):
       raise Exception(response.status)
     params = response_to_model_params(
         response=response, model_type_id=self.model_info.model_type_id, template=template)
-    #yaml file
+    # yaml file
     assert save_to.endswith('.yaml'), "File extension should be .yaml"
     with open(save_to, 'w') as f:
       yaml.dump(params, f, default_flow_style=False, sort_keys=False)
-    #updating the global model params
+    # updating the global model params
     self.training_params.update(params)
 
     return params
@@ -149,14 +164,14 @@ class Model(Lister, BaseClient):
       raise UserError(
           f"Run 'model.get_params' to get the params for the {self.model_info.model_type_id} model type"
       )
-    #getting all the keys in nested dictionary
+    # getting all the keys in nested dictionary
     all_keys = [key for key in self.training_params.keys()] + [
         key for key in self.training_params.values() if isinstance(key, dict) for key in key
     ]
-    #checking if the given params are valid
+    # checking if the given params are valid
     if not set(kwargs.keys()).issubset(all_keys):
       raise UserError("Invalid params")
-    #updating the global model params
+    # updating the global model params
     for key, value in kwargs.items():
       find_and_replace_key(self.training_params, key, value)
 
@@ -228,7 +243,7 @@ class Model(Lister, BaseClient):
         params_dict = yaml.safe_load(file)
     else:
       params_dict = self.training_params
-    #getting all the concepts for the model type
+    # getting all the concepts for the model type
     if self.model_info.model_type_id not in ["clusterer", "text-to-text"]:
       concepts = self._list_concepts()
     train_dict = params_parser(params_dict, concepts)
@@ -243,7 +258,7 @@ class Model(Lister, BaseClient):
 
     return response.model.model_version.id
 
-  def training_status(self, version_id: str, training_logs: bool = False) -> Dict[str, str]:
+  def training_status(self, version_id: str = None, training_logs: bool = False) -> Dict[str, str]:
     """Get the training status for the model version. Also stores training logs
 
     Args:
@@ -258,19 +273,20 @@ class Model(Lister, BaseClient):
         >>> model = Model(model_id='model_id', user_id='user_id', app_id='app_id')
         >>> model.training_status(version_id='version_id',training_logs=True)
     """
+    if not version_id and not self.model_info.model_version.id:
+      raise UserError(
+          "Model version ID is missing. Please provide a `model_version` with a valid `id` as an argument or as a URL in the following format: '{user_id}/{app_id}/models/{your_model_id}/model_version_id/{your_version_model_id}' when initializing."
+      )
+
+    if not self.model_info.model_type_id or not self.model_info.model_version.train_log:
+      self.load_info()
     if self.model_info.model_type_id not in TRAINABLE_MODEL_TYPES:
       raise UserError(f"Model type {self.model_info.model_type_id} is not trainable")
 
-    request = service_pb2.GetModelVersionRequest(
-        user_app_id=self.user_app_id, model_id=self.id, version_id=version_id)
-    response = self._grpc_request(self.STUB.GetModelVersion, request)
-    if response.status.code != status_code_pb2.SUCCESS:
-      raise Exception(response.status)
-
     if training_logs:
       try:
-        if response.model_version.train_log:
-          log_response = requests.get(response.model_version.train_log)
+        if self.model_info.model_version.train_log:
+          log_response = requests.get(self.model_info.model_version.train_log)
           log_response.raise_for_status()  # Check for any HTTP errors
           with open(version_id + '.log', 'wb') as file:
             for chunk in log_response.iter_content(chunk_size=4096):  # 4KB
@@ -280,7 +296,7 @@ class Model(Lister, BaseClient):
       except requests.exceptions.RequestException as e:
         raise Exception(f"An error occurred while getting training logs: {e}")
 
-    return response.model_version.status
+    return self.model_info.model_version.status
 
   def delete_version(self, version_id: str) -> None:
     """Deletes a model version for the Model.
@@ -412,7 +428,7 @@ class Model(Lister, BaseClient):
       response = self._grpc_request(self.STUB.PostModelOutputs, request)
 
       if response.status.code == status_code_pb2.MODEL_DEPLOYING and \
-        time.time() - start_time < 60 * 10: # 10 minutes
+              time.time() - start_time < 60 * 10:  # 10 minutes
         self.logger.info(f"{self.id} model is still deploying, please wait...")
         time.sleep(next(backoff_iterator))
         continue
@@ -617,18 +633,22 @@ class Model(Lister, BaseClient):
     return response.eval_metrics
 
   def evaluate(self,
-               dataset_id: str,
+               dataset: Dataset = None,
+               dataset_id: str = None,
                dataset_app_id: str = None,
                dataset_user_id: str = None,
+               dataset_version_id: str = None,
                eval_id: str = None,
                extended_metrics: dict = None,
                eval_info: dict = None) -> resources_pb2.EvalMetrics:
     """ Run evaluation
 
     Args:
-      dataset_id (str): Dataset Id.
+      dataset (Dataset): If Clarifai Dataset is set, it will ignore other arguments prefixed with 'dataset_'.
+      dataset_id (str): Dataset Id. Default is None.
       dataset_app_id (str): App ID for cross app evaluation, leave it as None to use Model App ID. Default is None.
       dataset_user_id (str): User ID for cross app evaluation, leave it as None to use Model User ID. Default is None.
+      dataset_version_id (str): Dataset version Id. Default is None.
       eval_id (str): Specific ID for the evaluation. You must specify this parameter to either overwrite the result with the dataset ID or format your evaluation in an informative manner. If you don't, it will use random ID from system. Default is None.
       extended_metrics (dict): user custom metrics result. Default is None.
       eval_info (dict): custom eval info. Default is empty dict.
@@ -638,6 +658,23 @@ class Model(Lister, BaseClient):
 
     """
     assert self.model_info.model_version.id, "Model version is empty. Please provide `model_version` as arguments or with a URL as the format '{user_id}/{app_id}/models/{your_model_id}/model_version_id/{your_version_model_id}' when initializing."
+
+    if dataset:
+      self.logger.info("Using dataset, ignore other arguments prefixed with 'dataset_'")
+      dataset_id = dataset.id
+      dataset_app_id = dataset.app_id
+      dataset_user_id = dataset.user_id
+      dataset_version_id = dataset.version.id
+    else:
+      self.logger.warning(
+          "Arguments prefixed with `dataset_` will be removed soon, please use dataset")
+
+    gt_dataset = resources_pb2.Dataset(
+        id=dataset_id,
+        app_id=dataset_app_id or self.auth_helper.app_id,
+        user_id=dataset_user_id or self.auth_helper.user_id,
+        version=resources_pb2.DatasetVersion(id=dataset_version_id))
+
     metrics = None
     if isinstance(extended_metrics, dict):
       metrics = Struct()
@@ -659,11 +696,7 @@ class Model(Lister, BaseClient):
             model_version=resources_pb2.ModelVersion(id=self.model_info.model_version.id),
         ),
         extended_metrics=metrics,
-        ground_truth_dataset=resources_pb2.Dataset(
-            id=dataset_id,
-            app_id=dataset_app_id or self.auth_helper.app_id,
-            user_id=dataset_user_id or self.auth_helper.user_id,
-        ),
+        ground_truth_dataset=gt_dataset,
         eval_info=eval_info_params,
     )
     request = service_pb2.PostEvaluationsRequest(
@@ -761,6 +794,157 @@ class Model(Lister, BaseClient):
 
     return result
 
+  def get_eval_by_dataset(self, dataset: Dataset) -> List[resources_pb2.EvalMetrics]:
+    """Get all eval data of dataset
+
+    Args:
+        dataset (Dataset): Clarifai dataset
+
+    Returns:
+        List[resources_pb2.EvalMetrics]
+    """
+    _id = dataset.id
+    app = dataset.app_id or self.app_id
+    user_id = dataset.user_id or self.user_id
+    version = dataset.version.id
+
+    list_eval: resources_pb2.EvalMetrics = self.list_evaluations()
+    outputs = []
+    for _eval in list_eval:
+      if _eval.status.code == status_code_pb2.MODEL_EVALUATED:
+        gt_ds = _eval.ground_truth_dataset
+        if (_id == gt_ds.id and user_id == gt_ds.user_id and app == gt_ds.app_id):
+          if not version or version == gt_ds.version.id:
+            outputs.append(_eval)
+
+    return outputs
+
+  def get_raw_eval(self,
+                   dataset: Dataset = None,
+                   eval_id: str = None,
+                   return_format: str = 'array') -> Union[resources_pb2.EvalTestSetEntry, Tuple[
+                       np.array, np.array, list, List[Input]], Tuple[List[dict], List[dict]]]:
+    """Get ground truths, predictions and input information. Do not pass dataset and eval_id at same time
+
+    Args:
+        dataset (Dataset): Clarifai dataset, get eval data of latest eval result of dataset.
+        eval_id (str): Evaluation ID, get eval data of specific eval id.
+        return_format (str, optional): Choice {proto, array, coco}. !Note that `coco` is only applicable for 'visual-detector'. Defaults to 'array'.
+
+    Returns:
+
+        Depends on `return_format`.
+
+        * if return_format == proto
+          `resources_pb2.EvalTestSetEntry`
+
+        * if return_format == array
+          `Tuple(np.array, np.array, List[str], List[Input])`: Tuple has 4 elements (y, y_pred, concept_ids, inputs).
+            y, y_pred, concept_ids can be used to compute metrics. 'inputs' can be use to download
+            - if model is 'classifier': 'y' and 'y_pred' are both arrays with a shape of (num_inputs,)
+            - if model is 'visual-detector': 'y' and 'y_pred' are arrays with a shape of (num_inputs,), where each element is array has shape (num_annotation, 6) consists of [x_min, y_min, x_max, y_max, concept_index, score]. The score is always 1 for 'y'
+
+        * if return_format == coco: Applicable only for 'visual-detector'
+          `Tuple[List[Dict], List[Dict]]`: Tuple has 2 elemnts where first element is COCO Ground Truth and last one is COCO Prediction Annotation
+
+    Example Usages:
+    ------
+    * Evaluate `visual-classifier` using sklearn
+
+    ```python
+    import os
+    from sklearn.metrics import accuracy_score
+    from sklearn.metrics import classification_report
+    import numpy as np
+    from clarifai.client.model import Model
+    from clarifai.client.dataset import Dataset
+    os.environ["CLARIFAI_PAT"] = "???"
+    model = Model(url="url/of/model/includes/version-id")
+    dataset = Dataset(dataset_id="dataset-id")
+    y, y_pred, clss, input_protos = model.get_raw_eval(dataset, return_format="array")
+    y = np.argmax(y, axis=1)
+    y_pred = np.argmax(y_pred, axis=1)
+    report = classification_report(y, y_pred, target_names=clss)
+    print(report)
+    acc = accuracy_score(y, y_pred)
+    print("acc ", acc)
+    ```
+
+    * Evaluate `visual-detector` using COCOeval
+
+    ```python
+    import os
+    import json
+    from pycocotools.coco import COCO
+    from pycocotools.cocoeval import COCOeval
+    from clarifai.client.model import Model
+    from clarifai.client.dataset import Dataset
+    os.environ["CLARIFAI_PAT"] = "???" # Insert your PAT
+    model = Model(url=model_url)
+    dataset = Dataset(url=dataset_url)
+    y, y_pred = model.get_raw_eval(dataset, return_format="coco")
+    # save as files to load in COCO API
+    def save_annot(d, path):
+      with open(path, "w") as fp:
+        json.dump(d, fp, indent=2)
+    gt_path = os.path.join("gt.json")
+    pred_path = os.path.join("pred.json")
+    save_annot(y, gt_path)
+    save_annot(y_pred, pred_path)
+
+    cocoGt = COCO(gt_path)
+    cocoPred = COCO(pred_path)
+    cocoEval = COCOeval(cocoGt, cocoPred, "bbox")
+    cocoEval.evaluate()
+    cocoEval.accumulate()
+    cocoEval.summarize() # Print out result of all classes with all area type
+    # Example:
+    # Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.863
+    # Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.973
+    # Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.939
+    # ...
+    ```
+
+    """
+    from clarifai.utils.evaluation.testset_annotation_parser import (
+        parse_eval_annotation_classifier, parse_eval_annotation_detector,
+        parse_eval_annotation_detector_coco)
+
+    valid_model_types = ["visual-classifier", "text-classifier", "visual-detector"]
+    supported_format = ['proto', 'array', 'coco']
+    assert return_format in supported_format, ValueError(
+        f"Expected return_format in {supported_format}, got {return_format}")
+    self.load_info()
+    model_type_id = self.model_info.model_type_id
+    assert model_type_id in valid_model_types, \
+      f"This method only supports model types {valid_model_types}, but your model type is {self.model_info.model_type_id}."
+    assert not (dataset and
+                eval_id), "Using both `dataset` and `eval_id`, but only one should be passed."
+    assert not dataset or not eval_id, "Please provide either `dataset` or `eval_id`, but nothing was passed."
+    if model_type_id.endswith("-classifier") and return_format == "coco":
+      raise ValueError(
+          f"return_format coco only applies for `visual-detector`, however your model is `{model_type_id}`"
+      )
+
+    if dataset:
+      eval_by_ds = self.get_eval_by_dataset(dataset)
+      if len(eval_by_ds) == 0:
+        raise Exception(f"Model is not valuated with dataset: {dataset}")
+      eval_id = eval_by_ds[0].id
+
+    detail_eval_data = self.get_eval_by_id(eval_id=eval_id, test_set=True, metrics_by_class=True)
+
+    if return_format == "proto":
+      return detail_eval_data.test_set
+    else:
+      if model_type_id.endswith("-classifier"):
+        return parse_eval_annotation_classifier(detail_eval_data)
+      elif model_type_id == "visual-detector":
+        if return_format == "array":
+          return parse_eval_annotation_detector(detail_eval_data)
+        elif return_format == "coco":
+          return parse_eval_annotation_detector_coco(detail_eval_data)
+
   def export(self, export_dir: str = None) -> None:
     """Export the model, stores the exported model as model.tar file
 
@@ -834,7 +1018,7 @@ class Model(Lister, BaseClient):
       while True:
         get_export_response = _get_export_response()
         if get_export_response.export.status.code == status_code_pb2.MODEL_EXPORTING and \
-          time.time() - start_time < 60 * 30: # 30 minutes
+                time.time() - start_time < 60 * 30:  # 30 minutes
           self.logger.info(
               f"Model ID {self.id} with version {self.model_info.model_version.id} is still exporting, please wait..."
           )
@@ -848,3 +1032,302 @@ class Model(Lister, BaseClient):
               Req ID: {get_export_response.status.req_id}""")
     elif get_export_response.export.status.code == status_code_pb2.MODEL_EXPORTED:
       _download_exported_model(get_export_response, os.path.join(export_dir, "model.tar"))
+
+  @staticmethod
+  def _make_pretrained_config_proto(input_field_maps: dict,
+                                    output_field_maps: dict,
+                                    url: str = None):
+    """Make PretrainedModelConfig for uploading new version
+
+    Args:
+        input_field_maps (dict): dict
+        output_field_maps (dict): dict
+        url (str, optional): direct download url. Defaults to None.
+    """
+
+    def _parse_fields_map(x):
+      """parse input, outputs to Struct"""
+      _fields_map = Struct()
+      _fields_map.update(x)
+      return _fields_map
+
+    input_fields_map = _parse_fields_map(input_field_maps)
+    output_fields_map = _parse_fields_map(output_field_maps)
+
+    return resources_pb2.PretrainedModelConfig(
+        input_fields_map=input_fields_map, output_fields_map=output_fields_map, model_zip_url=url)
+
+  @staticmethod
+  def _make_inference_params_proto(
+      inference_parameters: List[Dict]) -> List[resources_pb2.ModelTypeField]:
+    """Convert list of Clarifai inference parameters to proto for uploading new version
+
+    Args:
+        inference_parameters (List[Dict]): Each dict has keys {field_type, path, default_value, description}
+
+    Returns:
+        List[resources_pb2.ModelTypeField]
+    """
+
+    def _make_default_value_proto(dtype, value):
+      if dtype == 1:
+        return Value(bool_value=value)
+      elif dtype == 2 or dtype == 21:
+        return Value(string_value=value)
+      elif dtype == 3:
+        return Value(number_value=value)
+
+    iterative_proto_params = []
+    for param in inference_parameters:
+      dtype = param.get("field_type")
+      proto_param = resources_pb2.ModelTypeField(
+          path=param.get("path"),
+          field_type=dtype,
+          default_value=_make_default_value_proto(dtype=dtype, value=param.get("default_value")),
+          description=param.get("description"),
+      )
+      iterative_proto_params.append(proto_param)
+    return iterative_proto_params
+
+  def create_version_by_file(self,
+                             file_path: str,
+                             input_field_maps: dict,
+                             output_field_maps: dict,
+                             inference_parameter_configs: dict = None,
+                             model_version: str = None,
+                             part_id: int = 1,
+                             range_start: int = 0,
+                             no_cache: bool = False,
+                             no_resume: bool = False,
+                             description: str = "") -> 'Model':
+    """Create model version by uploading local file
+
+    Args:
+        file_path (str): path to built file.
+        input_field_maps (dict): a dict where the key is clarifai input field and the value is triton model input,
+          {clarifai_input_field: triton_input_filed}.
+        output_field_maps (dict): a dict where the keys are clarifai output fields and the values are triton model outputs,
+          {clarifai_output_field1: triton_output_filed1, clarifai_output_field2: triton_output_filed2,...}.
+        inference_parameter_configs (List[dict]): list of dicts - keys are path, field_type, default_value, description. Default is None
+        model_version (str, optional): Custom model version. Defaults to None.
+        part_id (int, optional): part id of file. Defaults to 1.
+        range_start (int, optional): range of uploaded size. Defaults to 0.
+        no_cache (bool, optional): not saving uploading cache that is used to resume uploading. Defaults to False.
+        no_resume (bool, optional): disable auto resume upload. Defaults to False.
+        description (str): Model description.
+
+    Return:
+      Model: instance of Model with new created version
+
+    """
+    file_size = os.path.getsize(file_path)
+    assert MIN_CHUNK_FOR_UPLOAD_FILE <= file_size <= MAX_CHUNK_FOR_UPLOAD_FILE, "The file size exceeds the allowable limit, which ranges from 5MiB to 5GiB."
+
+    pretrained_proto = Model._make_pretrained_config_proto(
+        input_field_maps=input_field_maps, output_field_maps=output_field_maps)
+    inference_param_proto = Model._make_inference_params_proto(
+        inference_parameter_configs) if inference_parameter_configs else None
+
+    if file_size >= 1e9:
+      chunk_size = 1024 * 50_000  # 50MB
+    else:
+      chunk_size = 1024 * 10_000  # 10MB
+
+    #self.logger.info(f"Chunk {chunk_size/1e6}MB, {file_size/chunk_size} steps")
+    #self.logger.info(f" Max bytes per stream {MAX_SIZE_PER_STREAM}")
+
+    cache_dir = os.path.join(file_path, '..', '.cache')
+    cache_upload_file = os.path.join(cache_dir, "upload.json")
+    last_percent = 0
+    if os.path.exists(cache_upload_file) and not no_resume:
+      with open(cache_upload_file, "r") as fp:
+        try:
+          cache_info = json.load(fp)
+          if isinstance(cache_info, dict):
+            part_id = cache_info.get("part_id", part_id)
+            chunk_size = cache_info.get("chunk_size", chunk_size)
+            range_start = cache_info.get("range_start", range_start)
+            model_version = cache_info.get("model_version", model_version)
+            last_percent = cache_info.get("last_percent", last_percent)
+        except Exception as e:
+          self.logger.error(f"Skipping loading the upload cache due to error {e}.")
+
+    def init_model_version_upload(model_version):
+      return service_pb2.PostModelVersionsUploadRequest(
+          upload_config=service_pb2.PostModelVersionsUploadConfig(
+              user_app_id=self.user_app_id,
+              model_id=self.id,
+              total_size=file_size,
+              model_version=resources_pb2.ModelVersion(
+                  id=model_version,
+                  pretrained_model_config=pretrained_proto,
+                  description=description,
+                  output_info=resources_pb2.OutputInfo(params_specs=inference_param_proto)),
+          ))
+
+    def _uploading(chunk, part_id, range_start, model_version):
+      return service_pb2.PostModelVersionsUploadRequest(
+          content_part=resources_pb2.UploadContentPart(
+              data=chunk, part_number=part_id, range_start=range_start))
+
+    finished_status = [status_code_pb2.SUCCESS, status_code_pb2.UPLOAD_DONE]
+    uploading_in_progress_status = [
+        status_code_pb2.UPLOAD_IN_PROGRESS, status_code_pb2.MODEL_UPLOADING
+    ]
+
+    def _save_cache(cache: dict):
+      if not no_cache:
+        os.makedirs(cache_dir, exist_ok=True)
+        with open(cache_upload_file, "w") as fp:
+          json.dump(cache, fp, indent=2)
+
+    def stream_request(fp, part_id, end_part_id, chunk_size, version):
+      yield init_model_version_upload(version)
+      for iter_part_id in range(part_id, end_part_id):
+        chunk = fp.read(chunk_size)
+        if not chunk:
+          return
+        yield _uploading(
+            chunk=chunk,
+            part_id=iter_part_id,
+            range_start=chunk_size * (iter_part_id - 1),
+            model_version=version)
+
+    tqdm_loader = tqdm(total=100)
+    if model_version:
+      desc = f"Uploading model `{self.id}` version `{model_version}` ..."
+    else:
+      desc = f"Uploading model `{self.id}` ..."
+    tqdm_loader.set_description(desc)
+
+    cache_uploading_info = {}
+    cache_uploading_info["part_id"] = part_id
+    cache_uploading_info["model_version"] = model_version
+    cache_uploading_info["range_start"] = range_start
+    cache_uploading_info["chunk_size"] = chunk_size
+    cache_uploading_info["last_percent"] = last_percent
+    tqdm_loader.update(last_percent)
+    last_part_id = part_id
+    n_chunks = file_size // chunk_size
+    n_chunk_per_stream = MAX_SIZE_PER_STREAM // chunk_size or 1
+
+    def stream_and_logging(request, tqdm_loader, cache_uploading_info, expected_steps: int = None):
+      for st_step, st_response in enumerate(self.auth_helper.get_stub().PostModelVersionsUpload(
+          request, metadata=self.auth_helper.metadata)):
+        if st_response.status.code in uploading_in_progress_status:
+          if cache_uploading_info["model_version"]:
+            assert st_response.model_version_id == cache_uploading_info[
+                "model_version"], RuntimeError
+          else:
+            cache_uploading_info["model_version"] = st_response.model_version_id
+          if st_step > 0:
+            cache_uploading_info["part_id"] += 1
+            cache_uploading_info["range_start"] += chunk_size
+            _save_cache(cache_uploading_info)
+
+            if st_response.status.percent_completed:
+              step_percent = st_response.status.percent_completed - cache_uploading_info["last_percent"]
+              cache_uploading_info["last_percent"] += step_percent
+              tqdm_loader.set_description(
+                  f"{st_response.status.description}, {st_response.status.details}, version id  {cache_uploading_info.get('model_version')}"
+              )
+              tqdm_loader.update(step_percent)
+        elif st_response.status.code not in finished_status + uploading_in_progress_status:
+          # TODO: Find better way to handle error
+          if expected_steps and st_step < expected_steps:
+            raise Exception(f"Failed to upload model, error: {st_response.status}")
+
+    with open(file_path, 'rb') as fp:
+      # seeking
+      for _ in range(1, last_part_id):
+        fp.read(chunk_size)
+      # Stream even part
+      end_part_id = n_chunks or 1
+      for iter_part_id in range(int(last_part_id), int(n_chunks), int(n_chunk_per_stream)):
+        end_part_id = iter_part_id + n_chunk_per_stream
+        if end_part_id >= n_chunks:
+          end_part_id = n_chunks
+        expected_steps = end_part_id - iter_part_id + 1  # init step
+        st_reqs = stream_request(
+            fp,
+            iter_part_id,
+            end_part_id=end_part_id,
+            chunk_size=chunk_size,
+            version=cache_uploading_info["model_version"])
+        stream_and_logging(st_reqs, tqdm_loader, cache_uploading_info, expected_steps)
+      # Stream last part
+      accum_size = (end_part_id - 1) * chunk_size
+      remained_size = file_size - accum_size if accum_size >= 0 else file_size
+      st_reqs = stream_request(
+          fp,
+          end_part_id,
+          end_part_id=end_part_id + 1,
+          chunk_size=remained_size,
+          version=cache_uploading_info["model_version"])
+      stream_and_logging(st_reqs, tqdm_loader, cache_uploading_info, 2)
+
+    # clean up cache
+    if not no_cache:
+      try:
+        os.remove(cache_upload_file)
+      except Exception:
+        _save_cache({})
+
+    if cache_uploading_info["last_percent"] <= 100:
+      tqdm_loader.update(100 - cache_uploading_info["last_percent"])
+      tqdm_loader.set_description("Upload done")
+
+    tqdm_loader.set_description(
+        f"Success uploading model {self.id}, new version {cache_uploading_info.get('model_version')}"
+    )
+
+    return Model.from_auth_helper(
+        auth=self.auth_helper,
+        model_id=self.id,
+        model_version=dict(id=cache_uploading_info.get('model_version')))
+
+  def create_version_by_url(self,
+                            url: str,
+                            input_field_maps: dict,
+                            output_field_maps: dict,
+                            inference_parameter_configs: List[dict] = None,
+                            description: str = "") -> 'Model':
+    """Upload a new version of an existing model in the Clarifai platform using direct download url.
+
+    Args:
+      url (str]): url of zip of model
+      input_field_maps (dict): a dict where the key is clarifai input field and the value is triton model input,
+          {clarifai_input_field: triton_input_filed}.
+      output_field_maps (dict): a dict where the keys are clarifai output fields and the values are triton model outputs,
+          {clarifai_output_field1: triton_output_filed1, clarifai_output_field2: triton_output_filed2,...}.
+      inference_parameter_configs (List[dict]): list of dicts - keys are path, field_type, default_value, description. Default is None
+      description (str): Model description.
+
+    Return:
+      Model: instance of Model with new created version
+    """
+
+    pretrained_proto = Model._make_pretrained_config_proto(
+        input_field_maps=input_field_maps, output_field_maps=output_field_maps, url=url)
+    inference_param_proto = Model._make_inference_params_proto(
+        inference_parameter_configs) if inference_parameter_configs else None
+    request = service_pb2.PostModelVersionsRequest(
+        user_app_id=self.user_app_id,
+        model_id=self.id,
+        model_versions=[
+            resources_pb2.ModelVersion(
+                pretrained_model_config=pretrained_proto,
+                description=description,
+                output_info=resources_pb2.OutputInfo(params_specs=inference_param_proto))
+        ])
+    response = self._grpc_request(self.STUB.PostModelVersions, request)
+
+    if response.status.code != status_code_pb2.SUCCESS:
+      raise Exception(f"Failed to upload model, error: {response.status}")
+    self.logger.info(
+        f"Success uploading model {self.id}, new version {response.model.model_version.id}")
+
+    return Model.from_auth_helper(
+        auth=self.auth_helper,
+        model_id=self.id,
+        model_version=dict(id=response.model.model_version.id))