clarifai 11.3.0rc2__py3-none-any.whl → 11.4.0__py3-none-any.whl

clarifai/utils/misc.py

@@ -8,95 +8,99 @@ from clarifai_grpc.grpc.api.status import status_code_pb2
 from clarifai.errors import UserError
 
 RETRYABLE_CODES = [
-    status_code_pb2.MODEL_DEPLOYING, status_code_pb2.MODEL_LOADING,
-    status_code_pb2.MODEL_BUSY_PLEASE_RETRY
+    status_code_pb2.MODEL_DEPLOYING,
+    status_code_pb2.MODEL_LOADING,
+    status_code_pb2.MODEL_BUSY_PLEASE_RETRY,
 ]
 
 DEFAULT_CONFIG = f'{os.environ["HOME"]}/.config/clarifai/config'
 
 
 def status_is_retryable(status_code: int) -> bool:
-  """Check if a status code is retryable."""
-  return status_code in RETRYABLE_CODES
+    """Check if a status code is retryable."""
+    return status_code in RETRYABLE_CODES
 
 
 class Chunker:
-  """Split an input sequence into small chunks."""
+    """Split an input sequence into small chunks."""
 
-  def __init__(self, seq: List, size: int) -> None:
-    self.seq = seq
-    self.size = size
+    def __init__(self, seq: List, size: int) -> None:
+        self.seq = seq
+        self.size = size
 
-  def chunk(self) -> List[List]:
-    """Chunk input sequence."""
-    return [self.seq[pos:pos + self.size] for pos in range(0, len(self.seq), self.size)]
+    def chunk(self) -> List[List]:
+        """Chunk input sequence."""
+        return [self.seq[pos : pos + self.size] for pos in range(0, len(self.seq), self.size)]
 
 
 class BackoffIterator:
-  """Iterator that returns a sequence of backoff values."""
+    """Iterator that returns a sequence of backoff values."""
 
-  def __init__(self, count=0):
-    self.count = count
+    def __init__(self, count=0):
+        self.count = count
 
-  def __iter__(self):
-    return self
+    def __iter__(self):
+        return self
 
-  def __next__(self):
-    self.count += 1
-    return 0.1 * (1.3**self.count)
+    def __next__(self):
+        self.count += 1
+        return 0.1 * (1.3**self.count)
 
 
 def get_from_dict_or_env(key: str, env_key: str, **data) -> str:
-  """Get a value from a dictionary or an environment variable."""
-  if key in data and data[key]:
-    return data[key]
-  else:
-    return get_from_env(key, env_key)
+    """Get a value from a dictionary or an environment variable."""
+    if key in data and data[key]:
+        return data[key]
+    else:
+        return get_from_env(key, env_key)
 
 
 def get_from_env(key: str, env_key: str) -> str:
-  """Get a value from a dictionary or an environment variable."""
-  if env_key in os.environ and os.environ[env_key]:
-    return os.environ[env_key]
-  else:
-    raise UserError(f"Did not find `{key}`, please add an environment variable"
-                    f" `{env_key}` which contains it, or pass"
-                    f"  `{key}` as a named parameter.")
+    """Get a value from a dictionary or an environment variable."""
+    if env_key in os.environ and os.environ[env_key]:
+        return os.environ[env_key]
+    else:
+        raise UserError(
+            f"Did not find `{key}`, please add an environment variable"
+            f" `{env_key}` which contains it, or pass"
+            f"  `{key}` as a named parameter."
+        )
 
 
-def concept_relations_accumulation(relations_dict: Dict[str, Any], subject_concept: str,
-                                   object_concept: str, predicate: str) -> Dict[str, Any]:
-  """Append the concept relation to relations dict based on its predicate.
+def concept_relations_accumulation(
+    relations_dict: Dict[str, Any], subject_concept: str, object_concept: str, predicate: str
+) -> Dict[str, Any]:
+    """Append the concept relation to relations dict based on its predicate.
 
     Args:
         relations_dict (dict): A dict of concept relations info.
     """
-  if predicate == 'hyponym':
-    if object_concept in relations_dict:
-      relations_dict[object_concept].append(subject_concept)
-    else:
-      relations_dict[object_concept] = [subject_concept]
-  elif predicate == 'hypernym':
-    if subject_concept in relations_dict:
-      relations_dict[subject_concept].append(object_concept)
+    if predicate == 'hyponym':
+        if object_concept in relations_dict:
+            relations_dict[object_concept].append(subject_concept)
+        else:
+            relations_dict[object_concept] = [subject_concept]
+    elif predicate == 'hypernym':
+        if subject_concept in relations_dict:
+            relations_dict[subject_concept].append(object_concept)
+        else:
+            relations_dict[subject_concept] = [object_concept]
     else:
-      relations_dict[subject_concept] = [object_concept]
-  else:
-    relations_dict[object_concept] = []
-    relations_dict[subject_concept] = []
-  return relations_dict
+        relations_dict[object_concept] = []
+        relations_dict[subject_concept] = []
+    return relations_dict
 
 
 def get_uuid(val: int) -> str:
-  """Generates a UUID."""
-  return uuid.uuid4().hex[:val]
+    """Generates a UUID."""
+    return uuid.uuid4().hex[:val]
 
 
 def clean_input_id(input_id: str) -> str:
-  """Clean input_id string into a valid input ID"""
-  input_id = re.sub('[., /]+', '_', input_id)
-  input_id = re.sub('[_]+', '_', input_id)
-  input_id = re.sub('[-]+', '-', input_id)
-  input_id = input_id.lower().strip('_-')
-  input_id = re.sub('[^a-z0-9-_]+', '', input_id)
-  return input_id
+    """Clean input_id string into a valid input ID"""
+    input_id = re.sub('[., /]+', '_', input_id)
+    input_id = re.sub('[_]+', '_', input_id)
+    input_id = re.sub('[-]+', '-', input_id)
+    input_id = input_id.lower().strip('_-')
+    input_id = re.sub('[^a-z0-9-_]+', '', input_id)
+    return input_id

clarifai/utils/model_train.py

@@ -7,156 +7,175 @@ from google.protobuf.struct_pb2 import Struct
 
 
 def response_to_templates(response: MultiModelTypeResponse, model_type_id: str) -> List[str]:
-  """Converts the response from the API to a list of templates for the given model type id."""
-  dict_response = MessageToDict(response)
-  templates = []
-  for model_type in dict_response['modelTypes']:
-    if model_type['id'] == model_type_id:
-      for modeltypefield in model_type['modelTypeFields']:
-        if modeltypefield['path'].split('.')[-1] == "template":
-          templates = [template['id'] for template in modeltypefield['modelTypeEnumOptions']]
-  return templates
-
-
-def response_to_model_params(response: MultiModelTypeResponse,
-                             model_type_id: str,
-                             template: str = None) -> Dict[str, Any]:
-  """Converts the response from the API to a dictionary of model params for the given model type id."""
-  dict_response = MessageToDict(response)
-  params = {}
-  if model_type_id != "clusterer":
-    params["dataset_id"] = ""
-    params["dataset_version_id"] = ""
-  if model_type_id not in ["clusterer", "text-to-text"]:
-    params["concepts"] = []
-  params["train_params"] = dict()
-
-  for model_type in dict_response['modelTypes']:
-    if model_type['id'] == model_type_id:
-      #iterate through the model type fields
-      for modeltypefield in model_type['modelTypeFields']:
-        _path = modeltypefield['path'].split('.')
-        #removing the fields which are not required
-        if (_path[0] in ["'eval_info'"]) or (_path[1] in ["dataset", "data"]) or (_path[-1] in [
-            "dataset_id", "dataset_version_id"
-        ]) or ("internalOnly" in modeltypefield.keys()):
-          continue
-        #checking the template model type fields
-        if _path[-1] != "template":
-          if _path[0] == 'train_info':
-            try:
-              params["train_params"][_path[-1]] = modeltypefield['defaultValue']
-            except Exception:
-              params["train_params"][_path[-1]] = None
-          if _path[0] == 'output_info':
-            params["inference_params"] = dict()
-            try:
-              params["inference_params"][_path[-1]] = modeltypefield['defaultValue']
-            except Exception:
-              params["inference_params"][_path[-1]] = None
-          if _path[0] == 'output_info' and _path[1] == 'output_config':
-            params["concepts_mutually_exclusive"] = (modeltypefield['defaultValue'])
-        else:
-          if 'modelTypeEnumOptions' in modeltypefield.keys():
-            #check given template is valid
-            all_templates = [template['id'] for template in modeltypefield['modelTypeEnumOptions']]
-            if template not in all_templates:
-              raise ValueError(f"Invalid template {template} for model type {model_type_id}. "
-                               f"Valid templates are {all_templates}")
-            for modeltypeenum in modeltypefield['modelTypeEnumOptions']:
-              #finding the given template
-              if modeltypeenum['id'] == template:
-                params['train_params']["template"] = modeltypeenum['id']
-                #iterate through the template fields
-                for modeltypeenumfield in modeltypeenum['modelTypeFields']:
-                  if "internalOnly" in modeltypeenumfield.keys():
+    """Converts the response from the API to a list of templates for the given model type id."""
+    dict_response = MessageToDict(response)
+    templates = []
+    for model_type in dict_response['modelTypes']:
+        if model_type['id'] == model_type_id:
+            for modeltypefield in model_type['modelTypeFields']:
+                if modeltypefield['path'].split('.')[-1] == "template":
+                    templates = [
+                        template['id'] for template in modeltypefield['modelTypeEnumOptions']
+                    ]
+    return templates
+
+
+def response_to_model_params(
+    response: MultiModelTypeResponse, model_type_id: str, template: str = None
+) -> Dict[str, Any]:
+    """Converts the response from the API to a dictionary of model params for the given model type id."""
+    dict_response = MessageToDict(response)
+    params = {}
+    if model_type_id != "clusterer":
+        params["dataset_id"] = ""
+        params["dataset_version_id"] = ""
+    if model_type_id not in ["clusterer", "text-to-text"]:
+        params["concepts"] = []
+    params["train_params"] = dict()
+
+    for model_type in dict_response['modelTypes']:
+        if model_type['id'] == model_type_id:
+            # iterate through the model type fields
+            for modeltypefield in model_type['modelTypeFields']:
+                _path = modeltypefield['path'].split('.')
+                # removing the fields which are not required
+                if (
+                    (_path[0] in ["'eval_info'"])
+                    or (_path[1] in ["dataset", "data"])
+                    or (_path[-1] in ["dataset_id", "dataset_version_id"])
+                    or ("internalOnly" in modeltypefield.keys())
+                ):
                     continue
-                  try:
-                    params["train_params"][modeltypeenumfield['path'].split('.')[
-                        -1]] = modeltypeenumfield['defaultValue']
-                  except Exception:
-                    params["train_params"][modeltypeenumfield['path'].split('.')[-1]] = None
-  #custom config
-  if "custom_config" in params['train_params'].keys():
-    # Write the content to the file
-    file_path = params['train_params']['template'] + ".py"
-    with open(file_path, "w") as script_file:
-      script_file.write(params['train_params']['custom_config'])
-    params['train_params']['custom_config'] = file_path
-
-  return params
+                # checking the template model type fields
+                if _path[-1] != "template":
+                    if _path[0] == 'train_info':
+                        try:
+                            params["train_params"][_path[-1]] = modeltypefield['defaultValue']
+                        except Exception:
+                            params["train_params"][_path[-1]] = None
+                    if _path[0] == 'output_info':
+                        params["inference_params"] = dict()
+                        try:
+                            params["inference_params"][_path[-1]] = modeltypefield['defaultValue']
+                        except Exception:
+                            params["inference_params"][_path[-1]] = None
+                    if _path[0] == 'output_info' and _path[1] == 'output_config':
+                        params["concepts_mutually_exclusive"] = modeltypefield['defaultValue']
+                elif 'modelTypeEnumOptions' in modeltypefield.keys():
+                    # check given template is valid
+                    all_templates = [
+                        template['id'] for template in modeltypefield['modelTypeEnumOptions']
+                    ]
+                    if template not in all_templates:
+                        raise ValueError(
+                            f"Invalid template {template} for model type {model_type_id}. "
+                            f"Valid templates are {all_templates}"
+                        )
+                    for modeltypeenum in modeltypefield['modelTypeEnumOptions']:
+                        # finding the given template
+                        if modeltypeenum['id'] == template:
+                            params['train_params']["template"] = modeltypeenum['id']
+                            # iterate through the template fields
+                            for modeltypeenumfield in modeltypeenum['modelTypeFields']:
+                                if "internalOnly" in modeltypeenumfield.keys():
+                                    continue
+                                try:
+                                    params["train_params"][
+                                        modeltypeenumfield['path'].split('.')[-1]
+                                    ] = modeltypeenumfield['defaultValue']
+                                except Exception:
+                                    params["train_params"][
+                                        modeltypeenumfield['path'].split('.')[-1]
+                                    ] = None
+    # custom config
+    if "custom_config" in params['train_params'].keys():
+        # Write the content to the file
+        file_path = params['train_params']['template'] + ".py"
+        with open(file_path, "w") as script_file:
+            script_file.write(params['train_params']['custom_config'])
+        params['train_params']['custom_config'] = file_path
+
+    return params
 
 
 def params_parser(params_dict: dict, concepts: List = None) -> Dict[str, Any]:
-  """Converts the params dictionary to a dictionary of model specific params for the given model"""
-  #dict parser
-  train_dict = {}
-  train_dict["train_info"] = dict()
-  train_dict["output_info"] = dict()
-
-  train_dict["train_info"]['params'] = Struct()
-  if 'custom_config' in params_dict['train_params'].keys():
-    # Open and read the Python file
-    with open(params_dict['train_params']['custom_config'], 'r') as python_file:
-      custom_config = python_file.read()
-    params_dict['train_params']['custom_config'] = custom_config
-
-  train_dict["train_info"]['params'].update(params_dict["train_params"])
-  if 'dataset_id' in params_dict.keys():
-    train_dict["train_info"]['params']['dataset_id'] = params_dict['dataset_id']
-    train_dict["train_info"]['params']['dataset_version_id'] = params_dict['dataset_version_id']
-  train_dict['train_info'] = resources_pb2.TrainInfo(**train_dict['train_info'])
-
-  if 'concepts' in params_dict.keys():
-    assert set(params_dict["concepts"]).issubset(
-        concepts), "Invalid concept IDs. Available concepts in the app are {}".format(concepts)
-    train_dict["output_info"]['data'] = resources_pb2.Data(
-        concepts=[resources_pb2.Concept(id=concept_id) for concept_id in params_dict["concepts"]])
-  if 'inference_params' in params_dict.keys():
-    train_dict["output_info"]['params'] = Struct()
-    train_dict['output_info']['params'].update(params_dict["inference_params"])
-  if 'concepts_mutually_exclusive' in params_dict.keys():
-    train_dict['output_info']['output_config'] = resources_pb2.OutputConfig(
-        concepts_mutually_exclusive=params_dict['concepts_mutually_exclusive'])
-  train_dict['output_info'] = resources_pb2.OutputInfo(**train_dict['output_info'])
-
-  return train_dict
-
-
-def response_to_param_info(response: MultiModelTypeResponse,
-                           model_type_id: str,
-                           param: str,
-                           template: str = None) -> Dict[str, Any]:
-  """Converts the response from the API to a dictionary of model param info for the given model type id."""
-  dict_response = MessageToDict(response)
-  for model_type in dict_response['modelTypes']:
-    if model_type['id'] == model_type_id:
-      #iterate through the model type fields
-      for modeltypefield in model_type['modelTypeFields']:
-        if modeltypefield['path'].split('.')[-1] == param:
-          if param == 'template':
-            del modeltypefield['placeholder']
-            del modeltypefield['modelTypeEnumOptions']
-            return modeltypefield
-          modeltypefield['param'] = modeltypefield.pop('path').split('.')[-1]
-          del modeltypefield['placeholder']
-          return modeltypefield
-        #checking the template model type fields
-        if modeltypefield['path'].split('.')[-1] == "template":
-          for modeltypeenum in modeltypefield['modelTypeEnumOptions']:
-            if modeltypeenum['id'] == template:
-              #iterate through the template fields
-              for modeltypeenumfield in modeltypeenum['modelTypeFields']:
-                if modeltypeenumfield['path'].split('.')[-1] == param:
-                  modeltypeenumfield['param'] = modeltypeenumfield.pop('path').split('.')[-1]
-                  del modeltypeenumfield['placeholder']
-                  return modeltypeenumfield
+    """Converts the params dictionary to a dictionary of model specific params for the given model"""
+    # dict parser
+    train_dict = {}
+    train_dict["train_info"] = dict()
+    train_dict["output_info"] = dict()
+
+    train_dict["train_info"]['params'] = Struct()
+    if 'custom_config' in params_dict['train_params'].keys():
+        # Open and read the Python file
+        with open(params_dict['train_params']['custom_config'], 'r') as python_file:
+            custom_config = python_file.read()
+        params_dict['train_params']['custom_config'] = custom_config
+
+    train_dict["train_info"]['params'].update(params_dict["train_params"])
+    if 'dataset_id' in params_dict.keys():
+        train_dict["train_info"]['params']['dataset_id'] = params_dict['dataset_id']
+        train_dict["train_info"]['params']['dataset_version_id'] = params_dict[
+            'dataset_version_id'
+        ]
+    train_dict['train_info'] = resources_pb2.TrainInfo(**train_dict['train_info'])
+
+    if 'concepts' in params_dict.keys():
+        assert set(params_dict["concepts"]).issubset(concepts), (
+            "Invalid concept IDs. Available concepts in the app are {}".format(concepts)
+        )
+        train_dict["output_info"]['data'] = resources_pb2.Data(
+            concepts=[
+                resources_pb2.Concept(id=concept_id) for concept_id in params_dict["concepts"]
+            ]
+        )
+    if 'inference_params' in params_dict.keys():
+        train_dict["output_info"]['params'] = Struct()
+        train_dict['output_info']['params'].update(params_dict["inference_params"])
+    if 'concepts_mutually_exclusive' in params_dict.keys():
+        train_dict['output_info']['output_config'] = resources_pb2.OutputConfig(
+            concepts_mutually_exclusive=params_dict['concepts_mutually_exclusive']
+        )
+    train_dict['output_info'] = resources_pb2.OutputInfo(**train_dict['output_info'])
+
+    return train_dict
+
+
+def response_to_param_info(
+    response: MultiModelTypeResponse, model_type_id: str, param: str, template: str = None
+) -> Dict[str, Any]:
+    """Converts the response from the API to a dictionary of model param info for the given model type id."""
+    dict_response = MessageToDict(response)
+    for model_type in dict_response['modelTypes']:
+        if model_type['id'] == model_type_id:
+            # iterate through the model type fields
+            for modeltypefield in model_type['modelTypeFields']:
+                if modeltypefield['path'].split('.')[-1] == param:
+                    if param == 'template':
+                        del modeltypefield['placeholder']
+                        del modeltypefield['modelTypeEnumOptions']
+                        return modeltypefield
+                    modeltypefield['param'] = modeltypefield.pop('path').split('.')[-1]
+                    del modeltypefield['placeholder']
+                    return modeltypefield
+                # checking the template model type fields
+                if modeltypefield['path'].split('.')[-1] == "template":
+                    for modeltypeenum in modeltypefield['modelTypeEnumOptions']:
+                        if modeltypeenum['id'] == template:
+                            # iterate through the template fields
+                            for modeltypeenumfield in modeltypeenum['modelTypeFields']:
+                                if modeltypeenumfield['path'].split('.')[-1] == param:
+                                    modeltypeenumfield['param'] = modeltypeenumfield.pop(
+                                        'path'
+                                    ).split('.')[-1]
+                                    del modeltypeenumfield['placeholder']
+                                    return modeltypeenumfield
 
 
 def find_and_replace_key(nested_dict: Dict, target_key: str, replacement_value: Any) -> None:
-  """Finds and replaces the target key with the replacement value in the nested dictionary."""
-  for key, value in nested_dict.items():
-    if key == target_key:
-      nested_dict[key] = replacement_value
-    elif isinstance(value, dict):
-      find_and_replace_key(value, target_key, replacement_value)
+    """Finds and replaces the target key with the replacement value in the nested dictionary."""
+    for key, value in nested_dict.items():
+        if key == target_key:
+            nested_dict[key] = replacement_value
+        elif isinstance(value, dict):
+            find_and_replace_key(value, target_key, replacement_value)