clarifai 9.10.1__py3-none-any.whl → 9.10.3__py3-none-any.whl

clarifai_utils/utils/model_train.py

@@ -1,157 +0,0 @@
-from typing import Any, Dict, List
-
-from clarifai_grpc.grpc.api import resources_pb2
-from clarifai_grpc.grpc.api.service_pb2 import MultiModelTypeResponse
-from google.protobuf.json_format import MessageToDict
-from google.protobuf.struct_pb2 import Struct
-
-
-def response_to_templates(response: MultiModelTypeResponse, model_type_id: str) -> List[str]:
-  """Converts the response from the API to a list of templates for the given model type id."""
-  dict_response = MessageToDict(response)
-  templates = []
-  for model_type in dict_response['modelTypes']:
-    if model_type['id'] == model_type_id:
-      for modeltypefield in model_type['modelTypeFields']:
-        if modeltypefield['path'].split('.')[-1] == "template":
-          templates = [template['id'] for template in modeltypefield['modelTypeEnumOptions']]
-  return templates
-
-
-def response_to_model_params(response: MultiModelTypeResponse,
-                             model_type_id: str,
-                             template: str = None) -> Dict[str, Any]:
-  """Converts the response from the API to a dictionary of model params for the given model type id."""
-  dict_response = MessageToDict(response)
-  params = {}
-  params["dataset_id"] = ""
-  params["dataset_version_id"] = ""
-  if model_type_id not in ["clusterer", "text-to-text"]:
-    params["concepts"] = []
-  params["train_params"] = dict()
-
-  for model_type in dict_response['modelTypes']:
-    if model_type['id'] == model_type_id:
-      #iterate through the model type fields
-      for modeltypefield in model_type['modelTypeFields']:
-        _path = modeltypefield['path'].split('.')
-        #removing the fields which are not required
-        if (_path[0] in ["'eval_info'"]) or (_path[1] in ["dataset", "data"]) or (_path[-1] in [
-            "dataset_id", "dataset_version_id"
-        ]):
-          continue
-        #checking the template model type fields
-        if _path[-1] != "template":
-          if _path[0] == 'train_info' or _path[0] == 'input_info':
-            try:
-              params["train_params"][_path[-1]] = modeltypefield['defaultValue']
-            except Exception:
-              params["train_params"][_path[-1]] = None
-          if _path[0] == 'output_info':
-            params["inference_params"] = dict()
-            try:
-              params["inference_params"][_path[-1]] = modeltypefield['defaultValue']
-            except Exception:
-              params["inference_params"][_path[-1]] = None
-        else:
-          if 'modelTypeEnumOptions' in modeltypefield.keys():
-            #check given template is valid
-            all_templates = [template['id'] for template in modeltypefield['modelTypeEnumOptions']]
-            if template not in all_templates:
-              raise ValueError(f"Invalid template {template} for model type {model_type_id}. "
-                               f"Valid templates are {all_templates}")
-            for modeltypeenum in modeltypefield['modelTypeEnumOptions']:
-              #finding the given template
-              if modeltypeenum['id'] == template:
-                params['train_params']["template"] = modeltypeenum['id']
-                #iterate through the template fields
-                for modeltypeenumfield in modeltypeenum['modelTypeFields']:
-                  try:
-                    params["train_params"][modeltypeenumfield['path'].split('.')[
-                        -1]] = modeltypeenumfield['defaultValue']
-                  except Exception:
-                    params["train_params"][modeltypeenumfield['path'].split('.')[-1]] = None
-  #custom config
-  if "custom_config" in params['train_params'].keys():
-    # Write the content to the file
-    file_path = params['train_params']['template'] + ".py"
-    with open(file_path, "w") as script_file:
-      script_file.write(params['train_params']['custom_config'])
-    params['train_params']['custom_config'] = file_path
-
-  return params
-
-
-def params_parser(params_dict: dict) -> Dict[str, Any]:
-  """Converts the params dictionary to a dictionary of model specific params for the given model"""
-  #dict parser
-  train_dict = {}
-  train_dict["train_info"] = dict()
-  train_dict["output_info"] = dict()
-
-  train_dict["train_info"]['params'] = Struct()
-  if 'custom_config' in params_dict['train_params'].keys():
-    # Open and read the Python file
-    with open(params_dict['train_params']['custom_config'], 'r') as python_file:
-      custom_config = python_file.read()
-    params_dict['train_params']['custom_config'] = custom_config
-
-  if 'base_embed_model' in params_dict['train_params'].keys():
-    train_dict["input_info"] = dict()
-    train_dict["input_info"]['base_embed_model'] = params_dict['train_params']['base_embed_model']
-    train_dict['input_info'] = resources_pb2.InputInfo(**train_dict['input_info'])
-    del params_dict['train_params']['base_embed_model']
-
-  train_dict["train_info"]['params'].update(params_dict["train_params"])
-  train_dict["train_info"]['params']['dataset_id'] = params_dict['dataset_id']
-  train_dict["train_info"]['params']['dataset_version_id'] = params_dict['dataset_version_id']
-  train_dict['train_info'] = resources_pb2.TrainInfo(**train_dict['train_info'])
-
-  if 'concepts' in params_dict.keys():
-    train_dict["output_info"]['data'] = resources_pb2.Data(
-        concepts=[resources_pb2.Concept(id=concept_id) for concept_id in params_dict["concepts"]])
-  if 'inference_params' in params_dict.keys():
-    train_dict["output_info"]['params'] = Struct()
-    train_dict['output_info']['params'].update(params_dict["inference_params"])
-  train_dict['output_info'] = resources_pb2.OutputInfo(**train_dict['output_info'])
-
-  return train_dict
-
-
-def response_to_param_info(response: MultiModelTypeResponse,
-                           model_type_id: str,
-                           param: str,
-                           template: str = None) -> Dict[str, Any]:
-  """Converts the response from the API to a dictionary of model param info for the given model type id."""
-  dict_response = MessageToDict(response)
-  for model_type in dict_response['modelTypes']:
-    if model_type['id'] == model_type_id:
-      #iterate through the model type fields
-      for modeltypefield in model_type['modelTypeFields']:
-        if modeltypefield['path'].split('.')[-1] == param:
-          if param == 'template':
-            del modeltypefield['placeholder']
-            del modeltypefield['modelTypeEnumOptions']
-            return modeltypefield
-          modeltypefield['param'] = modeltypefield.pop('path').split('.')[-1]
-          del modeltypefield['placeholder']
-          return modeltypefield
-        #checking the template model type fields
-        if modeltypefield['path'].split('.')[-1] == "template":
-          for modeltypeenum in modeltypefield['modelTypeEnumOptions']:
-            if modeltypeenum['id'] == template:
-              #iterate through the template fields
-              for modeltypeenumfield in modeltypeenum['modelTypeFields']:
-                if modeltypeenumfield['path'].split('.')[-1] == param:
-                  modeltypeenumfield['param'] = modeltypeenumfield.pop('path').split('.')[-1]
-                  del modeltypeenumfield['placeholder']
-                  return modeltypeenumfield
-
-
-def find_and_replace_key(nested_dict: Dict, target_key: str, replacement_value: Any) -> None:
-  """Finds and replaces the target key with the replacement value in the nested dictionary."""
-  for key, value in nested_dict.items():
-    if key == target_key:
-      nested_dict[key] = replacement_value
-    elif isinstance(value, dict):
-      find_and_replace_key(value, target_key, replacement_value)

clarifai_utils/versions.py

@@ -1,6 +0,0 @@
-import os
-
-CLIENT_VERSION = "9.10.1"
-OS_VER = os.sys.platform
-PYTHON_VERSION = '.'.join(
-    map(str, [os.sys.version_info.major, os.sys.version_info.minor, os.sys.version_info.micro]))

clarifai_utils/workflows/export.py

@@ -1,68 +0,0 @@
-from typing import Any, Dict
-
-import yaml
-from google.protobuf.json_format import MessageToDict
-
-VALID_YAML_KEYS = ["workflow", "id", "nodes", "node_inputs", "node_id", "model"]
-
-
-def clean_up_unused_keys(wf: dict):
-  """Removes unused keys from dict before exporting to yaml. Supports nested dicts."""
-  new_wf = dict()
-  for key, val in wf.items():
-    if key not in VALID_YAML_KEYS:
-      continue
-    if key == "model":
-      new_wf["model"] = {
-          "model_id": wf["model"]["id"],
-          "model_version_id": wf["model"]["model_version"]["id"]
-      }
-      # If the model is not from clarifai main, add the app_id and user_id to the model dict.
-      if wf["model"]["user_id"] != "clarifai" and wf["model"]["app_id"] != "main":
-        new_wf["model"].update({
-            "app_id": wf["model"]["app_id"],
-            "user_id": wf["model"]["user_id"]
-        })
-    elif isinstance(val, dict):
-      new_wf[key] = clean_up_unused_keys(val)
-    elif isinstance(val, list):
-      new_list = []
-      for i in val:
-        new_list.append(clean_up_unused_keys(i))
-      new_wf[key] = new_list
-    else:
-      new_wf[key] = val
-  return new_wf
-
-
-class Exporter:
-
-  def __init__(self, workflow):
-    self.wf = workflow
-
-  def __enter__(self):
-    return self
-
-  def parse(self) -> Dict[str, Any]:
-    """Reads a resources_pb2.Workflow object (e.g. from a GetWorkflow response)
-
-    Returns:
-        dict: A dict representation of the workflow.
-    """
-    if isinstance(self.wf, list):
-      self.wf = self.wf[0]
-    wf = {"workflow": MessageToDict(self.wf, preserving_proto_field_name=True)}
-    clean_wf = clean_up_unused_keys(wf)
-    self.wf_dict = clean_wf
-    return clean_wf
-
-  def export(self, out_path):
-    with open(out_path, 'w') as out_file:
-      yaml.dump(self.wf_dict["workflow"], out_file, default_flow_style=False)
-
-  def __exit__(self, *args):
-    self.close()
-
-  def close(self):
-    del self.wf
-    del self.wf_dict

clarifai_utils/workflows/utils.py

@@ -1,59 +0,0 @@
-from typing import Dict, Optional, Set
-
-from clarifai_grpc.grpc.api import resources_pb2
-from google.protobuf import struct_pb2
-from google.protobuf.json_format import MessageToDict
-
-
-def get_yaml_output_info_proto(yaml_model_output_info: Dict) -> Optional[resources_pb2.OutputInfo]:
-  """Converts a yaml model output info to an api model output info."""
-  if not yaml_model_output_info:
-    return None
-
-  return resources_pb2.OutputInfo(
-      params=convert_yaml_params_to_api_params(yaml_model_output_info.get('params')))
-
-
-def convert_yaml_params_to_api_params(yaml_params: Dict) -> Optional[struct_pb2.Struct]:
-  """Converts a yaml model output info params to an api model output info params."""
-  if not yaml_params:
-    return None
-
-  s = struct_pb2.Struct()
-  s.update(yaml_params)
-
-  return s
-
-
-def is_same_yaml_model(api_model: resources_pb2.Model, yaml_model: Dict) -> bool:
-  """Compares a model from the API with a model from a yaml file."""
-  api_model = MessageToDict(api_model, preserving_proto_field_name=True)
-
-  yaml_model_from_api = dict()
-  for k, _ in yaml_model.items():
-    if k == "output_info" and api_model["model_version"].get("output_info", "") != "":
-      yaml_model_from_api[k] = dict(params=api_model["model_version"]["output_info"].get("params"))
-    else:
-      yaml_model_from_api[k] = api_model.get(k)
-  yaml_model_from_api.update({"model_id": api_model.get("id")})
-
-  ignore_keys = {}
-
-  return is_dict_in_dict(yaml_model, yaml_model_from_api, ignore_keys)
-
-
-def is_dict_in_dict(d1: Dict, d2: Dict, ignore_keys: Set = None) -> bool:
-  """Compares two dicts recursively."""
-  for k, v in d1.items():
-    if ignore_keys and k in ignore_keys:
-      continue
-    if k not in d2:
-      return False
-    if isinstance(v, dict):
-      if not isinstance(d2[k], dict):
-        return False
-      return is_dict_in_dict(d1[k], d2[k], None)
-    elif v != d2[k]:
-      return False
-
-  return True

clarifai_utils/workflows/validate.py

@@ -1,67 +0,0 @@
-from schema import And, Optional, Regex, Schema, SchemaError, Use
-
-# Non-empty, up to 32-character ASCII strings with internal dashes and underscores.
-_id_validator = And(str, lambda s: 0 < len(s) <= 48, Regex(r'^[0-9A-Za-z]+([-_][0-9A-Za-z]+)*$'))
-
-# 32-character hex string, converted to lower-case.
-_hex_id_validator = And(str, Use(str.lower), Regex(r'^[0-9a-f]{32}'))
-
-
-def _model_does_not_have_model_version_id_and_other_fields(m):
-  """ Validate that model does not have model_version_id and other model fields."""
-  if ('model_version_id' in m) and _model_has_other_fields(m):
-    raise SchemaError(f"model should not set model_version_id and other model fields: {m};"
-                      f" please remove model_version_id or other model fields.")
-  return True
-
-
-def _model_has_other_fields(m):
-  return any(k not in ['model_id', 'model_version_id'] for k in m.keys())
-
-
-def _workflow_nodes_have_valid_dependencies(nodes):
-  """Validate that all inputs to a node are declared before it."""
-  node_ids = set()
-  for node in nodes:
-    for node_input in node.get("node_inputs", []):
-      if node_input["node_id"] not in node_ids:
-        raise SchemaError(f"missing input '{node_input['node_id']}' for node '{node['id']}'")
-    node_ids.add(node["id"])
-
-  return True
-
-
-_data_schema = Schema({
-    "workflow": {
-        "id":
-            _id_validator,
-        "nodes":
-            And(
-                len,
-                [{
-                    "id":
-                        And(str, len),  # Node IDs are not validated as IDs by the API.
-                    "model":
-                        And({
-                            "model_id": _id_validator,
-                            Optional("app_id"): _id_validator,
-                            Optional("user_id"): _id_validator,
-                            Optional("model_version_id"): _hex_id_validator,
-                            Optional("model_type_id"): _id_validator,
-                            Optional("description"): str,
-                            Optional("output_info"): {
-                                Optional("params"): dict,
-                            },
-                        }, _model_does_not_have_model_version_id_and_other_fields),
-                    Optional("node_inputs"):
-                        And(len, [{
-                            "node_id": And(str, len),
-                        }]),
-                }],
-                _workflow_nodes_have_valid_dependencies),
-    },
-})
-
-
-def validate(data):
-  return _data_schema.validate(data)