clarifai 9.8.1__py3-none-any.whl → 9.9.0__py3-none-any.whl

clarifai/models/model_serving/pb_model_repository.py

@@ -19,8 +19,7 @@ import os
 from pathlib import Path
 from typing import Callable, Type
 
-from .model_config.serializer import Serializer
-from .model_config.triton_config import TritonModelConfig
+from .model_config import Serializer, TritonModelConfig
 from .models import inference, pb_model, test
 
 
@@ -79,11 +78,9 @@ class TritonModelRepository:
             pass
         else:
           continue
-      # gen requirements & conda yaml
+      # gen requirements
       with open(os.path.join(repository_path, "requirements.txt"), "w") as f:
         f.write("clarifai>9.5.3\ntritonclient[all]")  # for model upload utils
-      with open(os.path.join(repository_path, "triton_conda.yaml"), "w") as conda_env:
-        conda_env.write("name: triton_conda-cp3.8-torch1.13.1-19f97078")
 
     if not os.path.isdir(model_version_path):
       os.mkdir(model_version_path)

clarifai/runners/example.py

@@ -0,0 +1,33 @@
+from clarifai_grpc.grpc.api import resources_pb2
+
+from clarifai.client.runner import Runner
+
+
+class MyRunner(Runner):
+  """A custom runner that adds "Hello World" to the end of the text and replaces the domain of the
+  image URL as an example.
+  """
+
+  def run_input(self, input: resources_pb2.Input) -> resources_pb2.Output:
+    """This is the method that will be called when the runner is run. It takes in an input and
+    returns an output.
+    """
+
+    output = resources_pb2.Output()
+
+    data = input.data
+
+    if data.text.raw != "":
+      output.data.text.raw = data.text.raw + "Hello World"
+    if data.image.url != "":
+      output.data.text.raw = data.image.url.replace("samples.clarifai.com", "newdomain.com")
+    return output
+
+
+if __name__ == '__main__':
+  # Make sure you set these env vars before running the example.
+  # CLARIFAI_PAT
+  # CLARIFAI_USER_ID
+
+  # You need to first create a runner in the Clarifai API and then use the ID here.
+  MyRunner(runner_id="sdk-test-runner").start()

clarifai/schema/search.py

@@ -0,0 +1,60 @@
+from schema import And, Optional, Regex, Schema
+
+
+def get_schema() -> Schema:
+  """Initialize the schema for rank and filter.
+
+        This schema validates:
+
+        - Rank and filter must be a list
+        - Each item in the list must be a dict
+        - The dict can contain these optional keys:
+            - 'image_url': Valid URL string
+            - 'text_raw': Non-empty string
+            - 'metadata': Dict
+            - 'image_bytes': Bytes
+            - 'geo_point': Dict with 'longitude', 'latitude' and 'geo_limit' as float, float and int respectively
+            - 'concepts': List where each item is a concept dict
+        - Concept dict requires at least one of:
+            - 'name': Non-empty string with dashes/underscores
+            - 'id': Non-empty string
+            - 'language': Non-empty string
+            - 'value': 0 or 1 integer
+
+        Returns:
+            Schema: The schema for rank and filter.
+        """
+  # Schema for a single concept
+  concept_schema = Schema({
+      Optional('value'):
+          And(int, lambda x: x in [0, 1]),
+      Optional('id'):
+          And(str, len),
+      Optional('language'):
+          And(str, len),
+      # Non-empty strings with internal dashes and underscores.
+      Optional('name'):
+          And(str, len, Regex(r'^[0-9A-Za-z]+([-_][0-9A-Za-z]+)*$'))
+  })
+
+  # Schema for a rank or filter item
+  rank_filter_item_schema = Schema({
+      Optional('image_url'):
+          And(str, Regex(r'^https?://')),
+      Optional('text_raw'):
+          And(str, len),
+      Optional('metadata'):
+          dict,
+      Optional('image_bytes'):
+          bytes,
+      Optional('geo_point'): {
+          'longitude': float,
+          'latitude': float,
+          'geo_limit': int
+      },
+      Optional("concepts"):
+          And(list, lambda x: all(concept_schema.is_valid(item) and len(item) > 0 for item in x)),
+  })
+
+  # Schema for rank and filter args
+  return Schema([rank_filter_item_schema])

clarifai/utils/logging.py

@@ -1,16 +1,66 @@
 import logging
-from typing import Optional
+from collections import defaultdict
+from typing import Dict, List, Optional
 
+from rich import print as rprint
 from rich.logging import RichHandler
 from rich.table import Table
 from rich.traceback import install
+from rich.tree import Tree
 
 install()
 
 
-def table_from_dict(data, column_names, title="") -> Table:
+def display_workflow_tree(nodes_data: List[Dict]) -> None:
+  """Displays a tree of the workflow nodes."""
+  # Create a mapping of node_id to the list of node_ids that are connected to it.
+  node_adj_mapping = defaultdict(list)
+  # Create a mapping of node_id to the node data info.
+  nodes_data_dict = {}
+  for node in nodes_data:
+    nodes_data_dict[node["id"]] = node
+    if node.get("node_inputs", "") == "":
+      node_adj_mapping["Input"].append(node["id"])
+    else:
+      for node_input in node["node_inputs"]:
+        node_adj_mapping[node_input["node_id"]].append(node["id"])
+
+  # Get all leaf nodes.
+  leaf_node_ids = set()
+  for node_id in list(nodes_data_dict.keys()):
+    if node_adj_mapping.get(node_id, "") == "":
+      leaf_node_ids.add(node_id)
+
+  def build_node_tree(node_id="Input"):
+    """Recursively builds a rich tree of the workflow nodes."""
+    # Set the style of the current node.
+    style_str = "green" if node_id in leaf_node_ids else "white"
+
+    # Create a Tree object for the current node.
+    if node_id != "Input":
+      node_table = table_from_dict(
+          [nodes_data_dict[node_id]["model"]],
+          column_names=["id", "model_type_id", "app_id", "user_id"],
+          title="Node: " + node_id)
+
+      tree = Tree(node_table, style=style_str, guide_style="underline2 white")
+    else:
+      tree = Tree(f"[green] {node_id}", style=style_str, guide_style="underline2 white")
+
+    # Recursively add the child nodes of the current node to the tree.
+    for child in node_adj_mapping.get(node_id, []):
+      tree.add(build_node_tree(child))
+
+    # Return the tree.
+    return tree
+
+  tree = build_node_tree("Input")
+  rprint(tree)
+
+
+def table_from_dict(data: List[Dict], column_names: List[str], title: str = "") -> Table:
   """Use this function for printing tables from a list of dicts."""
-  table = Table(title=title, show_header=True, header_style="bold blue")
+  table = Table(title=title, show_lines=False, show_header=True, header_style="blue")
   for column_name in column_names:
     table.add_column(column_name)
   for row in data:

clarifai/versions.py

@@ -1,6 +1,6 @@
 import os
 
-CLIENT_VERSION = '9.8.1'
+CLIENT_VERSION = '9.9.0'
 OS_VER = os.sys.platform
 PYTHON_VERSION = '.'.join(
     map(str, [os.sys.version_info.major, os.sys.version_info.minor, os.sys.version_info.micro]))

clarifai/workflows/export.py

@@ -0,0 +1,68 @@
+from typing import Any, Dict
+
+import yaml
+from google.protobuf.json_format import MessageToDict
+
+VALID_YAML_KEYS = ["workflow", "id", "nodes", "node_inputs", "node_id", "model"]
+
+
+def clean_up_unused_keys(wf: dict):
+  """Removes unused keys from dict before exporting to yaml. Supports nested dicts."""
+  new_wf = dict()
+  for key, val in wf.items():
+    if key not in VALID_YAML_KEYS:
+      continue
+    if key == "model":
+      new_wf["model"] = {
+          "model_id": wf["model"]["id"],
+          "model_version_id": wf["model"]["model_version"]["id"]
+      }
+      # If the model is not from clarifai main, add the app_id and user_id to the model dict.
+      if wf["model"]["user_id"] != "clarifai" and wf["model"]["app_id"] != "main":
+        new_wf["model"].update({
+            "app_id": wf["model"]["app_id"],
+            "user_id": wf["model"]["user_id"]
+        })
+    elif isinstance(val, dict):
+      new_wf[key] = clean_up_unused_keys(val)
+    elif isinstance(val, list):
+      new_list = []
+      for i in val:
+        new_list.append(clean_up_unused_keys(i))
+      new_wf[key] = new_list
+    else:
+      new_wf[key] = val
+  return new_wf
+
+
+class Exporter:
+
+  def __init__(self, workflow):
+    self.wf = workflow
+
+  def __enter__(self):
+    return self
+
+  def parse(self) -> Dict[str, Any]:
+    """Reads a resources_pb2.Workflow object (e.g. from a GetWorkflow response)
+
+    Returns:
+        dict: A dict representation of the workflow.
+    """
+    if isinstance(self.wf, list):
+      self.wf = self.wf[0]
+    wf = {"workflow": MessageToDict(self.wf, preserving_proto_field_name=True)}
+    clean_wf = clean_up_unused_keys(wf)
+    self.wf_dict = clean_wf
+    return clean_wf
+
+  def export(self, out_path):
+    with open(out_path, 'w') as out_file:
+      yaml.dump(self.wf_dict["workflow"], out_file, default_flow_style=False)
+
+  def __exit__(self, *args):
+    self.close()
+
+  def close(self):
+    del self.wf
+    del self.wf_dict

clarifai/workflows/utils.py

@@ -0,0 +1,59 @@
+from typing import Dict, Optional, Set
+
+from clarifai_grpc.grpc.api import resources_pb2
+from google.protobuf import struct_pb2
+from google.protobuf.json_format import MessageToDict
+
+
+def get_yaml_output_info_proto(yaml_model_output_info: Dict) -> Optional[resources_pb2.OutputInfo]:
+  """Converts a yaml model output info to an api model output info."""
+  if not yaml_model_output_info:
+    return None
+
+  return resources_pb2.OutputInfo(
+      params=convert_yaml_params_to_api_params(yaml_model_output_info.get('params')))
+
+
+def convert_yaml_params_to_api_params(yaml_params: Dict) -> Optional[struct_pb2.Struct]:
+  """Converts a yaml model output info params to an api model output info params."""
+  if not yaml_params:
+    return None
+
+  s = struct_pb2.Struct()
+  s.update(yaml_params)
+
+  return s
+
+
+def is_same_yaml_model(api_model: resources_pb2.Model, yaml_model: Dict) -> bool:
+  """Compares a model from the API with a model from a yaml file."""
+  api_model = MessageToDict(api_model, preserving_proto_field_name=True)
+
+  yaml_model_from_api = dict()
+  for k, _ in yaml_model.items():
+    if k == "output_info" and api_model["model_version"].get("output_info", "") != "":
+      yaml_model_from_api[k] = dict(params=api_model["model_version"]["output_info"].get("params"))
+    else:
+      yaml_model_from_api[k] = api_model.get(k)
+  yaml_model_from_api.update({"model_id": api_model.get("id")})
+
+  ignore_keys = {}
+
+  return is_dict_in_dict(yaml_model, yaml_model_from_api, ignore_keys)
+
+
+def is_dict_in_dict(d1: Dict, d2: Dict, ignore_keys: Set = None) -> bool:
+  """Compares two dicts recursively."""
+  for k, v in d1.items():
+    if ignore_keys and k in ignore_keys:
+      continue
+    if k not in d2:
+      return False
+    if isinstance(v, dict):
+      if not isinstance(d2[k], dict):
+        return False
+      return is_dict_in_dict(d1[k], d2[k], None)
+    elif v != d2[k]:
+      return False
+
+  return True

clarifai/workflows/validate.py

@@ -0,0 +1,67 @@
+from schema import And, Optional, Regex, Schema, SchemaError, Use
+
+# Non-empty, up to 32-character ASCII strings with internal dashes and underscores.
+_id_validator = And(str, lambda s: 0 < len(s) <= 48, Regex(r'^[0-9A-Za-z]+([-_][0-9A-Za-z]+)*$'))
+
+# 32-character hex string, converted to lower-case.
+_hex_id_validator = And(str, Use(str.lower), Regex(r'^[0-9a-f]{32}'))
+
+
+def _model_does_not_have_model_version_id_and_other_fields(m):
+  """ Validate that model does not have model_version_id and other model fields."""
+  if ('model_version_id' in m) and _model_has_other_fields(m):
+    raise SchemaError(f"model should not set model_version_id and other model fields: {m};"
+                      f" please remove model_version_id or other model fields.")
+  return True
+
+
+def _model_has_other_fields(m):
+  return any(k not in ['model_id', 'model_version_id'] for k in m.keys())
+
+
+def _workflow_nodes_have_valid_dependencies(nodes):
+  """Validate that all inputs to a node are declared before it."""
+  node_ids = set()
+  for node in nodes:
+    for node_input in node.get("node_inputs", []):
+      if node_input["node_id"] not in node_ids:
+        raise SchemaError(f"missing input '{node_input['node_id']}' for node '{node['id']}'")
+    node_ids.add(node["id"])
+
+  return True
+
+
+_data_schema = Schema({
+    "workflow": {
+        "id":
+            _id_validator,
+        "nodes":
+            And(
+                len,
+                [{
+                    "id":
+                        And(str, len),  # Node IDs are not validated as IDs by the API.
+                    "model":
+                        And({
+                            "model_id": _id_validator,
+                            Optional("app_id"): _id_validator,
+                            Optional("user_id"): _id_validator,
+                            Optional("model_version_id"): _hex_id_validator,
+                            Optional("model_type_id"): _id_validator,
+                            Optional("description"): str,
+                            Optional("output_info"): {
+                                Optional("params"): dict,
+                            },
+                        }, _model_does_not_have_model_version_id_and_other_fields),
+                    Optional("node_inputs"):
+                        And(len, [{
+                            "node_id": And(str, len),
+                        }]),
+                }],
+                _workflow_nodes_have_valid_dependencies),
+    },
+})
+
+
+def validate(data):
+  return _data_schema.validate(data)

{clarifai-9.8.1.dist-info → clarifai-9.9.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: clarifai
-Version: 9.8.1
+Version: 9.9.0
 Summary: Clarifai Python SDK
 Home-page: https://github.com/Clarifai/clarifai-python
 Author: Clarifai
@@ -18,6 +18,8 @@ Requires-Dist: tritonclient (==2.34.0)
 Requires-Dist: packaging
 Requires-Dist: tqdm (==4.64.1)
 Requires-Dist: rich (==13.4.2)
+Requires-Dist: PyYAML (==6.0.1)
+Requires-Dist: schema (==0.7.5)
 
 <h1 align="center">
   <a href="https://www.clarifai.com/"><img alt="Clarifai" title="Clarifai" src="https://upload.wikimedia.org/wikipedia/commons/b/bc/Clarifai_Logo_FC_Web.png"></a>
@@ -51,7 +53,7 @@ This is the official Python client for interacting with our powerful [API](https
 
 **Clarifai Community**: [https://clarifai.com/explore](https://clarifai.com/explore)
 
-**Python SDK Docs**: [https://clarifai-python.readthedocs.io/en/latest/index.html](https://clarifai-python.readthedocs.io/en/latest/index.html)
+**Python SDK Docs**: [https://docs.clarifai.com/python-sdk/api-reference](https://docs.clarifai.com/python-sdk/api-reference)
 
 
 ---
@@ -216,7 +218,23 @@ all_workflow = app.list_workflow()
 # List all workflow in community filtered by description
 all_face_community_workflows = App().list_workflows(filter_by={"query": "face"}, only_in_app=False) # Get all face related workflows
 ```
+#### Workflow Create
+Create a new workflow specified by a yaml config file.
+```python
+# Note: CLARIFAI_PAT must be set as env variable.
+from clarifai.client.app import App
+app = App(app_id="app_id", user_id="user_id")
+workflow = app.create_workflow(config_filepath="config.yml")
+```
 
+#### Workflow Export
+Export an existing workflow from Clarifai as a local yaml file.
+```python
+# Note: CLARIFAI_PAT must be set as env variable.
+from clarifai.client.workflow import Workflow
+workflow = Workflow("https://clarifai.com/clarifai/main/workflows/Demographics")
+workflow.export('demographics_workflow.yml')
+```
 
 ## More Examples
 See many more code examples in this [repo](https://github.com/Clarifai/examples).