clarifai 11.2.3__py3-none-any.whl → 11.2.3rc2__py3-none-any.whl

clarifai/runners/utils/serializers.py

@@ -0,0 +1,222 @@
+import json
+from typing import Dict, Iterable
+
+import numpy as np
+from clarifai_grpc.grpc.api import resources_pb2
+
+from clarifai.runners.utils import data_types
+
+
+class Serializer:
+
+  def serialize(self, data_proto, value):
+    pass
+
+  def deserialize(self, data_proto):
+    pass
+
+  def handles_list(self):
+    return False
+
+
+def is_repeated_field(field_name):
+  descriptor = resources_pb2.Data.DESCRIPTOR.fields_by_name.get(field_name)
+  return descriptor and descriptor.label == descriptor.LABEL_REPEATED
+
+
+class AtomicFieldSerializer(Serializer):
+
+  def __init__(self, field_name):
+    self.field_name = field_name
+
+  def serialize(self, data_proto, value):
+    try:
+      setattr(data_proto, self.field_name, value)
+    except TypeError as e:
+      raise TypeError(f"Incompatible type for {self.field_name}: {type(value)}") from e
+
+  def deserialize(self, data_proto):
+    return getattr(data_proto, self.field_name)
+
+
+class MessageSerializer(Serializer):
+
+  def __init__(self, field_name, message_class):
+    self.field_name = field_name
+    self.message_class = message_class
+    self.is_repeated_field = is_repeated_field(field_name)
+
+  def handles_list(self):
+    return self.is_repeated_field
+
+  def serialize(self, data_proto, value):
+    value = self.message_class.from_value(value).to_proto()
+    dst = getattr(data_proto, self.field_name)
+    try:
+      if self.is_repeated_field:
+        dst.add().CopyFrom(value)
+      else:
+        dst.CopyFrom(value)
+    except TypeError as e:
+      raise TypeError(f"Incompatible type for {self.field_name}: {type(value)}") from e
+
+  def serialize_list(self, data_proto, values):
+    assert self.is_repeated_field
+    dst = getattr(data_proto, self.field_name)
+    dst.extend([self.message_class.from_value(value).to_proto() for value in values])
+
+  def deserialize(self, data_proto):
+    src = getattr(data_proto, self.field_name)
+    if self.is_repeated_field:
+      values = [self.message_class.from_proto(x) for x in src]
+      if len(values) == 1:
+        return values[0]
+      return values if values else None
+    else:
+      if not data_proto.HasField(self.field_name):
+        return None
+      return self.message_class.from_proto(src)
+
+  def deserialize_list(self, data_proto):
+    assert self.is_repeated_field
+    src = getattr(data_proto, self.field_name)
+    return [self.message_class.from_proto(x) for x in src]
+
+
+class NDArraySerializer(Serializer):
+
+  def __init__(self, field_name, as_list=False):
+    self.field_name = field_name
+    self.as_list = as_list
+
+  def serialize(self, data_proto, value):
+    if self.as_list and not isinstance(value, Iterable):
+      raise TypeError(f"Expected list, got {type(value)}")
+    value = np.asarray(value)
+    if not np.issubdtype(value.dtype, np.number):
+      raise TypeError(f"Expected number array, got {value.dtype}")
+    proto = getattr(data_proto, self.field_name)
+    proto.buffer = value.tobytes()
+    proto.shape.extend(value.shape)
+    proto.dtype = str(value.dtype)
+
+  def deserialize(self, data_proto):
+    proto = getattr(data_proto, self.field_name)
+    if not proto.buffer:
+      return None
+    array = np.frombuffer(proto.buffer, dtype=np.dtype(proto.dtype)).reshape(proto.shape)
+    if self.as_list:
+      return array.tolist()
+    return array
+
+
+class JSONSerializer(Serializer):
+
+  def __init__(self, field_name, type=None):
+    self.field_name = field_name
+    self.type = type
+
+  def serialize(self, data_proto, value):
+    #if self.type is not None and not isinstance(value, self.type):
+    #  raise TypeError(f"Expected {self.type}, got {type(value)}")
+    try:
+      setattr(data_proto, self.field_name, json.dumps(value))
+    except TypeError as e:
+      raise TypeError(f"Incompatible type for {self.field_name}: {type(value)}") from e
+
+  def deserialize(self, data_proto):
+    value = getattr(data_proto, self.field_name)
+    if not value:
+      return None
+    return json.loads(value)
+
+
+class ListSerializer(Serializer):
+
+  def __init__(self, inner_serializer):
+    self.field_name = 'parts'
+    self.inner_serializer = inner_serializer
+
+  def handles_list(self):
+    # if handles_list() is called on this serializer, it means that we're
+    # trying to serialize a list of lists. In this case, we need to use
+    # parts[] for the outer list, so we return False here (we can't inline it).
+    return False
+
+  def serialize(self, data_proto, value):
+    if not isinstance(value, Iterable):
+      raise TypeError(f"Expected iterable, got {type(value)}")
+    if self.inner_serializer.handles_list():
+      self.inner_serializer.serialize_list(data_proto, value)
+    else:
+      for item in value:
+        part = data_proto.parts.add()
+        self.inner_serializer.serialize(part.data, item)
+
+  def deserialize(self, data_proto):
+    if self.inner_serializer.handles_list():
+      return self.inner_serializer.deserialize_list(data_proto)
+    return [self.inner_serializer.deserialize(part.data) for part in data_proto.parts]
+
+
+class TupleSerializer(Serializer):
+
+  def __init__(self, inner_serializers):
+    self.field_name = 'parts'
+    self.inner_serializers = inner_serializers
+
+  def serialize(self, data_proto, value):
+    if not isinstance(value, (tuple, list)):
+      raise TypeError(f"Expected tuple, got {type(value)}")
+    if len(value) != len(self.inner_serializers):
+      raise ValueError(f"Expected tuple of length {len(self.inner_serializers)}, got {len(value)}")
+    for i, (serializer, item) in enumerate(zip(self.inner_serializers, value)):
+      part = data_proto.parts.add()
+      part.id = str(i)
+      serializer.serialize(part.data, item)
+
+  def deserialize(self, data_proto):
+    if not data_proto.parts and self.inner_serializers:
+      return None
+    if len(data_proto.parts) != len(self.inner_serializers):
+      raise ValueError(
+          f"Expected tuple of length {len(self.inner_serializers)}, got {len(data_proto.parts)}")
+    return tuple(
+        serializer.deserialize(part.data)
+        for serializer, part in zip(self.inner_serializers, data_proto.parts))
+
+
+class NamedFieldsSerializer(Serializer):
+
+  def __init__(self, named_field_serializers: Dict[str, Serializer]):
+    self.field_name = 'parts'
+    self.named_field_serializers = named_field_serializers
+
+  def serialize(self, data_proto, value):
+    for name, serializer in self.named_field_serializers.items():
+      if name not in value:
+        raise TypeError(f"Missing field {name}")
+      part = self._get_part(data_proto, name, add=True)
+      serializer.serialize(part.data, value[name])
+
+  def deserialize(self, data_proto):
+    if not data_proto.parts and self.named_field_serializers:
+      return None
+    value = data_types.NamedFields()
+    for name, serializer in self.named_field_serializers.items():
+      part = self._get_part(data_proto, name)
+      value[name] = serializer.deserialize(part.data)
+    return value
+
+  def _get_part(self, data_proto, name, add=False):
+    for part in data_proto.parts:
+      if part.id == name:
+        return part
+    if add:
+      part = data_proto.parts.add()
+      part.id = name
+      return part
+    raise TypeError(f"Missing part with key {name}")
+
+
+# TODO dict serializer, maybe json only?

clarifai/utils/cli.py

@@ -1,14 +1,14 @@
 import importlib
 import os
 import pkgutil
-import sys
-import typing as t
-from collections import defaultdict
-from typing import OrderedDict
 
 import click
 import yaml
-from tabulate import tabulate
+
+from rich.console import Console
+from rich.panel import Panel
+from rich.style import Style
+from rich.text import Text
 
 
 def from_yaml(filename: str):
@@ -28,6 +28,19 @@ def dump_yaml(data, filename: str):
     click.echo(f"Error writing YAML file: {e}", err=True)
 
 
+def set_base_url(env):
+  environments = {
+      'prod': 'https://api.clarifai.com',
+      'staging': 'https://api-staging.clarifai.com',
+      'dev': 'https://api-dev.clarifai.com'
+  }
+
+  if env in environments:
+    return environments[env]
+  else:
+    raise ValueError("Invalid environment. Please choose from 'prod', 'staging', 'dev'.")
+
+
 # Dynamically find and import all command modules from the cli directory
 def load_command_modules():
   package_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'cli')
@@ -37,132 +50,21 @@ def load_command_modules():
       importlib.import_module(f'clarifai.cli.{module_name}')
 
 
-def display_co_resources(response,
-                         custom_columns={
-                             'ID': lambda c: c.id,
-                             'USER_ID': lambda c: c.user_id,
-                             'DESCRIPTION': lambda c: c.description,
-                         }):
+def display_co_resources(response, resource_type):
   """Display compute orchestration resources listing results using rich."""
 
-  formatter = TableFormatter(custom_columns)
-  print(formatter.format(list(response), fmt="plain"))
-
-
-class TableFormatter:
-
-  def __init__(self, custom_columns: OrderedDict):
-    """
-        Initializes the TableFormatter with column headers and custom column mappings.
-
-        :param headers: List of column headers for the table.
-        """
-    self.custom_columns = custom_columns
-
-  def format(self, objects, fmt='plain'):
-    """
-        Formats a list of objects into a table with custom columns.
-
-        :param objects: List of objects to format into a table.
-        :return: A string representing the table.
-        """
-    # Prepare the rows by applying the custom column functions to each object
-    rows = []
-    for obj in objects:
-      #   row = [self.custom_columns[header](obj) for header in self.headers]
-      row = [f(obj) for f in self.custom_columns.values()]
-      rows.append(row)
-
-    # Create the table
-    table = tabulate(rows, headers=self.custom_columns.keys(), tablefmt=fmt)
-    return table
-
-
-class AliasedGroup(click.Group):
-
-  def __init__(self,
-               name: t.Optional[str] = None,
-               commands: t.Optional[t.Union[t.MutableMapping[str, click.Command], t.Sequence[
-                   click.Command]]] = None,
-               **attrs: t.Any) -> None:
-    super().__init__(name, commands, **attrs)
-    self.alias_map = {}
-    self.command_to_aliases = defaultdict(list)
-
-  def add_alias(self, cmd: click.Command, alias: str) -> None:
-    self.alias_map[alias] = cmd
-    if alias != cmd.name:
-      self.command_to_aliases[cmd].append(alias)
-
-  def command(self, aliases=None, *args,
-              **kwargs) -> t.Callable[[t.Callable[..., t.Any]], click.Command]:
-    cmd_decorator = super().command(*args, **kwargs)
-    if aliases is None:
-      aliases = []
-
-    def aliased_decorator(f):
-      cmd = cmd_decorator(f)
-      if cmd.name:
-        self.add_alias(cmd, cmd.name)
-      for alias in aliases:
-        self.add_alias(cmd, alias)
-      return cmd
-
-    f = None
-    if args and callable(args[0]):
-      (f,) = args
-    if f is not None:
-      return aliased_decorator(f)
-    return aliased_decorator
-
-  def group(self, aliases=None, *args,
-            **kwargs) -> t.Callable[[t.Callable[..., t.Any]], click.Group]:
-    cmd_decorator = super().group(*args, **kwargs)
-    if aliases is None:
-      aliases = []
-
-    def aliased_decorator(f):
-      cmd = cmd_decorator(f)
-      if cmd.name:
-        self.add_alias(cmd, cmd.name)
-      for alias in aliases:
-        self.add_alias(cmd, alias)
-      return cmd
-
-    f = None
-    if args and callable(args[0]):
-      (f,) = args
-    if f is not None:
-      return aliased_decorator(f)
-    return aliased_decorator
-
-  def get_command(self, ctx: click.Context, cmd_name: str) -> t.Optional[click.Command]:
-    rv = click.Group.get_command(self, ctx, cmd_name)
-    if rv is not None:
-      return rv
-    return self.alias_map.get(cmd_name)
-
-  def format_commands(self, ctx, formatter):
-    sub_commands = self.list_commands(ctx)
-
-    rows = []
-    for sub_command in sub_commands:
-      cmd = self.get_command(ctx, sub_command)
-      if cmd is None or getattr(cmd, 'hidden', False):
-        continue
-      if cmd in self.command_to_aliases:
-        aliases = ', '.join(self.command_to_aliases[cmd])
-        sub_command = f'{sub_command} ({aliases})'
-      cmd_help = cmd.help
-      rows.append((sub_command, cmd_help))
-
-    if rows:
-      with formatter.section("Commands"):
-        formatter.write_dl(rows)
-
-
-def validate_context(ctx):
-  from clarifai.utils.logging import logger
-  if ctx.obj == {}:
-    logger.error("CLI config file missing. Run `clarifai login` to set up the CLI config.")
-    sys.exit(1)
+  console = Console()
+  panel = Panel(
+      Text(f"List of {resource_type}s", justify="center"),
+      title="",
+      style=Style(color="blue", bold=True),
+      border_style="green",
+      width=60)
+  console.print(panel)
+  for indx, item in enumerate(list(response)):
+    panel = Panel(
+        "\n".join([f"{'ID'}: {item.id}", f"{'Description'}: {item.description}"]),
+        title=f"{resource_type} {(indx + 1)}",
+        border_style="green",
+        width=60)
+    console.print(panel)

clarifai/utils/constants.py

@@ -1,7 +1,3 @@
-import os
-
 CLARIFAI_PAT_ENV_VAR = "CLARIFAI_PAT"
 CLARIFAI_SESSION_TOKEN_ENV_VAR = "CLARIFAI_SESSION_TOKEN"
 CLARIFAI_USER_ID_ENV_VAR = "CLARIFAI_USER_ID"
-
-DEFAULT_CONFIG = f'{os.environ["HOME"]}/.config/clarifai/config'

clarifai/utils/logging.py

@@ -10,6 +10,15 @@ import traceback
 from collections import defaultdict
 from typing import Any, Dict, List, Optional, Union
 
+from rich import print as rprint
+from rich.console import Console
+from rich.logging import RichHandler
+from rich.table import Table
+from rich.traceback import install
+from rich.tree import Tree
+
+install()
+
 # The default logger to use throughout the SDK is defined at bottom of this file.
 
 # For the json logger.
@@ -20,20 +29,6 @@ FIELD_BLACKLIST = [
     'msg', 'message', 'account', 'levelno', 'created', 'threadName', 'name', 'processName',
     'module', 'funcName', 'msecs', 'relativeCreated', 'pathname', 'args', 'thread', 'process'
 ]
-COLORS = {
-    'ARGUMENTS': '\033[90m',  # Gray
-    'DEBUG': '\033[90m',  # Gray
-    'INFO': '\033[32m',  # Green
-    'WARNING': '\033[33m',  # Yellow
-    'ERROR': '\033[31m',  # Red
-    'CRITICAL': '\033[31m',  # Red
-    'TIME': '\033[34m',
-    'RESET': '\033[0m'
-}
-LOG_FORMAT = f"[%(levelname)s] {COLORS.get('TIME')}%(asctime)s{COLORS.get('RESET')} %(message)s |" \
-             f"{COLORS.get('ARGUMENTS')} " \
-             f"%(optional_args)s " \
-             f"thread=%(thread)d {COLORS.get('RESET')}"
 
 # Create thread local storage that the format() call below uses.
 # This is only used by the json_logger in the appropriate CLARIFAI_DEPLOY levels.
@@ -64,9 +59,6 @@ def get_req_id_from_context():
 
 def display_workflow_tree(nodes_data: List[Dict]) -> None:
   """Displays a tree of the workflow nodes."""
-  from rich import print as rprint
-  from rich.tree import Tree
-
   # Create a mapping of node_id to the list of node_ids that are connected to it.
   node_adj_mapping = defaultdict(list)
   # Create a mapping of node_id to the node data info.
@@ -112,10 +104,8 @@ def display_workflow_tree(nodes_data: List[Dict]) -> None:
   rprint(tree)
 
 
-def table_from_dict(data: List[Dict], column_names: List[str],
-                    title: str = "") -> 'rich.Table':  #noqa F821
+def table_from_dict(data: List[Dict], column_names: List[str], title: str = "") -> Table:
   """Use this function for printing tables from a list of dicts."""
-  from rich.table import Table
   table = Table(title=title, show_lines=False, show_header=True, header_style="blue")
   for column_name in column_names:
     table.add_column(column_name)
@@ -144,18 +134,23 @@ def _configure_logger(name: str, logger_level: Union[int, str] = logging.NOTSET)
   # If ENABLE_JSON_LOGGER is not set, then use json logger if in k8s.
   enabled_json = os.getenv('ENABLE_JSON_LOGGER', None)
   in_k8s = 'KUBERNETES_SERVICE_HOST' in os.environ
-  handler = logging.StreamHandler()
-  handler.setLevel(logger_level)
   if enabled_json == 'true' or (in_k8s and enabled_json != 'false'):
     # Add the json handler and formatter
+    handler = logging.StreamHandler()
     formatter = JsonFormatter()
     handler.setFormatter(formatter)
+    logger.addHandler(handler)
   else:
-    # create formatter and add it to the handlers
-    formatter = TerminalFormatter(LOG_FORMAT)
+    # Add the new rich handler and formatter
+    try:
+      width, _ = os.get_terminal_size()
+    except OSError:
+      width = 255
+    handler = RichHandler(
+        rich_tracebacks=True, log_time_format="%Y-%m-%d %H:%M:%S.%f", console=Console(width=width))
+    formatter = logging.Formatter('%(message)s')
     handler.setFormatter(formatter)
-  # add the handlers to the logger
-  logger.addHandler(handler)
+    logger.addHandler(handler)
 
 
 def get_logger(logger_level: Union[int, str] = logging.NOTSET,
@@ -212,8 +207,6 @@ def display_concept_relations_tree(relations_dict: Dict[str, Any]) -> None:
     Args:
         relations_dict (dict): A dict of concept relations info.
     """
-  from rich import print as rprint
-  from rich.tree import Tree
   for parent, children in relations_dict.items():
     tree = Tree(parent)
     for child in children:
@@ -379,41 +372,5 @@ class JsonFormatter(logging.Formatter):
       )
 
 
-class TerminalFormatter(logging.Formatter):
-  """ If you have fields in your Formatter (see setup_logger where we setup the format strings) then
-  you can set them on the record using a filter. We do that for req_id here which is a request
-  specific field. This allows us to find requests easily between services.
-  """
-
-  def format(self, record):
-    record.optional_args = []
-
-    user_id = getattr(thread_log_info, 'user_id', None)
-    if user_id is not None:
-      record.optional_args.append("user_id=" + user_id)
-
-    app_id = getattr(thread_log_info, 'app_id', None)
-    if app_id is not None:
-      record.optional_args.append("app_id=" + app_id)
-
-    req_id = getattr(thread_log_info, 'req_id', None)
-    if req_id is not None:
-      record.optional_args.append("req_id=" + req_id)
-
-    record.optional_args = " ".join(record.optional_args)
-
-    color_code = COLORS.get(record.levelname, '')
-
-    record.levelname = f"{color_code}{record.levelname}{COLORS.get('RESET')}"
-    record.msg = f"{color_code}{str(record.msg)}{COLORS.get('RESET')}"
-
-    return super(TerminalFormatter, self).format(record)
-
-  def formatTime(self, record, datefmt=None):
-    # Note we didn't go with UTC here as it's easier to understand time in your time zone.
-    # The json logger leverages UTC though.
-    return datetime.datetime.fromtimestamp(record.created).strftime('%H:%M:%S.%f')
-
-
 # the default logger for the SDK.
 logger = get_logger(logger_level=os.environ.get("LOG_LEVEL", "INFO"), name="clarifai")

clarifai/utils/misc.py

@@ -12,8 +12,6 @@ RETRYABLE_CODES = [
     status_code_pb2.MODEL_BUSY_PLEASE_RETRY
 ]
 
-DEFAULT_CONFIG = f'{os.environ["HOME"]}/.config/clarifai/config'
-
 
 def status_is_retryable(status_code: int) -> bool:
   """Check if a status code is retryable."""

{clarifai-11.2.3.dist-info → clarifai-11.2.3rc2.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.2
 Name: clarifai
-Version: 11.2.3
+Version: 11.2.3rc2
 Summary: Clarifai Python SDK
 Home-page: https://github.com/Clarifai/clarifai-python
 Author: Clarifai
@@ -21,7 +21,7 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: clarifai-grpc>=11.2.6
-Requires-Dist: clarifai-protocol>=0.0.21
+Requires-Dist: clarifai-protocol>=0.0.22
 Requires-Dist: numpy>=1.22.0
 Requires-Dist: tqdm>=4.65.0
 Requires-Dist: rich>=13.4.2
@@ -42,7 +42,6 @@ Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page
 Dynamic: license
-Dynamic: license-file
 Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: requires-python