clarifai-grpc 6.4.0__py3-none-any.whl → 11.10.3__py3-none-any.whl

clarifai_grpc/__init__.py

@@ -0,0 +1,33 @@
+__version__ = "11.10.3"
+
+import os
+
+# pop off env var set to the old python implementation
+unset = False
+if os.environ.get('PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION', None) == 'python':
+    unset = True
+    os.environ.pop('PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION')
+# Don't use the clarifai logger since it is in the SDK package and depends on protobuf.
+import logging
+
+logger = logging.getLogger(__name__)
+
+if unset:
+    logger.warning(
+        "Unsetting PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python env var while importing clarifai package. It's best to unset this env var in your environemnt for faster performance."
+    )
+
+try:
+    from google.protobuf.internal import api_implementation
+
+    if api_implementation.Type() == 'python':
+        logger.warning(
+            "The python version of google protobuf is being used. We recommend that you upgrade the protobuf package >=4.21.0 to use the upd version which is much faster."
+        )
+    #     and not os.environ.get('CLARIFAI_SKIP_PROTOBUF_CHECK', 'false') == 'true'
+    # ):
+    #     raise Exception(
+    #         "We do not recommend running this library with the Python implementation of Protocol Buffers. Please check your installation to use the cpp or upb implementation. We recommend setting the environment variable PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=upb. You can skip this check by setting CLARIFAI_SKIP_PROTOBUF_CHECK=true"
+    #     )
+except ImportError:
+    pass

clarifai_grpc/channel/clarifai_channel.py

@@ -1,61 +1,167 @@
+import json
 import os
 
-import requests
+import grpc
 
 from clarifai_grpc.channel.grpc_json_channel import GRPCJSONChannel
-from clarifai_grpc.grpc.api import service_pb2_grpc
-
 
 RETRIES = 2  # if connections fail retry a couple times.
-CONNECTIONS = 20  # number of connections to maintain in pool.
+CONNECTIONS = (
+    20  # number of connections to maintain in pool, only usd for json channel, not direct GRPC.
+)
+MAX_MESSAGE_LENGTH = 1024 * 1024 * 1024  # 1GB
 
 wrap_response_deserializer = None
 
+grpc_json_config = json.dumps(
+    {
+        "methodConfig": [
+            {
+                "name": [{"service": "clarifai.api.V2"}],
+                "retryPolicy": {
+                    "maxAttempts": 5,
+                    "initialBackoff": "0.01s",
+                    "maxBackoff": "5s",
+                    "backoffMultiplier": 2,
+                    "retryableStatusCodes": ["UNAVAILABLE"],
+                },
+            }
+        ]
+    }
+)
+
 
 def _response_deserializer_for_json(response_deserializer):
-  return response_deserializer
+    return response_deserializer
 
 
 def _response_deserializer_for_grpc(response_deserializer):
-  return response_deserializer.FromString
+    return response_deserializer.FromString
 
 
 class ClarifaiChannel:
-  @classmethod
-  def get_json_channel(
-      cls,
-      base_url=os.environ.get('CLARIFAI_API_BASE', 'https://api.clarifai.com')
-  ):
-    global wrap_response_deserializer
-    wrap_response_deserializer = _response_deserializer_for_json
-
-    session = cls._make_requests_session()
-
-    return GRPCJSONChannel(session=session, base_url=base_url)
-
-  @staticmethod
-  def _make_requests_session():
-    http_adapter = requests.adapters.HTTPAdapter(
-      max_retries=RETRIES, pool_connections=CONNECTIONS, pool_maxsize=CONNECTIONS)
-
-    session = requests.Session()
-    session.mount('http://', http_adapter)
-    session.mount('https://', http_adapter)
-    return session
-
-  @staticmethod
-  def get_grpc_channel():
-    global wrap_response_deserializer
-    wrap_response_deserializer = _response_deserializer_for_grpc
-
-    return service_pb2_grpc.grpc.secure_channel(
-      'api-grpc.clarifai.com:18081',
-      service_pb2_grpc.grpc.ssl_channel_credentials()
-    )
-
-  @staticmethod
-  def get_insecure_grpc_channel():
-    global wrap_response_deserializer
-    wrap_response_deserializer = _response_deserializer_for_grpc
-
-    return service_pb2_grpc.grpc.insecure_channel('api-grpc.clarifai.com:18080')
+    @classmethod
+    def get_json_channel(
+        cls, base_url=os.environ.get("CLARIFAI_API_BASE", "https://api.clarifai.com")
+    ):
+        global wrap_response_deserializer
+        wrap_response_deserializer = _response_deserializer_for_json
+
+        session = cls._make_requests_session()
+
+        return GRPCJSONChannel(session=session, base_url=base_url)
+
+    @staticmethod
+    def _make_requests_session():
+        import requests  # noqa
+
+        http_adapter = requests.adapters.HTTPAdapter(
+            max_retries=RETRIES, pool_connections=CONNECTIONS, pool_maxsize=CONNECTIONS
+        )
+
+        session = requests.Session()
+        session.mount("http://", http_adapter)
+        session.mount("https://", http_adapter)
+        return session
+
+    @staticmethod
+    def get_grpc_channel(base=None, root_certificates_path=None):
+        global wrap_response_deserializer
+        wrap_response_deserializer = _response_deserializer_for_grpc
+
+        if not base:
+            base = os.environ.get("CLARIFAI_GRPC_BASE", "api.clarifai.com")
+        if base.startswith("http:") or base.startswith("https:"):
+            raise ValueError(
+                "For secure channels the 'base' passed via arguments or env variable CLARIFAI_GRPC_BASE should not start with http:// or https:// but be a direct api endpoint like 'api.clarifai.com'"
+            )
+
+        if root_certificates_path:
+            with open(root_certificates_path, "rb") as f:
+                root_certificates = f.read()
+            credentials = grpc.ssl_channel_credentials(root_certificates)
+        else:
+            credentials = grpc.ssl_channel_credentials()
+
+        return grpc.secure_channel(
+            base,
+            credentials,
+            options=[
+                ("grpc.service_config", grpc_json_config),
+                ("grpc.max_receive_message_length", MAX_MESSAGE_LENGTH),
+                ("grpc.max_send_message_length", MAX_MESSAGE_LENGTH),
+            ],
+        )
+
+    @staticmethod
+    def get_insecure_grpc_channel(base=None, port=18080):
+        global wrap_response_deserializer
+        wrap_response_deserializer = _response_deserializer_for_grpc
+
+        if not base:
+            base = os.environ.get("CLARIFAI_GRPC_BASE", None)
+
+        if not base:
+            raise ValueError("Please set 'base' via arguments or env variable CLARIFAI_GRPC_BASE")
+
+        channel_address = "{}:{}".format(base, port)
+
+        return grpc.insecure_channel(
+            channel_address,
+            options=[
+                ("grpc.service_config", grpc_json_config),
+                ("grpc.max_receive_message_length", MAX_MESSAGE_LENGTH),
+                ("grpc.max_send_message_length", MAX_MESSAGE_LENGTH),
+            ],
+        )
+
+    @staticmethod
+    def get_aio_grpc_channel(base=None, root_certificates_path=None):
+        global wrap_response_deserializer
+        wrap_response_deserializer = _response_deserializer_for_grpc
+
+        if not base:
+            base = os.environ.get("CLARIFAI_GRPC_BASE", "api.clarifai.com")
+        if base.startswith("http:") or base.startswith("https:"):
+            raise ValueError(
+                "For secure channels the 'base' passed via arguments or env variable CLARIFAI_GRPC_BASE should not start with http:// or https:// but be a direct api endpoint like 'api.clarifai.com'"
+            )
+
+        if root_certificates_path:
+            with open(root_certificates_path, "rb") as f:
+                root_certificates = f.read()
+            credentials = grpc.ssl_channel_credentials(root_certificates)
+        else:
+            credentials = grpc.ssl_channel_credentials()
+
+        return grpc.aio.secure_channel(
+            base,
+            credentials,
+            options=[
+                ("grpc.service_config", grpc_json_config),
+                ("grpc.max_receive_message_length", MAX_MESSAGE_LENGTH),
+                ("grpc.max_send_message_length", MAX_MESSAGE_LENGTH),
+            ],
+        )
+
+    @staticmethod
+    def get_aio_insecure_grpc_channel(base=None, port=18080):
+        global wrap_response_deserializer
+        wrap_response_deserializer = _response_deserializer_for_grpc
+
+        if not base:
+            base = os.environ.get("CLARIFAI_GRPC_BASE", None)
+
+        if not base:
+            raise ValueError("Please set 'base' via arguments or env variable CLARIFAI_GRPC_BASE")
+
+        channel_address = "{}:{}".format(base, port)
+
+        return grpc.aio.insecure_channel(
+            channel_address,
+            options=[
+                ("grpc.service_config", grpc_json_config),
+                ("grpc.max_receive_message_length", MAX_MESSAGE_LENGTH),
+                ("grpc.max_send_message_length", MAX_MESSAGE_LENGTH),
+            ],
+        )

clarifai_grpc/channel/custom_converters/custom_dict_to_message.py

@@ -3,43 +3,50 @@ from google.protobuf.message import Message  # noqa
 
 from clarifai_grpc.grpc.api.utils import extensions_pb2
 
-# Python 3 deprecates getargspec and introduces getfullargspec, which Python 2 doesn't have.
-try:
-  from inspect import getfullargspec as get_args
-except ImportError:
-  from inspect import getargspec as get_args
 
+def dict_to_protobuf(
+    protobuf_class,
+    js_dict,
+    ignore_unknown_fields=False,
+    descriptor_pool=None,
+    max_recursion_depth=100,
+):
+    """Parses a JSON dictionary representation into a message.
+
+    Args:
+      js_dict: Dict representation of a JSON message.
+      message: A protocol buffer message to merge into.
+      ignore_unknown_fields: If True, do not raise errors for unknown fields.
+      descriptor_pool: A Descriptor Pool for resolving types. If None use the
+        default.
+      max_recursion_depth: max recursion depth of JSON message to be
+        deserialized. JSON messages over this depth will fail to be
+        deserialized. Default value is 100.
+
+    Returns:
+      The same message passed as argument.
+    """
+    message = protobuf_class()
 
-def dict_to_protobuf(protobuf_class, js_dict, ignore_unknown_fields=False):
-  # type: (type(Message), dict, bool) -> Message
-  message = protobuf_class()
-
-  # Protobuf versions 3.6.* and 3.7.0 require a different number of parameters in the _Parser's
-  # constructor. In the case of 3.6.*, we pass only the argument ignore_unknown_fields, but in
-  # the case of 3.7.0, we pass in one additional None parameter. To be future proof(ish), pass in
-  # None to any subsequent parameter.
-  num_of_args = len(get_args(_Parser.__init__).args)
-  none_args = [None] * (num_of_args - 2)  # Subtract 2 for self and ignore_unknown_fields.
-  parser = _CustomParser(ignore_unknown_fields, *none_args)
+    parser = _CustomParser(ignore_unknown_fields, descriptor_pool, max_recursion_depth)
 
-  parser.ConvertMessage(js_dict, message)
-  return message
+    parser.ConvertMessage(js_dict, message, path="")
+    return message
 
 
 class _CustomParser(_Parser):
-
-  def _ConvertFieldValuePair(self, js, message):
-    """
-    Because of fields with custom extensions such as cl_default_float, we need
-    to adjust the original's method's JSON object parameter by setting them explicitly to the
-    default value.
-    """
-
-    message_descriptor = message.DESCRIPTOR
-    for f in message_descriptor.fields:
-      default_float = f.GetOptions().Extensions[extensions_pb2.cl_default_float]
-      if default_float:
-        if f.name not in js:
-          js[f.name] = default_float
-
-    super(_CustomParser, self)._ConvertFieldValuePair(js, message)
+    def _ConvertFieldValuePair(self, js, message, path):
+        """
+        Because of fields with custom extensions such as cl_default_float, we need
+        to adjust the original's method's JSON object parameter by setting them explicitly to the
+        default value.
+        """
+
+        message_descriptor = message.DESCRIPTOR
+        for f in message_descriptor.fields:
+            default_float = f.GetOptions().Extensions[extensions_pb2.cl_default_float]
+            if default_float:
+                if f.name not in js:
+                    js[f.name] = default_float
+
+        super(_CustomParser, self)._ConvertFieldValuePair(js, message, path)

clarifai_grpc/channel/custom_converters/custom_message_to_dict.py

@@ -8,81 +8,92 @@ from clarifai_grpc.grpc.api.utils import extensions_pb2
 
 
 def protobuf_to_dict(object_protobuf, use_integers_for_enums=True, ignore_show_empty=False):
-  # type: (Message, typing.Optional[bool], typing.Optional[bool]) -> dict
+    # type: (Message, typing.Optional[bool], typing.Optional[bool]) -> dict
 
-  # printer = _CustomPrinter(
-  printer = _CustomPrinter(
-      including_default_value_fields=False,
-      preserving_proto_field_name=True,
-      use_integers_for_enums=use_integers_for_enums,
-      ignore_show_empty=ignore_show_empty)
-  # pylint: disable=protected-access
-  return printer._MessageToJsonObject(object_protobuf)
+    # printer = _CustomPrinter(
+    printer = _CustomPrinter(
+        including_default_value_fields=False,
+        preserving_proto_field_name=True,
+        use_integers_for_enums=use_integers_for_enums,
+        ignore_show_empty=ignore_show_empty,
+    )
+    # pylint: disable=protected-access
+    return printer._MessageToJsonObject(object_protobuf)
 
 
 class _CustomPrinter(_Printer):
-
-  def __init__(self, including_default_value_fields, preserving_proto_field_name,
-               use_integers_for_enums, ignore_show_empty):
-    super(_CustomPrinter, self).__init__(including_default_value_fields,
-                                         preserving_proto_field_name, use_integers_for_enums)
-    self._ignore_show_empty = ignore_show_empty
-
-  def _RegularMessageToJsonObject(self, message, js):
-    """
-    Because of the fields with the custom extension `cl_show_if_empty`, we need to adjust the
-    original's method's return JSON object and keep these fields.
-    """
-
-    js = super(_CustomPrinter, self)._RegularMessageToJsonObject(message, js)
-
-    message_descriptor = message.DESCRIPTOR
-    for field in message_descriptor.fields:
-
-      if (self._ignore_show_empty and
-          not field.GetOptions().Extensions[extensions_pb2.cl_default_float]):
-        continue
-      if not field.GetOptions().Extensions[extensions_pb2.cl_show_if_empty]:
-        continue
-
-      # Singular message fields and oneof fields will not be affected.
-      if ((field.label != descriptor.FieldDescriptor.LABEL_REPEATED and
-           field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE) or
-          field.containing_oneof):
-        continue
-      if self.preserving_proto_field_name:
-        name = field.name
-      else:
-        name = field.json_name
-      if name in js:
-        # Skip the field which has been serialized already.
-        continue
-      if _IsMapEntry(field):
-        js[name] = {}
-      elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
-        js[name] = []
-      else:
-        js[name] = self._FieldToJsonObject(field, field.default_value)
-
-    return js
-
-  def _StructMessageToJsonObject(self, message):
-    """
-    Converts Struct message according to Proto3 JSON Specification.
-
-    However, by default, empty objects {} get converted to null. We overwrite this behavior so {}
-    get converted to {}.
-    """
-
-    fields = message.fields
-    ret = {}
-    for key in fields:
-      # When there's a Struct with an empty Struct field, this condition will hold True.
-      # Far as I know this is the only case this condition will be true. If not, this condition
-      # needs to be amended.
-      if fields[key].WhichOneof('kind') is None:
-        json_object = {}
-      else:
-        json_object = self._ValueMessageToJsonObject(fields[key])
-      ret[key] = json_object
-    return ret
+    def __init__(
+        self,
+        *,
+        including_default_value_fields,
+        preserving_proto_field_name,
+        use_integers_for_enums,
+        ignore_show_empty,
+    ):
+        super(_CustomPrinter, self).__init__(
+            always_print_fields_with_no_presence=including_default_value_fields,
+            preserving_proto_field_name=preserving_proto_field_name,
+            use_integers_for_enums=use_integers_for_enums,
+        )
+        self._ignore_show_empty = ignore_show_empty
+
+    def _RegularMessageToJsonObject(self, message, js):
+        """
+        Because of the fields with the custom extension `cl_show_if_empty`, we need to adjust the
+        original's method's return JSON object and keep these fields.
+        """
+
+        js = super(_CustomPrinter, self)._RegularMessageToJsonObject(message, js)
+
+        message_descriptor = message.DESCRIPTOR
+        for field in message_descriptor.fields:
+            if (
+                self._ignore_show_empty
+                and not field.GetOptions().Extensions[extensions_pb2.cl_default_float]
+            ):
+                continue
+            if not field.GetOptions().Extensions[extensions_pb2.cl_show_if_empty]:
+                continue
+
+            # Singular message fields and oneof fields will not be affected.
+            if (
+                field.label != descriptor.FieldDescriptor.LABEL_REPEATED
+                and field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE
+            ) or field.containing_oneof:
+                continue
+            if self.preserving_proto_field_name:
+                name = field.name
+            else:
+                name = field.json_name
+            if name in js:
+                # Skip the field which has been serialized already.
+                continue
+            if _IsMapEntry(field):
+                js[name] = {}
+            elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
+                js[name] = []
+            else:
+                js[name] = self._FieldToJsonObject(field, field.default_value)
+
+        return js
+
+    def _StructMessageToJsonObject(self, message):
+        """
+        Converts Struct message according to Proto3 JSON Specification.
+
+        However, by default, empty objects {} get converted to null. We overwrite this behavior so {}
+        get converted to {}.
+        """
+
+        fields = message.fields
+        ret = {}
+        for key in fields:
+            # When there's a Struct with an empty Struct field, this condition will hold True.
+            # Far as I know this is the only case this condition will be true. If not, this condition
+            # needs to be amended.
+            if fields[key].WhichOneof("kind") is None:
+                json_object = {}
+            else:
+                json_object = self._ValueMessageToJsonObject(fields[key])
+            ret[key] = json_object
+        return ret

clarifai_grpc/channel/errors.py

@@ -2,5 +2,10 @@ class ApiError(Exception):
     pass
 
 
+class NotImplementedCaller:
+    def __call__(self, *args, **kwargs):
+        raise NotImplementedError()
+
+
 class UsageError(Exception):
     pass

clarifai_grpc/channel/exceptions.py

@@ -1,2 +1,2 @@
 class ClarifaiException(Exception):
-  pass
+    pass