clarifai 9.10.2__py3-none-any.whl → 9.10.3__py3-none-any.whl

clarifai_utils/client/auth/helper.py

@@ -1,367 +0,0 @@
-import os
-import urllib.request
-from typing import Any, Dict
-
-from clarifai_grpc.channel.clarifai_channel import ClarifaiChannel
-from clarifai_grpc.grpc.api import resources_pb2, service_pb2_grpc
-
-DEFAULT_BASE = "https://api.clarifai.com"
-DEFAULT_UI = "https://clarifai.com"
-
-# Map from base domain to True / False for whether the base has https or http.
-# This is filled in get_stub() if it's not in there already.
-base_https_cache = {}
-ui_https_cache = {}
-
-
-def clear_cache() -> None:
-  """Clears the cache."""
-  base_https_cache.clear()
-  ui_https_cache.clear()
-
-
-def https_cache(cache: dict, url: str) -> str:
-  """This is a helper function to cache whether a url is http or https."""
-  HTTPS = True
-  HTTP = False
-
-  # If http or https is provided, we trust that it is correct.
-  # Note: this always stores the url without http:// or https://
-  if url.startswith("https://"):
-    url = url.replace("https://", "")
-    cache[url] = HTTPS
-  elif url.startswith("http://"):
-    url = url.replace("http://", "")
-    cache[url] = HTTP
-  elif url not in cache:
-    # We know our endpoints are https.
-    if ".clarifai.com" in url:
-      cache[url] = HTTPS
-    else:  # need to test it.
-      try:  # make request to https endpoint.
-        urllib.request.urlopen("https://%s/v2/auth/methods" % url, timeout=1)
-        cache[url] = HTTPS  # cache it.
-      except Exception as e:
-        if "SSL" in str(e):  # if ssl error then we know it's http.
-          cache[url] = HTTP
-          # For http urls we need host:port format.
-          if ":" not in url:
-            raise Exception("When providing an insecure url it must have both host:port format")
-        else:
-          raise Exception("Could not get a valid response from url: %s, is the API running there?"
-                          % url) from e
-  return url
-
-
-class ClarifaiAuthHelper:
-
-  def __init__(
-      self,
-      user_id: str,
-      app_id: str,
-      pat: str,
-      token: str = "",
-      base: str = DEFAULT_BASE,
-      ui: str = DEFAULT_UI,
-      validate: bool = True,
-  ):
-    """
-        A helper to get the authorization information needed to make API calls with the grpc
-        client to a specific app using a personal access token.
-
-        There are classmethods to make this object easily from either query_params provided by streamlit or from env vars.
-
-        Note: only one of personal access token (pat) or a session token (token) can be provided.
-        Always use PATs in your code and never session tokens, those are only provided internal UI code.
-
-    Args:
-      user_id: a user id who owns the resource you want to make calls to.
-      app_id: an app id for the application that owns the resource you want to interact with
-      pat: a personal access token.
-      token: a session token (internal use only, always use a PAT).
-      base: a url to the API endpoint to hit. Examples include api.clarifai.com,
-    https://api.clarifai.com (default), https://host:port, http://host:port, host:port (will be treated as http, not https). It's highly recommended to include the http:// or https:// otherwise we need to check the endpoint to determine if it has SSL during this __init__
-      ui: a url to the UI. Examples include clarifai.com,
-    https://clarifai.com (default), https://host:port, http://host:port, host:port (will be treated as http, not https). It's highly recommended to include the http:// or https:// otherwise we need to check the endpoint to determine if it has SSL during this __init__
-      validate: whether to validate the inputs. This is useful for overriding vars then validating
-    """
-
-    self.user_id = user_id
-    self.app_id = app_id
-    self._pat = pat
-    self._token = token
-
-    self.set_base(base)
-    self.set_ui(ui)
-    if validate:
-      self.validate()
-
-  def validate(self):
-    if self.user_id == "":
-      raise Exception(
-          "Need 'user_id' to not be empty in the query params or user CLARIFAI_USER_ID env var")
-    if self.app_id == "":
-      raise Exception(
-          "Need 'app_id' to not be empty in the query params or user CLARIFAI_APP_ID env var")
-    if self._pat != "" and self._token != "":
-      raise Exception(
-          "A personal access token OR a session token need to be provided, but you cannot provide both."
-      )
-    elif self._pat == "" and self._token == "":
-      raise Exception(
-          "Need 'pat' or 'token' in the query params or use one of the CLARIFAI_PAT or CLARIFAI_SESSION_TOKEN env vars"
-      )
-
-  @classmethod
-  def from_streamlit(cls, st: Any) -> "ClarifaiAuthHelper":
-    """ This is a convenient method to check the environment variables first to see if there are
-    required variables for auth, then override them with any additional query parameters that may
-    have been passed in.
-
-    Note: if a .streamlit/secrets.toml is present then st.secrets will auto populate the
-    corresponding environment variables and we will pick them up from there, OVERWRITING whatever
-    matching env var that may already be present.
-
-        Args:
-          st: the streamlit package typically as: 'import streamlit as st'
-        Returns:
-          auth: this class instantiated
-        """
-    # start with the env vars (potentially loaded from secrets.toml)
-    # Don't validate yet as we'll layer on the query params next.
-    auth = ClarifaiAuthHelper.from_env(validate=False)
-
-    # Then add in the query params.
-    try:
-      auth.add_streamlit_query_params(st.experimental_get_query_params())
-    except Exception as e:
-      st.error(e)
-      st.stop()
-      raise e
-
-    # Then validate.
-    try:
-      auth.validate()
-    except Exception as e:
-      st.error(e)
-      st.stop()
-      raise e
-
-    return auth
-
-  @classmethod
-  def from_streamlit_query_params(cls, query_params: Any = "") -> "ClarifaiAuthHelper":
-    """Initialize from streamlit queryparams. The following things will be looked for:
-          user_id: as 'user_id' in query_params
-          app_id: as 'app_id' in query_params
-          one of:
-            token: as 'token' in query_params
-            pat: as 'pat' in query_params
-          optionally:
-            base: as 'base' in query_params.
-            ui: as 'ui' in query_params.
-
-    """
-
-    # Setup an empty one (not from env).
-    auth = ClarifaiAuthHelper("", "", "", "", validate=False)
-
-    # Then add in the query params.
-    auth.add_streamlit_query_params(query_params)
-
-    # Then validate.
-    auth.validate()
-
-    return auth
-
-  def add_streamlit_query_params(self, query_params: Any = "") -> "ClarifaiAuthHelper":
-    """Initialize from streamlit queryparams. The following things will be looked for:
-          user_id: as 'user_id' in query_params
-          app_id: as 'app_id' in query_params
-          one of:
-            token: as 'token' in query_params
-            pat: as 'pat' in query_params
-          optionally:
-            base: as 'base' in query_params.
-
-    Args:
-      query_params: the streamlit.experimental_get_query_params() response or an empty dict to fall
-    back to using env vars.
-    """
-    error_description = """
-Please check the following required query params are in the url:
- - 'user_id': the user ID accessing the module.
- - 'app_id': the app the module is being accessed from.
- - 'token' or 'pat': to authenticate the calling user with a session token or personal access token.
-
-Additionally, these optional params are supported:
- - 'base': the base domain for the API such as https://api.clarifai.com
- - 'ui': the overall UI domain for redirects such as https://clarifai.com
-"""
-
-    if query_params == "":  # empty response from streamlit
-      query_params = {}
-    for k in ["user_id", "app_id", "token", "pat"]:
-      if k in query_params and len(query_params[k]) != 1:
-        err_str = "There should only be 1 query param value for key '%s'" % k
-        raise Exception(err_str + error_description)
-    if "user_id" in query_params:
-      self.user_id = query_params["user_id"][0]
-    if "app_id" in query_params:
-      self.app_id = query_params["app_id"][0]
-    if "token" in query_params:
-      self._token = query_params["token"][0]
-    if "pat" in query_params:
-      self._pat = query_params["pat"][0]
-    if "base" in query_params:
-      self.set_base(query_params["base"][0])
-    if "ui" in query_params:
-      self.set_ui(query_params["ui"][0])
-
-  @classmethod
-  def from_env(cls, validate: bool = True) -> "ClarifaiAuthHelper":
-    """Will look for the following env vars:
-        user_id: CLARIFAI_USER_ID env var.
-        app_id: CLARIFAI_APP_ID env var.
-        one of:
-          token: CLARIFAI_SESSION_TOKEN env var.
-          pat: CLARIFAI_PAT env var.
-        base: CLARIFAI_API_BASE env var.
-    """
-    user_id = os.environ.get("CLARIFAI_USER_ID", "")
-    app_id = os.environ.get("CLARIFAI_APP_ID", "")
-    token = os.environ.get("CLARIFAI_SESSION_TOKEN", "")
-    pat = os.environ.get("CLARIFAI_PAT", "")
-    base = os.environ.get("CLARIFAI_API_BASE", DEFAULT_BASE)
-    ui = os.environ.get("CLARIFAI_UI", DEFAULT_UI)
-    return cls(user_id, app_id, pat, token, base, ui, validate)
-
-  def get_user_app_id_proto(
-      self,
-      user_id: str = None,
-      app_id: str = None,
-  ) -> resources_pb2.UserAppIDSet:
-    """Get the gRPC metadata that contains either the session token or the PAT to use.
-
-    Args:
-      user_id: optional user_id to override the default
-      app_id: optional app_id to override the default.
-
-    Returns:
-      metadata: the metadata need to send with all grpc API calls in the API client.
-    """
-    user_id = self.user_id if user_id is None else user_id
-    app_id = self.app_id if app_id is None else app_id
-    return resources_pb2.UserAppIDSet(user_id=user_id, app_id=app_id)
-
-  @property
-  def metadata(self):
-    """Get the gRPC metadata that contains either the session token or the PAT to use.
-
-    Returns:
-      metadata: the metadata need to send with all grpc API calls in the API client.
-    """
-    if self._pat != "":
-      return (("authorization", "Key %s" % self._pat),)
-    elif self._token != "":
-      return (("x-clarifai-session-token", self._token),)
-    else:
-      raise Exception("'token' or 'pat' needed to be provided in the query params or env vars.")
-
-  def get_stub(self) -> service_pb2_grpc.V2Stub:
-    """Get the API gRPC stub using the right channel based on the API endpoint base.
-
-    Returns:
-      stub: The service_pb2_grpc.V2Stub stub for the API.
-    """
-    if self._base not in base_https_cache:
-      raise Exception("Cannot determine if base %s is https" % self._base)
-
-    https = base_https_cache[self._base]
-    if https:
-      channel = ClarifaiChannel.get_grpc_channel(base=self._base)
-    else:
-      if self._base.find(":") >= 0:
-        host, port = self._base.split(":")
-      else:
-        host = self._base
-        port = 80
-      channel = ClarifaiChannel.get_insecure_grpc_channel(base=host, port=port)
-    stub = service_pb2_grpc.V2Stub(channel)
-    return stub
-
-  @property
-  def ui(self) -> str:
-    """ Return the domain for the UI. """
-    if self._ui not in ui_https_cache:
-      raise Exception("Cannot determine if ui %s is https" % self._ui)
-    https = ui_https_cache[self._ui]
-    if https:
-      if not self._ui.startswith("https://"):
-        return "https://" + self._ui
-      return self._ui
-    if not self._ui.startswith("http://"):
-      return "http://" + self._ui
-    return self._ui
-
-  def set_base(self, base: str):
-    """ Set the base domain for the API. """
-    self._base = https_cache(base_https_cache, base)
-
-  def set_ui(self, ui: str):
-    """ Set the domain for the UI. """
-    self._ui = https_cache(ui_https_cache, ui)
-
-  @property
-  def base(self) -> str:
-    """ Return the base domain for the API. """
-    if self._base not in base_https_cache:
-      raise Exception("Cannot determine if base %s is https" % self._base)
-    https = base_https_cache[self._base]
-    if https:
-      if not self._base.startswith("https://"):
-        return "https://" + self._base
-      return self._base
-    if not self._base.startswith("http://"):
-      return "http://" + self._base
-    return self._base
-
-  @property
-  def pat(self) -> str:
-    """ Return the personal access token. """
-    return self._pat
-
-  def __str__(self):
-    return "ClarifaiAuthHelper:\n- base: %s\n- user_id: %s\n- app_id: %s\n" % (
-        self._base,
-        self.user_id,
-        self.app_id,
-    )
-
-  @classmethod
-  def required_env_vars(cls):
-    """ Return the list of the required environment variables. """
-    return ["CLARIFAI_USER_ID", "CLARIFAI_APP_ID", "CLARIFAI_PAT"]
-
-  @classmethod
-  def validate_secrets_dict(cls, toml_dict: Dict[str, Any]):
-    """ Validate the secrets.toml file has been filled with non-empty values for all the auth
-    parameters that are present.
-
-    We don't load the file here so that we don't need the tomli package dependency. You can simply
-    do:
-    import tomli
-
-    d = tomli.load(open("secrets.toml"))
-    ClarifaiAuthHelper.validate_secrets_dict(d)
-
-    """
-    # We don't validate the bases because they have sensible defaults
-    auth_keys = cls.required_env_vars()
-
-    for k, v in toml_dict.items():
-      if k in auth_keys:
-        if v == "":
-          raise Exception("'%s' in secrets.toml cannot be empty" % k)
-    # for all the keys that are not present, they have a non empty value.
-    return True

clarifai_utils/client/auth/register.py

@@ -1,23 +0,0 @@
-import abc
-import grpc
-
-from clarifai_grpc.grpc.api import service_pb2_grpc
-
-
-class V2Stub(abc.ABC):
-  """Abstract base class of clarifai api rpc client stubs"""
-
-
-class RpcCallable(abc.ABC):
-  """Abstract base class of clarifai api rpc callables"""
-
-
-# add grpc classes as subclasses of the abcs, so they also succeed in isinstance calls
-def _register_classes():
-  V2Stub.register(service_pb2_grpc.V2Stub)
-  for name in dir(grpc):
-    if name.endswith('Callable'):
-      RpcCallable.register(getattr(grpc, name))
-
-
-_register_classes()

clarifai_utils/client/auth/stub.py

@@ -1,127 +0,0 @@
-import logging
-import time
-from concurrent.futures import ThreadPoolExecutor
-
-import grpc
-from clarifai_grpc.grpc.api.status import status_code_pb2
-
-from clarifai.client.auth.helper import ClarifaiAuthHelper
-from clarifai.client.auth.register import RpcCallable, V2Stub
-
-throttle_status_codes = {
-    status_code_pb2.CONN_THROTTLED,
-    status_code_pb2.CONN_EXCEED_HOURLY_LIMIT,
-}
-
-retry_codes_grpc = {
-    grpc.StatusCode.UNAVAILABLE,
-}
-
-_threadpool = ThreadPoolExecutor(100)
-
-
-def create_stub(auth_helper: ClarifaiAuthHelper = None, max_retry_attempts: int = 10) -> V2Stub:
-  """
-  Create client stub that handles authorization and basic retries for
-  unavailable or throttled connections.
-
-  Args:
-    auth_helper:  ClarifaiAuthHelper to use for auth metadata (default: from env)
-    max_retry_attempts:  max attempts to retry rpcs with retryable failures
-  """
-  stub = AuthorizedStub(auth_helper)
-  if max_retry_attempts > 0:
-    return RetryStub(stub, max_retry_attempts)
-  return stub
-
-
-class AuthorizedStub(V2Stub):
-  """V2Stub proxy that inserts metadata authorization in rpc calls."""
-
-  def __init__(self, auth_helper: ClarifaiAuthHelper = None):
-    if auth_helper is None:
-      auth_helper = ClarifaiAuthHelper.from_env()
-    self.stub = auth_helper.get_stub()
-    self.metadata = auth_helper.metadata
-
-  def __getattr__(self, name):
-    value = getattr(self.stub, name)
-    if isinstance(value, RpcCallable):
-      value = _AuthorizedRpcCallable(value, self.metadata)
-    return value
-
-
-class _AuthorizedRpcCallable(RpcCallable):
-  """Adds metadata(authorization header) to rpc calls"""
-
-  def __init__(self, func, metadata):
-    self.f = func
-    self.metadata = metadata
-
-  def __repr__(self):
-    return repr(self.f)
-
-  def __call__(self, *args, **kwargs):
-    metadata = kwargs.pop('metadata', self.metadata)
-    return self.f(*args, **kwargs, metadata=metadata)
-
-  def future(self, *args, **kwargs):
-    metadata = kwargs.pop('metadata', self.metadata)
-    return self.f.future(*args, **kwargs, metadata=metadata)
-
-  def __getattr__(self, name):
-    return getattr(self.f, name)
-
-
-class RetryStub(V2Stub):
-  """
-  V2Stub proxy that retries requests (currently on unavailable server or throttle codes)
-  """
-
-  def __init__(self, stub, max_attempts=10, backoff_time=5):
-    self.stub = stub
-    self.max_attempts = max_attempts
-    self.backoff_time = backoff_time
-
-  def __getattr__(self, name):
-    value = getattr(self.stub, name)
-    if isinstance(value, RpcCallable):
-      value = _RetryRpcCallable(value, self.max_attempts, self.backoff_time)
-    return value
-
-
-class _RetryRpcCallable(RpcCallable):
-  """Retries rpc calls on unavailable server or throttle codes"""
-
-  def __init__(self, func, max_attempts, backoff_time):
-    self.f = func
-    self.max_attempts = max_attempts
-    self.backoff_time = backoff_time
-
-  def __repr__(self):
-    return repr(self.f)
-
-  def __call__(self, *args, **kwargs):
-    attempt = 0
-    while attempt < self.max_attempts:
-      attempt += 1
-      if attempt != 1:
-        time.sleep(self.backoff_time)  # TODO better backoff between attempts
-      try:
-        response = self.f(*args, **kwargs)
-        if (response.status.code in throttle_status_codes) and attempt < self.max_attempts:
-          logging.debug('Retrying with status %s' % str(response.status))
-        else:
-          return response
-      except grpc.RpcError as e:
-        if (e.code() in retry_codes_grpc) and attempt < self.max_attempts:
-          logging.debug('Retrying with status %s' % e.code())
-        else:
-          raise
-
-  def future(self, *args, **kwargs):
-    # TODO use single result event loop thread with asyncio
-    return _threadpool.submit(self, *args, **kwargs)
-
-  def __getattr__(self, name):
-    return getattr(self.f, name)

clarifai_utils/client/base.py

@@ -1,131 +0,0 @@
-import os
-from datetime import datetime
-from typing import Any, Callable
-
-from google.protobuf import struct_pb2
-from google.protobuf.timestamp_pb2 import Timestamp
-from google.protobuf.wrappers_pb2 import BoolValue
-
-from clarifai.client.auth import create_stub
-from clarifai.client.auth.helper import ClarifaiAuthHelper
-from clarifai.errors import ApiError, UserError
-
-
-class BaseClient:
-  """BaseClient is the base class for all the classes interacting with Clarifai endpoints.
-
-  Args:
-      **kwargs: Additional keyword arguments to be passed to the ClarifaiAuthHelper.
-          - user_id (str): A user ID for authentication.
-          - app_id (str): An app ID for the application to interact with.
-          - pat (str): A personal access token for authentication.
-          - base (str): The base URL for the API endpoint. Defaults to 'https://api.clarifai.com'.
-          - ui (str): The URL for the UI. Defaults to 'https://clarifai.com'.
-
-  Attributes:
-      auth_helper (ClarifaiAuthHelper): An instance of ClarifaiAuthHelper for authentication.
-      STUB (Stub): The gRPC Stub object for API interaction.
-      metadata (tuple): The gRPC metadata containing the personal access token.
-      userDataObject (UserAppIDSet): The protobuf object representing user and app IDs.
-      base (str): The base URL for the API endpoint.
-  """
-
-  def __init__(self, **kwargs):
-    pat = os.environ.get('CLARIFAI_PAT', "")
-    if pat == "":
-      raise UserError("CLARIFAI_PAT must be set as env vars")
-    self.auth_helper = ClarifaiAuthHelper(**kwargs, pat=pat, validate=False)
-    self.STUB = create_stub(self.auth_helper)
-    self.metadata = self.auth_helper.metadata
-    self.user_app_id = self.auth_helper.get_user_app_id_proto()
-    self.base = self.auth_helper.base
-
-  def _grpc_request(self, method: Callable, argument: Any):
-    """Makes a gRPC request to the API.
-
-    Args:
-        method (Callable): The gRPC method to call.
-        argument (Any): The argument to pass to the gRPC method.
-
-    Returns:
-        res (Any): The result of the gRPC method call.
-    """
-
-    try:
-      res = method(argument)
-      # MessageToDict(res) TODO global debug logger
-      return res
-    except ApiError:
-      raise Exception("ApiError")
-
-  def convert_string_to_timestamp(self, date_str) -> Timestamp:
-    """Converts a string to a Timestamp object.
-
-    Args:
-        date_str (str): The string to convert.
-
-    Returns:
-        Timestamp: The converted Timestamp object.
-    """
-    # Parse the string into a Python datetime object
-    try:
-      datetime_obj = datetime.strptime(date_str, '%Y-%m-%dT%H:%M:%S.%fZ')
-    except ValueError:
-      try:
-        datetime_obj = datetime.strptime(date_str, '%Y-%m-%dT%H:%M:%SZ')
-      except ValueError:
-        return Timestamp()
-
-    # Convert the datetime object to a Timestamp object
-    timestamp_obj = Timestamp()
-    timestamp_obj.FromDatetime(datetime_obj)
-
-    return timestamp_obj
-
-  def process_response_keys(self, old_dict, listing_resource=None):
-    """Converts keys in a response dictionary to resource proto format.
-
-    Args:
-        old_dict (dict): The dictionary to convert.
-
-    Returns:
-        new_dict (dict): The dictionary with processed keys.
-    """
-    if listing_resource:
-      old_dict[f'{listing_resource}_id'] = old_dict['id']
-      old_dict.pop('id')
-
-    def convert_recursive(item):
-      if isinstance(item, dict):
-        new_item = {}
-        for key, value in item.items():
-          if key == 'default_value':
-            # Map infer param value to proto value
-            value_map = dict(number_value=None, string_value=None, bool_value=None)
-
-            def map_fn(v):
-              return 'number_value' if isinstance(v, float) or isinstance(v, int) else \
-              'string_value' if isinstance(v, str) else \
-              'bool_value' if isinstance(v, bool) else None
-
-            value_map[map_fn(value)] = value
-            value = struct_pb2.Value(**value_map)
-          if key in ['created_at', 'modified_at', 'completed_at']:
-            value = self.convert_string_to_timestamp(value)
-          elif key in ['workflow_recommended']:
-            value = BoolValue(value=True)
-          elif key in ['fields_map', 'params']:
-            value_s = struct_pb2.Struct()
-            value_s.update(value)
-            value = value_s
-          elif key in ['metadata']:
-            continue  # TODO Fix "app_duplication"
-          new_item[key] = convert_recursive(value)
-        return new_item
-      elif isinstance(item, list):
-        return [convert_recursive(element) for element in item]
-      else:
-        return item
-
-    new_dict = convert_recursive(old_dict)
-    return new_dict