clarifai 11.3.0rc2__py3-none-any.whl → 11.4.0__py3-none-any.whl

clarifai/runners/utils/url_fetcher.py

@@ -6,44 +6,44 @@ from clarifai.utils.logging import logger
 
 
 def download_input(input):
-  _download_input_data(input.data)
-  if input.data.parts:
-    for i in range(len(input.data.parts)):
-      _download_input_data(input.data.parts[i].data)
+    _download_input_data(input.data)
+    if input.data.parts:
+        for i in range(len(input.data.parts)):
+            _download_input_data(input.data.parts[i].data)
 
 
 def _download_input_data(input_data):
-  """
-  This function will download any urls that are not already bytes.
-  """
-  if input_data.image.url and not input_data.image.base64:
-    # Download the image
-    with fsspec.open(input_data.image.url, 'rb') as f:
-      input_data.image.base64 = f.read()
-  if input_data.video.url and not input_data.video.base64:
-    # Download the video
-    with fsspec.open(input_data.video.url, 'rb') as f:
-      input_data.video.base64 = f.read()
-  if input_data.audio.url and not input_data.audio.base64:
-    # Download the audio
-    with fsspec.open(input_data.audio.url, 'rb') as f:
-      input_data.audio.base64 = f.read()
-  if input_data.text.url and not input_data.text.raw:
-    # Download the text
-    with fsspec.open(input_data.text.url, 'r') as f:
-      input_data.text.raw = f.read()
+    """
+    This function will download any urls that are not already bytes.
+    """
+    if input_data.image.url and not input_data.image.base64:
+        # Download the image
+        with fsspec.open(input_data.image.url, 'rb') as f:
+            input_data.image.base64 = f.read()
+    if input_data.video.url and not input_data.video.base64:
+        # Download the video
+        with fsspec.open(input_data.video.url, 'rb') as f:
+            input_data.video.base64 = f.read()
+    if input_data.audio.url and not input_data.audio.base64:
+        # Download the audio
+        with fsspec.open(input_data.audio.url, 'rb') as f:
+            input_data.audio.base64 = f.read()
+    if input_data.text.url and not input_data.text.raw:
+        # Download the text
+        with fsspec.open(input_data.text.url, 'r') as f:
+            input_data.text.raw = f.read()
 
 
 def ensure_urls_downloaded(request, max_threads=128):
-  """
-  This function will download any urls that are not already bytes and parallelize with a thread pool.
-  """
-  with concurrent.futures.ThreadPoolExecutor(max_workers=max_threads) as executor:
-    futures = []
-    for input in request.inputs:
-      futures.append(executor.submit(download_input, input))
-    for future in concurrent.futures.as_completed(futures):
-      try:
-        future.result()
-      except Exception as e:
-        logger.exception(f"Error downloading input: {e}")
+    """
+    This function will download any urls that are not already bytes and parallelize with a thread pool.
+    """
+    with concurrent.futures.ThreadPoolExecutor(max_workers=max_threads) as executor:
+        futures = []
+        for input in request.inputs:
+            futures.append(executor.submit(download_input, input))
+        for future in concurrent.futures.as_completed(futures):
+            try:
+                future.result()
+            except Exception as e:
+                logger.exception(f"Error downloading input: {e}")

clarifai/schema/search.py

@@ -2,71 +2,64 @@ from schema import And, Optional, Regex, Schema
 
 
 def get_schema() -> Schema:
-  """Initialize the schema for rank and filter.
+    """Initialize the schema for rank and filter.
 
-        This schema validates:
+    This schema validates:
 
-        - Rank and filter must be a list
-        - Each item in the list must be a dict
-        - The dict can contain these optional keys:
-            - 'image_url': Valid URL string
-            - 'text_raw': Non-empty string
-            - 'metadata': Dict
-            - 'image_bytes': Bytes
-            - 'geo_point': Dict with 'longitude', 'latitude' and 'geo_limit' as float, float and int respectively
-            - 'concepts': List where each item is a concept dict
-        - Concept dict requires at least one of:
-            - 'name': Non-empty string with dashes/underscores
-            - 'id': Non-empty string
-            - 'language': Non-empty string
-            - 'value': 0 or 1 integer
-        - 'input_types': List of 'image', 'video', 'text' or 'audio'
-        - 'input_dataset_ids': List of strings
-        - 'input_status_code': Integer
+    - Rank and filter must be a list
+    - Each item in the list must be a dict
+    - The dict can contain these optional keys:
+        - 'image_url': Valid URL string
+        - 'text_raw': Non-empty string
+        - 'metadata': Dict
+        - 'image_bytes': Bytes
+        - 'geo_point': Dict with 'longitude', 'latitude' and 'geo_limit' as float, float and int respectively
+        - 'concepts': List where each item is a concept dict
+    - Concept dict requires at least one of:
+        - 'name': Non-empty string with dashes/underscores
+        - 'id': Non-empty string
+        - 'language': Non-empty string
+        - 'value': 0 or 1 integer
+    - 'input_types': List of 'image', 'video', 'text' or 'audio'
+    - 'input_dataset_ids': List of strings
+    - 'input_status_code': Integer
 
-        Returns:
-            Schema: The schema for rank and filter.
-        """
-  # Schema for a single concept
-  concept_schema = Schema({
-      Optional('value'):
-          And(int, lambda x: x in [0, 1]),
-      Optional('id'):
-          And(str, len),
-      Optional('language'):
-          And(str, len),
-      # Non-empty strings with internal dashes and underscores.
-      Optional('name'):
-          And(str, len, Regex(r'^[0-9A-Za-z]+([-_][0-9A-Za-z]+)*$'))
-  })
+    Returns:
+        Schema: The schema for rank and filter.
+    """
+    # Schema for a single concept
+    concept_schema = Schema(
+        {
+            Optional('value'): And(int, lambda x: x in [0, 1]),
+            Optional('id'): And(str, len),
+            Optional('language'): And(str, len),
+            # Non-empty strings with internal dashes and underscores.
+            Optional('name'): And(str, len, Regex(r'^[0-9A-Za-z]+([-_][0-9A-Za-z]+)*$')),
+        }
+    )
 
-  # Schema for a rank or filter item
-  rank_filter_item_schema = Schema({
-      Optional('image_url'):
-          And(str, Regex(r'^https?://')),
-      Optional('text_raw'):
-          And(str, len),
-      Optional('metadata'):
-          dict,
-      Optional('image_bytes'):
-          bytes,
-      Optional('geo_point'): {
-          'longitude': float,
-          'latitude': float,
-          'geo_limit': int
-      },
-      Optional("concepts"):
-          And(list, lambda x: all(concept_schema.is_valid(item) and len(item) > 0 for item in x)),
+    # Schema for a rank or filter item
+    rank_filter_item_schema = Schema(
+        {
+            Optional('image_url'): And(str, Regex(r'^https?://')),
+            Optional('text_raw'): And(str, len),
+            Optional('metadata'): dict,
+            Optional('image_bytes'): bytes,
+            Optional('geo_point'): {'longitude': float, 'latitude': float, 'geo_limit': int},
+            Optional("concepts"): And(
+                list, lambda x: all(concept_schema.is_valid(item) and len(item) > 0 for item in x)
+            ),
+            ## input filters
+            Optional('input_types'): And(
+                list,
+                lambda input_types: all(
+                    input_type in ('image', 'video', 'text', 'audio') for input_type in input_types
+                ),
+            ),
+            Optional('input_dataset_ids'): list,
+            Optional('input_status_code'): int,
+        }
+    )
 
-      ## input filters
-      Optional('input_types'):
-          And(list, lambda input_types: all(input_type in ('image', 'video', 'text', 'audio')
-                                            for input_type in input_types)),
-      Optional('input_dataset_ids'):
-          list,
-      Optional('input_status_code'):
-          int,
-  })
-
-  # Schema for rank and filter args
-  return Schema([rank_filter_item_schema])
+    # Schema for rank and filter args
+    return Schema([rank_filter_item_schema])

clarifai/urls/helper.py

@@ -2,119 +2,142 @@ from urllib.parse import urlparse
 
 
 class ClarifaiUrlHelper(object):
-  """Lots of helper functionality for dealing with urls around modules."""
+    """Lots of helper functionality for dealing with urls around modules."""
 
-  def __init__(self, auth, module_manager_imv_id="module_manager_install"):
-    """
-    Args:
-      auth: a ClarifaiAuthHelper object.
-    """
-    self._auth = auth
-    self._module_manager_imv_id = module_manager_imv_id
+    def __init__(self, auth, module_manager_imv_id="module_manager_install"):
+        """
+        Args:
+          auth: a ClarifaiAuthHelper object.
+        """
+        self._auth = auth
+        self._module_manager_imv_id = module_manager_imv_id
 
-  @property
-  def auth(self):
-    return self._auth
+    @property
+    def auth(self):
+        return self._auth
 
-  def module_ui_url(self, user_id, app_id, module_id, module_version_id):
-    """This is the path to the module in community."""
-    return "%s/%s/%s/modules/%s/versions/%s" % (
-        self.auth.ui,
-        user_id,
-        app_id,
-        module_id,
-        module_version_id,
-    )
+    def module_ui_url(self, user_id, app_id, module_id, module_version_id):
+        """This is the path to the module in community."""
+        return "%s/%s/%s/modules/%s/versions/%s" % (
+            self.auth.ui,
+            user_id,
+            app_id,
+            module_id,
+            module_version_id,
+        )
 
-  def module_install_ui_url(self, dest_user_id, dest_app_id, module_url):
-    """This is a url that allows for installation of the module from the community at 'module_url'
+    def module_install_ui_url(self, dest_user_id, dest_app_id, module_url):
+        """This is a url that allows for installation of the module from the community at 'module_url'
         into the destination app_id of the destination user_id."""
-    return ("%s/%s/%s/installed_module_versions/%s/install?install=%s" %
-            (self.auth.ui, dest_user_id, dest_app_id, self._module_manager_imv_id, module_url))
+        return "%s/%s/%s/installed_module_versions/%s/install?install=%s" % (
+            self.auth.ui,
+            dest_user_id,
+            dest_app_id,
+            self._module_manager_imv_id,
+            module_url,
+        )
 
-  def imv_ui_url(self, dest_user_id, dest_app_id, imv_id):
-    return ("%s/%s/%s/installed_module_versions/%s" % (self.auth.ui, dest_user_id, dest_app_id,
-                                                       imv_id))
+    def imv_ui_url(self, dest_user_id, dest_app_id, imv_id):
+        return "%s/%s/%s/installed_module_versions/%s" % (
+            self.auth.ui,
+            dest_user_id,
+            dest_app_id,
+            imv_id,
+        )
 
-  def clarifai_url(self, user_id, app_id, resource_type, resource_id, version_id: str = None):
-    """This is the path to the resource in community.
+    def clarifai_url(self, user_id, app_id, resource_type, resource_id, version_id: str = None):
+        """This is the path to the resource in community.
 
-    Args:
-      user_id: the author of the resource.
-      app_id: the author's app the resource was created in.
-      resource_type: the type of resource. One of "modules", "models", "concepts", "inputs", "workflows", "tasks", "installed_module_versions"
-      resource_id: the resource ID
-      version_id: the version of the resource.
-    """
-    if resource_type not in [
-        "modules", "models", "concepts", "inputs", "workflows", "tasks",
-        "installed_module_versions"
-    ]:
-      raise ValueError(
-          "resource_type must be one of modules, models, concepts, inputs, workflows, tasks, installed_module_versions but was %s"
-          % resource_type)
-    if version_id is None:
-      return "%s/%s/%s/%s/%s" % (self.auth.ui, user_id, app_id, resource_type, resource_id)
-    return "%s/%s/%s/%s/%s/versions/%s" % (self.auth.ui, user_id, app_id, resource_type,
-                                           resource_id, version_id)
+        Args:
+          user_id: the author of the resource.
+          app_id: the author's app the resource was created in.
+          resource_type: the type of resource. One of "modules", "models", "concepts", "inputs", "workflows", "tasks", "installed_module_versions"
+          resource_id: the resource ID
+          version_id: the version of the resource.
+        """
+        if resource_type not in [
+            "modules",
+            "models",
+            "concepts",
+            "inputs",
+            "workflows",
+            "tasks",
+            "installed_module_versions",
+        ]:
+            raise ValueError(
+                "resource_type must be one of modules, models, concepts, inputs, workflows, tasks, installed_module_versions but was %s"
+                % resource_type
+            )
+        if version_id is None:
+            return "%s/%s/%s/%s/%s" % (self.auth.ui, user_id, app_id, resource_type, resource_id)
+        return "%s/%s/%s/%s/%s/versions/%s" % (
+            self.auth.ui,
+            user_id,
+            app_id,
+            resource_type,
+            resource_id,
+            version_id,
+        )
 
-  @classmethod
-  def split_clarifai_app_url(cls, url):
-    """
-    clarifai.com uses fully qualified urls to resources.
-    They are in the format of:
-    https://clarifai.com/{user_id}/{app_id}/
-    """
-    url = url.replace("https://", "", 1).replace("http://", "", 1)
-    o = urlparse(url)
-    path = o.path
-    path = path.lstrip("/")
-    parts = path.split("/")
-    if len(parts) != 3:
-      raise ValueError(
-          f"Provided url must have 2 parts after the domain name. The current parts are: {parts}")
-    return tuple(parts[1:])
+    @classmethod
+    def split_clarifai_app_url(cls, url):
+        """
+        clarifai.com uses fully qualified urls to resources.
+        They are in the format of:
+        https://clarifai.com/{user_id}/{app_id}/
+        """
+        url = url.replace("https://", "", 1).replace("http://", "", 1)
+        o = urlparse(url)
+        path = o.path
+        path = path.lstrip("/")
+        parts = path.split("/")
+        if len(parts) != 3:
+            raise ValueError(
+                f"Provided url must have 2 parts after the domain name. The current parts are: {parts}"
+            )
+        return tuple(parts[1:])
 
-  @classmethod
-  def split_clarifai_url(cls, url):
-    """
-    clarifai.com uses fully qualified urls to resources.
-    They are in the format of:
-    https://clarifai.com/{user_id}/{app_id}/{resource_type}/{resource_id}/{resource_version_type}/{resource_version_id}
-    Those last two are optional.
+    @classmethod
+    def split_clarifai_url(cls, url):
+        """
+        clarifai.com uses fully qualified urls to resources.
+        They are in the format of:
+        https://clarifai.com/{user_id}/{app_id}/{resource_type}/{resource_id}/{resource_version_type}/{resource_version_id}
+        Those last two are optional.
 
-    """
-    url = url.replace("https://", "", 1).replace("http://", "", 1)
-    o = urlparse(url)
-    path = o.path
-    path = path.lstrip("/")
-    parts = path.split("/")
-    if len(parts) != 5 and len(parts) != 7:
-      raise ValueError(
-          "Provided url must have 4 or 6 parts after the domain name. These are: {user_id}/{app_id}/{resource_type}/{resource_id}/{resource_version_type}/{resource_version_id}"
-      )
-    user_id, app_id, resource_type, resource_id = parts[1:5]
-    if len(parts) == 7:
-      resource_version_id = parts[6]
-    else:
-      resource_version_id = None
-    return user_id, app_id, resource_type, resource_id, resource_version_id
+        """
+        url = url.replace("https://", "", 1).replace("http://", "", 1)
+        o = urlparse(url)
+        path = o.path
+        path = path.lstrip("/")
+        parts = path.split("/")
+        if len(parts) != 5 and len(parts) != 7:
+            raise ValueError(
+                "Provided url must have 4 or 6 parts after the domain name. These are: {user_id}/{app_id}/{resource_type}/{resource_id}/{resource_version_type}/{resource_version_id}"
+            )
+        user_id, app_id, resource_type, resource_id = parts[1:5]
+        if len(parts) == 7:
+            resource_version_id = parts[6]
+        else:
+            resource_version_id = None
+        return user_id, app_id, resource_type, resource_id, resource_version_id
 
-  @classmethod
-  def split_module_ui_url(cls, install):
-    """Takes in a path like https://clarifai.com/zeiler/app/modules/module1/versions/2 to split it apart into it's IDs.
+    @classmethod
+    def split_module_ui_url(cls, install):
+        """Takes in a path like https://clarifai.com/zeiler/app/modules/module1/versions/2 to split it apart into it's IDs.
 
-    Returns:
-      user_id: the author of the module.
-      app_id: the author's app the module was created in.
-      module_id: the module ID
-      module_version_id: the version of the module.
-    """
-    user_id, app_id, resource_type, resource_id, resource_version_id = cls.split_clarifai_url(
-        install)
+        Returns:
+          user_id: the author of the module.
+          app_id: the author's app the module was created in.
+          module_id: the module ID
+          module_version_id: the version of the module.
+        """
+        user_id, app_id, resource_type, resource_id, resource_version_id = cls.split_clarifai_url(
+            install
+        )
 
-    if resource_type != "modules" or resource_version_id is None:
-      raise ValueError(
-          "Provided install url must have 6 parts after the domain name. These are {user_id}/{app_id}/modules/{module_id}/versions/{module_version_id}"
-      )
-    return user_id, app_id, resource_id, resource_version_id
+        if resource_type != "modules" or resource_version_id is None:
+            raise ValueError(
+                "Provided install url must have 6 parts after the domain name. These are {user_id}/{app_id}/modules/{module_id}/versions/{module_version_id}"
+            )
+        return user_id, app_id, resource_id, resource_version_id