clarifai 9.7.1__py3-none-any.whl → 9.7.2__py3-none-any.whl

clarifai_utils/dataset_export/dataset_export_inputs.py

@@ -1,205 +0,0 @@
-import os
-import tempfile
-import zipfile
-from io import BytesIO
-
-import requests
-from clarifai_grpc.grpc.api import resources_pb2
-from google.protobuf.json_format import MessageToDict
-from PIL import ImageFile
-from tqdm import tqdm
-
-from clarifai.auth.helper import ClarifaiAuthHelper
-
-
-class DatasetExportReader:
-  """
-  Unpacks the zipfile from DatasetVersionExport
-  - Downloads the temp archive onto disk
-  - Reads DatasetVersionExports archive in memory without extracting all
-  - Yield each api.Input object.
-  """
-
-  def __init__(self, session, archive_url=None, local_archive_path=None):
-
-    self.input_count = 0
-    self.temp_file = None
-    self.session = session
-
-    assert archive_url or local_archive_path, "Must use one input."
-
-    if archive_url:
-      print('url: %s' % archive_url)
-      self.temp_file = self._download_temp_archive(archive_url)
-      self.archive = zipfile.ZipFile(self.temp_file)
-    else:
-      print("path: %s" % local_archive_path)
-      self.archive = zipfile.ZipFile(local_archive_path)
-
-    self.file_name_list = self.archive.namelist()
-    assert "mimetype" in self.file_name_list, "Missing mimetype file in the dataset export archive."
-    assert self.archive.read("mimetype") == b"application/x.clarifai-data+protobuf"
-    self.file_name_list.remove("mimetype")
-
-    print("Obtained file name list. %d entries." % len(self.file_name_list))
-    self.split_dir = os.path.dirname(self.file_name_list[0]) if len(self.file_name_list) else ""
-
-  def _download_temp_archive(self, archive_url, chunk_size=128):
-    """
-    Downloads the temp archive of InputBatches.
-    """
-    r = self.session.get(archive_url, stream=True)
-    temp_file = tempfile.TemporaryFile()
-    for chunk in r.iter_content(chunk_size=chunk_size):
-      temp_file.write(chunk)
-
-    return temp_file
-
-  def __len__(self):
-    if not self.input_count:
-      if self.file_name_list is not None:
-        for filename in self.file_name_list:
-          self.input_count += int(filename.split('_n')[-1])
-
-    return self.input_count
-
-  def __iter__(self):
-    """
-    Loops through all InputBatches in the DatasetVersionExport and yields every api.Input object
-    """
-    if self.file_name_list is not None:
-      for filename in self.file_name_list:
-        db = resources_pb2.InputBatch().FromString(self.archive.read(filename))
-        for db_input in db.inputs:
-          yield db_input
-      print("DONE")
-
-  def __enter__(self):
-    return self
-
-  def __exit__(self, *args):
-    self.close()
-
-  def close(self):
-    print("closing file objects.")
-    self.archive.close()
-    if self.temp_file:
-      self.temp_file.close()
-
-
-class InputDownloader:
-  """
-  Takes an iterator or a list of api.Input instances as input,
-  and has a method for downloading all inputs (image/text/audio/video) of that data.
-  Has the ability of either writing to a new ZIP archive OR a filesystem directory.
-  """
-
-  def __init__(self, session, input_iterator):
-    self.input_iterator = input_iterator
-    self.num_inputs = 0
-    self.split_prefix = None
-    self.session = session
-    self.input_ext = dict(image=".jpg", text=".txt", audio=".wav", video=".mp4")
-    if isinstance(self.input_iterator, DatasetExportReader):
-      self.split_prefix = self.input_iterator.split_dir
-
-  def _save_image_to_archive(self, new_archive, hosted_url, file_name):
-    """
-    Use PIL ImageFile to return image parsed from the response bytestring (from requests) and append to zip file.
-    """
-    p = ImageFile.Parser()
-    p.feed(self.session.get(hosted_url).content)
-    image = p.close()
-    image_file = BytesIO()
-    image.save(image_file, 'JPEG')
-    new_archive.writestr(file_name, image_file.getvalue())
-
-  def _save_text_to_archive(self, new_archive, hosted_url, file_name):
-    """
-    Gets the text response bytestring (from requests) and append to zip file.
-    """
-    text_content = self.session.get(hosted_url).content
-    new_archive.writestr(file_name, text_content)
-
-  def _save_audio_to_archive(self, new_archive, hosted_url, file_name):
-    """
-    Gets the audio response bytestring (from requests) as chunks and append to zip file.
-    """
-    audio_response = requests.get(hosted_url, stream=True)
-    audio_stream = BytesIO()
-    # Retrieve the audio content in chunks and write to the BytesIO object
-    for chunk in audio_response.iter_content(chunk_size=128):
-      audio_stream.write(chunk)
-    new_archive.writestr(file_name, audio_stream.getvalue())
-
-  def _save_video_to_archive(self, new_archive, hosted_url, file_name):
-    """
-    Gets the video response bytestring (from requests) as chunks and append to zip file.
-    """
-    video_response = self.session.get(hosted_url)
-    video_stream = BytesIO()
-    # Retrieve the video content in chunks and write to the BytesIO object
-    for chunk in video_response.iter_content(chunk_size=128):
-      video_stream.write(chunk)
-    new_archive.writestr(file_name, video_stream.getvalue())
-
-  def _write_input_archive(self, save_path, split):
-    """
-    Writes the input archive into prefix dir.
-    """
-    try:
-      total = len(self.input_iterator)
-    except TypeError:
-      total = None
-    with zipfile.ZipFile(save_path, "a") as new_archive:
-      for input_ in tqdm(self.input_iterator, desc="Writing input archive", total=total):
-        # checks for input
-        data_dict = MessageToDict(input_.data)
-        input_type = list(
-            filter(lambda x: x in list(data_dict.keys()), list(self.input_ext.keys())))[0]
-        hosted = getattr(input_.data, input_type).hosted
-        if hosted.prefix:
-          assert 'orig' in hosted.sizes
-          hosted_url = f"{hosted.prefix}/orig/{hosted.suffix}"
-          file_name = os.path.join(split, input_.id + self.input_ext[input_type])
-          if input_type == "image":
-            self._save_image_to_archive(new_archive, hosted_url, file_name)
-          elif input_type == "text":
-            self._save_text_to_archive(new_archive, hosted_url, file_name)
-          elif input_type == "audio":
-            self._save_audio_to_archive(new_archive, hosted_url, file_name)
-          elif input_type == "video":
-            self._save_video_to_archive(new_archive, hosted_url, file_name)
-          self.num_inputs += 1
-
-  def _check_output_archive(self, save_path):
-    try:
-      archive = zipfile.ZipFile(save_path, 'r')
-    except zipfile.BadZipFile as e:
-      raise e
-    assert len(
-        archive.namelist()) == self.num_inputs, "Archive has %d inputs | expecting %d inputs" % (
-            len(archive.namelist()), self.num_inputs)
-
-  def download_input_archive(self, save_path, split=None):
-    """
-    Downloads the archive from the URL into an archive of inputs in the directory format {split}/{input_type}.
-    """
-    self._write_input_archive(save_path, split=split or self.split_prefix)
-    self._check_output_archive(save_path)
-
-
-if __name__ == "__main__":
-  import sys
-  if len(sys.argv) < 2:
-    print(f"usage: {sys.argv[0]} <archive-url> [<save-path>]")
-    sys.exit(2)
-  archive_url = sys.argv[1]
-  save_path = sys.argv[2] if len(sys.argv) > 2 else "output.zip"
-  metadata = getattr(ClarifaiAuthHelper.from_env(), "metadata")[0]
-  # Create a session object and set auth header
-  session = requests.Session()
-  session.headers.update({'Authorization': metadata[1]})
-
-  with DatasetExportReader(session=session, archive_url=archive_url) as reader:
-    InputDownloader(session, reader).download_input_archive(save_path=save_path)

clarifai_utils/listing/concepts.py

@@ -1,37 +0,0 @@
-from clarifai_grpc.grpc.api import resources_pb2, service_pb2
-from clarifai_grpc.grpc.api.status import status_code_pb2
-
-from clarifai.client import V2Stub
-
-
-def concepts_generator(stub: V2Stub, user_id: str, app_id: str, page_size: int = 64):
-  """
-    Lists all the concept in the given userAppID user_id, app_id app.
-
-    Args:
-      stub: client stub.
-      user_id: the user to list from.
-      app_id: the app in the user_id account to list from.
-      page_size: the pagination size to use while iterating.
-
-    Returns:
-      concepts: a list of Concept protos for all the concepts in the app.
-    """
-  userDataObject = resources_pb2.UserAppIDSet(user_id=user_id, app_id=app_id)
-
-  concept_success_status = {status_code_pb2.SUCCESS}
-
-  page = 1
-  while True:
-    response = stub.ListConcepts(
-        service_pb2.ListConceptsRequest(user_app_id=userDataObject, page=page, per_page=page_size),
-    )
-
-    if response.status.code not in concept_success_status:
-      raise Exception("ListConcepts failed with response %r" % response)
-    if len(response.concepts) == 0:
-      break
-    else:
-      for item in response.concepts:
-        yield item
-    page += 1

clarifai_utils/listing/datasets.py

@@ -1,37 +0,0 @@
-from clarifai_grpc.grpc.api import resources_pb2, service_pb2
-from clarifai_grpc.grpc.api.status import status_code_pb2
-
-from clarifai.client import V2Stub
-
-
-def datasets_generator(stub: V2Stub, user_id: str, app_id: str, page_size: int = 64):
-  """
-    Lists all the datasets in an application given a userAppID user_id, app_id app.
-
-    Args:
-      stub: client stub.
-      user_id: the user to list from.
-      app_id: the app in the user_id account to list from.
-      page_size: the pagination size to use while iterating.
-
-    Returns:
-      datasets: a list of Datasets protos for all the Datasets in the app.
-    """
-  userDataObject = resources_pb2.UserAppIDSet(user_id=user_id, app_id=app_id)
-
-  dataset_success_status = {status_code_pb2.SUCCESS}
-
-  page = 1
-  while True:
-    response = stub.ListDatasets(
-        service_pb2.ListDatasetsRequest(user_app_id=userDataObject, page=page, per_page=page_size),
-    )
-
-    if response.status.code not in dataset_success_status:
-      raise Exception("ListDatasets failed with response %r" % response)
-    if len(response.datasets) == 0:
-      break
-    else:
-      for item in response.datasets:
-        yield item
-    page += 1

clarifai_utils/listing/inputs.py

@@ -1,111 +0,0 @@
-from clarifai_grpc.grpc.api import resources_pb2, service_pb2
-from clarifai_grpc.grpc.api.status import status_code_pb2
-
-from clarifai.client import V2Stub
-
-
-def generator_setup(user_id, app_id):
-  userDataObject = resources_pb2.UserAppIDSet(user_id=user_id, app_id=app_id)
-
-  input_success_status = {
-      status_code_pb2.INPUT_DOWNLOAD_SUCCESS,
-      status_code_pb2.INPUT_DOWNLOAD_PENDING,
-      status_code_pb2.INPUT_DOWNLOAD_IN_PROGRESS,
-  }
-
-  response_success_status = {status_code_pb2.SUCCESS, status_code_pb2.MIXED_STATUS}
-
-  return (userDataObject, input_success_status, response_success_status)
-
-
-def inputs_generator(
-    stub: V2Stub,
-    user_id: str,
-    app_id: str,
-    page_size: int = 64,
-    only_successful_inputs: bool = True,
-):
-  """
-    Lists all the inputs in the given userAppID user_id, app_id app. This uses the more efficient
-    streaming input listing technique.
-
-    Args:
-      stub: grpc client stub.
-      user_id: the user to list from.
-      app_id: the app in the user_id account to list from.
-      page_size: the pagination size to use while iterating.
-      only_successful_inputs: only accept inputs with non-failure statuses.
-    """
-  userDataObject, input_success_status, response_success_status = generator_setup(
-      user_id=user_id, app_id=app_id)
-
-  # userDataObject = resources_pb2.UserAppIDSet(user_id=user_id, app_id=app_id)
-
-  # input_success_status = {
-  #     status_code_pb2.INPUT_DOWNLOAD_SUCCESS,
-  #     status_code_pb2.INPUT_DOWNLOAD_PENDING,
-  #     status_code_pb2.INPUT_DOWNLOAD_IN_PROGRESS,
-  # }
-
-  # response_success_status = {status_code_pb2.SUCCESS, status_code_pb2.MIXED_STATUS}
-
-  last_id = ""
-
-  while True:
-    response = stub.StreamInputs(
-        service_pb2.StreamInputsRequest(
-            user_app_id=userDataObject, per_page=page_size, last_id=last_id),)
-    if response.status.code not in response_success_status:
-      raise Exception("Stream inputs failed with response %r" % response)
-    if len(response.inputs) == 0:
-      break
-    else:
-      for inp in response.inputs:
-        last_id = inp.id
-        if only_successful_inputs:
-          if inp.status.code in input_success_status:
-            yield inp
-        else:
-          yield inp
-
-
-def dataset_inputs_generator(
-    stub: V2Stub,
-    user_id: str,
-    app_id: str,
-    page_size: int = 64,
-    only_successful_inputs: bool = True,
-    dataset_id: str = None,
-):
-  """
-    Lists all the inputs in the given userAppID user_id, app_id app. This uses the more efficient
-    streaming input listing technique.
-
-    Args:
-      stub: grpc client stub.
-      user_id: the user to list from.
-      app_id: the app in the user_id account to list from.
-      page_size: the pagination size to use while iterating.
-      only_successful_inputs: only accept inputs with non-failure statuses.
-    """
-  userDataObject, input_success_status, response_success_status = generator_setup(
-      user_id=user_id, app_id=app_id)
-  page = 1
-
-  while True:
-    response = stub.ListDatasetInputs(
-        service_pb2.ListDatasetInputsRequest(
-            user_app_id=userDataObject, per_page=page_size, page=page, dataset_id=dataset_id),)
-    if response.status.code not in response_success_status:
-      raise Exception("List Dataset inputs failed with response %r" % response)
-    if len(response.dataset_inputs) == 0:
-      break
-    else:
-      if only_successful_inputs:
-        for inp in response.dataset_inputs:
-          if inp.input.status.code in input_success_status:
-            yield inp.input
-      else:
-        for inp in response.dataset_inputs:
-          yield inp.input
-    page += 1

clarifai_utils/listing/installed_module_versions.py

@@ -1,40 +0,0 @@
-from clarifai_grpc.grpc.api import resources_pb2, service_pb2
-from clarifai_grpc.grpc.api.status import status_code_pb2
-
-from clarifai.client import V2Stub
-
-
-def installed_module_versions_generator(stub: V2Stub,
-                                        user_id: str,
-                                        app_id: str,
-                                        page_size: int = 64):
-  """
-  Lists all the installed module versions in the given userAppID user_id, app_id app.
-
-  Args:
-    stub: client stub.
-    user_id: the user to list from.
-    app_id: the app in the user_id account to list from.
-    page_size: the pagination size to use while iterating.
-
-  Returns:
-    imvs: a list of InstalledModuleVersion protos for all the installed modules versions in the app.
-  """
-  userDataObject = resources_pb2.UserAppIDSet(user_id=user_id, app_id=app_id)
-
-  imv_success_status = {status_code_pb2.SUCCESS}
-
-  page = 1
-  while True:
-    response = stub.ListInstalledModuleVersions(
-        service_pb2.ListInstalledModuleVersionsRequest(
-            user_app_id=userDataObject, page=page, per_page=page_size),)
-
-    if response.status.code not in imv_success_status:
-      raise Exception("ListInstalledModuleVersions failed with response %r" % response)
-    for item in response.installed_module_versions:
-      yield item
-    page += 1
-    # if we don't get a full page back we know we're done.
-    if len(response.installed_module_versions) < page_size:
-      break

clarifai_utils/listing/lister.py

@@ -1,200 +0,0 @@
-from clarifai_grpc.grpc.api import resources_pb2
-
-from clarifai.client import V2Stub
-from clarifai.listing.concepts import concepts_generator
-from clarifai.listing.datasets import datasets_generator
-from clarifai.listing.inputs import dataset_inputs_generator, inputs_generator
-from clarifai.listing.installed_module_versions import installed_module_versions_generator
-from clarifai.listing.models import models_generator
-from clarifai.listing.module_versions import module_versions_generator
-from clarifai.listing.modules import modules_generator
-
-
-class ClarifaiResourceLister(object):
-
-  def __init__(self, stub: V2Stub, user_id: str, app_id: str, page_size: int = 16):
-    """
-        Helper class for common listing of resources in an Clarifai App.
-
-        Args:
-          stub: grpc client V2Strub for our API.
-          user_id: the user to list from.
-          app_id: the app in the user_id account to list from.
-          page_size: the pagination size to use while iterating.
-        """
-    self.stub = stub
-    self.user_id = user_id
-    self.app_id = app_id
-    self.user_app_id_proto = resources_pb2.UserAppIDSet(user_id=user_id, app_id=app_id)
-    self.default_page_size = page_size
-
-  def list_all_models(self, page_size: int = None, only_in_app: bool = False):
-    """
-        This lists all the Models accessible in app. Not recommended for large apps.
-
-        Params:
-          page_size: how many elements per page to fetch
-          only_in_app: if only models created in the app should be returned
-
-        Returns:
-          inputs: a list of Model protos for all the inputs in the app.
-        """
-    return [item for item in self.models_generator(page_size, only_in_app)]
-
-  def models_generator(self, page_size: int = None, only_in_app: bool = False):
-    """
-        This lists all the models in an app. Not recommended for large apps.
-
-        Params:
-          page_size: how many elements per page to fetch
-          only_in_app: if only models created in the app should be returned
-
-        Returns:
-          gen: a generator that yields a single Model proto at a time.
-        """
-    page_size = self.default_page_size if page_size is None else page_size
-    return models_generator(self.stub, self.user_id, self.app_id, page_size, only_in_app)
-
-  def list_all_datasets(self, page_size: int = None):
-    """
-        This lists all Datasets in app. Not recommended for large apps.
-
-        Params:
-          page_size: how many elements per page to fetch
-
-        Returns:
-          inputs: a list of Dataset protos for all the inputs in the app.
-        """
-    return [item for item in self.datasets_generator(page_size)]
-
-  def datasets_generator(self, page_size: int = None):
-    """
-        This lists all the Datasets in an app. Not recommended for large apps.
-
-        Params:
-          page_size: how many elements per page to fetch
-
-        Returns:
-          gen: a generator that yields a single Dataset proto at a time.
-        """
-    page_size = self.default_page_size if page_size is None else page_size
-    return datasets_generator(self.stub, self.user_id, self.app_id, page_size)
-
-  def list_all_concepts(self, page_size: int = None):
-    """
-        This lists all the concepts in an app. Not recommended for large apps.
-
-        Returns:
-          concepts: a list of Concept protos for all the concepts in the app.
-        """
-    return [item for item in self.concepts_generator(page_size)]
-
-  def concepts_generator(self, page_size: int = None):
-    """
-        This lists all the concepts in an app. Not recommended for large apps.
-
-        Returns:
-          gen: a generator that yields a single Concept proto at a time.
-        """
-    page_size = self.default_page_size if page_size is None else page_size
-    return concepts_generator(self.stub, self.user_id, self.app_id, page_size)
-
-  def list_all_inputs(self, page_size: int = None):
-    """
-        This lists all the inputs in an app. Not recommended for large apps.
-
-        Returns:
-          inputs: a list of Input protos for all the inputs in the app.
-        """
-    return [item for item in self.inputs_generator(page_size)]
-
-  def inputs_generator(self, page_size: int = None):
-    """
-        This lists all the inputs in an app. Not recommended for large apps.
-
-        Returns:
-          gen: a generator that yields a single Input proto at a time.
-        """
-    page_size = self.default_page_size if page_size is None else page_size
-    return inputs_generator(self.stub, self.user_id, self.app_id, page_size)
-
-  def list_all_dataset_inputs(self, page_size: int = None, dataset_id: str = None):
-    """
-        This lists all the inputs in a dataset. Not recommended for large datasets.
-
-        Returns:
-          inputs: a list of Input protos for all the inputs in the app.
-        """
-    return [item for item in self.dataset_inputs_generator(page_size, dataset_id)]
-
-  def dataset_inputs_generator(self, page_size: int = None, dataset_id: str = None):
-    """
-        This lists all the inputs in a dataset. Not recommended for large datasets.
-
-        Returns:
-          gen: a generator that yields a single Input proto at a time.
-        """
-    page_size = self.default_page_size if page_size is None else page_size
-    return dataset_inputs_generator(
-        stub=self.stub,
-        user_id=self.user_id,
-        app_id=self.app_id,
-        page_size=page_size,
-        dataset_id=dataset_id)
-
-  def list_all_installed_module_versions(self, page_size: int = None):
-    """
-    This lists all the installed_module_versions in an app. Not recommended for large apps.
-
-    Returns:
-      installed_module_versions: a list of InstalledModuleVersion protos for all the installed_module_versions in the app.
-    """
-    return [item for item in self.installed_module_versions_generator(page_size)]
-
-  def installed_module_versions_generator(self, page_size: int = None):
-    """
-    This lists all the installed_module_versions in an app. Not recommended for large apps.
-
-    Returns:
-      gen: a generator that yields a single InstalledModuleVersion proto at a time.
-    """
-    page_size = self.default_page_size if page_size is None else page_size
-    return installed_module_versions_generator(self.stub, self.user_id, self.app_id, page_size)
-
-  def list_all_modules(self, page_size: int = None):
-    """
-    This lists all the modules in an app. Not recommended for large apps.
-
-    Returns:
-      modules: a list of Module protos for all the modules in the app.
-    """
-    return [item for item in self.module_generator(page_size)]
-
-  def module_generator(self, page_size: int = None):
-    """
-    This lists all the module in an app. Not recommended for large apps.
-
-    Returns:
-      gen: a generator that yields a single Module proto at a time.
-    """
-    page_size = self.default_page_size if page_size is None else page_size
-    return modules_generator(self.stub, self.user_id, self.app_id, page_size)
-
-  def list_all_module_versions(self, module_id: str, page_size: int = None):
-    """
-    This lists all the module_versions in an app. Not recommended for large apps.
-
-    Returns:
-      module_versions: a list of ModuleVersion protos for all the module_versions in the app.
-    """
-    return [item for item in self.module_versions_generator(module_id, page_size)]
-
-  def module_versions_generator(self, module_id: str, page_size: int = None):
-    """
-    This lists all the module_versions in an app. Not recommended for large apps.
-
-    Returns:
-      gen: a generator that yields a single ModuleVersion proto at a time.
-    """
-    page_size = self.default_page_size if page_size is None else page_size
-    return module_versions_generator(self.stub, self.user_id, self.app_id, module_id, page_size)

clarifai_utils/listing/models.py

@@ -1,46 +0,0 @@
-from clarifai_grpc.grpc.api import resources_pb2, service_pb2
-from clarifai_grpc.grpc.api.status import status_code_pb2
-
-from clarifai.client import V2Stub
-
-
-def models_generator(stub: V2Stub,
-                     user_id: str,
-                     app_id: str,
-                     page_size: int = 64,
-                     only_in_app: bool = False):
-  """
-    Lists all the models accessible in an application given a userAppID user_id, app_id app.
-
-    Args:
-      stub: client stub.
-      user_id: the user to list from.
-      app_id: the app in the user_id account to list from.
-      page_size: the pagination size to use while iterating.
-      only_in_app: if the models returned should only be ones that have been created in the app
-
-    Returns:
-      models: a list of Model protos for all the models in the app.
-    """
-  userDataObject = resources_pb2.UserAppIDSet(user_id=user_id, app_id=app_id)
-
-  model_success_status = {status_code_pb2.SUCCESS}
-
-  page = 1
-  while True:
-    response = stub.ListModels(
-        service_pb2.ListModelsRequest(user_app_id=userDataObject, page=page, per_page=page_size),)
-
-    if response.status.code not in model_success_status:
-      raise Exception("ListModels failed with response %r" % response)
-    if len(response.models) == 0:
-      break
-    else:
-      if only_in_app:
-        for item in response.models:
-          if item.app_id == app_id:
-            yield item
-      else:
-        for item in response.models:
-          yield item
-    page += 1