clarifai 9.10.2__py3-none-any.whl → 9.10.3__py3-none-any.whl

clarifai_utils/datasets/export/inputs_annotations.py

@@ -1,222 +0,0 @@
-import json
-import os
-import tempfile
-import zipfile
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from io import BytesIO
-from typing import Any, Dict, Iterator, List, Optional
-
-import requests
-from clarifai_grpc.grpc.api import resources_pb2
-from google.protobuf.json_format import MessageToDict
-from PIL import ImageFile
-from tqdm import tqdm
-
-from clarifai.errors import UserError
-from clarifai.utils.logging import get_logger
-
-logger = get_logger("INFO", __name__)
-
-
-class DatasetExportReader:
-
-  def __init__(self,
-               session: requests.Session,
-               archive_url: Optional[str] = None,
-               local_archive_path: Optional[str] = None):
-    """Download/Reads the zipfile archive and yields every api.Input object.
-
-    Args:
-        session: requests.Session object
-        archive_url: URL of the DatasetVersionExport archive
-        local_archive_path: Path to the DatasetVersionExport archive
-    """
-    self.input_count = 0
-    self.temp_file = None
-    self.session = session
-
-    assert archive_url or local_archive_path, UserError(
-        "Either archive_url or local_archive_path must be provided.")
-    assert not (archive_url and local_archive_path), UserError(
-        "Only one of archive_url or local_archive_path must be provided.")
-
-    if archive_url:
-      logger.info('url: %s' % archive_url)
-      self.temp_file = self._download_temp_archive(archive_url)
-      self.archive = zipfile.ZipFile(self.temp_file)
-    else:
-      logger.info("path: %s" % local_archive_path)
-      self.archive = zipfile.ZipFile(local_archive_path)
-
-    self.file_name_list = self.archive.namelist()
-    assert "mimetype" in self.file_name_list, "Missing mimetype file in the dataset export archive."
-    assert self.archive.read("mimetype") == b"application/x.clarifai-data+protobuf"
-    self.file_name_list.remove("mimetype")
-
-    logger.info("Obtained file name list. %d entries." % len(self.file_name_list))
-    self.split_dir = os.path.dirname(self.file_name_list[0]) if len(self.file_name_list) else ""
-
-  def _download_temp_archive(self, archive_url: str,
-                             chunk_size: int = 128) -> tempfile.TemporaryFile:
-    """Downloads the temp archive of InputBatches."""
-    r = self.session.get(archive_url, stream=True)
-    temp_file = tempfile.TemporaryFile()
-    for chunk in r.iter_content(chunk_size=chunk_size):
-      temp_file.write(chunk)
-
-    return temp_file
-
-  def __len__(self) -> int:
-    if not self.input_count:
-      if self.file_name_list is not None:
-        for filename in self.file_name_list:
-          self.input_count += int(filename.split('_n')[-1])
-
-    return self.input_count
-
-  def __iter__(self) -> Iterator[resources_pb2.Input]:
-    """Loops through all InputBatches in the DatasetVersionExport and yields every api.Input object"""
-    if self.file_name_list is not None:
-      for filename in self.file_name_list:
-        db = resources_pb2.InputBatch().FromString(self.archive.read(filename))
-        for db_input in db.inputs:
-          yield db_input
-
-  def __enter__(self) -> 'DatasetExportReader':
-    return self
-
-  def __exit__(self, *args: Any) -> None:
-    self.close()
-
-  def close(self) -> None:
-    logger.info("closing file objects.")
-    self.archive.close()
-    if self.temp_file:
-      self.temp_file.close()
-
-
-class InputAnnotationDownloader:
-
-  def __init__(self,
-               session: requests.Session,
-               input_iterator: DatasetExportReader,
-               num_workers: int = 4):
-    """Downloads the archive from the URL into an archive of inputs, annotations in the directory format
-    {split}/inputs and {split}/annotations.
-
-    Args:
-        session: requests.Session object
-        input_iterator: Iterable of DatasetExportReader object
-        num_workers: Number of threads to use for downloading
-    """
-    self.input_iterator = input_iterator
-    self.num_workers = min(num_workers, 10)  # Max 10 threads
-    self.num_inputs_annotations = 0
-    self.split_prefix = None
-    self.session = session
-    self.input_ext = dict(image=".png", text=".txt", audio=".mp3", video=".mp4")
-    if isinstance(self.input_iterator, DatasetExportReader):
-      self.split_prefix = self.input_iterator.split_dir
-
-  def _save_image_to_archive(self, new_archive: zipfile.ZipFile, hosted_url: str,
-                             file_name: str) -> None:
-    """Use PIL ImageFile to return image parsed from the response bytestring (from requests) and append to zip file."""
-    p = ImageFile.Parser()
-    p.feed(self.session.get(hosted_url).content)
-    image = p.close()
-    image_file = BytesIO()
-    image.save(image_file, 'PNG')
-    new_archive.writestr(file_name, image_file.getvalue())
-
-  def _save_text_to_archive(self, new_archive: zipfile.ZipFile, hosted_url: str,
-                            file_name: str) -> None:
-    """Gets the text response bytestring (from requests) and append to zip file."""
-    text_content = self.session.get(hosted_url).content
-    new_archive.writestr(file_name, text_content)
-
-  def _save_audio_to_archive(self, new_archive: zipfile.ZipFile, hosted_url: str,
-                             file_name: str) -> None:
-    """Gets the audio response bytestring (from requests) as chunks and append to zip file."""
-    audio_response = self.session.get(hosted_url, stream=True)
-    audio_stream = BytesIO()
-    # Retrieve the audio content in chunks and write to the BytesIO object
-    for chunk in audio_response.iter_content(chunk_size=128):
-      audio_stream.write(chunk)
-    new_archive.writestr(file_name, audio_stream.getvalue())
-
-  def _save_video_to_archive(self, new_archive: zipfile.ZipFile, hosted_url: str,
-                             file_name: str) -> None:
-    """Gets the video response bytestring (from requests) as chunks and append to zip file."""
-    video_response = self.session.get(hosted_url)
-    video_stream = BytesIO()
-    # Retrieve the video content in chunks and write to the BytesIO object
-    for chunk in video_response.iter_content(chunk_size=128):
-      video_stream.write(chunk)
-    new_archive.writestr(file_name, video_stream.getvalue())
-
-  def _save_annotation_to_archive(self, new_archive: zipfile.ZipFile, annot_data: List[Dict],
-                                  file_name: str) -> None:
-    """Gets the annotation response bytestring (from requests) and append to zip file."""
-    # Serialize the dictionary to a JSON string
-    json_str = json.dumps(annot_data)
-    # Convert the JSON string to bytes
-    bytes_object = json_str.encode()
-
-    new_archive.writestr(file_name, bytes_object)
-
-  def _write_archive(self, input_, new_archive, split: Optional[str] = None) -> None:
-    """Writes the input, annotation archive into prefix dir."""
-    data_dict = MessageToDict(input_.data)
-    input_type = list(filter(lambda x: x in list(data_dict.keys()),
-                             list(self.input_ext.keys())))[0]
-    hosted = getattr(input_.data, input_type).hosted
-    if hosted.prefix:
-      assert 'orig' in hosted.sizes
-      hosted_url = f"{hosted.prefix}/orig/{hosted.suffix}"
-      file_name = os.path.join(split, "inputs", input_.id + self.input_ext[input_type])
-      if input_type == "image":
-        self._save_image_to_archive(new_archive, hosted_url, file_name)
-      elif input_type == "text":
-        self._save_text_to_archive(new_archive, hosted_url, file_name)
-      elif input_type == "audio":
-        self._save_audio_to_archive(new_archive, hosted_url, file_name)
-      elif input_type == "video":
-        self._save_video_to_archive(new_archive, hosted_url, file_name)
-      self.num_inputs_annotations += 1
-
-    if data_dict.get("concepts") or data_dict.get("regions"):
-      file_name = os.path.join(split, "annotations", input_.id + ".json")
-      annot_data = data_dict.get("concepts") or data_dict.get("regions")
-
-      self._save_annotation_to_archive(new_archive, annot_data, file_name)
-      self.num_inputs_annotations += 1
-
-  def _check_output_archive(self, save_path: str) -> None:
-    try:
-      archive = zipfile.ZipFile(save_path, 'r')
-    except zipfile.BadZipFile as e:
-      raise e
-    assert len(
-        archive.namelist()
-    ) == self.num_inputs_annotations, "Archive has %d inputs+annotations | expecting %d inputs+annotations" % (
-        len(archive.namelist()), self.num_inputs_annotations)
-
-  def download_archive(self, save_path: str, split: Optional[str] = None) -> None:
-    """Downloads the archive from the URL into an archive of inputs, annotations in the directory format
-    {split}/inputs and {split}/annotations.
-    """
-    with zipfile.ZipFile(save_path, "a") as new_archive:
-      with ThreadPoolExecutor(max_workers=self.num_workers) as executor:
-        with tqdm(total=len(self.input_iterator), desc='Downloading Dataset') as progress:
-          # Submit all jobs to the executor and store the returned futures
-          futures = [
-              executor.submit(self._write_archive, input_, new_archive, split)
-              for input_ in self.input_iterator
-          ]
-
-          for _ in as_completed(futures):
-            progress.update()
-
-    self._check_output_archive(save_path)
-    logger.info("Downloaded %d inputs+annotations to %s" % (self.num_inputs_annotations,
-                                                            save_path))

clarifai_utils/datasets/upload/base.py

@@ -1,66 +0,0 @@
-from collections import defaultdict
-from typing import Iterator, List, Tuple, TypeVar, Union
-
-from clarifai_grpc.grpc.api import resources_pb2
-
-from clarifai.client.input import Inputs
-from clarifai.datasets.upload.features import (TextFeatures, VisualClassificationFeatures,
-                                               VisualDetectionFeatures, VisualSegmentationFeatures)
-
-OutputFeaturesType = TypeVar(
-    'OutputFeaturesType',
-    bound=Union[TextFeatures, VisualClassificationFeatures, VisualDetectionFeatures,
-                VisualSegmentationFeatures])
-
-
-class ClarifaiDataset:
-  """Clarifai datasets base class."""
-
-  def __init__(self, datagen_object: Iterator, dataset_id: str, split: str) -> None:
-    self.datagen_object = datagen_object
-    self.dataset_id = dataset_id
-    self.split = split
-    self.all_input_ids = {}
-    self._all_input_protos = {}
-    self._all_annotation_protos = defaultdict(list)
-    self.input_object = Inputs()
-
-  def __len__(self) -> int:
-    """Get size of all input protos"""
-    return len(self.datagen_object)
-
-  def _to_list(self, input_protos: Iterator) -> List:
-    """Parse protos iterator to list."""
-    return list(input_protos)
-
-  def _extract_protos(self) -> None:
-    """Create input image protos for each data generator item."""
-    raise NotImplementedError()
-
-  def get_protos(self, input_ids: List[int]
-                ) -> Tuple[List[resources_pb2.Input], List[resources_pb2.Annotation]]:
-    """Get input and annotation protos based on input_ids.
-    Args:
-      input_ids: List of input IDs to retrieve the protos for.
-    Returns:
-      Input and Annotation proto iterators for the specified input IDs.
-    """
-    input_protos, annotation_protos = self._extract_protos(input_ids)
-
-    return input_protos, annotation_protos
-
-
-class ClarifaiDataLoader:
-  """Clarifai data loader base class."""
-
-  def __init__(self, split: str) -> None:
-    pass
-
-  def load_data(self) -> None:
-    raise NotImplementedError()
-
-  def __len__(self) -> int:
-    raise NotImplementedError()
-
-  def __getitem__(self, index: int) -> OutputFeaturesType:
-    raise NotImplementedError()

clarifai_utils/datasets/upload/examples/README.md

@@ -1,31 +0,0 @@
-## Dataset upload from local directory
-
-Examples of how to upload your local directory datasets into clarifai app using `module_dir` feature from `Dataset`.
-
-**Note:**
-**Note:**
-
-- Ensure that the `CLARIFAI_PAT` environment variable is set.
-- Ensure that the appropriate base workflow is being set for indexing respective input type.
-
-
-## Image Classification - Cifar10
-```python
-from clarifai.client.dataset import Dataset
-dataset = Dataset(user_id="user_id", app_id="app_id", dataset_id="dataset_id")
-dataset.upload_dataset(task="visual_classification", split="train", module_dir="path_to_cifar10_module")
-```
-
-## Image Classification - [Food-101](https://data.vision.ee.ethz.ch/cvl/datasets_extra/food-101/)
-```python
-from clarifai.client.dataset import Dataset
-dataset = Dataset(user_id="user_id", app_id="app_id", dataset_id="dataset_id")
-dataset.upload_dataset(task="visual_classification", split="train", module_dir="path_to_food-101_module")
-```
-
-## Text Classification - IMDB Reviews
-```python
-from clarifai.client.dataset import Dataset
-dataset = Dataset(user_id="user_id", app_id="app_id", dataset_id="dataset_id")
-dataset.upload_dataset(task="text_clf", split="train", module_dir="path_to_imdb_reviews_module")
-```

clarifai_utils/datasets/upload/examples/image_classification/cifar10/cifar_small_test.csv

@@ -1,10 +0,0 @@
-image_path,label
-images/test_batch_700.jpg,horse
-images/test_batch_701.jpg,bird
-images/test_batch_702.jpg,deer
-images/test_batch_703.jpg,ship
-images/test_batch_704.jpg,horse
-images/test_batch_705.jpg,deer
-images/test_batch_706.jpg,bird
-images/test_batch_707.jpg,truck
-images/test_batch_709.jpg,ship

clarifai_utils/datasets/upload/examples/image_classification/cifar10/cifar_small_train.csv

@@ -1,10 +0,0 @@
-image_path,label
-images/test_batch_700.jpg,horse
-images/test_batch_701.jpg,bird
-images/test_batch_702.jpg,deer
-images/test_batch_703.jpg,ship
-images/test_batch_704.jpg,horse
-images/test_batch_705.jpg,deer
-images/test_batch_706.jpg,bird
-images/test_batch_707.jpg,truck
-images/test_batch_709.jpg,ship

clarifai_utils/datasets/upload/examples/image_classification/cifar10/dataset.py

@@ -1,46 +0,0 @@
-#! Cifar10 Dataset
-
-import csv
-import os
-
-from clarifai.datasets.upload.base import ClarifaiDataLoader
-from clarifai.datasets.upload.features import VisualClassificationFeatures
-
-
-class Cifar10DataLoader(ClarifaiDataLoader):
-  """Cifar10 Dataset."""
-
-  def __init__(self, split: str = "train"):
-    """Initialize dataset params.
-    Args:
-      split: "train" or "test"
-    """
-    self.split = split
-    self.data_dirs = {
-        "train": os.path.join(os.path.dirname(__file__), "cifar_small_train.csv"),
-        "test": os.path.join(os.path.dirname(__file__), "cifar_small_test.csv")
-    }
-    self.data = self.load_data()
-
-  def load_data(self):
-    data = []
-    with open(self.data_dirs[self.split]) as _file:
-      reader = csv.reader(_file)
-      next(reader, None)  # skip header
-      for review in reader:
-        data.append((review[0], review[1]))
-    return data
-
-  def __getitem__(self, index):
-    item = self.data[index]
-    return VisualClassificationFeatures(
-        image_path=os.path.join(os.path.dirname(__file__), item[0]),
-        label=item[1],
-        id=os.path.basename(item[0]).split(".")[0],
-        metadata={
-            "split": self.split,
-            "image_path": item[0]
-        })
-
-  def __len__(self):
-    return len(self.data)

clarifai_utils/datasets/upload/examples/image_classification/food-101/dataset.py

@@ -1,39 +0,0 @@
-import os
-
-from clarifai.datasets.upload.base import ClarifaiDataLoader
-from clarifai.datasets.upload.features import VisualClassificationFeatures
-
-
-class Food101DataLoader(ClarifaiDataLoader):
-  """Food-101 Image Classification Dataset."""
-
-  def __init__(self, split: str = "train"):
-    """Initialize dataset params.
-    Args:
-      split: "train" or "test"
-    """
-    self.split = split
-    self.image_dir = {"train": os.path.join(os.path.dirname(__file__), "images")}
-    self.load_data()
-
-  def load_data(self):
-    """Load data for the food-101 dataset."""
-    self.data = []
-    class_names = os.listdir(self.image_dir[self.split])
-    for class_name in class_names:
-      for image in os.listdir(os.path.join(self.image_dir[self.split], class_name)):
-        image_path = os.path.join(self.image_dir[self.split], class_name, image)
-        self.data.append({
-            "image_path": image_path,
-            "class_name": class_name,
-        })
-
-  def __getitem__(self, idx):
-    data_item = self.data[idx]
-    image_path = data_item["image_path"]
-    class_name = data_item["class_name"]
-    return VisualClassificationFeatures(
-        image_path=image_path, label=class_name, id=os.path.basename(image_path).split(".")[0])
-
-  def __len__(self):
-    return len(self.data)

clarifai_utils/datasets/upload/examples/text_classification/imdb_dataset/dataset.py

@@ -1,42 +0,0 @@
-import csv
-import os
-
-from clarifai.datasets.upload.base import ClarifaiDataLoader
-from clarifai.datasets.upload.features import TextFeatures
-
-
-class IMDBMovieReviewsDataLoader(ClarifaiDataLoader):
-  """IMDB 50K Movie Reviews Dataset."""
-
-  def __init__(self, split: str = "train"):
-    """Initialize dataset params.
-    Args:
-        split: "train" or "test"
-    """
-    self.split = split
-    self.data_dirs = {
-        "train": os.path.join(os.path.dirname(__file__), "train.csv"),
-        "test": os.path.join(os.path.dirname(__file__), "test.csv")
-    }
-    self.data = []
-
-    self.load_data()
-
-  def load_data(self):
-    with open(self.data_dirs[self.split]) as _file:
-      reader = csv.reader(_file)
-      next(reader, None)  # skip header
-      for review in reader:
-        self.data.append({
-            "text": review[0],
-            "labels": review[1],
-            "id": None,
-            "metadata": dict(split=self.split)
-        })
-
-  def __getitem__(self, idx):
-    item = self.data[idx]
-    return TextFeatures(text=item["text"], labels=item["labels"], id=item["id"])
-
-  def __len__(self):
-    return len(self.data)