clarifai 9.7.0__py3-none-any.whl → 9.7.2__py3-none-any.whl

clarifai_utils/data_upload/convert_csv.py

@@ -1,182 +0,0 @@
-import csv
-import os
-import sys
-from itertools import chain
-
-import pandas as pd
-from sklearn.model_selection import train_test_split
-
-
-class DataPreprocessor:
-
-  def __init__(self, filename, multi_val, separator):
-    self.filename = filename
-    self.multi_val = multi_val
-    self.separator = separator
-    self.df = pd.read_csv(filename)
-
-  def process_data(self):
-    text_col = self.get_column("Enter the number of the column that contains the text: ")
-    label_col = self.get_column(
-        "Enter the number of the column that contains the labels: ", exclude=text_col)
-
-    self.split_values(label_col)
-    self.df[label_col] = self.df[label_col].apply(lambda x: [x] if isinstance(x, str) else x)
-
-    # Use chain.from_iterable to expand the multi-values if applicable
-    unique_labels = list(set(chain.from_iterable(self.df[label_col].values)))
-
-    print(
-        "\nThe following unqiue labels have been found in the '{}' column and will be used in the dataset:".
-        format(label_col))
-    for i, label in enumerate(unique_labels, start=1):
-      print('{}) {}'.format(i, label))
-
-    self.convert_for_classification(text_col, label_col, unique_labels)
-
-  def get_column(self, prompt, exclude=None):
-    available_columns = self.df.columns.drop(exclude) if exclude else self.df.columns
-    if len(available_columns) == 1:
-      print(f'\nThe column named \'{available_columns[0]}\' will be used as the labels column.')
-      return available_columns[0]
-    else:
-      for i, col in enumerate(available_columns):
-        print(f'{i+1}) {col}')
-      col_index = int(input(prompt)) - 1
-      return available_columns[col_index]
-
-  def split_values(self, label_col):
-    if self.multi_val.lower() == 'y':
-      self.df[label_col] = self.df[label_col].apply(
-          lambda x: str(x).split(self.separator) if not isinstance(x, float) else x)
-
-  def convert_for_classification(self, text_col, label_col, unique_labels):
-    # Binary classification
-    if len(unique_labels) == 2:
-      print("Converting the CSV to be used with binary classification")
-      self.df['input.data.text.raw'] = self.df[text_col]
-      self.df['input.data.concepts[0].id'] = label_col
-      self.df['input.data.concepts[0].value'] = self.df[label_col].apply(
-          lambda x: 1 if unique_labels[0] in x else 0)
-      self.df = self.df[[
-          'input.data.text.raw', 'input.data.concepts[0].id', 'input.data.concepts[0].value'
-      ]]
-
-    # Multi-class classification
-    else:
-      print("Converting the CSV to be used with multi-class classification")
-      self.df['input.data.text.raw'] = self.df[text_col].apply(
-          lambda x: x[0] if isinstance(x, list) else x)
-      for i in range(len(unique_labels)):
-        self.df[f'input.data.concepts[{i}].id'] = self.df[label_col].apply(
-            lambda x: unique_labels[i] if unique_labels[i] in x else '')
-        self.df[f'input.data.concepts[{i}].value'] = self.df[label_col].apply(
-            lambda x: 1 if unique_labels[i] in x else '')
-
-      self.df = self.df[['input.data.text.raw'] +
-                        [f'input.data.concepts[{i}].id' for i in range(len(unique_labels))] +
-                        [f'input.data.concepts[{i}].value' for i in range(len(unique_labels))]]
-
-      # Reorder the columns
-      cols = self.df.columns.tolist()
-      new_cols = cols[:1]  # The first column 'input.data.text.raw'
-      pairs = [[cols[i], cols[i + len(unique_labels)]] for i in range(1, len(unique_labels) + 1)]
-      for pair in pairs:
-        new_cols.extend(pair)
-      self.df = self.df[new_cols]
-
-      # Remove special characters from column names
-      self.df.columns = self.df.columns.str.replace("^[\[]|[\]]$", "", regex=True)
-
-
-class DatasetSplitter:
-
-  def __init__(self, df, split_dataset, shuffle_dataset, seed=555):
-    self.df = df
-    self.split_dataset = split_dataset
-    self.shuffle_dataset = shuffle_dataset
-    self.seed = seed if seed != '' else 555
-
-  def split_and_save(self, filename_base):
-    if self.split_dataset.lower() == 'y':
-      split_type = self.get_split_type()
-
-      if split_type == 1:
-        train_pct = self.get_percentage(
-            'What percentage of the dataset should be used for training? Enter a number between 1 and 99: ',
-            99)
-        test_pct = 100 - train_pct
-        print(f'Data will be split {train_pct}% train, {test_pct}% test')  # Added print statement
-      elif split_type == 2:
-        train_pct = self.get_percentage(
-            'What percentage of the dataset should be used for training? Enter a number between 1 and 98: ',
-            98)
-        max_val_pct = 99 - train_pct  # Max percentage for validation is now reduced by 1
-        val_pct = self.get_percentage(
-            f'What percentage of the dataset should be used for validation? Enter a number between 1 and {max_val_pct}: ',
-            max_val_pct)
-        test_pct = 100 - train_pct - val_pct
-        print(f'Data will be split {train_pct}% train, {val_pct}% validation, {test_pct}% test'
-             )  # Added print statement
-
-      train_df, test_df = train_test_split(
-          self.df,
-          test_size=test_pct / 100,
-          random_state=self.seed,
-          shuffle=self.shuffle_dataset.lower() == 'y')
-      train_df.to_csv(filename_base + '-train.csv', index=False, quoting=csv.QUOTE_MINIMAL)
-      test_df.to_csv(filename_base + '-test.csv', index=False, quoting=csv.QUOTE_MINIMAL)
-
-      if split_type == 2:
-        train_df, val_df = train_test_split(
-            train_df,
-            test_size=val_pct / 100,
-            random_state=self.seed,
-            shuffle=self.shuffle_dataset.lower() == 'y')
-        val_df.to_csv(filename_base + '-validation.csv', index=False, quoting=csv.QUOTE_MINIMAL)
-    else:
-      self.df.to_csv(filename_base + '.csv', index=False, quoting=csv.QUOTE_MINIMAL)
-
-  def get_split_type(self):
-    split_type = int(
-        input(
-            'How would you like to split the dataset?\n1) Train and test datasets\n2) Train, validate, and test datasets\n'
-        ))
-    while split_type not in [1, 2]:
-      split_type = int(
-          input(
-              'Invalid option. Enter 1 for "Train and test" or 2 for "Train, validate, and test": '
-          ))
-    return split_type
-
-  def get_percentage(self, prompt, max_pct):
-    pct = int(input(prompt))
-    while not 1 <= pct <= max_pct:
-      pct = int(input(f'Invalid input. Please enter a number between 1 and {max_pct}: '))
-    return pct
-
-
-def main():
-  filename = sys.argv[1]
-  multi_val = input('Do any columns have multiple values? (y/[n]) ')
-  separator = input('Enter the separator: ') if multi_val.lower() == 'y' else None
-
-  preprocessor = DataPreprocessor(filename, multi_val, separator)
-  preprocessor.process_data()
-
-  split_dataset = input('Would you like to split this dataset? (y/[n]) ')
-  shuffle_dataset = 'n'
-  seed = '555'
-
-  if split_dataset.lower() == 'y':
-    shuffle_dataset = input('Would you like to shuffle the dataset before splitting? (y/[n]) ')
-    if shuffle_dataset.lower() == 'y':
-      seed = input('Enter a seed integer or hit enter to use the default [555]: ')
-
-  splitter = DatasetSplitter(preprocessor.df, split_dataset, shuffle_dataset, seed)
-  splitter.split_and_save(os.path.splitext(filename)[0] + '-clarifai')
-  print("Done!")
-
-
-if __name__ == "__main__":
-  main()

clarifai_utils/data_upload/datasets/base.py

@@ -1,87 +0,0 @@
-from collections import defaultdict
-from typing import Iterator, List, Tuple
-
-from clarifai_grpc.grpc.api import resources_pb2
-from google.protobuf.struct_pb2 import Struct
-
-
-class ClarifaiDataset:
-  """
-  Clarifai datasets base class.
-  """
-
-  def __init__(self, datagen_object: Iterator, dataset_id: str, split: str) -> None:
-    self.datagen_object = datagen_object
-    self.dataset_id = dataset_id
-    self.split = split
-    self.input_ids = []
-    self._all_input_protos = {}
-    self._all_annotation_protos = defaultdict(list)
-
-  def __len__(self) -> int:
-    """
-    Get size of all input protos
-    """
-    return len(self._all_input_protos)
-
-  def _to_list(self, input_protos: Iterator) -> List:
-    """
-    Parse protos iterator to list.
-    """
-    return list(input_protos)
-
-  def create_input_protos(self, image_path: str, label: str, input_id: str, dataset_id: str,
-                          metadata: Struct) -> resources_pb2.Input:
-    """
-    Create input protos for each image, label input pair.
-    Args:
-    	`image_path`: path to image.
-    	`label`: image label
-    	`input_id: unique input id
-    	`dataset_id`: Clarifai dataset id
-    	`metadata`: input metadata
-    Returns:
-    	An input proto representing a single row input
-    """
-    raise NotImplementedError()
-
-  def _extract_protos(self) -> None:
-    """
-    Create input image protos for each data generator item.
-    """
-    raise NotImplementedError()
-
-  def get_protos(self, input_ids: List[str]
-                ) -> Tuple[List[resources_pb2.Input], List[resources_pb2.Annotation]]:
-    """
-    Get input and annotation protos based on input_ids.
-    Args:
-      `input_ids`: List of input IDs to retrieve the protos for.
-    Returns:
-      Input and Annotation proto iterators for the specified input IDs.
-    """
-    input_protos = [self._all_input_protos.get(input_id) for input_id in input_ids]
-    annotation_protos = []
-    if len(self._all_annotation_protos) > 0:
-      annotation_protos = [self._annotation_protos.get(input_id) for input_id in input_ids]
-      annotation_protos = [
-          ann_proto for ann_protos in annotation_protos for ann_proto in ann_protos
-      ]
-
-    return input_protos, annotation_protos
-
-
-class Chunker:
-  """
-  Split an input sequence into small chunks.
-  """
-
-  def __init__(self, seq: List, size: int) -> None:
-    self.seq = seq
-    self.size = size
-
-  def chunk(self) -> List[List]:
-    """
-    Chunk input sequence.
-    """
-    return [self.seq[pos:pos + self.size] for pos in range(0, len(self.seq), self.size)]

clarifai_utils/data_upload/datasets/image.py

@@ -1,253 +0,0 @@
-import os
-from typing import Iterator, List, Union
-
-from clarifai_grpc.grpc.api import resources_pb2
-from google.protobuf.struct_pb2 import Struct
-from tqdm import tqdm
-
-from .base import ClarifaiDataset
-
-
-class VisualClassificationDataset(ClarifaiDataset):
-
-  def __init__(self, datagen_object: Iterator, dataset_id: str, split: str) -> None:
-    super().__init__(datagen_object, dataset_id, split)
-    self._extract_protos()
-
-  def create_input_protos(self, image_path: str, labels: List[Union[str, int]], input_id: str,
-                          dataset_id: str, geo_info: Union[List[float], None],
-                          metadata: Struct) -> resources_pb2.Input:
-    """
-    Create input protos for each image, label input pair.
-    Args:
-      `image_path`: image path.
-      `labels`: image label(s)
-      `input_id: unique input id
-      `dataset_id`: Clarifai dataset id
-      `geo_info`: image longitude, latitude info
-      `metadata`: image metadata
-    Returns:
-      An input proto representing a single row input
-    """
-    geo_pb = resources_pb2.Geo(geo_point=resources_pb2.GeoPoint(
-        longitude=geo_info[0], latitude=geo_info[1])) if geo_info is not None else None
-
-    input_proto = resources_pb2.Input(
-        id=input_id,
-        dataset_ids=[dataset_id],
-        data=resources_pb2.Data(
-            image=resources_pb2.Image(base64=open(image_path, 'rb').read(),),
-            geo=geo_pb,
-            concepts=[
-                resources_pb2.Concept(
-                  id=f"id-{''.join(_label.split(' '))}", name=_label, value=1.)\
-                for _label in labels
-            ],
-            metadata=metadata))
-
-    return input_proto
-
-  def _extract_protos(self) -> None:
-    """
-    Create input image protos for each data generator item.
-    """
-    for i, item in tqdm(enumerate(self.datagen_object), desc="Creating input protos..."):
-      metadata = Struct()
-      image_path = item.image_path
-      label = item.label if isinstance(item.label, list) else [item.label]  # clarifai concept
-      input_id = f"{self.dataset_id}-{self.split}-{i}" if item.id is None else f"{self.split}-{str(item.id)}"
-      geo_info = item.geo_info
-      metadata.update({"filename": os.path.basename(image_path), "split": self.split})
-
-      self.input_ids.append(input_id)
-      input_proto = self.create_input_protos(image_path, label, input_id, self.dataset_id,
-                                             geo_info, metadata)
-      self._all_input_protos[input_id] = input_proto
-
-
-class VisualDetectionDataset(ClarifaiDataset):
-  """
-  Visual detection dataset proto class.
-  """
-
-  def __init__(self, datagen_object: Iterator, dataset_id: str, split: str) -> None:
-    super().__init__(datagen_object, dataset_id, split)
-    self._extract_protos()
-
-  def create_input_protos(self, image_path: str, input_id: str, dataset_id: str,
-                          geo_info: Union[List[float], None],
-                          metadata: Struct) -> resources_pb2.Input:
-    """
-    Create input protos for each image, label input pair.
-    Args:
-      `image_path`: file path to image
-      `input_id: unique input id
-      `dataset_id`: Clarifai dataset id
-      `geo_info`: image longitude, latitude info
-      `metadata`: image metadata
-    Returns:
-      An input proto representing a single row input
-    """
-    geo_pb = resources_pb2.Geo(geo_point=resources_pb2.GeoPoint(
-        longitude=geo_info[0], latitude=geo_info[1])) if geo_info is not None else None
-    input_image_proto = resources_pb2.Input(
-        id=input_id,
-        dataset_ids=[dataset_id],
-        data=resources_pb2.Data(
-            image=resources_pb2.Image(base64=open(image_path, 'rb').read(),),
-            geo=geo_pb,
-            metadata=metadata))
-
-    return input_image_proto
-
-  def create_annotation_proto(self, label: str, annotations: List, input_id: str,
-                              dataset_id: str) -> resources_pb2.Annotation:
-    """
-    Create an input proto for each bounding box, label input pair.
-    Args:
-      `label`: annotation label
-      `annotations`: a list of a single bbox's coordinates.
-      `input_id: unique input id
-      `dataset_id`: Clarifai dataset id
-    Returns:
-      An input proto representing a single image input
-    """
-    input_annot_proto = resources_pb2.Annotation(
-        input_id=input_id,
-        data=resources_pb2.Data(regions=[
-            resources_pb2.Region(
-                region_info=resources_pb2.RegionInfo(bounding_box=resources_pb2.BoundingBox(
-                    # Annotations ordering: [xmin, ymin, xmax, ymax]
-                    # top_row must be less than bottom row
-                    # left_col must be less than right col
-                    top_row=annotations[1],  #y_min
-                    left_col=annotations[0],  #x_min
-                    bottom_row=annotations[3],  #y_max
-                    right_col=annotations[2]  #x_max
-                )),
-                data=resources_pb2.Data(concepts=[
-                    resources_pb2.Concept(
-                        id=f"id-{''.join(label.split(' '))}", name=label, value=1.)
-                ]))
-        ]))
-
-    return input_annot_proto
-
-  def _extract_protos(self) -> None:
-    """
-    Create input image protos for each data generator item.
-    """
-    for i, item in tqdm(enumerate(self.datagen_object), desc="Creating input protos..."):
-      metadata = Struct()
-      image = item.image_path
-      labels = item.classes  # list:[l1,...,ln]
-      bboxes = item.bboxes  # [[xmin,ymin,xmax,ymax],...,[xmin,ymin,xmax,ymax]]
-      input_id = f"{self.dataset_id}-{self.split}-{i}" if item.id is None else f"{self.split}-{str(item.id)}"
-      metadata.update({"filename": os.path.basename(image), "split": self.split})
-      geo_info = item.geo_info
-
-      self.input_ids.append(input_id)
-      input_image_proto = self.create_input_protos(image, input_id, self.dataset_id, geo_info,
-                                                   metadata)
-      self._all_input_protos[input_id] = input_image_proto
-
-      # iter over bboxes and classes
-      # one id could have more than one bbox and label
-      for i in range(len(bboxes)):
-        input_annot_proto = self.create_annotation_proto(labels[i], bboxes[i], input_id,
-                                                         self.dataset_id)
-        self._all_annotation_protos[input_id].append(input_annot_proto)
-
-
-class VisualSegmentationDataset(ClarifaiDataset):
-  """
-  Visual segmentation dataset proto class.
-  """
-
-  def __init__(self, datagen_object: Iterator, dataset_id: str, split: str) -> None:
-    super().__init__(datagen_object, dataset_id, split)
-    self._extract_protos()
-
-  def create_input_protos(self, image_path: str, input_id: str, dataset_id: str,
-                          geo_info: Union[List[float], None],
-                          metadata: Struct) -> resources_pb2.Input:
-    """
-    Create input protos for each image, label input pair.
-    Args:
-      `image_path`: absolute image file path
-      `input_id: unique input id
-      `dataset_id`: Clarifai dataset id
-      `geo_info`: image longitude, latitude info
-      `metadata`: image metadata
-    Returns:
-      An input proto representing a single input item
-    """
-    geo_pb = resources_pb2.Geo(geo_point=resources_pb2.GeoPoint(
-        longitude=geo_info[0], latitude=geo_info[1])) if geo_info is not None else None
-    input_image_proto = resources_pb2.Input(
-        id=input_id,
-        dataset_ids=[dataset_id],
-        data=resources_pb2.Data(
-            image=resources_pb2.Image(base64=open(image_path, 'rb').read(),),
-            geo=geo_pb,
-            metadata=metadata))
-
-    return input_image_proto
-
-  def create_mask_proto(self, label: str, polygons: List[List[float]], input_id: str,
-                        dataset_id: str) -> resources_pb2.Annotation:
-    """
-    Create an input mask proto for an input polygon/mask and label.
-    Args:
-      `label`: image label
-      `polygons`: Polygon x,y points iterable
-      `input_id: unique input id
-      `dataset_id`: Clarifai dataset id
-    Returns:
-      An input proto corresponding to a single image
-    """
-    input_mask_proto = resources_pb2.Annotation(
-        input_id=input_id,
-        data=resources_pb2.Data(regions=[
-            resources_pb2.Region(
-                region_info=resources_pb2.RegionInfo(polygon=resources_pb2.Polygon(
-                    points=[
-                        resources_pb2.Point(
-                            row=_point[1],  # row is y point
-                            col=_point[0],  # col is x point
-                            visibility="VISIBLE") for _point in polygons
-                    ])),
-                data=resources_pb2.Data(concepts=[
-                    resources_pb2.Concept(
-                        id=f"id-{''.join(label.split(' '))}", name=label, value=1.)
-                ]))
-        ]))
-
-    return input_mask_proto
-
-  def _extract_protos(self) -> None:
-    """
-    Create input image and annotation protos for each data generator item.
-    """
-    for i, item in tqdm(enumerate(self.datagen_object), desc="Creating input protos..."):
-      metadata = Struct()
-      image = item.image_path  # image path
-      labels = item.classes  # list of class labels
-      _polygons = item.polygons  # list of polygons: [[[x,y],...,[x,y]],...]
-      input_id = f"{self.dataset_id}-{self.split}-{i}" if item.id is None else f"{self.split}-{str(item.id)}"
-      metadata.update({"filename": os.path.basename(image), "split": self.split})
-      geo_info = item.geo_info
-
-      self.input_ids.append(input_id)
-      input_image_proto = self.create_input_protos(image, input_id, self.dataset_id, geo_info,
-                                                   metadata)
-      self._all_input_protos[input_id] = input_image_proto
-
-      ## Iterate over each masked image and create a proto for upload to clarifai
-      ## The length of masks/polygons-list and labels must be equal
-      for i, _polygon in enumerate(_polygons):
-        try:
-          input_mask_proto = self.create_mask_proto(labels[i], _polygon, input_id, self.dataset_id)
-          self._all_annotation_protos[input_id].append(input_mask_proto)
-        except IndexError:
-          continue

clarifai_utils/data_upload/datasets/text.py

@@ -1,60 +0,0 @@
-from typing import Iterator, List
-
-from clarifai_grpc.grpc.api import resources_pb2
-from google.protobuf.struct_pb2 import Struct
-from tqdm import tqdm
-
-from .base import ClarifaiDataset
-
-
-class TextClassificationDataset(ClarifaiDataset):
-  """
-  Upload text classification datasets to clarifai datasets
-  """
-
-  def __init__(self, datagen_object: Iterator, dataset_id: str, split: str) -> None:
-    super().__init__(datagen_object, dataset_id, split)
-    self._extract_protos()
-
-  def create_input_protos(self, text_input: str, labels: List[str], input_id: str, dataset_id: str,
-                          metadata: Struct) -> resources_pb2.Input:
-    """
-    Create input protos for each text, label input pairs.
-    Args:
-    	`text_input`: text string.
-    	`labels`: text labels
-    	`input_id: unique input id
-    	`dataset_id`: Clarifai dataset id
-    	`metadata`:input metadata
-    Returns:
-    	An input proto representing a single row input
-    """
-    input_proto = resources_pb2.Input(
-        id=input_id,
-        dataset_ids=[dataset_id],
-        data=resources_pb2.Data(
-            text=resources_pb2.Text(raw=text_input),
-            concepts=[
-                resources_pb2.Concept(
-                    id=f"id-{''.join(_label.split(' '))}", name=_label, value=1.)
-                for _label in labels
-            ],
-            metadata=metadata))
-
-    return input_proto
-
-  def _extract_protos(self) -> None:
-    """
-    Creates input protos for each data generator item.
-    """
-    for i, item in tqdm(enumerate(self.datagen_object), desc="Loading text data"):
-      metadata = Struct()
-      text = item.text
-      labels = item.labels if isinstance(item.labels, list) else [item.labels]  # clarifai concept
-      input_id = f"{self.dataset_id}-{self.split}-{i}" if item.id is None else f"{self.split}-{str(item.id)}"
-      metadata.update({"split": self.split})
-
-      self.input_ids.append(input_id)
-      input_proto = self.create_input_protos(text, labels, input_id, self.dataset_id, metadata)
-
-      self._all_input_protos[input_id] = input_proto

clarifai_utils/data_upload/datasets/zoo/README.md

@@ -1,55 +0,0 @@
-## Datasets Zoo
-
-A collection of data preprocessing modules for popular public datasets to allow for compatible upload into Clarifai user app datasets.
-
-## Usage
-
-If a dataset module exists in the zoo, uploading the specific dataset can be easily done by simply creating a python script (or via commandline) and specifying the dataset module name in the `from_zoo` parameter of the  `UploadConfig` class .i.e.
-
-```python
-from clarifai.data_upload.upload import UploadConfig
-
-upload_obj = UploadConfig(
-	user_id="",
-	app_id="",
-	pat="", # Clarifai user PAT (not Clarifai app PAT)
-	dataset_id="",
-	task="",
-	from_zoo="coco_detection",
-	split="val" # train, val or test depending on the dataset
-	)
-# execute data upload to Clarifai app dataset
-upload_obj.upload_to_clarifai()
-```
-
-## Zoo Datasets
-
- | dataset name | task | module name (.py) | splits |
- | --- | --- | --- | --- |
- | [COCO 2017](https://cocodataset.org/#download) | Detection | `coco_detection` | `train`, `val` |
- |        | Segmentation | `coco_segmentation` | `train`, `val` |
- |       | Captions | `coco_captions` | `train`, `val` |
- |[xVIEW](http://xviewdataset.org/)  | Detection | `xview_detection` | `train`
- | [ImageNet](https://www.image-net.org/)  | Classification | `imagenet_classification` | `train`
-## Contributing Modules
-
-A dataset (preprocessing) module is a python script that contains a dataset class which implements data download (to download the dataset from a source to local disk dir) & extraction and dataloader methods.
-
-The class naming convention is `<datasetname>Dataset`. The dataset class must accept `split` as the only argument in the `__init__` method and the `dataloader` method must be a generator that yields either of `VisualClassificationFeatures()`, `VisualDetectionFeatures()`, `VisualSegmentationFeatures()` or `TextFeatures()` as defined in [clarifai/data_upload/datasets/features.py](datasets/features.py). Other methods can be added as seen fit but `dataloader()` is the main method and must strictly be named `dataloader`.
-Reference can be taken from the existing dataset modules in the zoo for development.
-
-## Notes
-
-* Dataset in the zoo by default first create a `data` directory in the local directory where the call to `UploadConfig(...).upload_to_clarifai()` is made and then download the data into this `data` directory, preprocess the data and finally execute upload to a Clarifai app dataset. For instance with the COCO dataset modules above, the coco2017 dataset is by default downloaded first into a `data` directory, extracted and then preprocessing is performed on it and finally uploaded to Clarifai.
-
-* Taking the above into consideration, to avoid the scripts re-downloading data you already have locally, create a `data` directory in the same directory where you'll make a call to `UploadConfig(...).upload_to_clarifai()` and move your extracted data there. **Ensure that the extracted folder/file names and file structure MATCH those when the downloaded zips are extracted.**
-
-* COCO Format: To reuse the coco modules above on your coco format data, ensure the criteria in the two points above is adhered to first. If so, pass the coco module name from any of the above in the zoo to the `from_zoo=` parameter in `UploadConfig()` and finally invoke the `upload_to_clarifai()` method.
-
-* xVIEW Dataset: To upload, you have to register and download images,label from [xviewdataset](http://xviewdataset.org/#dataset) follow the above mentioned steps to place extracted folder in `data` directory. Finally pass the xview module name to `from_zoo=` parameter in `UploadConfig()` and invoke the `upload_to_clarifai()` method.
-
-* ImageNet Dataset: ImageNet Dataset should be downloaded and placed in the 'data' folder along with the [label mapping file](https://www.kaggle.com/competitions/imagenet-object-localization-challenge/data?select=LOC_synset_mapping.txt).
-
-		<data>/
-      	├── train/
-      	├── LOC_synset_mapping.txt