clarifai 9.7.0__py3-none-any.whl → 9.7.2__py3-none-any.whl

clarifai_utils/data_upload/datasets/zoo/coco_segmentation.py

@@ -1,160 +0,0 @@
-#! COCO 2017 Image Segmentation dataset
-
-import gc
-import os
-import zipfile
-from functools import reduce
-from glob import glob
-
-import cv2
-import numpy as np
-import requests
-from pycocotools import mask as maskUtils
-from pycocotools.coco import COCO
-from tqdm import tqdm
-
-from ..features import VisualSegmentationFeatures
-
-
-class COCOSegmentationDataset:
-  """COCO 2017 Image Segmentation Dataset."""
-
-  def __init__(self, split: str = "train"):
-    """
-    Initialize coco dataset.
-    Args:
-      filenames: the coco zip filenames: Dict[str, str] to be downloaded if download=True,
-      data_dir: the local coco dataset directory
-      split: "train" or "val"
-    """
-    self.filenames = {
-        "train": "train2017.zip",
-        "val": "val2017.zip",
-        "annotations": "annotations_trainval2017.zip"
-    }
-    self.split = split
-    self.url = "http://images.cocodataset.org/zips/"  # coco base image-zip url
-    self.data_dir = os.path.join(os.curdir, "data")  # data storage dir
-    self.extracted_coco_dirs = {"train": None, "val": None, "annotations": None}
-
-  def coco_download(self, save_dir):
-    """Download coco dataset."""
-    if not os.path.exists(save_dir):
-      os.mkdir(save_dir)
-
-    #check if train, val and annotation dirs exist
-    #so that the coco2017 data isn't downloaded
-    for key, filename in self.filenames.items():
-      existing_files = glob(f"{save_dir}/{key}*")
-      if existing_files:
-        print(f"{key} dataset already downloded and extracted")
-        continue
-
-      print("-" * 80)
-      print(f"Downloading {filename}")
-      print("-" * 80)
-
-      if "annotations" in filename:
-        self.url = "http://images.cocodataset.org/annotations/"
-
-      response = requests.get(self.url + filename, stream=True)
-      response.raise_for_status()
-      with open(os.path.join(save_dir, filename), "wb") as _file:
-        for chunk in tqdm(response.iter_content(chunk_size=5124000)):
-          if chunk:
-            _file.write(chunk)
-      print("Coco data download complete...")
-
-      #extract files
-      zf = zipfile.ZipFile(os.path.join(save_dir, filename))
-      print(f" Extracting {filename} file")
-      zf.extractall(path=save_dir)
-      # Delete coco zip
-      print(f" Deleting {filename}")
-      os.remove(path=os.path.join(save_dir, filename))
-
-  def dataloader(self):
-    """
-    Transform coco data into clarifai proto compatible format for upload.
-    Returns:
-      VisualSegmentationFeatures type generator.
-    """
-    if isinstance(self.filenames, dict) and len(self.filenames) == 3:
-      self.coco_download(self.data_dir)
-      self.extracted_coco_dirs["train"] = [os.path.join(self.data_dir, i) \
-      for i in os.listdir(self.data_dir) if "train" in i][0]
-      self.extracted_coco_dirs["val"] = [os.path.join(self.data_dir, i) \
-      for i in os.listdir(self.data_dir) if "val" in i][0]
-
-      self.extracted_coco_dirs["annotations"] = [os.path.join(self.data_dir, i) \
-      for i in os.listdir(self.data_dir) if "annotations" in i][0]
-    else:
-      raise Exception(f"`filenames` must be a dict of atleast 3 coco zip file names; \
-      train, val and annotations. Found {len(self.filenames)} items instead.")
-
-    annot_file = glob(self.extracted_coco_dirs["annotations"] + "/" +\
-     f"instances_{self.split}*")[0]
-    coco = COCO(annot_file)
-    categories = coco.loadCats(coco.getCatIds())
-    cat_id_map = {category["id"]: category["name"] for category in categories}
-    cat_img_ids = {}
-    for cat_id in list(cat_id_map.keys()):
-      cat_img_ids[cat_id] = coco.getImgIds(catIds=[cat_id])
-
-    img_ids = []
-    for i in list(cat_img_ids.values()):
-      img_ids.extend(i)
-
-    #get annotations for each image id
-    for _id in set(img_ids):
-      annots = []  # polygons
-      class_names = []
-      labels = [i for i in list(filter(lambda x: _id in cat_img_ids[x], cat_img_ids))]
-      image_path = glob(self.extracted_coco_dirs[self.split]+"/"+\
-      f"{str(_id).zfill(12)}*")[0]
-
-      image_height, image_width = cv2.imread(image_path).shape[:2]
-      for cat_id in labels:
-        annot_ids = coco.getAnnIds(imgIds=_id, catIds=[cat_id])
-        if len(annot_ids) > 0:
-          img_annotations = coco.loadAnns(annot_ids)
-          for ann in img_annotations:
-            # get polygons
-            if type(ann['segmentation']) == list:
-              for seg in ann['segmentation']:
-                poly = np.array(seg).reshape((int(len(seg) / 2), 2))
-                poly[:, 0], poly[:, 1] = poly[:, 0] / image_width, poly[:, 1] / image_height
-                annots.append(poly.tolist())  #[[x=col, y=row],...]
-                class_names.append(cat_id_map[cat_id])
-            else:  # seg: {"counts":[...]}
-              if type(ann['segmentation']['counts']) == list:
-                rle = maskUtils.frPyObjects([ann['segmentation']], image_height, image_width)
-              else:
-                rle = ann['segmentation']
-              mask = maskUtils.decode(rle)  #binary mask
-              #convert mask to polygons and add to annots
-              contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
-              polygons = []
-              for cont in contours:
-                if cont.size >= 6:
-                  polygons.append(cont.astype(float).flatten().tolist())
-              # store polygons in (x,y) pairs
-              polygons_flattened = reduce(lambda x, y: x + y, polygons)
-              del polygons
-              del contours
-              del mask
-              gc.collect()
-
-              polygons = np.array(polygons_flattened).reshape((int(len(polygons_flattened) / 2),
-                                                               2))
-              polygons[:, 0] = polygons[:, 0] / image_width
-              polygons[:, 1] = polygons[:, 1] / image_height
-
-              annots.append(polygons.tolist())  #[[x=col, y=row],...,[x=col, y=row]]
-              class_names.append(cat_id_map[cat_id])
-        else:  # if no annotations for given image_id-cat_id pair
-          continue
-      assert len(class_names) == len(annots), f"Num classes must match num annotations\
-      for a single image. Found {len(class_names)} classes and {len(annots)} polygons."
-
-      yield VisualSegmentationFeatures(image_path, class_names, annots, id=_id)

clarifai_utils/data_upload/examples/README.md

@@ -1,5 +0,0 @@
-## Data upload from packages
-
-Examples of how to upload your local directory datasets into clarifai app datasets using data_upload utils `from_module` feature.
-
-The `examples.py` script imports the data upload module and executes the upload of the IMDB_reviews text_classification and [food-101 image classification](https://data.vision.ee.ethz.ch/cvl/datasets_extra/food-101/) datasets by passing the package paths as a parameter in UploadConfig().

clarifai_utils/data_upload/examples/image_classification/cifar10/dataset.py

@@ -1,40 +0,0 @@
-#! Cifar10 Dataset
-
-import csv
-import os
-
-from clarifai.data_upload.datasets.features import VisualClassificationFeatures
-
-
-class Cifar10Dataset:
-  """Cifar10 Dataset."""
-
-  def __init__(self, split: str = "train"):
-    """
-    Initialize dataset params.
-    Args:
-      data_dir: the local dataset directory.
-      split: "train" or "test"
-    """
-    self.split = split
-    self.data_dirs = {
-        "train": os.path.join(os.path.dirname(__file__), "cifar_small_train.csv"),
-        "test": os.path.join(os.path.dirname(__file__), "cifar_small_test.csv")
-    }
-
-  def dataloader(self):
-    """
-    Transform text data into clarifai proto compatible
-    format for upload.
-    Returns:
-      TextFeatures type generator.
-    """
-    ## Your preprocessing code here
-    with open(self.data_dirs[self.split]) as _file:
-      reader = csv.reader(_file)
-      next(reader, None)  # skip header
-      for review in reader:
-        yield VisualClassificationFeatures(
-            image_path='examples/image_classification/cifar10/' + review[0],
-            label=review[1],
-            id=None)

clarifai_utils/data_upload/examples/image_classification/food-101/dataset.py

@@ -1,39 +0,0 @@
-#! Food-101 image classification dataset
-
-import os
-
-from clarifai.data_upload.datasets.features import VisualClassificationFeatures
-
-
-class Food101Dataset:
-  """Food-101 Image Classification Dataset.
-  url: https://data.vision.ee.ethz.ch/cvl/datasets_extra/food-101/
-  """
-
-  def __init__(self, split: str = "train"):
-    """
-    Initialize dataset params.
-    Args:
-      data_dir: the local dataset directory.
-      split: "train" or "test"
-    """
-    self.split = split
-    self.image_dir = {"train": os.path.join(os.path.dirname(__file__), "images")}
-
-  def dataloader(self):
-    """
-    Transform food-101 dataset into clarifai proto compatible
-    format for upload.
-    Returns:
-      VisualClassificationFeatures type generator.
-    """
-    ## Your preprocessing code here
-    class_names = os.listdir(self.image_dir[self.split])
-    for class_name in class_names:
-      for image in os.listdir(os.path.join(self.image_dir[self.split], class_name)):
-        image_path = os.path.join(self.image_dir[self.split], class_name, image)
-        yield VisualClassificationFeatures(
-            image_path=image_path,
-            label=class_name,
-            id=None  # or image_id
-        )

clarifai_utils/data_upload/examples/image_detection/voc/annotations/2007_000464.xml

@@ -1,39 +0,0 @@
-<annotation>
-	<folder>VOC2012</folder>
-	<filename>2007_000464.jpg</filename>
-	<source>
-		<database>The VOC2007 Database</database>
-		<annotation>PASCAL VOC2007</annotation>
-		<image>flickr</image>
-	</source>
-	<size>
-		<width>375</width>
-		<height>500</height>
-		<depth>3</depth>
-	</size>
-	<segmented>1</segmented>
-	<object>
-		<name>cow</name>
-		<pose>Left</pose>
-		<truncated>0</truncated>
-		<difficult>0</difficult>
-		<bndbox>
-			<xmin>71</xmin>
-			<ymin>252</ymin>
-			<xmax>216</xmax>
-			<ymax>314</ymax>
-		</bndbox>
-	</object>
-	<object>
-		<name>cow</name>
-		<pose>Left</pose>
-		<truncated>0</truncated>
-		<difficult>0</difficult>
-		<bndbox>
-			<xmin>58</xmin>
-			<ymin>202</ymin>
-			<xmax>241</xmax>
-			<ymax>295</ymax>
-		</bndbox>
-	</object>
-</annotation>

clarifai_utils/data_upload/examples/image_detection/voc/annotations/2008_000853.xml

@@ -1,28 +0,0 @@
-<annotation>
-	<folder>VOC2012</folder>
-	<filename>2008_000853.jpg</filename>
-	<source>
-		<database>The VOC2008 Database</database>
-		<annotation>PASCAL VOC2008</annotation>
-		<image>flickr</image>
-	</source>
-	<size>
-		<width>375</width>
-		<height>500</height>
-		<depth>3</depth>
-	</size>
-	<segmented>1</segmented>
-	<object>
-		<name>cat</name>
-		<pose>Frontal</pose>
-		<truncated>0</truncated>
-		<occluded>1</occluded>
-		<bndbox>
-			<xmin>37</xmin>
-			<ymin>345</ymin>
-			<xmax>186</xmax>
-			<ymax>417</ymax>
-		</bndbox>
-		<difficult>0</difficult>
-	</object>
-</annotation>

clarifai_utils/data_upload/examples/image_detection/voc/annotations/2008_003182.xml

@@ -1,54 +0,0 @@
-<annotation>
-	<folder>VOC2012</folder>
-	<filename>2008_003182.jpg</filename>
-	<source>
-		<database>The VOC2008 Database</database>
-		<annotation>PASCAL VOC2008</annotation>
-		<image>flickr</image>
-	</source>
-	<size>
-		<width>333</width>
-		<height>500</height>
-		<depth>3</depth>
-	</size>
-	<segmented>0</segmented>
-	<object>
-		<name>person</name>
-		<pose>Left</pose>
-		<truncated>1</truncated>
-		<occluded>0</occluded>
-		<bndbox>
-			<xmin>181</xmin>
-			<ymin>187</ymin>
-			<xmax>319</xmax>
-			<ymax>500</ymax>
-		</bndbox>
-		<difficult>0</difficult>
-	</object>
-	<object>
-		<name>horse</name>
-		<pose>Unspecified</pose>
-		<truncated>1</truncated>
-		<occluded>1</occluded>
-		<bndbox>
-			<xmin>71</xmin>
-			<ymin>181</ymin>
-			<xmax>333</xmax>
-			<ymax>500</ymax>
-		</bndbox>
-		<difficult>0</difficult>
-	</object>
-	<object>
-		<name>bottle</name>
-		<pose>Unspecified</pose>
-		<truncated>0</truncated>
-		<occluded>1</occluded>
-		<bndbox>
-			<xmin>211</xmin>
-			<ymin>312</ymin>
-			<xmax>235</xmax>
-			<ymax>358</ymax>
-		</bndbox>
-		<difficult>0</difficult>
-	</object>
-</annotation>

clarifai_utils/data_upload/examples/image_detection/voc/annotations/2008_008526.xml

@@ -1,67 +0,0 @@
-<annotation>
-	<folder>VOC2012</folder>
-	<filename>2008_008526.jpg</filename>
-	<source>
-		<database>The VOC2008 Database</database>
-		<annotation>PASCAL VOC2008</annotation>
-		<image>flickr</image>
-	</source>
-	<size>
-		<width>500</width>
-		<height>375</height>
-		<depth>3</depth>
-	</size>
-	<segmented>0</segmented>
-	<object>
-		<name>sofa</name>
-		<pose>Frontal</pose>
-		<truncated>1</truncated>
-		<occluded>1</occluded>
-		<bndbox>
-			<xmin>1</xmin>
-			<ymin>152</ymin>
-			<xmax>500</xmax>
-			<ymax>375</ymax>
-		</bndbox>
-		<difficult>0</difficult>
-	</object>
-	<object>
-		<name>bottle</name>
-		<pose>Unspecified</pose>
-		<truncated>0</truncated>
-		<occluded>1</occluded>
-		<bndbox>
-			<xmin>162</xmin>
-			<ymin>232</ymin>
-			<xmax>197</xmax>
-			<ymax>338</ymax>
-		</bndbox>
-		<difficult>0</difficult>
-	</object>
-	<object>
-		<name>person</name>
-		<pose>Frontal</pose>
-		<truncated>1</truncated>
-		<occluded>1</occluded>
-		<bndbox>
-			<xmin>66</xmin>
-			<ymin>138</ymin>
-			<xmax>352</xmax>
-			<ymax>375</ymax>
-		</bndbox>
-		<difficult>0</difficult>
-	</object>
-	<object>
-		<name>person</name>
-		<pose>Frontal</pose>
-		<truncated>1</truncated>
-		<occluded>1</occluded>
-		<bndbox>
-			<xmin>234</xmin>
-			<ymin>141</ymin>
-			<xmax>500</xmax>
-			<ymax>375</ymax>
-		</bndbox>
-		<difficult>0</difficult>
-	</object>
-</annotation>

clarifai_utils/data_upload/examples/image_detection/voc/annotations/2009_004315.xml

@@ -1,28 +0,0 @@
-<annotation>
-	<filename>2009_004315.jpg</filename>
-	<folder>VOC2012</folder>
-	<object>
-		<name>bird</name>
-		<bndbox>
-			<xmax>461</xmax>
-			<xmin>219</xmin>
-			<ymax>327</ymax>
-			<ymin>51</ymin>
-		</bndbox>
-		<difficult>0</difficult>
-		<occluded>0</occluded>
-		<pose>Left</pose>
-		<truncated>0</truncated>
-	</object>
-	<segmented>0</segmented>
-	<size>
-		<depth>3</depth>
-		<height>463</height>
-		<width>500</width>
-	</size>
-	<source>
-		<annotation>PASCAL VOC2009</annotation>
-		<database>The VOC2009 Database</database>
-		<image>flickr</image>
-	</source>
-</annotation>

clarifai_utils/data_upload/examples/image_detection/voc/annotations/2009_004382.xml

@@ -1,28 +0,0 @@
-<annotation>
-	<filename>2009_004382.jpg</filename>
-	<folder>VOC2012</folder>
-	<object>
-		<name>cat</name>
-		<bndbox>
-			<xmax>264</xmax>
-			<xmin>50</xmin>
-			<ymax>500</ymax>
-			<ymin>1</ymin>
-		</bndbox>
-		<difficult>0</difficult>
-		<occluded>0</occluded>
-		<pose>Frontal</pose>
-		<truncated>0</truncated>
-	</object>
-	<segmented>0</segmented>
-	<size>
-		<depth>3</depth>
-		<height>500</height>
-		<width>313</width>
-	</size>
-	<source>
-		<annotation>PASCAL VOC2009</annotation>
-		<database>The VOC2009 Database</database>
-		<image>flickr</image>
-	</source>
-</annotation>

clarifai_utils/data_upload/examples/image_detection/voc/annotations/2011_000430.xml

@@ -1,28 +0,0 @@
-<annotation>
-	<filename>2011_000430.jpg</filename>
-	<folder>VOC2012</folder>
-	<object>
-		<name>dog</name>
-		<bndbox>
-			<xmax>500</xmax>
-			<xmin>17</xmin>
-			<ymax>365</ymax>
-			<ymin>33</ymin>
-		</bndbox>
-		<difficult>0</difficult>
-		<occluded>0</occluded>
-		<pose>Left</pose>
-		<truncated>1</truncated>
-	</object>
-	<segmented>0</segmented>
-	<size>
-		<depth>3</depth>
-		<height>375</height>
-		<width>500</width>
-	</size>
-	<source>
-		<annotation>PASCAL VOC2011</annotation>
-		<database>The VOC2011 Database</database>
-		<image>flickr</image>
-	</source>
-</annotation>

clarifai_utils/data_upload/examples/image_detection/voc/annotations/2011_001610.xml

@@ -1,46 +0,0 @@
-<annotation>
-	<filename>2011_001610.jpg</filename>
-	<folder>VOC2012</folder>
-	<object>
-		<name>person</name>
-		<bndbox>
-			<xmax>283</xmax>
-			<xmin>113</xmin>
-			<ymax>390</ymax>
-			<ymin>112</ymin>
-		</bndbox>
-		<difficult>0</difficult>
-		<occluded>1</occluded>
-		<pose>Frontal</pose>
-		<truncated>1</truncated>
-		<part>
-			<name>hand</name>
-			<bndbox>
-				<xmin>128</xmin>
-				<ymin>231</ymin>
-				<xmax>183</xmax>
-				<ymax>283</ymax>
-			</bndbox>
-		</part>
-		<part>
-			<name>head</name>
-			<bndbox>
-				<xmin>123</xmin>
-				<ymin>115</ymin>
-				<xmax>209</xmax>
-				<ymax>226</ymax>
-			</bndbox>
-		</part>
-	</object>
-	<segmented>0</segmented>
-	<size>
-		<depth>3</depth>
-		<height>500</height>
-		<width>375</width>
-	</size>
-	<source>
-		<annotation>PASCAL VOC2011</annotation>
-		<database>The VOC2011 Database</database>
-		<image>flickr</image>
-	</source>
-</annotation>