clarifai 9.10.2__py3-none-any.whl → 9.10.3__py3-none-any.whl

clarifai_utils/datasets/upload/loaders/imagenet_classification.py

@@ -1,59 +0,0 @@
-#! ImageNet Classification dataset
-
-import os
-
-from clarifai.datasets.upload.base import ClarifaiDataLoader
-from ..features import VisualClassificationFeatures
-
-
-class ImageNetDataLoader(ClarifaiDataLoader):
-  """ImageNet Dataset."""
-
-  def __init__(self, split: str = "train"):
-    """
-    Initialize dataset params.
-    Args:
-      data_dir: the local dataset directory.
-      split: "train" or "test"
-    """
-    self.split = split
-    self.data_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
-                                 "data")  # data storage directory
-    self.label_map = dict()
-    self.concepts = []
-    self.image_paths = []
-
-    self.load_data()
-
-  def load_data(self):
-    #Creating label map
-    with open(os.path.join(self.data_dir, "LOC_synset_mapping.txt")) as _file:
-      for _id in _file:
-        #Removing the spaces,upper quotes and Converting to set to remove repetitions. Then converting to list for compatibility.
-        self.label_map[_id.split(" ")[0]] = list({
-            "".join(("".join((label.rstrip().lstrip().split(" ")))).split("'"))
-            for label in _id[_id.find(" ") + 1:].split(",")
-        })
-
-    for _folder in os.listdir(os.path.join(self.data_dir, self.split)):
-      try:
-        concept = self.label_map[_folder]  #concepts
-      except Exception:
-        continue
-      folder_path = os.path.join(self.data_dir, self.split) + "/" + _folder
-      for _img in os.listdir(folder_path):
-        if _img.lower().endswith(('.png', '.jpg', '.jpeg', '.tiff')):
-          self.concepts.append(concept)
-          self.image_paths.append(folder_path + "/" + _img)
-
-    assert len(self.concepts) == len(self.image_paths)
-    "Number of concepts and images are not equal"
-
-  def __len__(self):
-    return len(self.image_paths)
-
-  def __getitem__(self, idx):
-    return VisualClassificationFeatures(
-        image_path=self.image_paths[idx],
-        label=self.concepts[idx],
-        id=self.image_paths[idx].split('.')[0].split('/')[-1])

clarifai_utils/datasets/upload/loaders/xview_detection.py

@@ -1,148 +0,0 @@
-import glob
-import json
-import os
-from collections import defaultdict
-from concurrent.futures import ThreadPoolExecutor
-from multiprocessing import cpu_count
-from typing import DefaultDict, Dict, List
-
-import cv2
-from tqdm import tqdm
-
-from clarifai.datasets.upload.base import ClarifaiDataLoader
-
-from ..features import VisualDetectionFeatures
-
-
-class xviewDetectionDataLoader(ClarifaiDataLoader):
-  """xview Image Detection Dataset"""
-
-  xview_concepts = [
-      'Fixed-wing Aircraft', 'Small Aircraft', 'Cargo Plane', 'Helicopter', 'Passenger Vehicle',
-      'Small Car', 'Bus', 'Pickup Truck', 'Utility Truck', 'Truck', 'Cargo Truck', 'Truck w-Box',
-      'Truck Tractor', 'Trailer', 'Truck w-Flatbed', 'Truck w-Liquid', 'Crane Truck',
-      'Railway Vehicle', 'Passenger Car', 'Cargo Car', 'Flat Car', 'Tank car', 'Locomotive',
-      'Maritime Vessel', 'Motorboat', 'Sailboat', 'Tugboat', 'Barge', 'Fishing Vessel', 'Ferry',
-      'Yacht', 'Container Ship', 'Oil Tanker', 'Engineering Vehicle', 'Tower crane',
-      'Container Crane', 'Reach Stacker', 'Straddle Carrier', 'Mobile Crane', 'Dump Truck',
-      'Haul Truck', 'Scraper-Tractor', 'Front loader-Bulldozer', 'Excavator', 'Cement Mixer',
-      'Ground Grader', 'Hut-Tent', 'Shed', 'Building', 'Aircraft Hangar', 'Damaged Building',
-      'Facility', 'Construction Site', 'Vehicle Lot', 'Helipad', 'Storage Tank',
-      'Shipping container lot', 'Shipping Container', 'Pylon', 'Tower'
-  ]
-
-  def __init__(self, split: str = "train") -> None:
-    """Initialize and Compress xview dataset.
-    Args:
-    split: "train"
-    """
-
-    self.data_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data")
-    self.img_dir = os.path.join(self.data_dir, "train_images")
-    self.img_comp_dir = os.path.join(self.data_dir, "train_images_comp")
-    self.label_file = os.path.join(self.data_dir, "xview_train.geojson")
-
-    self.preprocess()
-    self.all_data = self.xview_data_parser()
-
-    self.load_data()
-
-  def compress_tiff(self, img_path: str) -> None:
-    """Compress tiff image"""
-    img_comp_path = os.path.join(self.img_comp_dir, os.path.basename(img_path))
-    img_arr = cv2.imread(img_path)
-    cv2.imwrite(
-        img_comp_path, img_arr, params=(cv2.IMWRITE_TIFF_COMPRESSION, 8))  # 8: Adobe Deflate
-
-  def preprocess(self):
-    """Compress the tiff images to comply with clarifai grpc image encoding limit(<20MB) Uses ADOBE_DEFLATE compression algorithm"""
-    all_img_ids = glob.glob(os.path.join(self.img_dir, "*.tif"))
-
-    if not os.path.exists(self.img_comp_dir):
-      os.mkdir(self.img_comp_dir)
-
-    num_workers = cpu_count()
-    futures = []
-
-    with ThreadPoolExecutor(max_workers=num_workers) as executor:
-      with tqdm(total=len(all_img_ids), desc="compressing ...") as progress:
-
-        for img_path in all_img_ids:
-          future = executor.submit(self.compress_tiff, img_path)
-          future.add_done_callback(lambda _: progress.update())
-          futures.append(future)
-
-        results = []
-        for future in futures:
-          result = future.result()
-          results.append(result)
-
-  def xview_classes2indices(self, classes: List) -> List:
-    """remap xview classes 11-94 to 0-59"""
-    indices = [
-        -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 1, 2, -1, 3, -1, 4, 5, 6, 7, 8, -1, 9, 10,
-        11, 12, 13, 14, 15, -1, -1, 16, 17, 18, 19, 20, 21, 22, -1, 23, 24, 25, -1, 26, 27, -1, 28,
-        -1, 29, 30, 31, 32, 33, 34, 35, 36, 37, -1, 38, 39, 40, 41, 42, 43, 44, 45, -1, -1, -1, -1,
-        46, 47, 48, 49, -1, 50, 51, -1, 52, -1, -1, -1, 53, 54, -1, 55, -1, -1, 56, -1, 57, -1, 58,
-        59
-    ]
-    return [indices[int(c)] for c in classes]
-
-  def xview_indices2concepts(self, indices: List) -> List:
-    """remap classes to concept names"""
-    return [self.xview_concepts[i] for i in indices]
-
-  def xview_data_parser(self) -> DefaultDict[str, Dict[List, List]]:
-    """Parse geojson data into nested dict of imageid w.r.t bounding boxes, concepts"""
-    all_data = defaultdict(lambda: dict(bboxes=[], concepts=[]))
-
-    with open(self.label_file) as f:
-      geojson_data = json.loads(f.read())
-
-    for feature in tqdm(
-        geojson_data['features'], total=len(geojson_data['features']),
-        desc="Parsing geojson data"):
-      image_id = feature['properties']['image_id'].split(".")[0]
-      xview_classid = feature['properties']['type_id']
-      bbox = list(map(int, feature['properties']['bounds_imcoords'].split(",")))
-      concept = self.xview_indices2concepts(self.xview_classes2indices([xview_classid]))
-
-      all_data[image_id]['bboxes'].append(bbox)
-      all_data[image_id]['concepts'].append(concept[0])
-
-    return all_data
-
-  def load_data(self):
-    """Load image paths"""
-    self.image_paths = []
-    all_img_ids = glob.glob(os.path.join(self.img_comp_dir, "*.tif"))
-    self.image_paths = all_img_ids
-
-  def __len__(self):
-    return len(self.image_paths)
-
-  def __getitem__(self, index: int):
-    """Get dataset for a given index.
-    Returns:
-        VisualDetectionFeature type.
-    """
-    _id = os.path.splitext(os.path.basename(self.image_paths[index]))[0]
-    image_path = self.image_paths[index]
-
-    image_height, image_width = cv2.imread(image_path).shape[:2]
-    annots = []
-    class_names = []
-    for bbox, concept in zip(self.all_data[_id]['bboxes'], self.all_data[_id]['concepts']):
-      x_min = max(min(bbox[0] / image_width, 1.0), 0.0)  #left_col
-      y_min = max(min(bbox[1] / image_height, 1.0), 0.0)  #top_row
-      x_max = max(min(bbox[2] / image_width, 1.0), 0.0)  #right_col
-      y_max = max(min(bbox[3] / image_height, 1.0), 0.0)  #bottom_row
-      if (x_min >= x_max) or (y_min >= y_max):
-        continue
-      annots.append([x for x in [x_min, y_min, x_max, y_max]])
-      class_names.append(concept)
-
-    assert len(class_names) == len(annots), f"Num classes must match num bbox annotations\
-        for a single image. Found {len(class_names)} classes and {len(annots)} bboxes."
-
-    return VisualDetectionFeatures(image_path, class_names, annots, id=_id)

clarifai_utils/datasets/upload/text.py

@@ -1,53 +0,0 @@
-from concurrent.futures import ThreadPoolExecutor
-from typing import Iterator, List, Tuple
-
-from clarifai_grpc.grpc.api import resources_pb2
-from google.protobuf.struct_pb2 import Struct
-
-from .base import ClarifaiDataset
-
-
-class TextClassificationDataset(ClarifaiDataset):
-  """Upload text classification datasets to clarifai datasets"""
-
-  def __init__(self, datagen_object: Iterator, dataset_id: str, split: str) -> None:
-    super().__init__(datagen_object, dataset_id, split)
-
-  def _extract_protos(self, batch_input_ids: List[int]
-                     ) -> Tuple[List[resources_pb2.Input], List[resources_pb2.Annotation]]:
-    """Create input image and annotation protos for batch of input ids.
-    Args:
-      batch_input_ids: List of input IDs to retrieve the protos for.
-    Returns:
-      input_protos: List of input protos.
-      annotation_protos: List of annotation protos.
-    """
-    input_protos, annotation_protos = [], []
-
-    def process_datagen_item(id):
-      datagen_item = self.datagen_object[id]
-      metadata = Struct()
-      text = datagen_item.text
-      labels = datagen_item.labels if isinstance(
-          datagen_item.labels, list) else [datagen_item.labels]  # clarifai concept
-      input_id = f"{self.dataset_id}-{self.split}-{id}" if datagen_item.id is None else f"{self.dataset_id}-{self.split}-{str(datagen_item.id)}"
-      if datagen_item.metadata is not None:
-        metadata.update(datagen_item.metadata)
-      else:
-        metadata.update({"split": self.split})
-
-      self.all_input_ids[id] = input_id
-      input_protos.append(
-          self.input_object.get_text_input(
-              input_id=input_id,
-              raw_text=text,
-              dataset_id=self.dataset_id,
-              labels=labels,
-              metadata=metadata))
-
-    with ThreadPoolExecutor(max_workers=4) as executor:
-      futures = [executor.submit(process_datagen_item, id) for id in batch_input_ids]
-      for job in futures:
-        job.result()
-
-    return input_protos, annotation_protos

clarifai_utils/datasets/upload/utils.py

@@ -1,63 +0,0 @@
-import importlib
-import inspect
-import os
-from typing import Union
-
-from clarifai.datasets.upload.base import ClarifaiDataLoader
-
-
-def load_module_dataloader(module_dir: Union[str, os.PathLike], split: str) -> ClarifaiDataLoader:
-  """Validate and import dataset module data generator.
-  Args:
-    `module_dir`: relative path to the module directory
-    The directory must contain a `dataset.py` script and the data itself.
-    `split`: "train" or "val"/"test" dataset split
-  Module Directory Structure:
-  ---------------------------
-      <folder_name>/
-      ├──__init__.py
-      ├──<Your local dir dataset>/
-      └──dataset.py
-  dataset.py must implement a class named following the convention,
-  <dataset_name>DataLoader and this class must inherit from base ClarifaiDataLoader()
-  """
-  module_path = os.path.join(module_dir, "dataset.py")
-  spec = importlib.util.spec_from_file_location("dataset", module_path)
-
-  if not spec:
-    raise ImportError(f"Module not found at {module_path}")
-
-  # Load the module using the spec
-  dataset = importlib.util.module_from_spec(spec)
-  # Execute the module to make its contents available
-  spec.loader.exec_module(dataset)
-
-  # get main module class
-  main_module_cls = None
-  for name, obj in dataset.__dict__.items():
-    if inspect.isclass(obj) and "DataLoader" in name:
-      main_module_cls = obj
-    else:
-      continue
-
-  return main_module_cls(split)
-
-
-def load_dataloader(name: str, split: str) -> ClarifaiDataLoader:
-  """Get dataset generator object from dataset loaders.
-  Args:
-    `name`: dataset module name in datasets/upload/loaders/.
-    `split`: "train" or "val"/"test" dataset split
-  Returns:
-    Data generator object
-  """
-  loader_dataset = importlib.import_module(f"clarifai.datasets.upload.loaders.{name}")
-  # get main module class
-  main_module_cls = None
-  for name, obj in loader_dataset.__dict__.items():
-    if inspect.isclass(obj) and "DataLoader" in name:
-      main_module_cls = obj
-    else:
-      continue
-
-  return main_module_cls(split)

clarifai_utils/errors.py

@@ -1,89 +0,0 @@
-# -*- coding: utf-8 -*-
-import json
-import time
-
-import requests  # noqa
-from google.protobuf.json_format import MessageToDict
-
-from clarifai.versions import CLIENT_VERSION, OS_VER, PYTHON_VERSION
-
-
-class TokenError(Exception):
-  pass
-
-
-class ApiError(Exception):
-  """ API Server error """
-
-  def __init__(self, resource: str, params: dict, method: str,
-               response: requests.Response = None) -> None:
-    self.resource = resource
-    self.params = params
-    self.method = method
-    self.response = response
-
-    self.error_code = 'N/A'
-    self.error_desc = 'N/A'
-    self.error_details = 'N/A'
-    response_json = 'N/A'
-
-    if response is not None:
-      response_json_dict = MessageToDict(response)
-
-      self.error_code = response_json_dict.get('status', {}).get('code', None)
-      self.error_desc = response_json_dict.get('status', {}).get('description', None)
-      self.error_details = response_json_dict.get('status', {}).get('details', None)
-      response_json = json.dumps(response_json_dict['status'], indent=2)
-
-    current_ts_str = str(time.time())
-
-    msg = """%(method)s %(resource)s FAILED(%(time_ts)s).  error_code: %(error_code)s, error_description: %(error_desc)s, error_details: %(error_details)s
- >> Python client %(client_version)s with Python %(python_version)s on %(os_version)s
- >> %(method)s %(resource)s
- >> REQUEST(%(time_ts)s) %(request)s
- >> RESPONSE(%(time_ts)s) %(response)s""" % {
-        'method': method,
-        'resource': resource,
-        'error_code': self.error_code,
-        'error_desc': self.error_desc,
-        'error_details': self.error_details,
-        'request': json.dumps(params, indent=2),
-        'response': response_json,
-        'time_ts': current_ts_str,
-        'client_version': CLIENT_VERSION,
-        'python_version': PYTHON_VERSION,
-        'os_version': OS_VER
-    }
-
-    super(ApiError, self).__init__(msg)
-
-
-class ApiClientError(Exception):
-  """ API Client Error """
-
-
-class UserError(Exception):
-  """ User Error """
-
-
-class AuthError(Exception):
-  """Raised when a client has missing or invalid authentication."""
-
-
-def _base_url(url: str) -> str:
-  """
-  Extracts the base URL from the url, which is everything before the 4th slash character.
-  https://www.clarifai.com/v2/models/1/output -> https://www.clarifai.com/v2/
-  """
-  try:
-    return url[:_find_nth(url, '/', 4) + 1]
-  except Exception:
-    return ''
-
-
-def _find_nth(haystack: str, needle: str, n: int) -> int:
-  start = haystack.find(needle)
-  while start >= 0 and n > 1:
-    start = haystack.find(needle, start + len(needle))
-    n -= 1
-  return start