clarifai 11.4.3rc1__py3-none-any.whl → 11.4.4__py3-none-any.whl

clarifai/runners/models/test_model_builder.py

@@ -1,89 +0,0 @@
-import unittest
-from unittest.mock import patch, mock_open
-from clarifai.runners.models.model_builder import (
-    ModelBuilder,
-    DEFAULT_DOWNLOAD_CHECKPOINT_WHEN,
-)
-
-mock_data = """
-checkpoints:
-  type: huggingface
-  repo_id: test_repo
-model:
-  user_id: test_user
-  app_id: test_app
-  id: test_model
-  model_type_id: test_type
-inference_compute_info:
-  cpu_limit: "2"
-  cpu_memory: "4Gi"
-  num_accelerators: 0
-"""
-
-
-class TestModelBuilder(unittest.TestCase):
-    @patch("builtins.open", new_callable=mock_open, read_data=mock_data)
-    @patch("os.path.exists")
-    @patch("os.listdir")
-    def test_validate_folder(self, mock_listdir, mock_exists, mock_open_file):
-        mock_exists.return_value = True
-        mock_listdir.return_value = ["config.yaml", "1", "requirements.txt"]
-
-        builder = ModelBuilder(folder="test_folder", validate_api_ids=False)
-        validated_folder = builder._validate_folder("test_folder")
-        self.assertIn("test_folder", validated_folder)
-
-    def test_default_download_checkpoint_when(self):
-        self.assertEqual(DEFAULT_DOWNLOAD_CHECKPOINT_WHEN, "runtime")
-
-    @patch(
-        "builtins.open",
-        new_callable=mock_open,
-        read_data="checkpoints:\n  type: huggingface\n  repo_id: test_repo\n",
-    )
-    @patch("os.path.exists")
-    @patch("yaml.safe_load")
-    def test_load_config(self, mock_yaml_load, mock_exists, mock_open_file):
-        mock_exists.return_value = True
-        mock_yaml_load.return_value = {
-            "checkpoints": {"type": "huggingface", "repo_id": "test_repo"}
-        }
-        config = ModelBuilder._load_config("config.yaml")
-        self.assertIn("checkpoints", config)
-        self.assertEqual(config["checkpoints"]["type"], "huggingface")
-
-    @patch("shutil.copy")
-    @patch("os.path.exists")
-    def test_backup_config(self, mock_exists, mock_copy):
-        mock_exists.side_effect = [True, False]  # config exists, backup does not
-        ModelBuilder._backup_config("config.yaml")
-        mock_copy.assert_called_once_with("config.yaml", "config.yaml.bak")
-
-    @patch("builtins.open", new_callable=mock_open)
-    @patch("yaml.safe_dump")
-    def test_save_config(self, mock_yaml_dump, mock_open_file):
-        config = {"key": "value"}
-        ModelBuilder._save_config("config.yaml", config)
-        mock_yaml_dump.assert_called_once_with(config, mock_open_file())
-
-    def test_validate_config_model(self):
-        builder = ModelBuilder(folder="test_folder", validate_api_ids=False)
-        builder.config = {"model": {}}
-        try:
-            builder._validate_config_model()
-        except AssertionError:
-            self.fail("_validate_config_model raised AssertionError unexpectedly!")
-
-    @patch("os.path.exists")
-    @patch("os.listdir")
-    def test_validate_folder_missing_config(self, mock_listdir, mock_exists):
-        mock_exists.return_value = True
-        mock_listdir.return_value = ["1", "requirements.txt"]
-
-        builder = ModelBuilder(folder="test_folder", validate_api_ids=False)
-        with self.assertRaises(AssertionError):
-            builder._validate_folder("test_folder")
-
-
-if __name__ == "__main__":
-    unittest.main()

clarifai/runners/utils/data_handler.py

@@ -1,231 +0,0 @@
-from typing import Dict, List, Tuple, Union
-
-import numpy as np
-from clarifai_grpc.grpc.api import resources_pb2
-from clarifai_grpc.grpc.api.status import status_code_pb2, status_pb2
-from PIL import Image
-
-from clarifai.client.auth.helper import ClarifaiAuthHelper
-
-from .data_utils import bytes_to_image, image_to_bytes
-
-
-class BaseDataHandler:
-
-  def __init__(self,
-               proto: Union[resources_pb2.Input, resources_pb2.Output],
-               auth: ClarifaiAuthHelper = None):
-    self._proto = proto
-    self._auth = auth
-
-  #
-  def to_python(self):
-    return dict(text=self.text, image=self.image, audio=self.audio)
-
-  # ---------------- Start get/setters ---------------- #
-  # Proto
-  @property
-  def proto(self):
-    return self._proto
-
-  # Status
-  @property
-  def status(self) -> status_pb2.Status:
-    return self._proto.status
-
-  def set_status(self, code: str, description: str = ""):
-    self._proto.status.code = code
-    self._proto.status.description = description
-
-  # Text
-  @property
-  def text(self) -> Union[None, str]:
-    data = self._proto.data.text
-    text = None
-    if data.ByteSize():
-      if data.raw:
-        text = data.raw
-      else:
-        raise NotImplementedError
-    return text
-
-  def set_text(self, text: str):
-    self._proto.data.text.raw = text
-
-  # Image
-  @property
-  def image(self, format: str = "np") -> Union[None, Image.Image, np.ndarray]:
-    data = self._proto.data.image
-    image = None
-    if data.ByteSize():
-      data: resources_pb2.Image = data
-      if data.base64:
-        image = data.base64
-      elif data.url:
-        raise NotImplementedError
-      image = bytes_to_image(image)
-      image = image if not format == "np" else np.asarray(image).astype("uint8")
-    return image
-
-  def set_image(self, image: Union[Image.Image, np.ndarray]):
-    if isinstance(image, np.ndarray):
-      image = Image.fromarray(image)
-    self._proto.data.image.base64 = image_to_bytes(image)
-
-  # Audio
-  @property
-  def audio(self) -> bytes:
-    data = self._proto.data.audio
-    audio = None
-    if data.ByteSize():
-      if data.base64:
-        audio = data.base64
-    return audio
-
-  def set_audio(self, audio: bytes):
-    self._proto.data.audio.base64 = audio
-
-  # Bboxes
-  @property
-  def bboxes(self, real_coord: bool = False, image_width: int = None,
-             image_height: int = None) -> Tuple[List, List, List]:
-    if real_coord:
-      assert (image_height or image_width
-             ), "image_height and image_width are required when when return real coordinates"
-    xyxy = []
-    scores = []
-    concepts = []
-    for _, each in enumerate(self._proto.data.regions):
-      box = each.region_info
-      score = each.value
-      concept = each.data.concepts[0].id
-      x1 = box.left_col
-      y1 = box.top_row
-      x2 = box.right_col
-      y2 = box.bottom_row
-      if real_coord:
-        x1 = x1 * image_width
-        y1 = y1 * image_height
-        x2 = x2 * image_width
-        y2 = y2 * image_height
-      xyxy.append([x1, y1, x2, y2])
-      scores.append(score)
-      concepts.append(concept)
-
-    return xyxy, scores, concepts
-
-  def set_bboxes(self,
-                 boxes: list,
-                 scores: list,
-                 concepts: list,
-                 real_coord: bool = False,
-                 image_width: int = None,
-                 image_height: int = None):
-    if real_coord:
-      assert (image_height and
-              image_width), "image_height and image_width are required when `real_coord` is set"
-      bboxes = [[x[1] / image_height, x[0] / image_width, x[3] / image_height, x[2] / image_width]
-                for x in boxes]  # normalize the bboxes to [0,1] and [y1 x1 y2 x2]
-      bboxes = np.clip(bboxes, 0, 1.0)
-
-    regions = []
-    for ith, bbox in enumerate(bboxes):
-      score = scores[ith]
-      concept = concepts[ith]
-      if any([each > 1.0 for each in bbox]):
-        assert ValueError(
-            "Box coordinates is not normalized between [0, 1]. Please set format_box to True and provide image_height and image_width to normalize"
-        )
-      region = resources_pb2.RegionInfo(bounding_box=resources_pb2.BoundingBox(
-          top_row=bbox[0],  # y_min
-          left_col=bbox[1],  # x_min
-          bottom_row=bbox[2],  # y_max
-          right_col=bbox[3],  # x_max
-      ))
-      data = resources_pb2.Data(concepts=resources_pb2.Concept(id=concept, value=score))
-      regions.append(resources_pb2.Region(region_info=region, data=data))
-
-    self._proto.data.regions = regions
-
-  # Concepts
-  @property
-  def concepts(self) -> Dict[str, float]:
-    con_scores = {}
-    for each in self.proto.data.concepts:
-      con_scores.update({each.id: each.value})
-    return con_scores
-
-  def set_concepts(self, concept_score_pairs: Dict[str, float]):
-    concepts = []
-    for concept, score in concept_score_pairs.items():
-      con_score = resources_pb2.Concept(id=concept, name=concept, value=score)
-      concepts.append(con_score)
-    if concepts:
-      self._proto.data.ClearField("concepts")
-      for each in concepts:
-        self._proto.data.concepts.append(each)
-
-  # Embeddings
-  @property
-  def embeddings(self) -> List[List[float]]:
-    return [each.vector for each in self.proto.data.embeddings]
-
-  def set_embeddings(self, list_vectors: List[List[float]]):
-    if list_vectors[0]:
-      self._proto.data.ClearField("embeddings")
-    for vec in list_vectors:
-      self._proto.data.embeddings.append(
-          resources_pb2.Embedding(vector=vec, num_dimensions=len(vec)))
-
-  # ---------------- End get/setters ---------------- #
-
-  # Constructors
-  @classmethod
-  def from_proto(cls, proto):
-    clss = cls(proto=proto)
-    return clss
-
-  @classmethod
-  def from_data(
-      cls,
-      status_code: int = status_code_pb2.SUCCESS,
-      status_description: str = "",
-      text: str = None,
-      image: Union[Image.Image, np.ndarray] = None,
-      audio: bytes = None,
-      boxes: dict = None,
-      concepts: Dict[str, float] = {},
-      embeddings: List[List[float]] = [],
-  ) -> 'OutputDataHandler':
-    clss = cls(proto=resources_pb2.Output())
-    if isinstance(image, Image.Image) or isinstance(image, np.ndarray):
-      clss.set_image(image)
-    if text:
-      clss.set_text(text)
-    if audio:
-      clss.set_audio(audio)
-    if boxes:
-      clss.set_bboxes(**boxes)
-    if concepts:
-      clss.set_concepts(concepts)
-    if embeddings:
-      clss.set_embeddings(embeddings)
-
-    clss.set_status(code=status_code, description=status_description)
-    return clss
-
-
-class InputDataHandler(BaseDataHandler):
-
-  def __init__(self,
-               proto: resources_pb2.Input = resources_pb2.Input(),
-               auth: ClarifaiAuthHelper = None):
-    super().__init__(proto=proto, auth=auth)
-
-
-class OutputDataHandler(BaseDataHandler):
-
-  def __init__(self,
-               proto: resources_pb2.Output = resources_pb2.Output(),
-               auth: ClarifaiAuthHelper = None):
-    super().__init__(proto=proto, auth=auth)

clarifai/utils/config.py~

@@ -1,145 +0,0 @@
-import click
-import sys
-import os
-import typing as t
-import yaml
-
-from dataclasses import dataclass, field
-from clarifai.utils.cli import AliasedGroup, TableFormatter, load_command_modules, load_command_modules
-
-
-
-@dataclass
-class AccessToken():
-  type: str
-  value: str
-
-  def __str__(self):
-    return f'{self.type}:{self.value}' if self.type == 'env' else '********'
-
-  def to_serializable_dict(self):
-    return self.__dict__
-
-  @classmethod
-  def from_serializable_dict(cls, _dict):
-    return cls(**_dict)
-
-  # Dictionary protocol methods:
-
-  def __getitem__(self, key: str) -> t.Any:
-    if key == 'type':
-      return self.type
-    elif key == 'value':
-      return self.value
-    else:
-      raise KeyError(key)
-
-  def __setitem__(self, key: str, value: t.Any) -> None:
-    if key == 'type':
-      self.type = value
-    elif key == 'value':
-      self.value = value
-    else:
-      raise KeyError(key)
-
-  def __delitem__(self, key: str) -> None:
-    raise TypeError("Cannot delete attributes from AccessToken")
-
-  def __iter__(self):
-    return iter(['type', 'value'])
-
-  def __len__(self) -> int:
-    return 2
-
-  def __contains__(self, key: str) -> bool:
-    return key in ['type', 'value']
-
-  def keys(self):
-    return ['type', 'value']
-
-  def values(self):
-    return [self.type, self.value]
-
-  def items(self):
-    return [('type', self.type), ('value', self.value)]
-
-  def get(self, key: str, default: t.Any = None) -> t.Any:
-    try:
-      return self[key]
-    except KeyError:
-      return default
-
-
-@dataclass
-class Context():
-  name: str
-  user_id: str
-  base_url: str
-  access_token: AccessToken = field(default_factory=lambda: AccessToken('env', 'CLARIFAI_PAT'))
-  env: t.Dict[str, str] = field(default_factory=dict)
-
-  pat: str = None
-
-  def _resolve_pat(self) -> str:
-    if self.access_token['type'].lower() == 'env':
-      return os.getenv(self.access_token['value'], '')
-    elif self.access_token['type'].lower() == 'raw':
-      return self.access_token['value']
-    else:
-      raise Exception('Only "env" and "raw" methods are supported')
-
-  def __post_init__(self):
-    self.pat = self._resolve_pat()
-    self.access_token = AccessToken(**self.access_token)
-
-  def to_serializable_dict(self):
-    result = {
-        'name': self.name,
-        'user_id': self.user_id,
-        'base_url': self.base_url,
-        'access_token': self.access_token.to_serializable_dict(),
-    }
-    if self.env:
-      result['env'] = self.env
-    return result
-
-
-@dataclass
-class Config():
-  current_context: str
-  filename: str
-  contexts: dict[str, Context] = field(default_factory=dict)
-
-  def __post_init__(self):
-    for k, v in self.contexts.items():
-      if 'name' not in v:
-        v['name'] = k
-    self.contexts = {k: Context(**v) for k, v in self.contexts.items()}
-
-  @classmethod
-  def from_yaml(cls, filename: str = DEFAULT_CONFIG):
-    with open(filename, 'r') as f:
-      cfg = yaml.safe_load(f)
-    return cls(**cfg, filename=filename)
-
-  def to_dict(self):
-    return {
-        'current_context': self.current_context,
-        'contexts': {k: v.to_serializable_dict()
-                     for k, v in self.contexts.items()}
-    }
-
-  def to_yaml(self, filename: str = None):
-    if filename is None:
-      filename = self.filename
-    dir = os.path.dirname(filename)
-    if len(dir):
-      os.makedirs(dir, exist_ok=True)
-    _dict = self.to_dict()
-    for k, v in _dict['contexts'].items():
-      v.pop('name', None)
-    with open(filename, 'w') as f:
-      yaml.safe_dump(_dict, f)
-
-  def current(self) -> Context:
-    return self.contexts[self.current_context]