clarifai 10.8.9__tar.gz → 10.9.0__tar.gz

{clarifai-10.8.9/clarifai.egg-info → clarifai-10.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: clarifai
-Version: 10.8.9
+Version: 10.9.0
 Summary: Clarifai Python SDK
 Home-page: https://github.com/Clarifai/clarifai-python
 Author: Clarifai

clarifai-10.9.0/clarifai/__init__.py

@@ -0,0 +1 @@
+__version__ = "10.9.0"

{clarifai-10.8.9 → clarifai-10.9.0}/clarifai/datasets/upload/loaders/imagenet_classification.py

@@ -24,6 +24,10 @@ class ImageNetDataLoader(ClarifaiDataLoader):
 
     self.load_data()
 
+  @property
+  def task(self):
+    return "visual_classification"
+
   def load_data(self):
     #Creating label map
     with open(os.path.join(self.data_dir, "LOC_synset_mapping.txt")) as _file:
@@ -54,5 +58,5 @@ class ImageNetDataLoader(ClarifaiDataLoader):
   def __getitem__(self, idx):
     return VisualClassificationFeatures(
         image_path=self.image_paths[idx],
-        label=self.concepts[idx],
+        labels=self.concepts[idx],
         id=self.image_paths[idx].split('.')[0].split('/')[-1])

{clarifai-10.8.9 → clarifai-10.9.0}/clarifai/utils/evaluation/helpers.py

@@ -58,13 +58,16 @@ class _BaseEvalResultHandler:
   model: Model
   eval_data: List[resources_pb2.EvalMetrics] = field(default_factory=list)
 
-  def evaluate_and_wait(self, dataset: Dataset):
+  def evaluate_and_wait(self, dataset: Dataset, eval_info: dict = None):
     from tqdm import tqdm
     dataset_id = dataset.id
     dataset_app_id = dataset.app_id
     dataset_user_id = dataset.user_id
     _ = self.model.evaluate(
-        dataset_id=dataset_id, dataset_app_id=dataset_app_id, dataset_user_id=dataset_user_id)
+        dataset_id=dataset_id,
+        dataset_app_id=dataset_app_id,
+        dataset_user_id=dataset_user_id,
+        eval_info=eval_info)
     latest_eval = self.model.list_evaluations()[0]
     excepted = 10
     desc = f"Please wait for the evaluation process between model {self.get_model_name()} and dataset {dataset_user_id}/{dataset_app_id}/{dataset_id} to complete."
@@ -83,7 +86,10 @@ class _BaseEvalResultHandler:
           f"Model has failed to evaluate \n {latest_eval.status}.\nPlease check your dataset inputs!"
       )
 
-  def find_eval_id(self, datasets: List[Dataset] = [], attempt_evaluate: bool = False):
+  def find_eval_id(self,
+                   datasets: List[Dataset] = [],
+                   attempt_evaluate: bool = False,
+                   eval_info: dict = None):
     list_eval_outputs = self.model.list_evaluations()
     self.eval_data = []
     for dataset in datasets:
@@ -117,7 +123,7 @@ class _BaseEvalResultHandler:
       # if not evaluated, but user wants to proceed it
       if not _is_found:
         if attempt_evaluate:
-          self.eval_data.append(self.evaluate_and_wait(dataset))
+          self.eval_data.append(self.evaluate_and_wait(dataset, eval_info=eval_info))
         # otherwise raise error
         else:
           raise Exception(

{clarifai-10.8.9 → clarifai-10.9.0}/clarifai/utils/evaluation/main.py

@@ -53,6 +53,7 @@ class EvalResultCompare:
                models: Union[List[Model], List[str]],
                datasets: Union[Dataset, List[Dataset], str, List[str]],
                attempt_evaluate: bool = False,
+               eval_info: dict = None,
                auth_kwargs: dict = {}):
     assert isinstance(models, list), ValueError("Expected list")
 
@@ -97,7 +98,7 @@ class EvalResultCompare:
         assert self.model_type == model_type, f"Can not compare when model types are different, {self.model_type} != {model_type}"
       m = make_handler_by_type(model_type)(model=model)
       logger.info(f"* {m.get_model_name(pretify=True)}")
-      m.find_eval_id(datasets=datasets, attempt_evaluate=attempt_evaluate)
+      m.find_eval_id(datasets=datasets, attempt_evaluate=attempt_evaluate, eval_info=eval_info)
       self._eval_handlers.append(m)
 
   @property

{clarifai-10.8.9 → clarifai-10.9.0/clarifai.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: clarifai
-Version: 10.8.9
+Version: 10.9.0
 Summary: Clarifai Python SDK
 Home-page: https://github.com/Clarifai/clarifai-python
 Author: Clarifai

{clarifai-10.8.9 → clarifai-10.9.0}/tests/test_app.py

@@ -1,6 +1,5 @@
 import logging
 import os
-import time
 import uuid
 
 import pytest
@@ -79,6 +78,11 @@ class TestApp:
     all_apps = list(client.list_apps())
     assert len(all_apps) > 0
 
+  # TODO To resolve `Insufficient scopes` error
+  # def test_app_input_count(self, app):
+  #   input_count = app.get_input_count()
+  #   assert input_count == 41
+
   def test_get_model(self, client):
     model = client.app(app_id=MAIN_APP_ID).model(model_id=GENERAL_MODEL_ID)
     versions = list(model.list_versions())
@@ -126,8 +130,9 @@ class TestApp:
 
   def test_get_dataset(self, create_app):
     dataset = create_app.dataset(dataset_id=CREATE_DATASET_ID)
+    dataset.create_version()
     versions = list(dataset.list_versions())
-    assert len(versions) == 0  #test for list_versions
+    assert len(versions) == 1  #test for list_versions
     assert dataset.id == CREATE_DATASET_ID and dataset.app_id == CREATE_APP_ID and dataset.user_id == CREATE_APP_USER_ID
 
   def test_get_module(self, create_app):
@@ -144,16 +149,6 @@ class TestApp:
     all_concept_relations = list(create_app.search_concept_relations(show_tree=True))
     assert len(all_concept_relations) == 1
 
-  def test_export_dataset(self, create_app):
-    dataset = create_app.dataset(dataset_id=CREATE_DATASET_ID)
-    dataset_demo_version = dataset.create_version()
-    versions = list(dataset.list_versions())
-    time.sleep(5)
-    dataset_demo_version.export(save_path='tests/output_demo.zip')
-    assert len(versions) == 1  #test for create_version
-    assert os.path.exists('tests/output_demo.zip') is True
-    os.remove('tests/output_demo.zip')
-
   def test_patch_app(self, caplog):
     with caplog.at_level(logging.INFO):
       User(user_id=CREATE_APP_USER_ID).patch_app(

{clarifai-10.8.9 → clarifai-10.9.0}/tests/test_data_upload.py

@@ -1,11 +1,14 @@
 import logging
 import os
+import time
 import uuid
 
 import pytest
 from google.protobuf.struct_pb2 import Struct
 
 from clarifai.client.user import User
+from clarifai.datasets.upload.loaders.coco_detection import COCODetectionDataLoader
+from clarifai.datasets.upload.loaders.imagenet_classification import ImageNetDataLoader
 from clarifai.datasets.upload.utils import load_module_dataloader
 
 CREATE_APP_USER_ID = os.environ["CLARIFAI_USER_ID"]
@@ -25,6 +28,8 @@ AUDIO_FILE_PATH = os.path.dirname(__file__) + "/assets/sample.mp3"
 CSV_FILE_PATH = os.path.dirname(__file__) + "/assets/sample.csv"
 FOLDER_PATH = os.path.dirname(__file__) + "/assets/test"
 MODULE_DIR = os.path.dirname(__file__) + "/assets/voc"
+COCO_DET_DIR = os.path.dirname(__file__) + "/assets/coco_detection"
+IMAGENET_DIR = os.path.dirname(__file__) + "/assets/imagenet_classification"
 TEXTS_FOLDER_PATH = os.path.dirname(__file__) + "/assets/sample_texts"
 
 
@@ -230,14 +235,42 @@ class Testdataupload:
 
   def test_upload_dataset(self, caplog):
     dataloader = load_module_dataloader(module_dir=MODULE_DIR, split="train")
-    self.dataset.upload_dataset(dataloader)
+    self.dataset.upload_dataset(dataloader, get_upload_status=True, log_warnings=True)
     uploaded_inputs = list(self.input_object.list_inputs())
     annotations = list(self.input_object.list_annotations(batch_input=uploaded_inputs))
     with caplog.at_level(logging.INFO):
       self.input_object.delete_inputs(uploaded_inputs)
       assert "Inputs Deleted" in caplog.text  # Testing delete inputs action
     assert len(uploaded_inputs) == 10  # 3 inputs are uploaded from the folder
-    assert len(annotations) == 28  # Test for list annotatoins
+    assert len(annotations) == 28  # Test for list annotations
+
+  def test_upload_coco_dataset(self, caplog):
+    dataloader = COCODetectionDataLoader(
+        images_dir=os.path.join(COCO_DET_DIR, "images"),
+        label_filepath=os.path.join(COCO_DET_DIR, "instances_default.json"))
+    self.dataset.upload_dataset(dataloader)
+    uploaded_inputs = list(self.input_object.list_inputs())
+    annotations = list(self.input_object.list_annotations(batch_input=uploaded_inputs))
+    with caplog.at_level(logging.INFO):
+      self.input_object.delete_inputs(uploaded_inputs)
+      assert "Inputs Deleted" in caplog.text  # Testing delete inputs action
+    assert len(uploaded_inputs) == 3  # 3 inputs are uploaded from the folder
+    assert len(annotations) == 7  # Test for list annotations
+
+  def test_upload_imagenet_dataset(self):
+    dataloader = ImageNetDataLoader(data_dir=IMAGENET_DIR)
+    self.dataset.upload_dataset(dataloader)
+    uploaded_inputs = list(self.input_object.list_inputs())
+    annotations = list(self.input_object.list_annotations(batch_input=uploaded_inputs))
+    assert len(uploaded_inputs) == 10  # 10 inputs are uploaded from the folder
+    assert len(annotations) == 10  # Test for list annotations
+
+  def test_export_dataset(self):
+    dataset_demo_version = self.dataset.create_version()
+    time.sleep(5)
+    dataset_demo_version.export(save_path='tests/output_demo.zip')
+    assert os.path.exists('tests/output_demo.zip') is True
+    os.remove('tests/output_demo.zip')
 
   @classmethod
   def teardown_class(self):

clarifai-10.9.0/tests/test_eval.py

@@ -0,0 +1,129 @@
+import logging
+import os
+import time
+import uuid
+
+import pytest
+from clarifai_grpc.grpc.api.status import status_code_pb2
+
+from clarifai.client.user import User
+from clarifai.utils.evaluation import EvalResultCompare
+
+CREATE_APP_USER_ID = os.environ["CLARIFAI_USER_ID"]
+NOW = uuid.uuid4().hex[:10]
+CREATE_APP_ID = f"ci_input_app_{NOW}"
+CREATE_DATASET_ID = "ci_input_test_dataset"
+CREATE_DATASET_NEW_ID = "ci_input_test_dataset_new"
+CREATE_MODEL_ID = "ci_input_test_model_1"
+CSV_FILE_PATH = os.path.dirname(__file__) + "/assets/sample.csv"
+
+
+def create_app():
+  client = User(user_id=CREATE_APP_USER_ID)
+  return client.create_app(app_id=CREATE_APP_ID, base_workflow="Universal")
+
+
+@pytest.mark.requires_secrets
+class TestEval:
+  """Tests for model evaluating.
+  """
+
+  @classmethod
+  def setup_class(self):
+    self.app = create_app()
+    self.input_object = self.app.inputs()
+    self.dataset = self.app.create_dataset(dataset_id=CREATE_DATASET_ID)
+    self.dataset_new = self.app.create_dataset(dataset_id=CREATE_DATASET_NEW_ID)
+    self.model = self.app.create_model(
+        model_id=CREATE_MODEL_ID, model_type_id='embedding-classifier')
+
+  def test_evaluate(self, caplog):
+    # Prepare dataset
+    self.dataset.upload_from_csv(
+        csv_path=CSV_FILE_PATH, input_type='text', csv_type='raw', labels=True)
+    dataset_version = self.dataset.create_version()
+    self.dataset_new.upload_from_csv(
+        csv_path=CSV_FILE_PATH, input_type='text', csv_type='raw', labels=True)
+    self.dataset_new.create_version()
+    concepts = [concept.id for concept in self.app.list_concepts()]
+    # Prepare for training
+    self.model.get_params(save_to='tests/assets/model_params_for_eval.yaml')
+    assert len(concepts) == 2  #test data upload for training
+    self.model.update_params(
+        dataset_id=CREATE_DATASET_ID,
+        concepts=concepts,
+        dataset_version_id=dataset_version.version.id)
+    with caplog.at_level(logging.INFO):
+      model_version_id = self.model.train()
+      assert "Model Training Started" in caplog.text  #test model training
+
+    while True:
+      status = self.model.training_status(version_id=model_version_id, training_logs=True)
+      if status.code == 21106:  #MODEL_TRAINING_FAILED
+        break
+      elif status.code == 21100:  #MODEL_TRAINED
+        break
+      else:
+        time.sleep(2)
+
+    # Test eval
+    ## Test list eval
+    all_evals = self.model.list_evaluations()
+    assert len(all_evals) == 0
+
+    ## Test evaluate
+    self.model.evaluate(dataset=self.dataset, eval_id="one", eval_info={"use_kfold": False})
+    all_evals = self.model.list_evaluations()
+    assert len(all_evals) == 1
+
+    ## Test get_eval_by_id
+    time.time()
+    while True:
+      response1 = self.model.get_eval_by_id("one")
+      if (response1.status.code == status_code_pb2.MODEL_EVALUATED):
+        break
+      else:
+        time.sleep(2)
+
+    ## Test get_eval_by_dataset
+    dataset_eval = self.model.get_eval_by_dataset(dataset=self.dataset)
+    assert dataset_eval[0].id == "one" and dataset_eval[0].status.code == status_code_pb2.MODEL_EVALUATED
+
+    ## Test get_raw_eval
+    with pytest.raises(Exception) as e:
+      self.model.get_raw_eval(dataset=self.dataset)
+    assert "method only supports model types ['visual-classifier', 'text-classifier', 'visual-detector']" in str(
+        e.value).lower()
+
+    ## Test get_latest_eval
+    current_eval = self.model.get_latest_eval(
+        label_counts=True,
+        test_set=True,
+        binary_metrics=True,
+        confusion_matrix=True,
+        metrics_by_area=True,
+        metrics_by_class=True)
+    assert current_eval.id == "one" and current_eval.status.code == status_code_pb2.MODEL_EVALUATED
+
+    ## Test EvalResultCompare
+    eval_result = EvalResultCompare(
+        models=[self.model],
+        datasets=[self.dataset, self.dataset_new],
+        attempt_evaluate=True,
+        eval_info={
+            "use_kfold": False
+        })
+    eval_result.all('tests/eval/', overwrite=True)
+    assert os.path.exists('tests/eval/conf-0.5') is True
+
+    # cleanup
+    with caplog.at_level(logging.INFO):
+      self.model.delete_version(version_id=model_version_id)
+      assert "Model Version Deleted" in caplog.text  #test model version deletion
+    os.remove('tests/assets/model_params_for_eval.yaml')
+
+  @classmethod
+  def teardown_class(self):
+    self.app.delete_model(model_id=CREATE_MODEL_ID)
+    self.app.delete_dataset(dataset_id=CREATE_DATASET_ID)
+    User(user_id=CREATE_APP_USER_ID).delete_app(app_id=CREATE_APP_ID)

clarifai-10.8.9/clarifai/__init__.py

@@ -1 +0,0 @@
-__version__ = "10.8.9"

clarifai-10.8.9/tests/test_eval.py

@@ -1,90 +0,0 @@
-import logging
-import os
-import uuid
-
-import pytest
-
-from clarifai.client.user import User
-
-CREATE_APP_USER_ID = os.environ["CLARIFAI_USER_ID"]
-NOW = uuid.uuid4().hex[:10]
-CREATE_APP_ID = f"ci_input_app_{NOW}"
-CREATE_DATASET_ID = "ci_input_test_dataset"
-CREATE_MODEL_ID = "ci_input_test_model_1"
-CSV_FILE_PATH = os.path.dirname(__file__) + "/assets/sample.csv"
-
-
-def create_app():
-  client = User(user_id=CREATE_APP_USER_ID)
-  return client.create_app(app_id=CREATE_APP_ID, base_workflow="Empty")
-
-
-@pytest.mark.requires_secrets
-class TestEval:
-  """Tests for model evaluating.
-  """
-
-  @classmethod
-  def setup_class(self):
-    self.app = create_app()
-    self.input_object = self.app.inputs()
-    self.dataset = self.app.create_dataset(dataset_id=CREATE_DATASET_ID)
-    self.model = self.app.create_model(model_id=CREATE_MODEL_ID, model_type_id='text-classifier')
-
-  def test_evaluate(self, caplog):
-    # Prepare dataset
-    self.dataset.upload_from_csv(
-        csv_path=CSV_FILE_PATH, input_type='text', csv_type='raw', labels=True)
-    dataset_version = self.dataset.create_version()
-    concepts = [concept.id for concept in self.app.list_concepts()]
-    # Prepare for training
-    self.model.get_params(
-        template='HF_GPTNeo_125m_lora', save_to='tests/assets/model_params_for_eval.yaml')
-    param_info = self.model.get_param_info(param='tokenizer_config')
-    assert param_info['param'] == 'tokenizer_config'  #test get param info
-    assert len(concepts) == 2  #test data upload for training
-    self.model.update_params(
-        dataset_id=CREATE_DATASET_ID,
-        concepts=concepts,
-        dataset_version_id=dataset_version.version.id)
-    with caplog.at_level(logging.INFO):
-      model_version_id = self.model.train()
-      assert "Model Training Started" in caplog.text  #test model training
-
-    # Test eval
-    ## Test list eval
-    with pytest.raises(Exception) as e:
-      self.model.list_evaluations()
-    assert "model version is empty" in str(e.value).lower()
-
-    self.model.model_info.model_version.id = model_version_id
-    with pytest.raises(Exception) as e:
-      self.model.list_evaluations()
-    assert "model not yet trained" in str(e.value).lower()
-    ## Test evaluate
-    with pytest.raises(Exception) as e:
-      self.model.evaluate(dataset_id=CREATE_DATASET_ID)
-    assert "model not yet trained" in str(e.value).lower()
-
-    ## Test get eval
-    with pytest.raises(Exception) as e:
-      self.model.get_latest_eval(
-          label_counts=True,
-          test_set=True,
-          binary_metrics=True,
-          confusion_matrix=True,
-          metrics_by_area=True,
-          metrics_by_class=True)
-    assert "model not yet trained" in str(e.value).lower()
-
-    # cleanup
-    with caplog.at_level(logging.INFO):
-      self.model.delete_version(version_id=model_version_id)
-      assert "Model Version Deleted" in caplog.text  #test model version deletion
-    os.remove('tests/assets/model_params_for_eval.yaml')
-
-  @classmethod
-  def teardown_class(self):
-    self.app.delete_dataset(dataset_id=CREATE_DATASET_ID)
-    self.app.delete_model(model_id=CREATE_MODEL_ID)
-    User(user_id=CREATE_APP_USER_ID).delete_app(app_id=CREATE_APP_ID)