clarifai 10.8.9__tar.gz → 10.9.1__tar.gz

{clarifai-10.8.9/clarifai.egg-info → clarifai-10.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: clarifai
-Version: 10.8.9
+Version: 10.9.1
 Summary: Clarifai Python SDK
 Home-page: https://github.com/Clarifai/clarifai-python
 Author: Clarifai
@@ -20,7 +20,7 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: clarifai-grpc>=10.8.7
+Requires-Dist: clarifai-grpc>=10.8.8
 Requires-Dist: clarifai-protocol>=0.0.6
 Requires-Dist: numpy>=1.22.0
 Requires-Dist: tqdm>=4.65.0

clarifai-10.9.1/clarifai/__init__.py

@@ -0,0 +1 @@
+__version__ = "10.9.1"

{clarifai-10.8.9 → clarifai-10.9.1}/clarifai/datasets/upload/loaders/imagenet_classification.py

@@ -24,6 +24,10 @@ class ImageNetDataLoader(ClarifaiDataLoader):
 
     self.load_data()
 
+  @property
+  def task(self):
+    return "visual_classification"
+
   def load_data(self):
     #Creating label map
     with open(os.path.join(self.data_dir, "LOC_synset_mapping.txt")) as _file:
@@ -54,5 +58,5 @@ class ImageNetDataLoader(ClarifaiDataLoader):
   def __getitem__(self, idx):
     return VisualClassificationFeatures(
         image_path=self.image_paths[idx],
-        label=self.concepts[idx],
+        labels=self.concepts[idx],
         id=self.image_paths[idx].split('.')[0].split('/')[-1])

clarifai-10.9.1/clarifai/runners/models/model_run_locally.py

@@ -0,0 +1,194 @@
+import argparse
+import importlib.util
+import inspect
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import traceback
+import venv
+
+from clarifai_grpc.grpc.api import resources_pb2, service_pb2
+from clarifai_grpc.grpc.api.status import status_code_pb2, status_pb2
+from clarifai_protocol import BaseRunner
+
+from clarifai.runners.models.model_upload import ModelUploader
+from clarifai.utils.logging import logger
+
+
+class ModelRunLocally:
+
+  def __init__(self, model_path):
+    self.model_path = model_path
+    self.requirements_file = os.path.join(self.model_path, "requirements.txt")
+    self.venv_dir, self.temp_dir = self.create_temp_venv()
+    self.python_executable = os.path.join(self.venv_dir, "bin", "python")
+
+  def create_temp_venv(self):
+    """Create a temporary virtual environment."""
+    logger.info("Creating temporary virtual environment...")
+    temp_dir = tempfile.mkdtemp()
+    venv_dir = os.path.join(temp_dir, "venv")
+    venv.create(venv_dir, with_pip=True)
+
+    logger.info(f"Created temporary virtual environment at {venv_dir}")
+    return venv_dir, temp_dir
+
+  def install_requirements(self):
+    """Install the dependencies from requirements.txt and Clarifai."""
+    pip_executable = os.path.join(self.venv_dir, "bin", "pip")
+    try:
+      logger.info(
+          f"Installing requirements from {self.requirements_file}... in the virtual environment {self.venv_dir}"
+      )
+      subprocess.check_call([pip_executable, "install", "-r", self.requirements_file])
+      logger.info("Installing Clarifai package...")
+      subprocess.check_call([pip_executable, "install", "clarifai"])
+      logger.info("Requirements installed successfully!")
+    except subprocess.CalledProcessError as e:
+      logger.error(f"Error installing requirements: {e}")
+      self.clean_up()
+      sys.exit(1)
+
+  def _get_model_runner(self):
+    """Dynamically import the runner class from the model file."""
+
+    # import the runner class that to be implement by the user
+    runner_path = os.path.join(self.model_path, "1", "model.py")
+
+    # arbitrary name given to the module to be imported
+    module = "runner_module"
+
+    spec = importlib.util.spec_from_file_location(module, runner_path)
+    runner_module = importlib.util.module_from_spec(spec)
+    sys.modules[module] = runner_module
+    spec.loader.exec_module(runner_module)
+
+    # Find all classes in the model.py file that are subclasses of BaseRunner
+    classes = [
+        cls for _, cls in inspect.getmembers(runner_module, inspect.isclass)
+        if issubclass(cls, BaseRunner) and cls.__module__ == runner_module.__name__
+    ]
+
+    #  Ensure there is exactly one subclass of BaseRunner in the model.py file
+    if len(classes) != 1:
+      raise Exception("Expected exactly one subclass of BaseRunner, found: {}".format(
+          len(classes)))
+
+    MyRunner = classes[0]
+    return MyRunner
+
+  def _build_request(self):
+    """Create a mock inference request for testing the model."""
+
+    uploader = ModelUploader(self.model_path)
+    model_version_proto = uploader.get_model_version_proto()
+    model_version_proto.id = "model_version"
+
+    return service_pb2.PostModelOutputsRequest(
+        model=resources_pb2.Model(model_version=model_version_proto),
+        inputs=[
+            resources_pb2.Input(data=resources_pb2.Data(
+                text=resources_pb2.Text(raw="How many people live in new york?"),
+                image=resources_pb2.Image(url="https://samples.clarifai.com/metro-north.jpg"),
+                audio=resources_pb2.Audio(url="https://samples.clarifai.com/GoodMorning.wav"),
+            ))
+        ],
+    )
+
+  def _run_model_inference(self, runner):
+    """Perform inference using the runner."""
+    request = self._build_request()
+
+    try:
+      return runner.predict(request)
+    except Exception as e:
+      logger.error(f"Model Prediction failed: {e}")
+      traceback.print_exc()
+      return service_pb2.MultiOutputResponse(status=status_pb2.Status(
+          code=status_code_pb2.MODEL_PREDICTION_FAILED,
+          description="Prediction failed",
+          details="",
+          internal_details=str(e),
+      ))
+
+  def _run_test(self):
+    """Test the model locally by making a prediction."""
+    # validate that we have checkpoints downloaded before constructing MyRunner
+    uploader = ModelUploader(self.model_path)
+    uploader.download_checkpoints()
+    # construct MyRunner which will call load_model()
+    MyRunner = self._get_model_runner()
+    runner = MyRunner(
+        runner_id="n/a",
+        nodepool_id="n/a",
+        compute_cluster_id="n/a",
+    )
+    runner.load_model()
+
+    # send an inference.
+    response = self._run_model_inference(runner)
+    if response.outputs[0].status.code != status_code_pb2.SUCCESS:
+      logger.error(f"Moddel Prediction failed: {response}")
+    else:
+      logger.info(f"Model Prediction succeeded: {response}")
+
+  def test_model(self):
+    """Test the model by running it locally in the virtual environment."""
+    command = [
+        self.python_executable,
+        "-c",
+        f"import sys; sys.path.append('{os.path.dirname(os.path.abspath(__file__))}'); "
+        f"from model_run_locally import ModelRunLocally; ModelRunLocally('{self.model_path}')._run_test()",
+    ]
+    try:
+      logger.info("Testing the model locally...")
+      subprocess.check_call(command)
+      logger.info("Model tested successfully!")
+    except subprocess.CalledProcessError as e:
+      logger.error(f"Error testing the model: {e}")
+      sys.exit(1)
+
+  # run the model server
+  def run_model_server(self):
+    """Run the Clarifai Runners's model server."""
+
+    command = [
+        self.python_executable, "-m", "clarifai.runners.server", "--model_path", self.model_path,
+        "--start_dev_server"
+    ]
+    try:
+      logger.info(f"Starting model server with model at {self.model_path}...")
+      subprocess.check_call(command)
+      logger.info("Model server started successfully!")
+    except subprocess.CalledProcessError as e:
+      logger.error(f"Error running model server: {e}")
+      self.clean_up()
+      sys.exit(1)
+
+  def clean_up(self):
+    """Clean up the temporary virtual environment."""
+    if os.path.exists(self.temp_dir):
+      logger.info("Cleaning up temporary virtual environment...")
+      shutil.rmtree(self.temp_dir)
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+      '--model_path', type=str, required=True, help='Path of the model folder to upload')
+  args = parser.parse_args()
+
+  model_path = args.model_path
+  manager = ModelRunLocally(model_path)
+
+  try:
+    manager.install_requirements()
+    manager.test_model()
+  finally:
+    manager.clean_up()
+
+
+if __name__ == "__main__":
+  main()

{clarifai-10.8.9 → clarifai-10.9.1}/clarifai/runners/models/model_upload.py

@@ -10,7 +10,7 @@ from google.protobuf import json_format
 from rich import print
 
 from clarifai.client import BaseClient
-from clarifai.runners.utils.loader import HuggingFaceLoarder
+from clarifai.runners.utils.loader import HuggingFaceLoader
 from clarifai.urls.helper import ClarifaiUrlHelper
 from clarifai.utils.logging import logger
 
@@ -59,6 +59,27 @@ class ModelUploader:
       config = yaml.safe_load(file)
     return config
 
+  def _validate_config_checkpoints(self):
+    if not self.config.get("checkpoints"):
+      logger.info("No checkpoints specified in the config file")
+      return None
+
+    assert "type" in self.config.get("checkpoints"), "No loader type specified in the config file"
+    loader_type = self.config.get("checkpoints").get("type")
+    if not loader_type:
+      logger.info("No loader type specified in the config file for checkpoints")
+    assert loader_type == "huggingface", "Only huggingface loader supported for now"
+    if loader_type == "huggingface":
+      assert "repo_id" in self.config.get("checkpoints"), "No repo_id specified in the config file"
+      repo_id = self.config.get("checkpoints").get("repo_id")
+
+      # prefer env var for HF_TOKEN but if not provided then use the one from config.yaml if any.
+      if 'HF_TOKEN' in os.environ:
+        hf_token = os.environ['HF_TOKEN']
+      else:
+        hf_token = self.config.get("checkpoints").get("hf_token", None)
+      return repo_id, hf_token
+
   @property
   def client(self):
     if self._client is None:
@@ -180,27 +201,9 @@ class ModelUploader:
     return f"{self.folder}.tar.gz"
 
   def download_checkpoints(self):
-    if not self.config.get("checkpoints"):
-      logger.info("No checkpoints specified in the config file")
-      return
-
-    assert "type" in self.config.get("checkpoints"), "No loader type specified in the config file"
-    loader_type = self.config.get("checkpoints").get("type")
-    if not loader_type:
-      logger.info("No loader type specified in the config file for checkpoints")
-    assert loader_type == "huggingface", "Only huggingface loader supported for now"
-    if loader_type == "huggingface":
-      assert "repo_id" in self.config.get("checkpoints"), "No repo_id specified in the config file"
-      repo_id = self.config.get("checkpoints").get("repo_id")
-
-      # prefer env var for HF_TOKEN but if not provided then use the one from config.yaml if any.
-      if 'HF_TOKEN' in os.environ:
-        hf_token = os.environ['HF_TOKEN']
-      else:
-        hf_token = self.config.get("checkpoints").get("hf_token", None)
-        assert hf_token != 'hf_token', "The default 'hf_token' is not valid. Please provide a valid token or leave that field out of config.yaml if not needed."
-      loader = HuggingFaceLoarder(repo_id=repo_id, token=hf_token)
-
+    repo_id, hf_token = self._validate_config_checkpoints()
+    if repo_id and hf_token:
+      loader = HuggingFaceLoader(repo_id=repo_id, token=hf_token)
       success = loader.download_checkpoints(self.checkpoint_path)
 
       if not success:
@@ -232,7 +235,7 @@ class ModelUploader:
     concepts = config.get('concepts')
     logger.info(f"Updated config.yaml with {len(concepts)} concepts.")
 
-  def _get_model_version_proto(self):
+  def get_model_version_proto(self):
 
     model_version_proto = resources_pb2.ModelVersion(
         pretrained_model_config=resources_pb2.PretrainedModelConfig(),
@@ -242,8 +245,7 @@ class ModelUploader:
     model_type_id = self.config.get('model').get('model_type_id')
     if model_type_id in self.CONCEPTS_REQUIRED_MODEL_TYPE:
 
-      loader = HuggingFaceLoarder()
-      labels = loader.fetch_labels(self.checkpoint_path)
+      labels = HuggingFaceLoader.fetch_labels(self.checkpoint_path)
       # sort the concepts by id and then update the config file
       labels = sorted(labels.items(), key=lambda x: int(x[0]))
 
@@ -258,6 +260,21 @@ class ModelUploader:
     file_path = f"{self.folder}.tar.gz"
     logger.info(f"Will tar it into file: {file_path}")
 
+    model_type_id = self.config.get('model').get('model_type_id')
+    repo_id, hf_token = self._validate_config_checkpoints()
+
+    loader = HuggingFaceLoader(repo_id=repo_id, token=hf_token)
+
+    if not download_checkpoints and not loader.validate_download(self.checkpoint_path) and (
+        model_type_id in self.CONCEPTS_REQUIRED_MODEL_TYPE) and 'concepts' not in self.config:
+      logger.error(
+          f"Model type {model_type_id} requires concepts to be specified in the config file or download the model checkpoints to infer the concepts."
+      )
+      input("Press Enter to download the checkpoints to infer the concepts and continue...")
+      self.download_checkpoints()
+
+    model_version_proto = self.get_model_version_proto()
+
     if download_checkpoints:
       tar_cmd = f"tar --exclude=*~ -czvf {self.tar_file} -C {self.folder} ."
     else:  # we don't want to send the checkpoints up even if they are in the folder.
@@ -268,8 +285,6 @@ class ModelUploader:
     os.system(tar_cmd)
     logger.info("Tarring complete, about to start upload.")
 
-    model_version_proto = self._get_model_version_proto()
-
     file_size = os.path.getsize(self.tar_file)
     logger.info(f"Size of the tar is: {file_size} bytes")
 

{clarifai-10.8.9 → clarifai-10.9.1}/clarifai/runners/utils/loader.py

@@ -6,7 +6,9 @@ import subprocess
 from clarifai.utils.logging import logger
 
 
-class HuggingFaceLoarder:
+class HuggingFaceLoader:
+
+  HF_DOWNLOAD_TEXT = "The 'huggingface_hub' package is not installed. Please install it using 'pip install huggingface_hub'."
 
   def __init__(self, repo_id=None, token=None):
     self.repo_id = repo_id
@@ -14,9 +16,7 @@ class HuggingFaceLoarder:
     if token:
       try:
         if importlib.util.find_spec("huggingface_hub") is None:
-          raise ImportError(
-              "The 'huggingface_hub' package is not installed. Please install it using 'pip install huggingface_hub'."
-          )
+          raise ImportError(self.HF_DOWNLOAD_TEXT)
         os.environ['HF_TOKEN'] = token
         subprocess.run(f'huggingface-cli login --token={os.environ["HF_TOKEN"]}', shell=True)
       except Exception as e:
@@ -27,9 +27,7 @@ class HuggingFaceLoarder:
     try:
       from huggingface_hub import snapshot_download
     except ImportError:
-      raise ImportError(
-          "The 'huggingface_hub' package is not installed. Please install it using 'pip install huggingface_hub'."
-      )
+      raise ImportError(self.HF_DOWNLOAD_TEXT)
     if os.path.exists(checkpoint_path) and self.validate_download(checkpoint_path):
       logger.info("Checkpoints already exist")
       return True
@@ -54,20 +52,26 @@ class HuggingFaceLoarder:
 
   def validate_hf_model(self,):
     # check if model exists on HF
-
-    from huggingface_hub import file_exists, repo_exists
+    try:
+      from huggingface_hub import file_exists, repo_exists
+    except ImportError:
+      raise ImportError(self.HF_DOWNLOAD_TEXT)
     return repo_exists(self.repo_id) and file_exists(self.repo_id, 'config.json')
 
   def validate_download(self, checkpoint_path: str):
     # check if model exists on HF
-    from huggingface_hub import list_repo_files
+    try:
+      from huggingface_hub import list_repo_files
+    except ImportError:
+      raise ImportError(self.HF_DOWNLOAD_TEXT)
     checkpoint_dir_files = [
         f for dp, dn, fn in os.walk(os.path.expanduser(checkpoint_path)) for f in fn
     ]
     return (len(checkpoint_dir_files) >= len(list_repo_files(self.repo_id))) and len(
         list_repo_files(self.repo_id)) > 0
 
-  def fetch_labels(self, checkpoint_path: str):
+  @staticmethod
+  def fetch_labels(checkpoint_path: str):
     # Fetch labels for classification, detection and segmentation models
     config_path = os.path.join(checkpoint_path, 'config.json')
     with open(config_path, 'r') as f:

{clarifai-10.8.9 → clarifai-10.9.1}/clarifai/utils/evaluation/helpers.py

@@ -58,13 +58,16 @@ class _BaseEvalResultHandler:
   model: Model
   eval_data: List[resources_pb2.EvalMetrics] = field(default_factory=list)
 
-  def evaluate_and_wait(self, dataset: Dataset):
+  def evaluate_and_wait(self, dataset: Dataset, eval_info: dict = None):
     from tqdm import tqdm
     dataset_id = dataset.id
     dataset_app_id = dataset.app_id
     dataset_user_id = dataset.user_id
     _ = self.model.evaluate(
-        dataset_id=dataset_id, dataset_app_id=dataset_app_id, dataset_user_id=dataset_user_id)
+        dataset_id=dataset_id,
+        dataset_app_id=dataset_app_id,
+        dataset_user_id=dataset_user_id,
+        eval_info=eval_info)
     latest_eval = self.model.list_evaluations()[0]
     excepted = 10
     desc = f"Please wait for the evaluation process between model {self.get_model_name()} and dataset {dataset_user_id}/{dataset_app_id}/{dataset_id} to complete."
@@ -83,7 +86,10 @@ class _BaseEvalResultHandler:
           f"Model has failed to evaluate \n {latest_eval.status}.\nPlease check your dataset inputs!"
       )
 
-  def find_eval_id(self, datasets: List[Dataset] = [], attempt_evaluate: bool = False):
+  def find_eval_id(self,
+                   datasets: List[Dataset] = [],
+                   attempt_evaluate: bool = False,
+                   eval_info: dict = None):
     list_eval_outputs = self.model.list_evaluations()
     self.eval_data = []
     for dataset in datasets:
@@ -117,7 +123,7 @@ class _BaseEvalResultHandler:
       # if not evaluated, but user wants to proceed it
       if not _is_found:
         if attempt_evaluate:
-          self.eval_data.append(self.evaluate_and_wait(dataset))
+          self.eval_data.append(self.evaluate_and_wait(dataset, eval_info=eval_info))
         # otherwise raise error
         else:
           raise Exception(

{clarifai-10.8.9 → clarifai-10.9.1}/clarifai/utils/evaluation/main.py

@@ -53,6 +53,7 @@ class EvalResultCompare:
                models: Union[List[Model], List[str]],
                datasets: Union[Dataset, List[Dataset], str, List[str]],
                attempt_evaluate: bool = False,
+               eval_info: dict = None,
                auth_kwargs: dict = {}):
     assert isinstance(models, list), ValueError("Expected list")
 
@@ -97,7 +98,7 @@ class EvalResultCompare:
         assert self.model_type == model_type, f"Can not compare when model types are different, {self.model_type} != {model_type}"
       m = make_handler_by_type(model_type)(model=model)
       logger.info(f"* {m.get_model_name(pretify=True)}")
-      m.find_eval_id(datasets=datasets, attempt_evaluate=attempt_evaluate)
+      m.find_eval_id(datasets=datasets, attempt_evaluate=attempt_evaluate, eval_info=eval_info)
       self._eval_handlers.append(m)
 
   @property

{clarifai-10.8.9 → clarifai-10.9.1/clarifai.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: clarifai
-Version: 10.8.9
+Version: 10.9.1
 Summary: Clarifai Python SDK
 Home-page: https://github.com/Clarifai/clarifai-python
 Author: Clarifai
@@ -20,7 +20,7 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: clarifai-grpc>=10.8.7
+Requires-Dist: clarifai-grpc>=10.8.8
 Requires-Dist: clarifai-protocol>=0.0.6
 Requires-Dist: numpy>=1.22.0
 Requires-Dist: tqdm>=4.65.0

{clarifai-10.8.9 → clarifai-10.9.1}/clarifai.egg-info/SOURCES.txt

@@ -72,6 +72,7 @@ clarifai/runners/dockerfile_template/Dockerfile.cuda.template
 clarifai/runners/models/__init__.py
 clarifai/runners/models/base_typed_model.py
 clarifai/runners/models/model_class.py
+clarifai/runners/models/model_run_locally.py
 clarifai/runners/models/model_runner.py
 clarifai/runners/models/model_servicer.py
 clarifai/runners/models/model_upload.py

{clarifai-10.8.9 → clarifai-10.9.1}/clarifai.egg-info/requires.txt

@@ -1,4 +1,4 @@
-clarifai-grpc>=10.8.7
+clarifai-grpc>=10.8.8
 clarifai-protocol>=0.0.6
 numpy>=1.22.0
 tqdm>=4.65.0

{clarifai-10.8.9 → clarifai-10.9.1}/requirements.txt

@@ -1,4 +1,4 @@
-clarifai-grpc>=10.8.7
+clarifai-grpc>=10.8.8
 clarifai-protocol>=0.0.6
 numpy>=1.22.0
 tqdm>=4.65.0

{clarifai-10.8.9 → clarifai-10.9.1}/tests/test_app.py

@@ -1,6 +1,5 @@
 import logging
 import os
-import time
 import uuid
 
 import pytest
@@ -79,6 +78,11 @@ class TestApp:
     all_apps = list(client.list_apps())
     assert len(all_apps) > 0
 
+  # TODO To resolve `Insufficient scopes` error
+  # def test_app_input_count(self, app):
+  #   input_count = app.get_input_count()
+  #   assert input_count == 41
+
   def test_get_model(self, client):
     model = client.app(app_id=MAIN_APP_ID).model(model_id=GENERAL_MODEL_ID)
     versions = list(model.list_versions())
@@ -126,8 +130,9 @@ class TestApp:
 
   def test_get_dataset(self, create_app):
     dataset = create_app.dataset(dataset_id=CREATE_DATASET_ID)
+    dataset.create_version()
     versions = list(dataset.list_versions())
-    assert len(versions) == 0  #test for list_versions
+    assert len(versions) == 1  #test for list_versions
     assert dataset.id == CREATE_DATASET_ID and dataset.app_id == CREATE_APP_ID and dataset.user_id == CREATE_APP_USER_ID
 
   def test_get_module(self, create_app):
@@ -144,16 +149,6 @@ class TestApp:
     all_concept_relations = list(create_app.search_concept_relations(show_tree=True))
     assert len(all_concept_relations) == 1
 
-  def test_export_dataset(self, create_app):
-    dataset = create_app.dataset(dataset_id=CREATE_DATASET_ID)
-    dataset_demo_version = dataset.create_version()
-    versions = list(dataset.list_versions())
-    time.sleep(5)
-    dataset_demo_version.export(save_path='tests/output_demo.zip')
-    assert len(versions) == 1  #test for create_version
-    assert os.path.exists('tests/output_demo.zip') is True
-    os.remove('tests/output_demo.zip')
-
   def test_patch_app(self, caplog):
     with caplog.at_level(logging.INFO):
       User(user_id=CREATE_APP_USER_ID).patch_app(

{clarifai-10.8.9 → clarifai-10.9.1}/tests/test_data_upload.py

@@ -1,11 +1,14 @@
 import logging
 import os
+import time
 import uuid
 
 import pytest
 from google.protobuf.struct_pb2 import Struct
 
 from clarifai.client.user import User
+from clarifai.datasets.upload.loaders.coco_detection import COCODetectionDataLoader
+from clarifai.datasets.upload.loaders.imagenet_classification import ImageNetDataLoader
 from clarifai.datasets.upload.utils import load_module_dataloader
 
 CREATE_APP_USER_ID = os.environ["CLARIFAI_USER_ID"]
@@ -25,6 +28,8 @@ AUDIO_FILE_PATH = os.path.dirname(__file__) + "/assets/sample.mp3"
 CSV_FILE_PATH = os.path.dirname(__file__) + "/assets/sample.csv"
 FOLDER_PATH = os.path.dirname(__file__) + "/assets/test"
 MODULE_DIR = os.path.dirname(__file__) + "/assets/voc"
+COCO_DET_DIR = os.path.dirname(__file__) + "/assets/coco_detection"
+IMAGENET_DIR = os.path.dirname(__file__) + "/assets/imagenet_classification"
 TEXTS_FOLDER_PATH = os.path.dirname(__file__) + "/assets/sample_texts"
 
 
@@ -230,14 +235,42 @@ class Testdataupload:
 
   def test_upload_dataset(self, caplog):
     dataloader = load_module_dataloader(module_dir=MODULE_DIR, split="train")
-    self.dataset.upload_dataset(dataloader)
+    self.dataset.upload_dataset(dataloader, get_upload_status=True, log_warnings=True)
     uploaded_inputs = list(self.input_object.list_inputs())
     annotations = list(self.input_object.list_annotations(batch_input=uploaded_inputs))
     with caplog.at_level(logging.INFO):
       self.input_object.delete_inputs(uploaded_inputs)
       assert "Inputs Deleted" in caplog.text  # Testing delete inputs action
     assert len(uploaded_inputs) == 10  # 3 inputs are uploaded from the folder
-    assert len(annotations) == 28  # Test for list annotatoins
+    assert len(annotations) == 28  # Test for list annotations
+
+  def test_upload_coco_dataset(self, caplog):
+    dataloader = COCODetectionDataLoader(
+        images_dir=os.path.join(COCO_DET_DIR, "images"),
+        label_filepath=os.path.join(COCO_DET_DIR, "instances_default.json"))
+    self.dataset.upload_dataset(dataloader)
+    uploaded_inputs = list(self.input_object.list_inputs())
+    annotations = list(self.input_object.list_annotations(batch_input=uploaded_inputs))
+    with caplog.at_level(logging.INFO):
+      self.input_object.delete_inputs(uploaded_inputs)
+      assert "Inputs Deleted" in caplog.text  # Testing delete inputs action
+    assert len(uploaded_inputs) == 3  # 3 inputs are uploaded from the folder
+    assert len(annotations) == 7  # Test for list annotations
+
+  def test_upload_imagenet_dataset(self):
+    dataloader = ImageNetDataLoader(data_dir=IMAGENET_DIR)
+    self.dataset.upload_dataset(dataloader)
+    uploaded_inputs = list(self.input_object.list_inputs())
+    annotations = list(self.input_object.list_annotations(batch_input=uploaded_inputs))
+    assert len(uploaded_inputs) == 10  # 10 inputs are uploaded from the folder
+    assert len(annotations) == 10  # Test for list annotations
+
+  def test_export_dataset(self):
+    dataset_demo_version = self.dataset.create_version()
+    time.sleep(5)
+    dataset_demo_version.export(save_path='tests/output_demo.zip')
+    assert os.path.exists('tests/output_demo.zip') is True
+    os.remove('tests/output_demo.zip')
 
   @classmethod
   def teardown_class(self):

clarifai-10.9.1/tests/test_eval.py

@@ -0,0 +1,129 @@
+import logging
+import os
+import time
+import uuid
+
+import pytest
+from clarifai_grpc.grpc.api.status import status_code_pb2
+
+from clarifai.client.user import User
+from clarifai.utils.evaluation import EvalResultCompare
+
+CREATE_APP_USER_ID = os.environ["CLARIFAI_USER_ID"]
+NOW = uuid.uuid4().hex[:10]
+CREATE_APP_ID = f"ci_input_app_{NOW}"
+CREATE_DATASET_ID = "ci_input_test_dataset"
+CREATE_DATASET_NEW_ID = "ci_input_test_dataset_new"
+CREATE_MODEL_ID = "ci_input_test_model_1"
+CSV_FILE_PATH = os.path.dirname(__file__) + "/assets/sample.csv"
+
+
+def create_app():
+  client = User(user_id=CREATE_APP_USER_ID)
+  return client.create_app(app_id=CREATE_APP_ID, base_workflow="Universal")
+
+
+@pytest.mark.requires_secrets
+class TestEval:
+  """Tests for model evaluating.
+  """
+
+  @classmethod
+  def setup_class(self):
+    self.app = create_app()
+    self.input_object = self.app.inputs()
+    self.dataset = self.app.create_dataset(dataset_id=CREATE_DATASET_ID)
+    self.dataset_new = self.app.create_dataset(dataset_id=CREATE_DATASET_NEW_ID)
+    self.model = self.app.create_model(
+        model_id=CREATE_MODEL_ID, model_type_id='embedding-classifier')
+
+  def test_evaluate(self, caplog):
+    # Prepare dataset
+    self.dataset.upload_from_csv(
+        csv_path=CSV_FILE_PATH, input_type='text', csv_type='raw', labels=True)
+    dataset_version = self.dataset.create_version()
+    self.dataset_new.upload_from_csv(
+        csv_path=CSV_FILE_PATH, input_type='text', csv_type='raw', labels=True)
+    self.dataset_new.create_version()
+    concepts = [concept.id for concept in self.app.list_concepts()]
+    # Prepare for training
+    self.model.get_params(save_to='tests/assets/model_params_for_eval.yaml')
+    assert len(concepts) == 2  #test data upload for training
+    self.model.update_params(
+        dataset_id=CREATE_DATASET_ID,
+        concepts=concepts,
+        dataset_version_id=dataset_version.version.id)
+    with caplog.at_level(logging.INFO):
+      model_version_id = self.model.train()
+      assert "Model Training Started" in caplog.text  #test model training
+
+    while True:
+      status = self.model.training_status(version_id=model_version_id, training_logs=True)
+      if status.code == 21106:  #MODEL_TRAINING_FAILED
+        break
+      elif status.code == 21100:  #MODEL_TRAINED
+        break
+      else:
+        time.sleep(2)
+
+    # Test eval
+    ## Test list eval
+    all_evals = self.model.list_evaluations()
+    assert len(all_evals) == 0
+
+    ## Test evaluate
+    self.model.evaluate(dataset=self.dataset, eval_id="one", eval_info={"use_kfold": False})
+    all_evals = self.model.list_evaluations()
+    assert len(all_evals) == 1
+
+    ## Test get_eval_by_id
+    time.time()
+    while True:
+      response1 = self.model.get_eval_by_id("one")
+      if (response1.status.code == status_code_pb2.MODEL_EVALUATED):
+        break
+      else:
+        time.sleep(2)
+
+    ## Test get_eval_by_dataset
+    dataset_eval = self.model.get_eval_by_dataset(dataset=self.dataset)
+    assert dataset_eval[0].id == "one" and dataset_eval[0].status.code == status_code_pb2.MODEL_EVALUATED
+
+    ## Test get_raw_eval
+    with pytest.raises(Exception) as e:
+      self.model.get_raw_eval(dataset=self.dataset)
+    assert "method only supports model types ['visual-classifier', 'text-classifier', 'visual-detector']" in str(
+        e.value).lower()
+
+    ## Test get_latest_eval
+    current_eval = self.model.get_latest_eval(
+        label_counts=True,
+        test_set=True,
+        binary_metrics=True,
+        confusion_matrix=True,
+        metrics_by_area=True,
+        metrics_by_class=True)
+    assert current_eval.id == "one" and current_eval.status.code == status_code_pb2.MODEL_EVALUATED
+
+    ## Test EvalResultCompare
+    eval_result = EvalResultCompare(
+        models=[self.model],
+        datasets=[self.dataset, self.dataset_new],
+        attempt_evaluate=True,
+        eval_info={
+            "use_kfold": False
+        })
+    eval_result.all('tests/eval/', overwrite=True)
+    assert os.path.exists('tests/eval/conf-0.5') is True
+
+    # cleanup
+    with caplog.at_level(logging.INFO):
+      self.model.delete_version(version_id=model_version_id)
+      assert "Model Version Deleted" in caplog.text  #test model version deletion
+    os.remove('tests/assets/model_params_for_eval.yaml')
+
+  @classmethod
+  def teardown_class(self):
+    self.app.delete_model(model_id=CREATE_MODEL_ID)
+    self.app.delete_dataset(dataset_id=CREATE_DATASET_ID)
+    User(user_id=CREATE_APP_USER_ID).delete_app(app_id=CREATE_APP_ID)

clarifai-10.8.9/clarifai/__init__.py

@@ -1 +0,0 @@
-__version__ = "10.8.9"

clarifai-10.8.9/tests/test_eval.py

@@ -1,90 +0,0 @@
-import logging
-import os
-import uuid
-
-import pytest
-
-from clarifai.client.user import User
-
-CREATE_APP_USER_ID = os.environ["CLARIFAI_USER_ID"]
-NOW = uuid.uuid4().hex[:10]
-CREATE_APP_ID = f"ci_input_app_{NOW}"
-CREATE_DATASET_ID = "ci_input_test_dataset"
-CREATE_MODEL_ID = "ci_input_test_model_1"
-CSV_FILE_PATH = os.path.dirname(__file__) + "/assets/sample.csv"
-
-
-def create_app():
-  client = User(user_id=CREATE_APP_USER_ID)
-  return client.create_app(app_id=CREATE_APP_ID, base_workflow="Empty")
-
-
-@pytest.mark.requires_secrets
-class TestEval:
-  """Tests for model evaluating.
-  """
-
-  @classmethod
-  def setup_class(self):
-    self.app = create_app()
-    self.input_object = self.app.inputs()
-    self.dataset = self.app.create_dataset(dataset_id=CREATE_DATASET_ID)
-    self.model = self.app.create_model(model_id=CREATE_MODEL_ID, model_type_id='text-classifier')
-
-  def test_evaluate(self, caplog):
-    # Prepare dataset
-    self.dataset.upload_from_csv(
-        csv_path=CSV_FILE_PATH, input_type='text', csv_type='raw', labels=True)
-    dataset_version = self.dataset.create_version()
-    concepts = [concept.id for concept in self.app.list_concepts()]
-    # Prepare for training
-    self.model.get_params(
-        template='HF_GPTNeo_125m_lora', save_to='tests/assets/model_params_for_eval.yaml')
-    param_info = self.model.get_param_info(param='tokenizer_config')
-    assert param_info['param'] == 'tokenizer_config'  #test get param info
-    assert len(concepts) == 2  #test data upload for training
-    self.model.update_params(
-        dataset_id=CREATE_DATASET_ID,
-        concepts=concepts,
-        dataset_version_id=dataset_version.version.id)
-    with caplog.at_level(logging.INFO):
-      model_version_id = self.model.train()
-      assert "Model Training Started" in caplog.text  #test model training
-
-    # Test eval
-    ## Test list eval
-    with pytest.raises(Exception) as e:
-      self.model.list_evaluations()
-    assert "model version is empty" in str(e.value).lower()
-
-    self.model.model_info.model_version.id = model_version_id
-    with pytest.raises(Exception) as e:
-      self.model.list_evaluations()
-    assert "model not yet trained" in str(e.value).lower()
-    ## Test evaluate
-    with pytest.raises(Exception) as e:
-      self.model.evaluate(dataset_id=CREATE_DATASET_ID)
-    assert "model not yet trained" in str(e.value).lower()
-
-    ## Test get eval
-    with pytest.raises(Exception) as e:
-      self.model.get_latest_eval(
-          label_counts=True,
-          test_set=True,
-          binary_metrics=True,
-          confusion_matrix=True,
-          metrics_by_area=True,
-          metrics_by_class=True)
-    assert "model not yet trained" in str(e.value).lower()
-
-    # cleanup
-    with caplog.at_level(logging.INFO):
-      self.model.delete_version(version_id=model_version_id)
-      assert "Model Version Deleted" in caplog.text  #test model version deletion
-    os.remove('tests/assets/model_params_for_eval.yaml')
-
-  @classmethod
-  def teardown_class(self):
-    self.app.delete_dataset(dataset_id=CREATE_DATASET_ID)
-    self.app.delete_model(model_id=CREATE_MODEL_ID)
-    User(user_id=CREATE_APP_USER_ID).delete_app(app_id=CREATE_APP_ID)