clarifai 11.2.0__tar.gz → 11.2.2__tar.gz

{clarifai-11.2.0/clarifai.egg-info → clarifai-11.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: clarifai
-Version: 11.2.0
+Version: 11.2.2
 Summary: Clarifai Python SDK
 Home-page: https://github.com/Clarifai/clarifai-python
 Author: Clarifai
@@ -20,7 +20,7 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: clarifai-grpc>=11.2.5
+Requires-Dist: clarifai-grpc>=11.2.6
 Requires-Dist: clarifai-protocol>=0.0.20
 Requires-Dist: numpy>=1.22.0
 Requires-Dist: tqdm>=4.65.0

clarifai-11.2.2/clarifai/__init__.py

@@ -0,0 +1 @@
+__version__ = "11.2.2"

{clarifai-11.2.0 → clarifai-11.2.2}/clarifai/cli/base.py

@@ -43,17 +43,10 @@ def cli(ctx):
   """Clarifai CLI"""
   ctx.ensure_object(dict)
   config_path = f"{os.getenv('HOME')}/.clarifai/config.yaml"
-  config_dir = os.path.dirname(config_path)
   if os.path.exists(config_path):
     ctx.obj = from_yaml(config_path)
   else:
     ctx.obj = {}
-    if ctx.invoked_subcommand != 'login':
-      logger.error("Error getting config: CLI config file missing")
-      logger.info("Running `login` to set up the CLI configuration.")
-      if not os.path.exists(config_dir):
-        os.makedirs(config_dir)
-      ctx.invoke(login)
 
 
 @cli.command()
@@ -74,16 +67,16 @@ def login(ctx, config, env, user_id):
 
   if 'pat' in ctx.obj:
     os.environ["CLARIFAI_PAT"] = ctx.obj['pat']
-    click.echo("Loaded PAT from config file.")
+    logger.info("Loaded PAT from config file.")
   elif 'CLARIFAI_PAT' in os.environ:
     ctx.obj['pat'] = os.environ["CLARIFAI_PAT"]
-    click.echo("Loaded PAT from environment variable.")
+    logger.info("Loaded PAT from environment variable.")
   else:
     _pat = click.prompt(
         "Get your PAT from https://clarifai.com/settings/security and pass it here", type=str)
     os.environ["CLARIFAI_PAT"] = _pat
     ctx.obj['pat'] = _pat
-    click.echo("PAT saved successfully.")
+    logger.info("PAT saved successfully.")
 
   if user_id:
     ctx.obj['user_id'] = user_id
@@ -97,7 +90,7 @@ def login(ctx, config, env, user_id):
     user_id = click.prompt("Pass the User ID here", type=str)
     os.environ["CLARIFAI_USER_ID"] = user_id
     ctx.obj['user_id'] = user_id
-    click.echo("User ID saved successfully.")
+    logger.info("User ID saved successfully.")
 
   if env:
     ctx.obj['env'] = env
@@ -113,9 +106,14 @@ def login(ctx, config, env, user_id):
     ctx.obj['env'] = 'prod'
     ctx.obj['base_url'] = set_base_url(ctx.obj['env'])
     os.environ["CLARIFAI_API_BASE"] = ctx.obj['base_url']
-    click.echo("Base URL saved successfully.")
+    logger.info("Base URL saved successfully.")
+
+  config_path = f"{os.getenv('HOME')}/.clarifai/config.yaml"
+  config_dir = os.path.dirname(config_path)
+  if not os.path.exists(config_dir):
+    os.makedirs(config_dir)
 
-  dump_yaml(ctx.obj, f"{os.getenv('HOME')}/.clarifai/config.yaml")
+  dump_yaml(ctx.obj, config_path)
 
 
 # Import the CLI commands to register them

{clarifai-11.2.0 → clarifai-11.2.2}/clarifai/cli/compute_cluster.py

@@ -1,7 +1,8 @@
 import click
+
 from clarifai.cli.base import cli
 from clarifai.client.user import User
-from clarifai.utils.cli import display_co_resources
+from clarifai.utils.cli import display_co_resources, validate_context
 
 
 @cli.group(['computecluster', 'cc'])
@@ -24,6 +25,8 @@ def computecluster():
 @click.pass_context
 def create(ctx, config, compute_cluster_id):
   """Create a new Compute Cluster with the given config file."""
+
+  validate_context(ctx)
   user = User(user_id=ctx.obj['user_id'], pat=ctx.obj['pat'], base_url=ctx.obj['base_url'])
   if compute_cluster_id:
     user.create_compute_cluster(config, compute_cluster_id=compute_cluster_id)
@@ -37,6 +40,8 @@ def create(ctx, config, compute_cluster_id):
 @click.pass_context
 def list(ctx, page_no, per_page):
   """List all compute clusters for the user."""
+
+  validate_context(ctx)
   user = User(user_id=ctx.obj['user_id'], pat=ctx.obj['pat'], base_url=ctx.obj['base_url'])
   response = user.list_compute_clusters(page_no, per_page)
   display_co_resources(response, "Compute Cluster")
@@ -47,5 +52,7 @@ def list(ctx, page_no, per_page):
 @click.pass_context
 def delete(ctx, compute_cluster_id):
   """Deletes a compute cluster for the user."""
+
+  validate_context(ctx)
   user = User(user_id=ctx.obj['user_id'], pat=ctx.obj['pat'], base_url=ctx.obj['base_url'])
   user.delete_compute_clusters([compute_cluster_id])

{clarifai-11.2.0 → clarifai-11.2.2}/clarifai/cli/deployment.py

@@ -1,7 +1,7 @@
 import click
 from clarifai.cli.base import cli
 from clarifai.client.nodepool import Nodepool
-from clarifai.utils.cli import display_co_resources, from_yaml
+from clarifai.utils.cli import display_co_resources, from_yaml, validate_context
 
 
 @cli.group(['deployment', 'dpl'])
@@ -29,6 +29,8 @@ def deployment():
 @click.pass_context
 def create(ctx, nodepool_id, config, deployment_id):
   """Create a new Deployment with the given config file."""
+
+  validate_context(ctx)
   if not nodepool_id:
     deployment_config = from_yaml(config)
     nodepool_id = deployment_config['deployment']['nodepools'][0]['id']
@@ -56,6 +58,7 @@ def create(ctx, nodepool_id, config, deployment_id):
 def list(ctx, nodepool_id, page_no, per_page):
   """List all deployments for the nodepool."""
 
+  validate_context(ctx)
   nodepool = Nodepool(
       nodepool_id=nodepool_id,
       user_id=ctx.obj['user_id'],
@@ -76,6 +79,7 @@ def list(ctx, nodepool_id, page_no, per_page):
 def delete(ctx, nodepool_id, deployment_id):
   """Deletes a deployment for the nodepool."""
 
+  validate_context(ctx)
   nodepool = Nodepool(
       nodepool_id=nodepool_id,
       user_id=ctx.obj['user_id'],

{clarifai-11.2.0 → clarifai-11.2.2}/clarifai/cli/model.py

@@ -96,11 +96,14 @@ def download_checkpoints(model_path, out_path, stage):
     help=
     'Keep the Docker image after testing the model locally (applicable for container mode). Defaults to False.'
 )
-def test_locally(model_path, keep_env=False, keep_image=False, mode='env'):
-  """Test model locally.
-
-  MODEL_PATH: Path to the model directory. If not specified, the current directory is used by default.
-  """
+@click.option(
+    '--skip_dockerfile',
+    is_flag=True,
+    help=
+    'Flag to skip generating a dockerfile so that you can manually edit an already created dockerfile. Apply for `--mode conatainer`.',
+)
+def test_locally(model_path, keep_env=False, keep_image=False, mode='env', skip_dockerfile=False):
+  """Test model locally."""
   try:
     from clarifai.runners.models import model_run_locally
     if mode == 'env' and keep_image:
@@ -114,7 +117,11 @@ def test_locally(model_path, keep_env=False, keep_image=False, mode='env'):
     elif mode == "container":
       click.echo("Testing model locally inside a container...")
       model_run_locally.main(
-          model_path, inside_container=True, run_model_server=False, keep_image=keep_image)
+          model_path,
+          inside_container=True,
+          run_model_server=False,
+          keep_image=keep_image,
+          skip_dockerfile=skip_dockerfile)
     click.echo("Model tested successfully.")
   except Exception as e:
     click.echo(f"Failed to test model locally: {e}", err=True)
@@ -154,11 +161,14 @@ def test_locally(model_path, keep_env=False, keep_image=False, mode='env'):
     help=
     'Keep the Docker image after testing the model locally (applicable for container mode). Defaults to False.'
 )
-def run_locally(model_path, port, mode, keep_env, keep_image):
-  """Run the model locally and start a gRPC server to serve the model.
-
-  MODEL_PATH: Path to the model directory. If not specified, the current directory is used by default.
-  """
+@click.option(
+    '--skip_dockerfile',
+    is_flag=True,
+    help=
+    'Flag to skip generating a dockerfile so that you can manually edit an already created dockerfile. Apply for `--mode conatainer`.',
+)
+def run_locally(model_path, port, mode, keep_env, keep_image, skip_dockerfile=False):
+  """Run the model locally and start a gRPC server to serve the model."""
   try:
     from clarifai.runners.models import model_run_locally
     if mode == 'env' and keep_image:
@@ -176,7 +186,8 @@ def run_locally(model_path, port, mode, keep_env, keep_image):
           inside_container=True,
           run_model_server=True,
           port=port,
-          keep_image=keep_image)
+          keep_image=keep_image,
+          skip_dockerfile=skip_dockerfile)
     click.echo(f"Model server started locally from {model_path} in {mode} mode.")
   except Exception as e:
     click.echo(f"Failed to starts model server locally: {e}", err=True)
@@ -230,7 +241,8 @@ def predict(ctx, config, model_id, user_id, app_id, model_url, file_path, url, b
   import json
 
   from clarifai.client.model import Model
-  from clarifai.utils.cli import from_yaml
+  from clarifai.utils.cli import from_yaml, validate_context
+  validate_context(ctx)
   if config:
     config = from_yaml(config)
     model_id, user_id, app_id, model_url, file_path, url, bytes, input_type, compute_cluster_id, nodepool_id, deployment_id, inference_params, output_config = (

{clarifai-11.2.0 → clarifai-11.2.2}/clarifai/cli/nodepool.py

@@ -1,7 +1,7 @@
 import click
 from clarifai.cli.base import cli
 from clarifai.client.compute_cluster import ComputeCluster
-from clarifai.utils.cli import display_co_resources, dump_yaml, from_yaml
+from clarifai.utils.cli import display_co_resources, dump_yaml, from_yaml, validate_context
 
 
 @cli.group(['nodepool', 'np'])
@@ -27,6 +27,7 @@ def nodepool():
 def create(ctx, compute_cluster_id, config, nodepool_id):
   """Create a new Nodepool with the given config file."""
 
+  validate_context(ctx)
   nodepool_config = from_yaml(config)
   if not compute_cluster_id:
     if 'compute_cluster' not in nodepool_config['nodepool']:
@@ -63,6 +64,7 @@ def create(ctx, compute_cluster_id, config, nodepool_id):
 def list(ctx, compute_cluster_id, page_no, per_page):
   """List all nodepools for the user."""
 
+  validate_context(ctx)
   compute_cluster = ComputeCluster(
       compute_cluster_id=compute_cluster_id,
       user_id=ctx.obj['user_id'],
@@ -83,6 +85,7 @@ def list(ctx, compute_cluster_id, page_no, per_page):
 def delete(ctx, compute_cluster_id, nodepool_id):
   """Deletes a nodepool for the user."""
 
+  validate_context(ctx)
   compute_cluster = ComputeCluster(
       compute_cluster_id=compute_cluster_id,
       user_id=ctx.obj['user_id'],

{clarifai-11.2.0 → clarifai-11.2.2}/clarifai/client/app.py

@@ -629,7 +629,7 @@ class App(Lister, BaseClient):
 
     Args:
         model_id (str): The model ID for the model to interact with.
-        model_version_id (str): The model version ID for the model version to interact with.
+        model_version (Dict): The model version ID for the model version to interact with.
 
     Returns:
         Model: A Model object for the existing model ID.

{clarifai-11.2.0 → clarifai-11.2.2}/clarifai/runners/models/model_builder.py

@@ -171,7 +171,14 @@ class ModelBuilder:
 
       # get from config.yaml otherwise fall back to HF_TOKEN env var.
       hf_token = self.config.get("checkpoints").get("hf_token", os.environ.get("HF_TOKEN", None))
-      return loader_type, repo_id, hf_token, when
+
+      allowed_file_patterns = self.config.get("checkpoints").get('allowed_file_patterns', None)
+      if isinstance(allowed_file_patterns, str):
+        allowed_file_patterns = [allowed_file_patterns]
+      ignore_file_patterns = self.config.get("checkpoints").get('ignore_file_patterns', None)
+      if isinstance(ignore_file_patterns, str):
+        ignore_file_patterns = [ignore_file_patterns]
+      return loader_type, repo_id, hf_token, when, allowed_file_patterns, ignore_file_patterns
 
   def _check_app_exists(self):
     resp = self.client.STUB.GetApp(service_pb2.GetAppRequest(user_app_id=self.client.user_app_id))
@@ -216,7 +223,7 @@ class ModelBuilder:
         assert model_type_id in CONCEPTS_REQUIRED_MODEL_TYPE, f"Model type {model_type_id} not supported for concepts"
 
     if self.config.get("checkpoints"):
-      loader_type, _, hf_token, _ = self._validate_config_checkpoints()
+      loader_type, _, hf_token, _, _, _ = self._validate_config_checkpoints()
 
       if loader_type == "huggingface" and hf_token:
         is_valid_token = HuggingFaceLoader.validate_hftoken(hf_token)
@@ -480,8 +487,10 @@ class ModelBuilder:
     if not self.config.get("checkpoints"):
       logger.info("No checkpoints specified in the config file")
       return path
+    clarifai_model_type_id = self.config.get('model').get('model_type_id')
 
-    loader_type, repo_id, hf_token, when = self._validate_config_checkpoints()
+    loader_type, repo_id, hf_token, when, allowed_file_patterns, ignore_file_patterns = self._validate_config_checkpoints(
+    )
     if stage not in ["build", "upload", "runtime"]:
       raise Exception("Invalid stage provided, must be one of ['build', 'upload', 'runtime']")
     if when != stage:
@@ -490,14 +499,18 @@ class ModelBuilder:
       )
       return path
 
-    success = True
+    success = False
     if loader_type == "huggingface":
-      loader = HuggingFaceLoader(repo_id=repo_id, token=hf_token)
+      loader = HuggingFaceLoader(
+          repo_id=repo_id, token=hf_token, model_type_id=clarifai_model_type_id)
       # for runtime default to /tmp path
       if stage == "runtime" and checkpoint_path_override is None:
         checkpoint_path_override = self.default_runtime_checkpoint_path()
       path = checkpoint_path_override if checkpoint_path_override else self.checkpoint_path
-      success = loader.download_checkpoints(path)
+      success = loader.download_checkpoints(
+          path,
+          allowed_file_patterns=allowed_file_patterns,
+          ignore_file_patterns=ignore_file_patterns)
 
     if loader_type:
       if not success:
@@ -569,7 +582,7 @@ class ModelBuilder:
     logger.debug(f"Will tar it into file: {file_path}")
 
     model_type_id = self.config.get('model').get('model_type_id')
-    loader_type, repo_id, hf_token, when = self._validate_config_checkpoints()
+    loader_type, repo_id, hf_token, when, _, _ = self._validate_config_checkpoints()
 
     if (model_type_id in CONCEPTS_REQUIRED_MODEL_TYPE) and 'concepts' not in self.config:
       logger.info(
@@ -617,7 +630,7 @@ class ModelBuilder:
       # First check for the env variable, then try querying huggingface. If all else fails, use the default.
       checkpoint_size = os.environ.get('CHECKPOINT_SIZE_BYTES', 0)
       if not checkpoint_size:
-        _, repo_id, _, _ = self._validate_config_checkpoints()
+        _, repo_id, _, _, _, _ = self._validate_config_checkpoints()
         checkpoint_size = HuggingFaceLoader.get_huggingface_checkpoint_total_size(repo_id)
       if not checkpoint_size:
         checkpoint_size = self.DEFAULT_CHECKPOINT_SIZE

{clarifai-11.2.0 → clarifai-11.2.2}/clarifai/runners/models/model_run_locally.py

@@ -473,21 +473,25 @@ def main(model_path,
          inside_container=False,
          port=8080,
          keep_env=False,
-         keep_image=False):
+         keep_image=False,
+         skip_dockerfile: bool = False):
 
   manager = ModelRunLocally(model_path)
   # get whatever stage is in config.yaml to force download now
   # also always write to where upload/build wants to, not the /tmp folder that runtime stage uses
-  _, _, _, when = manager.builder._validate_config_checkpoints()
+  _, _, _, when, _, _ = manager.builder._validate_config_checkpoints()
   manager.builder.download_checkpoints(
       stage=when, checkpoint_path_override=manager.builder.checkpoint_path)
   if inside_container:
     if not manager.is_docker_installed():
       sys.exit(1)
-    manager.builder.create_dockerfile()
+    if not skip_dockerfile:
+      manager.builder.create_dockerfile()
     image_tag = manager._docker_hash()
-    image_name = f"{manager.config['model']['id']}:{image_tag}"
-    container_name = manager.config['model']['id']
+    model_id = manager.config['model']['id'].lower()
+    # must be in lowercase
+    image_name = f"{model_id}:{image_tag}"
+    container_name = model_id
     if not manager.docker_image_exists(image_name):
       manager.build_docker_image(image_name=image_name)
     try:

{clarifai-11.2.0 → clarifai-11.2.2}/clarifai/runners/utils/loader.py

@@ -6,6 +6,7 @@ import shutil
 
 import requests
 
+from clarifai.runners.utils.const import CONCEPTS_REQUIRED_MODEL_TYPE
 from clarifai.utils.logging import logger
 
 
@@ -13,9 +14,10 @@ class HuggingFaceLoader:
 
   HF_DOWNLOAD_TEXT = "The 'huggingface_hub' package is not installed. Please install it using 'pip install huggingface_hub'."
 
-  def __init__(self, repo_id=None, token=None):
+  def __init__(self, repo_id=None, token=None, model_type_id=None):
     self.repo_id = repo_id
     self.token = token
+    self.clarifai_model_type_id = model_type_id
     if token:
       if self.validate_hftoken(token):
         try:
@@ -43,13 +45,17 @@ class HuggingFaceLoader:
           f"Error setting up Hugging Face token, please make sure you have the correct token: {e}")
       return False
 
-  def download_checkpoints(self, checkpoint_path: str):
+  def download_checkpoints(self,
+                           checkpoint_path: str,
+                           allowed_file_patterns=None,
+                           ignore_file_patterns=None):
     # throw error if huggingface_hub wasn't installed
     try:
       from huggingface_hub import snapshot_download
     except ImportError:
       raise ImportError(self.HF_DOWNLOAD_TEXT)
-    if os.path.exists(checkpoint_path) and self.validate_download(checkpoint_path):
+    if os.path.exists(checkpoint_path) and self.validate_download(
+        checkpoint_path, allowed_file_patterns, ignore_file_patterns):
       logger.info("Checkpoints already exist")
       return True
     else:
@@ -61,10 +67,16 @@ class HuggingFaceLoader:
           return False
 
         self.ignore_patterns = self._get_ignore_patterns()
+        if ignore_file_patterns:
+          if self.ignore_patterns:
+            self.ignore_patterns.extend(ignore_file_patterns)
+          else:
+            self.ignore_patterns = ignore_file_patterns
         snapshot_download(
             repo_id=self.repo_id,
             local_dir=checkpoint_path,
             local_dir_use_symlinks=False,
+            allow_patterns=allowed_file_patterns,
             ignore_patterns=self.ignore_patterns)
         # Remove the `.cache` folder if it exists
         cache_path = os.path.join(checkpoint_path, ".cache")
@@ -75,7 +87,8 @@ class HuggingFaceLoader:
         logger.error(f"Error downloading model checkpoints {e}")
         return False
       finally:
-        is_downloaded = self.validate_download(checkpoint_path)
+        is_downloaded = self.validate_download(checkpoint_path, allowed_file_patterns,
+                                               ignore_file_patterns)
         if not is_downloaded:
           logger.error("Error validating downloaded model checkpoints")
           return False
@@ -109,9 +122,13 @@ class HuggingFaceLoader:
       from huggingface_hub import file_exists, repo_exists
     except ImportError:
       raise ImportError(self.HF_DOWNLOAD_TEXT)
-    return repo_exists(self.repo_id) and file_exists(self.repo_id, 'config.json')
+    if self.clarifai_model_type_id in CONCEPTS_REQUIRED_MODEL_TYPE:
+      return repo_exists(self.repo_id) and file_exists(self.repo_id, 'config.json')
+    else:
+      return repo_exists(self.repo_id)
 
-  def validate_download(self, checkpoint_path: str):
+  def validate_download(self, checkpoint_path: str, allowed_file_patterns: list,
+                        ignore_file_patterns: list):
     # check if model exists on HF
     try:
       from huggingface_hub import list_repo_files
@@ -120,7 +137,20 @@ class HuggingFaceLoader:
     # Get the list of files on the repo
     repo_files = list_repo_files(self.repo_id, token=self.token)
 
+    # Get the list of files on the repo that are allowed
+    if allowed_file_patterns:
+
+      def should_allow(file_path):
+        return any(fnmatch.fnmatch(file_path, pattern) for pattern in allowed_file_patterns)
+
+      repo_files = [f for f in repo_files if should_allow(f)]
+
     self.ignore_patterns = self._get_ignore_patterns()
+    if ignore_file_patterns:
+      if self.ignore_patterns:
+        self.ignore_patterns.extend(ignore_file_patterns)
+      else:
+        self.ignore_patterns = ignore_file_patterns
     # Get the list of files on the repo that are not ignored
     if getattr(self, "ignore_patterns", None):
       patterns = self.ignore_patterns

{clarifai-11.2.0 → clarifai-11.2.2}/clarifai/utils/cli.py

@@ -1,6 +1,7 @@
 import importlib
 import os
 import pkgutil
+import sys
 
 import click
 import yaml
@@ -10,6 +11,8 @@ from rich.panel import Panel
 from rich.style import Style
 from rich.text import Text
 
+from clarifai.utils.logging import logger
+
 
 def from_yaml(filename: str):
   try:
@@ -68,3 +71,9 @@ def display_co_resources(response, resource_type):
         border_style="green",
         width=60)
     console.print(panel)
+
+
+def validate_context(ctx):
+  if ctx.obj == {}:
+    logger.error("CLI config file missing. Run `clarifai login` to set up the CLI config.")
+    sys.exit(1)

{clarifai-11.2.0 → clarifai-11.2.2/clarifai.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: clarifai
-Version: 11.2.0
+Version: 11.2.2
 Summary: Clarifai Python SDK
 Home-page: https://github.com/Clarifai/clarifai-python
 Author: Clarifai
@@ -20,7 +20,7 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: clarifai-grpc>=11.2.5
+Requires-Dist: clarifai-grpc>=11.2.6
 Requires-Dist: clarifai-protocol>=0.0.20
 Requires-Dist: numpy>=1.22.0
 Requires-Dist: tqdm>=4.65.0

{clarifai-11.2.0 → clarifai-11.2.2}/clarifai.egg-info/requires.txt

@@ -1,4 +1,4 @@
-clarifai-grpc>=11.2.5
+clarifai-grpc>=11.2.6
 clarifai-protocol>=0.0.20
 numpy>=1.22.0
 tqdm>=4.65.0

{clarifai-11.2.0 → clarifai-11.2.2}/requirements.txt

@@ -1,4 +1,4 @@
-clarifai-grpc>=11.2.5
+clarifai-grpc>=11.2.6
 clarifai-protocol>=0.0.20
 numpy>=1.22.0
 tqdm>=4.65.0

clarifai-11.2.0/clarifai/__init__.py

@@ -1 +0,0 @@
-__version__ = "11.2.0"