clarifai 11.2.3__py3-none-any.whl → 11.2.3rc2__py3-none-any.whl

clarifai/cli/model.py

@@ -5,7 +5,7 @@ from clarifai.cli.base import cli
 
 @cli.group(['model'])
 def model():
-  """Manage models: upload, test, local dev, predict, etc"""
+  """Manage models: upload, test locally, run locally, predict, and more"""
 
 
 @model.command()
@@ -76,6 +76,31 @@ def download_checkpoints(model_path, out_path, stage):
     required=False,
     default=".",
 )
+@click.option(
+    '--out_path',
+    type=click.Path(exists=False),
+    required=False,
+    default=None,
+    help='Path to write the method signature defitions to. If not provided, use stdout.')
+def signatures(model_path, out_path):
+  """Generate method signatures for the model."""
+
+  from clarifai.runners.models.model_builder import ModelBuilder
+  builder = ModelBuilder(model_path, download_validation_only=True)
+  signatures = builder.method_signatures_yaml()
+  if out_path:
+    with open(out_path, 'w') as f:
+      f.write(signatures)
+  else:
+    click.echo(signatures)
+
+
+@model.command()
+@click.option(
+    '--model_path',
+    type=click.Path(exists=True),
+    required=True,
+    help='Path to the model directory.')
 @click.option(
     '--mode',
     type=click.Choice(['env', 'container'], case_sensitive=False),
@@ -96,14 +121,11 @@ def download_checkpoints(model_path, out_path, stage):
     help=
     'Keep the Docker image after testing the model locally (applicable for container mode). Defaults to False.'
 )
-@click.option(
-    '--skip_dockerfile',
-    is_flag=True,
-    help=
-    'Flag to skip generating a dockerfile so that you can manually edit an already created dockerfile. Apply for `--mode conatainer`.',
-)
-def test_locally(model_path, keep_env=False, keep_image=False, mode='env', skip_dockerfile=False):
-  """Test model locally."""
+def test_locally(model_path, keep_env=False, keep_image=False, mode='env'):
+  """Test model locally.
+
+  MODEL_PATH: Path to the model directory. If not specified, the current directory is used by default.
+  """
   try:
     from clarifai.runners.models import model_run_locally
     if mode == 'env' and keep_image:
@@ -117,11 +139,7 @@ def test_locally(model_path, keep_env=False, keep_image=False, mode='env', skip_
     elif mode == "container":
       click.echo("Testing model locally inside a container...")
       model_run_locally.main(
-          model_path,
-          inside_container=True,
-          run_model_server=False,
-          keep_image=keep_image,
-          skip_dockerfile=skip_dockerfile)
+          model_path, inside_container=True, run_model_server=False, keep_image=keep_image)
     click.echo("Model tested successfully.")
   except Exception as e:
     click.echo(f"Failed to test model locally: {e}", err=True)
@@ -161,14 +179,11 @@ def test_locally(model_path, keep_env=False, keep_image=False, mode='env', skip_
     help=
     'Keep the Docker image after testing the model locally (applicable for container mode). Defaults to False.'
 )
-@click.option(
-    '--skip_dockerfile',
-    is_flag=True,
-    help=
-    'Flag to skip generating a dockerfile so that you can manually edit an already created dockerfile. Apply for `--mode conatainer`.',
-)
-def run_locally(model_path, port, mode, keep_env, keep_image, skip_dockerfile=False):
-  """Run the model locally and start a gRPC server to serve the model."""
+def run_locally(model_path, port, mode, keep_env, keep_image):
+  """Run the model locally and start a gRPC server to serve the model.
+
+  MODEL_PATH: Path to the model directory. If not specified, the current directory is used by default.
+  """
   try:
     from clarifai.runners.models import model_run_locally
     if mode == 'env' and keep_image:
@@ -186,8 +201,7 @@ def run_locally(model_path, port, mode, keep_env, keep_image, skip_dockerfile=Fa
           inside_container=True,
           run_model_server=True,
           port=port,
-          keep_image=keep_image,
-          skip_dockerfile=skip_dockerfile)
+          keep_image=keep_image)
     click.echo(f"Model server started locally from {model_path} in {mode} mode.")
   except Exception as e:
     click.echo(f"Failed to starts model server locally: {e}", err=True)
@@ -241,8 +255,7 @@ def predict(ctx, config, model_id, user_id, app_id, model_url, file_path, url, b
   import json
 
   from clarifai.client.model import Model
-  from clarifai.utils.cli import from_yaml, validate_context
-  validate_context(ctx)
+  from clarifai.utils.cli import from_yaml
   if config:
     config = from_yaml(config)
     model_id, user_id, app_id, model_url, file_path, url, bytes, input_type, compute_cluster_id, nodepool_id, deployment_id, inference_params, output_config = (

clarifai/cli/nodepool.py

@@ -1,29 +1,32 @@
 import click
-
 from clarifai.cli.base import cli
-from clarifai.utils.cli import (AliasedGroup, display_co_resources, dump_yaml, from_yaml,
-                                validate_context)
+from clarifai.client.compute_cluster import ComputeCluster
+from clarifai.utils.cli import display_co_resources, dump_yaml, from_yaml
 
 
-@cli.group(['nodepool', 'np'], cls=AliasedGroup)
+@cli.group(['nodepool', 'np'])
 def nodepool():
   """Manage Nodepools: create, delete, list"""
+  pass
 
 
-@nodepool.command(['c'])
-@click.argument('compute_cluster_id')
-@click.argument('nodepool_id')
+@nodepool.command()
+@click.option(
+    '-cc_id',
+    '--compute_cluster_id',
+    required=False,
+    help='Compute Cluster ID for the compute cluster to interact with.')
 @click.option(
     '--config',
     type=click.Path(exists=True),
     required=True,
     help='Path to the nodepool config file.')
+@click.option(
+    '-np_id', '--nodepool_id', required=False, help='New Nodepool ID for the nodepool to create.')
 @click.pass_context
-def create(ctx, compute_cluster_id, nodepool_id, config):
+def create(ctx, compute_cluster_id, config, nodepool_id):
   """Create a new Nodepool with the given config file."""
-  from clarifai.client.compute_cluster import ComputeCluster
 
-  validate_context(ctx)
   nodepool_config = from_yaml(config)
   if not compute_cluster_id:
     if 'compute_cluster' not in nodepool_config['nodepool']:
@@ -39,74 +42,50 @@ def create(ctx, compute_cluster_id, nodepool_id, config):
 
   compute_cluster = ComputeCluster(
       compute_cluster_id=compute_cluster_id,
-      user_id=ctx.obj.current.user_id,
-      pat=ctx.obj.current.pat,
-      base_url=ctx.obj.current.api_base)
+      user_id=ctx.obj['user_id'],
+      pat=ctx.obj['pat'],
+      base_url=ctx.obj['base_url'])
   if nodepool_id:
     compute_cluster.create_nodepool(config, nodepool_id=nodepool_id)
   else:
     compute_cluster.create_nodepool(config)
 
 
-@nodepool.command(['ls'])
-@click.argument('compute_cluster_id', default="")
+@nodepool.command()
+@click.option(
+    '-cc_id',
+    '--compute_cluster_id',
+    required=True,
+    help='Compute Cluster ID for the compute cluster to interact with.')
 @click.option('--page_no', required=False, help='Page number to list.', default=1)
-@click.option('--per_page', required=False, help='Number of items per page.', default=128)
+@click.option('--per_page', required=False, help='Number of items per page.', default=16)
 @click.pass_context
 def list(ctx, compute_cluster_id, page_no, per_page):
-  """List all nodepools for the user across all compute clusters. If compute_cluster_id is provided
-  it will list only within that compute cluster. """
-  from clarifai.client.compute_cluster import ComputeCluster
-  from clarifai.client.user import User
-
-  validate_context(ctx)
-
-  cc_id = compute_cluster_id
+  """List all nodepools for the user."""
 
-  if cc_id:
-    compute_cluster = ComputeCluster(
-        compute_cluster_id=cc_id,
-        user_id=ctx.obj.current.user_id,
-        pat=ctx.obj.current.pat,
-        base_url=ctx.obj.current.api_base)
-    response = compute_cluster.list_nodepools(page_no, per_page)
-  else:
-    user = User(
-        user_id=ctx.obj.current.user_id,
-        pat=ctx.obj.current.pat,
-        base_url=ctx.obj.current.api_base)
-    ccs = user.list_compute_clusters(page_no, per_page)
-    response = []
-    for cc in ccs:
-      compute_cluster = ComputeCluster(
-          compute_cluster_id=cc.id,
-          user_id=ctx.obj.current.user_id,
-          pat=ctx.obj.current.pat,
-          base_url=ctx.obj.current.api_base)
-      response.extend([i for i in compute_cluster.list_nodepools(page_no, per_page)])
-
-  display_co_resources(
-      response,
-      custom_columns={
-          'ID': lambda c: c.id,
-          'USER_ID': lambda c: c.compute_cluster.user_id,
-          'COMPUTE_CLUSTER_ID': lambda c: c.compute_cluster.id,
-          'DESCRIPTION': lambda c: c.description,
-      })
+  compute_cluster = ComputeCluster(
+      compute_cluster_id=compute_cluster_id,
+      user_id=ctx.obj['user_id'],
+      pat=ctx.obj['pat'],
+      base_url=ctx.obj['base_url'])
+  response = compute_cluster.list_nodepools(page_no, per_page)
+  display_co_resources(response, "Nodepool")
 
 
-@nodepool.command(['rm'])
-@click.argument('compute_cluster_id')
-@click.argument('nodepool_id')
+@nodepool.command()
+@click.option(
+    '-cc_id',
+    '--compute_cluster_id',
+    required=True,
+    help='Compute Cluster ID for the compute cluster to interact with.')
+@click.option('-np_id', '--nodepool_id', help='Nodepool ID of the user to delete.')
 @click.pass_context
 def delete(ctx, compute_cluster_id, nodepool_id):
   """Deletes a nodepool for the user."""
-  from clarifai.client.compute_cluster import ComputeCluster
 
-  validate_context(ctx)
   compute_cluster = ComputeCluster(
       compute_cluster_id=compute_cluster_id,
-      user_id=ctx.obj.current.user_id,
-      pat=ctx.obj.current.pat,
-      base_url=ctx.obj.current.api_base)
+      user_id=ctx.obj['user_id'],
+      pat=ctx.obj['pat'],
+      base_url=ctx.obj['base_url'])
   compute_cluster.delete_nodepools([nodepool_id])

clarifai/client/app.py

@@ -629,7 +629,7 @@ class App(Lister, BaseClient):
 
     Args:
         model_id (str): The model ID for the model to interact with.
-        model_version (Dict): The model version ID for the model version to interact with.
+        model_version_id (str): The model version ID for the model version to interact with.
 
     Returns:
         Model: A Model object for the existing model ID.

clarifai/client/auth/stub.py

@@ -1,4 +1,5 @@
 import itertools
+import logging
 import time
 from concurrent.futures import ThreadPoolExecutor
 
@@ -7,7 +8,7 @@ from clarifai_grpc.grpc.api.status import status_code_pb2
 
 from clarifai.client.auth.helper import ClarifaiAuthHelper
 from clarifai.client.auth.register import RpcCallable, V2Stub
-from clarifai.utils.logging import logger
+
 throttle_status_codes = {
     status_code_pb2.CONN_THROTTLED,
     status_code_pb2.CONN_EXCEED_HOURLY_LIMIT,
@@ -25,7 +26,7 @@ def validate_response(response, attempt, max_attempts):
   def handle_simple_response(response):
     if hasattr(response, 'status') and hasattr(response.status, 'code'):
       if (response.status.code in throttle_status_codes) and attempt < max_attempts:
-        logger.debug('Retrying with status %s' % str(response.status))
+        logging.debug('Retrying with status %s' % str(response.status))
         return None  # Indicates a retry is needed
       else:
         return response
@@ -41,7 +42,7 @@ def validate_response(response, attempt, max_attempts):
         return itertools.chain([validated_response], response)
       return None  # Indicates a retry is needed
     except grpc.RpcError as e:
-      logger.error('Error processing streaming response: %s' % str(e))
+      logging.error('Error processing streaming response: %s' % str(e))
       return None  # Indicates an error
   else:
     # Handle simple response validation
@@ -142,7 +143,7 @@ class _RetryRpcCallable(RpcCallable):
           return v
       except grpc.RpcError as e:
         if (e.code() in retry_codes_grpc) and attempt < self.max_attempts:
-          logger.debug('Retrying with status %s' % e.code())
+          logging.debug('Retrying with status %s' % e.code())
         else:
           raise
 

clarifai/client/cli/base_cli.py

@@ -0,0 +1,88 @@
+import click
+import os
+import yaml
+
+@click.group()
+@click.pass_context
+def cli(ctx):
+    """Clarifai CLI"""
+    ctx.ensure_object(dict)
+    config_path = 'config.yaml'
+    if os.path.exists(config_path):
+        ctx.obj = _from_yaml(config_path)
+        print("Loaded config from file.")
+        print(f"Config: {ctx.obj}")
+    else:
+        ctx.obj = {}
+
+def _from_yaml(filename: str):
+    try:
+        with open(filename, 'r') as f:
+            return yaml.safe_load(f)
+    except yaml.YAMLError as e:
+        click.echo(f"Error reading YAML file: {e}", err=True)
+        return {}
+
+def _dump_yaml(data, filename: str):
+    try:
+        with open(filename, 'w') as f:
+            yaml.dump(data, f)
+    except Exception as e:
+        click.echo(f"Error writing YAML file: {e}", err=True)
+
+def _set_base_url(env):
+    environments = {'prod': 'https://api.clarifai.com', 'staging': 'https://api-staging.clarifai.com', 'dev': 'https://api-dev.clarifai.com'}
+    return environments.get(env, 'https://api.clarifai.com')
+
+
+@cli.command()
+@click.option('--config', type=click.Path(), required=False, help='Path to the config file')
+@click.option('-e', '--env', required=False, help='Environment', type=click.Choice(['prod', 'staging', 'dev']))
+@click.option('--user_id', required=False, help='User ID')
+@click.pass_context
+def login(ctx, config, env, user_id):
+    """Login command to set PAT and other configurations."""
+
+    if config and os.path.exists(config):
+        ctx.obj = _from_yaml(config)
+        
+    if 'pat' in ctx.obj:
+        os.environ["CLARIFAI_PAT"] = ctx.obj['pat']
+        click.echo("Loaded PAT from config file.")
+    elif 'CLARIFAI_PAT' in os.environ:
+        ctx.obj['pat'] = os.environ["CLARIFAI_PAT"]
+        click.echo("Loaded PAT from environment variable.")
+    else:
+        _pat = click.prompt("Get your PAT from https://clarifai.com/settings/security and pass it here", type=str)
+        os.environ["CLARIFAI_PAT"] = _pat
+        ctx.obj['pat'] = _pat
+        click.echo("PAT saved successfully.")
+
+    if user_id:
+        ctx.obj['user_id'] = user_id
+        os.environ["CLARIFAI_USER_ID"] = ctx.obj['user_id']
+    elif 'user_id' in ctx.obj or 'CLARIFAI_USER_ID' in os.environ:
+        ctx.obj['user_id'] = ctx.obj.get('user_id', os.environ["CLARIFAI_USER_ID"])
+        os.environ["CLARIFAI_USER_ID"] = ctx.obj['user_id']
+
+    if env:
+        ctx.obj['env'] = env
+        ctx.obj['base_url'] = _set_base_url(env)
+        os.environ["CLARIFAI_API_BASE"] = ctx.obj['base_url']
+    elif 'env' in ctx.obj:
+        ctx.obj['env'] = ctx.obj.get('env', "prod")
+        ctx.obj['base_url'] = _set_base_url(ctx.obj['env'])
+        os.environ["CLARIFAI_API_BASE"] = ctx.obj['base_url']
+    elif 'CLARIFAI_API_BASE' in os.environ:
+        ctx.obj['base_url'] = os.environ["CLARIFAI_API_BASE"]
+    
+    _dump_yaml(ctx.obj, 'config.yaml')
+    
+    click.echo("Login successful.")
+
+# Import the model CLI commands to register them
+from clarifai.client.cli.model_cli import model  # Ensure this is the correct import path
+
+
+if __name__ == '__main__':
+    cli()

clarifai/client/cli/model_cli.py

@@ -0,0 +1,29 @@
+import click
+from clarifai.client.cli.base_cli import cli
+
+@cli.group()
+def model():
+    """Manage models: upload, test locally"""
+    pass
+
+@model.command()
+@click.argument('model_path', type=click.Path(exists=True))
+@click.option('--download_checkpoints', is_flag=True, help='Flag to download checkpoints before uploading and including them in the tar file that is uploaded. Defaults to False, which will attempt to download them at docker build time.', )
+@click.option('--skip_dockerfile', is_flag =True, help='Flag to skip generating a dockerfile so that you can manually edit an already created dockerfile.', )
+def upload(model_path, download_checkpoints, skip_dockerfile):
+    """Upload a model to Clarifai."""
+    from clarifai.runners.models import model_upload
+
+    model_upload.main(model_path, download_checkpoints, skip_dockerfile)
+    
+@model.command()
+@click.argument('model_path', type=click.Path(exists=True))
+def test_locally(model_path):
+    """Test model locally."""
+    try:
+        from clarifai.runners.models import run_test_locally
+        run_test_locally.main(model_path)
+        click.echo(f"Model tested locally from {model_path}.")
+    except Exception as e:
+        click.echo(f"Failed to test model locally: {e}", err=True)
+

clarifai/client/dataset.py

@@ -1,3 +1,4 @@
+import logging
 import os
 import time
 import uuid
@@ -353,7 +354,7 @@ class Dataset(Lister, BaseClient):
         break
       if failed_input_ids:
         retry_input_ids = [dataset_obj.all_input_ids[id] for id in failed_input_ids]
-        logger.warning(
+        logging.warning(
             f"Retrying upload for {len(failed_input_ids)} inputs in current batch: {retry_input_ids}\n"
         )
         failed_retrying_inputs, _, retry_response = self._upload_inputs_annotations(
@@ -493,7 +494,7 @@ class Dataset(Lister, BaseClient):
       add_file_handler(self.logger, f"Dataset_Upload{str(int(datetime.now().timestamp()))}.log")
 
     if retry_duplicates and duplicate_input_ids:
-      logger.warning(f"Retrying upload for {len(duplicate_input_ids)} duplicate inputs...\n")
+      logging.warning(f"Retrying upload for {len(duplicate_input_ids)} duplicate inputs...\n")
       duplicate_inputs_indexes = [input["Index"] for input in duplicate_input_ids]
       self.upload_dataset(
           dataloader=dataloader,
@@ -504,7 +505,7 @@ class Dataset(Lister, BaseClient):
 
     if failed_input_ids:
       #failed_inputs= ([input["Input_ID"] for input in failed_input_ids])
-      logger.warning(f"Retrying upload for {len(failed_input_ids)} failed inputs...\n")
+      logging.warning(f"Retrying upload for {len(failed_input_ids)} failed inputs...\n")
       failed_input_indexes = [input["Index"] for input in failed_input_ids]
       self.upload_dataset(
           dataloader=dataloader, log_retry_ids=failed_input_indexes, is_log_retry=True, **kwargs)