clarifai 11.5.6__tar.gz → 11.6.1__tar.gz

{clarifai-11.5.6/clarifai.egg-info → clarifai-11.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: clarifai
-Version: 11.5.6
+Version: 11.6.1
 Home-page: https://github.com/Clarifai/clarifai-python
 Author: Clarifai
 Author-email: support@clarifai.com
@@ -19,8 +19,8 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: clarifai-grpc>=11.5.5
-Requires-Dist: clarifai-protocol>=0.0.24
+Requires-Dist: clarifai-grpc>=11.6.0
+Requires-Dist: clarifai-protocol>=0.0.25
 Requires-Dist: numpy>=1.22.0
 Requires-Dist: tqdm>=4.65.0
 Requires-Dist: PyYAML>=6.0.1

clarifai-11.6.1/clarifai/__init__.py

@@ -0,0 +1 @@
+__version__ = "11.6.1"

{clarifai-11.5.6 → clarifai-11.6.1}/clarifai/cli/base.py

@@ -6,6 +6,7 @@ import sys
 import click
 import yaml
 
+from clarifai import __version__
 from clarifai.utils.cli import AliasedGroup, TableFormatter, load_command_modules
 from clarifai.utils.config import Config, Context
 from clarifai.utils.constants import DEFAULT_BASE, DEFAULT_CONFIG, DEFAULT_UI
@@ -14,6 +15,7 @@ from clarifai.utils.logging import logger
 
 # @click.group(cls=CustomMultiGroup)
 @click.group(cls=AliasedGroup)
+@click.version_option(version=__version__)
 @click.option('--config', default=DEFAULT_CONFIG)
 @click.pass_context
 def cli(ctx, config):

{clarifai-11.5.6 → clarifai-11.6.1}/clarifai/cli/model.py

@@ -1,5 +1,6 @@
 import os
 import shutil
+import tempfile
 
 import click
 
@@ -16,13 +17,19 @@ from clarifai.utils.constants import (
     DEFAULT_LOCAL_DEV_NODEPOOL_ID,
 )
 from clarifai.utils.logging import logger
+from clarifai.utils.misc import (
+    clone_github_repo,
+    format_github_repo_url,
+)
 
 
 @cli.group(
     ['model'], context_settings={'max_content_width': shutil.get_terminal_size().columns - 10}
 )
 def model():
-    """Manage models: upload, test, local dev, predict, etc"""
+    """Manage & Develop Models: init, download-checkpoints, signatures, upload\n
+    Run & Test Models Locally: local-runner, local-grpc, local-test\n
+    Model Inference: list, predict"""
 
 
 @model.command()
@@ -38,7 +45,22 @@ def model():
     required=False,
     help='Model type: "mcp" for MCPModelClass, "openai" for OpenAIModelClass, or leave empty for default ModelClass.',
 )
-def init(model_path, model_type_id):
+@click.option(
+    '--github-pat',
+    required=False,
+    help='GitHub Personal Access Token for authentication when cloning private repositories.',
+)
+@click.option(
+    '--github-repo',
+    required=False,
+    help='GitHub repository URL or "user/repo" format to clone a repository from. If provided, the entire repository contents will be copied to the target directory instead of using default templates.',
+)
+@click.option(
+    '--branch',
+    required=False,
+    help='Git branch to clone from the GitHub repository. If not specified, the default branch will be used.',
+)
+def init(model_path, model_type_id, github_pat, github_repo, branch):
     """Initialize a new model directory structure.
 
     Creates the following structure in the specified directory:
@@ -47,65 +69,113 @@ def init(model_path, model_type_id):
     ├── requirements.txt
     └── config.yaml
 
+    If --github-repo is provided, the entire repository contents will be copied to the target
+    directory instead of using default templates. The --github-pat option can be used for authentication
+    when cloning private repositories. The --branch option can be used to specify a specific
+    branch to clone from.
+
     MODEL_PATH: Path where to create the model directory structure. If not specified, the current directory is used by default.
     """
-    from clarifai.cli.templates.model_templates import (
-        get_config_template,
-        get_model_template,
-        get_requirements_template,
-    )
-
     # Resolve the absolute path
     model_path = os.path.abspath(model_path)
 
     # Create the model directory if it doesn't exist
     os.makedirs(model_path, exist_ok=True)
 
-    # Create the 1/ subdirectory
-    model_version_dir = os.path.join(model_path, "1")
-    os.makedirs(model_version_dir, exist_ok=True)
-
-    # Create model.py
-    model_py_path = os.path.join(model_version_dir, "model.py")
-    if os.path.exists(model_py_path):
-        logger.warning(f"File {model_py_path} already exists, skipping...")
-    else:
-        model_template = get_model_template(model_type_id)
-        with open(model_py_path, 'w') as f:
-            f.write(model_template)
-        logger.info(f"Created {model_py_path}")
-
-    # Create requirements.txt
-    requirements_path = os.path.join(model_path, "requirements.txt")
-    if os.path.exists(requirements_path):
-        logger.warning(f"File {requirements_path} already exists, skipping...")
-    else:
-        requirements_template = get_requirements_template(model_type_id)
-        with open(requirements_path, 'w') as f:
-            f.write(requirements_template)
-        logger.info(f"Created {requirements_path}")
-
-    # Create config.yaml
-    config_path = os.path.join(model_path, "config.yaml")
-    if os.path.exists(config_path):
-        logger.warning(f"File {config_path} already exists, skipping...")
-    else:
-        config_model_type_id = "text-to-text"  # default
-
-        config_template = get_config_template(config_model_type_id)
-        with open(config_path, 'w') as f:
-            f.write(config_template)
-        logger.info(f"Created {config_path}")
-
-    logger.info(f"Model initialization complete in {model_path}")
-    logger.info("Next steps:")
-    logger.info("1. Search for '# TODO: please fill in' comments in the generated files")
-    logger.info("2. Update the model configuration in config.yaml")
-    logger.info("3. Add your model dependencies to requirements.txt")
-    logger.info("4. Implement your model logic in 1/model.py")
-
+    # Handle GitHub repository cloning if provided
+    if github_repo:
+        logger.info(f"Initializing model from GitHub repository: {github_repo}")
+
+        # Check if it's a local path or normalize the GitHub repo URL
+        if os.path.exists(github_repo):
+            repo_url = github_repo
+        else:
+            repo_url = format_github_repo_url(github_repo)
+
+        # Create a temporary directory for cloning
+        with tempfile.TemporaryDirectory() as temp_dir:
+            clone_dir = os.path.join(temp_dir, "repo")
+
+            # Clone the repository
+            if not clone_github_repo(repo_url, clone_dir, github_pat, branch):
+                logger.error(
+                    "Failed to clone repository. Falling back to template-based initialization."
+                )
+                github_repo = None  # Fall back to template mode
+            else:
+                # Copy the entire repository content to target directory (excluding .git)
+                for item in os.listdir(clone_dir):
+                    if item == '.git':
+                        continue
+
+                    source_path = os.path.join(clone_dir, item)
+                    target_path = os.path.join(model_path, item)
+
+                    if os.path.isdir(source_path):
+                        shutil.copytree(source_path, target_path, dirs_exist_ok=True)
+                    else:
+                        shutil.copy2(source_path, target_path)
+
+                logger.info("Model initialization complete with GitHub repository")
+                logger.info("Next steps:")
+                logger.info("1. Review the model configuration")
+                logger.info("2. Install any required dependencies manually")
+                logger.info("3. Test the model locally using 'clarifai model local-test'")
+                return
+
+    # Fall back to template-based initialization if no GitHub repo or if GitHub repo failed
+    if not github_repo:
+        from clarifai.cli.templates.model_templates import (
+            get_config_template,
+            get_model_template,
+            get_requirements_template,
+        )
 
-@model.command()
+        # Create the 1/ subdirectory
+        model_version_dir = os.path.join(model_path, "1")
+        os.makedirs(model_version_dir, exist_ok=True)
+
+        # Create model.py
+        model_py_path = os.path.join(model_version_dir, "model.py")
+        if os.path.exists(model_py_path):
+            logger.warning(f"File {model_py_path} already exists, skipping...")
+        else:
+            model_template = get_model_template(model_type_id)
+            with open(model_py_path, 'w') as f:
+                f.write(model_template)
+            logger.info(f"Created {model_py_path}")
+
+        # Create requirements.txt
+        requirements_path = os.path.join(model_path, "requirements.txt")
+        if os.path.exists(requirements_path):
+            logger.warning(f"File {requirements_path} already exists, skipping...")
+        else:
+            requirements_template = get_requirements_template(model_type_id)
+            with open(requirements_path, 'w') as f:
+                f.write(requirements_template)
+            logger.info(f"Created {requirements_path}")
+
+        # Create config.yaml
+        config_path = os.path.join(model_path, "config.yaml")
+        if os.path.exists(config_path):
+            logger.warning(f"File {config_path} already exists, skipping...")
+        else:
+            config_model_type_id = "text-to-text"  # default
+
+            config_template = get_config_template(config_model_type_id)
+            with open(config_path, 'w') as f:
+                f.write(config_template)
+            logger.info(f"Created {config_path}")
+
+        logger.info(f"Model initialization complete in {model_path}")
+        logger.info("Next steps:")
+        logger.info("1. Search for '# TODO: please fill in' comments in the generated files")
+        logger.info("2. Update the model configuration in config.yaml")
+        logger.info("3. Add your model dependencies to requirements.txt")
+        logger.info("4. Implement your model logic in 1/model.py")
+
+
+@model.command(help="Upload a trained model.")
 @click.argument("model_path", type=click.Path(exists=True), required=False, default=".")
 @click.option(
     '--stage',
@@ -120,17 +190,25 @@ def init(model_path, model_type_id):
     is_flag=True,
     help='Flag to skip generating a dockerfile so that you can manually edit an already created dockerfile.',
 )
-def upload(model_path, stage, skip_dockerfile):
+@click.pass_context
+def upload(ctx, model_path, stage, skip_dockerfile):
     """Upload a model to Clarifai.
 
     MODEL_PATH: Path to the model directory. If not specified, the current directory is used by default.
     """
     from clarifai.runners.models.model_builder import upload_model
 
-    upload_model(model_path, stage, skip_dockerfile)
+    validate_context(ctx)
+    upload_model(
+        model_path,
+        stage,
+        skip_dockerfile,
+        pat=ctx.obj.current.pat,
+        base_url=ctx.obj.current.api_base,
+    )
 
 
-@model.command()
+@model.command(help="Download model checkpoint files.")
 @click.argument(
     "model_path",
     type=click.Path(exists=True),
@@ -164,7 +242,7 @@ def download_checkpoints(model_path, out_path, stage):
     builder.download_checkpoints(stage=stage, checkpoint_path_override=out_path)
 
 
-@model.command()
+@model.command(help="Generate model method signatures.")
 @click.argument(
     "model_path",
     type=click.Path(exists=True),
@@ -195,7 +273,7 @@ def signatures(model_path, out_path):
         click.echo(signatures)
 
 
-@model.command()
+@model.command(name="local-test", help="Execute all model unit tests locally.")
 @click.argument(
     "model_path",
     type=click.Path(exists=True),
@@ -254,7 +332,7 @@ def test_locally(model_path, keep_env=False, keep_image=False, mode='env', skip_
         click.echo(f"Failed to test model locally: {e}", err=True)
 
 
-@model.command()
+@model.command(name="local-grpc", help="Run the model locally via a gRPC server.")
 @click.argument(
     "model_path",
     type=click.Path(exists=True),
@@ -322,7 +400,7 @@ def run_locally(model_path, port, mode, keep_env, keep_image, skip_dockerfile=Fa
         click.echo(f"Failed to starts model server locally: {e}", err=True)
 
 
-@model.command()
+@model.command(name="local-runner", help="Run the model locally for dev, debug, or local compute.")
 @click.argument(
     "model_path",
     type=click.Path(exists=True),
@@ -486,7 +564,13 @@ def local_dev(ctx, model_path):
         )
         if y.lower() != 'y':
             raise click.Abort()
-        model = app.create_model(model_id, model_type_id=DEFAULT_LOCAL_DEV_MODEL_TYPE)
+        try:
+            model_type_id = ctx.obj.current.model_type_id
+        except AttributeError:
+            model_type_id = DEFAULT_LOCAL_DEV_MODEL_TYPE
+
+        model = app.create_model(model_id, model_type_id=model_type_id)
+        ctx.obj.current.CLARIFAI_MODEL_TYPE_ID = model_type_id
         ctx.obj.current.CLARIFAI_MODEL_ID = model_id
         ctx.obj.to_yaml()  # save to yaml file.
 
@@ -605,6 +689,7 @@ def local_dev(ctx, model_path):
             f"config.yaml not found in {model_path}. Please ensure you are passing the correct directory."
         )
     config = ModelBuilder._load_config(config_file)
+    model_type_id = config.get('model', {}).get('model_type_id', DEFAULT_LOCAL_DEV_MODEL_TYPE)
     # The config.yaml doens't match what we created above.
     if 'model' in config and model_id != config['model'].get('id'):
         logger.info(f"Current model section of config.yaml: {config.get('model', {})}")
@@ -614,7 +699,7 @@ def local_dev(ctx, model_path):
         if y.lower() != 'y':
             raise click.Abort()
         config = ModelBuilder._set_local_dev_model(
-            config, user_id, app_id, model_id, DEFAULT_LOCAL_DEV_MODEL_TYPE
+            config, user_id, app_id, model_id, model_type_id
         )
         ModelBuilder._backup_config(config_file)
         ModelBuilder._save_config(config_file, config)
@@ -655,7 +740,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
     )
 
 
-@model.command()
+@model.command(help="Perform a prediction using the model.")
 @click.option(
     '--config',
     type=click.Path(exists=True),
@@ -837,7 +922,7 @@ def predict(
 )
 @click.pass_context
 def list_model(ctx, user_id, app_id):
-    """List models of user/community
+    """List models of user/community.
 
     USER_ID: User id. If not specified, the current user is used by default. Set "all" to get all public models in Clarifai platform.
     """

{clarifai-11.5.6 → clarifai-11.6.1}/clarifai/cli/pipeline.py

@@ -27,6 +27,146 @@ def upload(path):
     upload_pipeline(path)
 
 
+@pipeline.command()
+@click.option(
+    '--config',
+    type=click.Path(exists=True),
+    required=False,
+    help='Path to the pipeline run config file.',
+)
+@click.option('--pipeline_id', required=False, help='Pipeline ID to run.')
+@click.option('--pipeline_version_id', required=False, help='Pipeline Version ID to run.')
+@click.option(
+    '--pipeline_version_run_id',
+    required=False,
+    help='Pipeline Version Run ID. If not provided, a UUID will be generated.',
+)
+@click.option('--user_id', required=False, help='User ID of the pipeline.')
+@click.option('--app_id', required=False, help='App ID that contains the pipeline.')
+@click.option('--nodepool_id', required=False, help='Nodepool ID to run the pipeline on.')
+@click.option(
+    '--compute_cluster_id', required=False, help='Compute Cluster ID to run the pipeline on.'
+)
+@click.option('--pipeline_url', required=False, help='Pipeline URL to run.')
+@click.option(
+    '--timeout',
+    type=int,
+    default=3600,
+    help='Maximum time to wait for completion in seconds. Default 3600 (1 hour).',
+)
+@click.option(
+    '--monitor_interval',
+    type=int,
+    default=10,
+    help='Interval between status checks in seconds. Default 10.',
+)
+@click.option(
+    '--log_file',
+    type=click.Path(),
+    required=False,
+    help='Path to file where logs should be written. If not provided, logs are displayed on console.',
+)
+@click.option(
+    '--monitor',
+    is_flag=True,
+    default=False,
+    help='Monitor an existing pipeline run instead of starting a new one. Requires pipeline_version_run_id.',
+)
+@click.pass_context
+def run(
+    ctx,
+    config,
+    pipeline_id,
+    pipeline_version_id,
+    pipeline_version_run_id,
+    user_id,
+    app_id,
+    nodepool_id,
+    compute_cluster_id,
+    pipeline_url,
+    timeout,
+    monitor_interval,
+    log_file,
+    monitor,
+):
+    """Run a pipeline and monitor its progress."""
+    import json
+
+    from clarifai.client.pipeline import Pipeline
+    from clarifai.utils.cli import from_yaml, validate_context
+
+    validate_context(ctx)
+
+    if config:
+        config_data = from_yaml(config)
+        pipeline_id = config_data.get('pipeline_id', pipeline_id)
+        pipeline_version_id = config_data.get('pipeline_version_id', pipeline_version_id)
+        pipeline_version_run_id = config_data.get(
+            'pipeline_version_run_id', pipeline_version_run_id
+        )
+        user_id = config_data.get('user_id', user_id)
+        app_id = config_data.get('app_id', app_id)
+        nodepool_id = config_data.get('nodepool_id', nodepool_id)
+        compute_cluster_id = config_data.get('compute_cluster_id', compute_cluster_id)
+        pipeline_url = config_data.get('pipeline_url', pipeline_url)
+        timeout = config_data.get('timeout', timeout)
+        monitor_interval = config_data.get('monitor_interval', monitor_interval)
+        log_file = config_data.get('log_file', log_file)
+        monitor = config_data.get('monitor', monitor)
+
+    # compute_cluster_id and nodepool_id are mandatory regardless of whether pipeline_url is provided
+    if not compute_cluster_id or not nodepool_id:
+        raise ValueError("--compute_cluster_id and --nodepool_id are mandatory parameters.")
+
+    # When monitor flag is used, pipeline_version_run_id is mandatory
+    if monitor and not pipeline_version_run_id:
+        raise ValueError("--pipeline_version_run_id is required when using --monitor flag.")
+
+    if pipeline_url:
+        # When using pipeline_url, other parameters are optional (will be parsed from URL)
+        required_params_provided = True
+    else:
+        # When not using pipeline_url, all individual parameters are required
+        required_params_provided = all([pipeline_id, user_id, app_id, pipeline_version_id])
+
+    if not required_params_provided:
+        raise ValueError(
+            "Either --user_id & --app_id & --pipeline_id & --pipeline_version_id or --pipeline_url must be provided."
+        )
+
+    if pipeline_url:
+        pipeline = Pipeline(
+            url=pipeline_url,
+            pat=ctx.obj.current.pat,
+            base_url=ctx.obj.current.api_base,
+            pipeline_version_run_id=pipeline_version_run_id,
+            nodepool_id=nodepool_id,
+            compute_cluster_id=compute_cluster_id,
+            log_file=log_file,
+        )
+    else:
+        pipeline = Pipeline(
+            pipeline_id=pipeline_id,
+            pipeline_version_id=pipeline_version_id,
+            pipeline_version_run_id=pipeline_version_run_id,
+            user_id=user_id,
+            app_id=app_id,
+            nodepool_id=nodepool_id,
+            compute_cluster_id=compute_cluster_id,
+            pat=ctx.obj.current.pat,
+            base_url=ctx.obj.current.api_base,
+            log_file=log_file,
+        )
+
+    if monitor:
+        # Monitor existing pipeline run instead of starting new one
+        result = pipeline.monitor_only(timeout=timeout, monitor_interval=monitor_interval)
+    else:
+        # Start new pipeline run and monitor it
+        result = pipeline.run(timeout=timeout, monitor_interval=monitor_interval)
+    click.echo(json.dumps(result, indent=2, default=str))
+
+
 @pipeline.command()
 @click.argument(
     "pipeline_path",

{clarifai-11.5.6 → clarifai-11.6.1}/clarifai/cli/pipeline_step.py

@@ -8,7 +8,7 @@ from clarifai.utils.logging import logger
 
 
 @cli.group(
-    ['pipelinestep'],
+    ['pipelinestep', 'ps'],
     context_settings={'max_content_width': shutil.get_terminal_size().columns - 10},
 )
 def pipeline_step():

{clarifai-11.5.6 → clarifai-11.6.1}/clarifai/client/__init__.py

@@ -7,6 +7,7 @@ from clarifai.client.input import Inputs
 from clarifai.client.lister import Lister
 from clarifai.client.model import Model
 from clarifai.client.module import Module
+from clarifai.client.pipeline import Pipeline
 from clarifai.client.search import Search
 from clarifai.client.user import User
 from clarifai.client.workflow import Workflow
@@ -18,6 +19,7 @@ __all__ = [
     'App',
     'Model',
     'Workflow',
+    'Pipeline',
     'Module',
     'Lister',
     'Dataset',

{clarifai-11.5.6 → clarifai-11.6.1}/clarifai/client/model.py

@@ -66,6 +66,7 @@ class Model(Lister, BaseClient):
         compute_cluster_id: str = None,
         nodepool_id: str = None,
         deployment_id: str = None,
+        deployment_user_id: str = None,
         **kwargs,
     ):
         """Initializes a Model object.
@@ -78,6 +79,10 @@ class Model(Lister, BaseClient):
             pat (str): A personal access token for authentication. Can be set as env var CLARIFAI_PAT
             token (str): A session token for authentication. Accepts either a session token or a pat. Can be set as env var CLARIFAI_SESSION_TOKEN
             root_certificates_path (str): Path to the SSL root certificates file, used to establish secure gRPC connections.
+            compute_cluster_id (str): Compute cluster ID for runner selector.
+            nodepool_id (str): Nodepool ID for runner selector.
+            deployment_id (str): Deployment ID for runner selector.
+            deployment_user_id (str): User ID to use for runner selector (organization or user). If not provided, defaults to PAT owner user_id.
             **kwargs: Additional keyword arguments to be passed to the Model.
         """
         if url and model_id:
@@ -115,10 +120,13 @@ class Model(Lister, BaseClient):
         )
         Lister.__init__(self)
 
+        self.deployment_user_id = deployment_user_id
+
         self._set_runner_selector(
             compute_cluster_id=compute_cluster_id,
             nodepool_id=nodepool_id,
             deployment_id=deployment_id,
+            deployment_user_id=deployment_user_id,
         )
 
     @classmethod
@@ -633,9 +641,13 @@ class Model(Lister, BaseClient):
         compute_cluster_id: str = None,
         nodepool_id: str = None,
         deployment_id: str = None,
+        deployment_user_id: str = None,
     ):
-        # Get UserID
-        if any([deployment_id, nodepool_id, compute_cluster_id]):
+        # Get UserID for runner selector
+        user_id = None
+        if deployment_user_id:
+            user_id = deployment_user_id
+        elif any([deployment_id, nodepool_id, compute_cluster_id]):
             from clarifai.client.user import User
 
             user_id = (
@@ -643,13 +655,11 @@ class Model(Lister, BaseClient):
                 .get_user_info(user_id='me')
                 .user.id
             )
-
         runner_selector = None
         if deployment_id and (compute_cluster_id or nodepool_id):
             raise UserError(
                 "You can only specify one of deployment_id or compute_cluster_id and nodepool_id."
             )
-
         if deployment_id:
             runner_selector = Deployment.get_runner_selector(
                 user_id=user_id, deployment_id=deployment_id
@@ -658,7 +668,6 @@ class Model(Lister, BaseClient):
             runner_selector = Nodepool.get_runner_selector(
                 user_id=user_id, compute_cluster_id=compute_cluster_id, nodepool_id=nodepool_id
             )
-
         # set the runner selector
         self._runner_selector = runner_selector