clarifai 11.8.2__tar.gz → 11.8.3__tar.gz

{clarifai-11.8.2/clarifai.egg-info → clarifai-11.8.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: clarifai
-Version: 11.8.2
+Version: 11.8.3
 Home-page: https://github.com/Clarifai/clarifai-python
 Author: Clarifai
 Author-email: support@clarifai.com

clarifai-11.8.3/clarifai/__init__.py

@@ -0,0 +1 @@
+__version__ = "11.8.3"

{clarifai-11.8.2 → clarifai-11.8.3}/clarifai/cli/model.py

@@ -231,6 +231,44 @@ def init(
                     repo_url = format_github_repo_url(github_url)
                     repo_url = f"https://github.com/{owner}/{repo}"
 
+                try:
+                    # Create a temporary directory for cloning
+                    with tempfile.TemporaryDirectory(prefix="clarifai_model_") as clone_dir:
+                        # Clone the repository with explicit branch parameter
+                        if not clone_github_repo(repo_url, clone_dir, github_pat, branch):
+                            logger.error(f"Failed to clone repository from {repo_url}")
+                            github_url = None  # Fall back to template mode
+
+                        else:
+                            # Copy the entire repository content to target directory (excluding .git)
+                            for item in os.listdir(clone_dir):
+                                if item == '.git':
+                                    continue
+
+                                source_path = os.path.join(clone_dir, item)
+                                target_path = os.path.join(model_path, item)
+
+                                if os.path.isdir(source_path):
+                                    shutil.copytree(source_path, target_path, dirs_exist_ok=True)
+                                else:
+                                    shutil.copy2(source_path, target_path)
+
+                            logger.info(f"Successfully cloned repository to {model_path}")
+                            logger.info(
+                                "Model initialization complete with GitHub repository clone"
+                            )
+                            logger.info("Next steps:")
+                            logger.info("1. Review the model configuration")
+                            logger.info("2. Install any required dependencies manually")
+                            logger.info(
+                                "3. Test the model locally using 'clarifai model local-test'"
+                            )
+                            return
+
+                except Exception as e:
+                    logger.error(f"Failed to clone GitHub repository: {e}")
+                    github_url = None  # Fall back to template mode
+
     if toolkit:
         logger.info(f"Initializing model from GitHub repository: {github_url}")
 
@@ -240,31 +278,31 @@ def init(
         else:
             repo_url = format_github_repo_url(github_url)
 
-    try:
-        # Create a temporary directory for cloning
-        with tempfile.TemporaryDirectory(prefix="clarifai_model_") as clone_dir:
-            # Clone the repository with explicit branch parameter
-            if not clone_github_repo(repo_url, clone_dir, github_pat, branch):
-                logger.error(f"Failed to clone repository from {repo_url}")
-                github_url = None  # Fall back to template mode
-
-            else:
-                # Copy the entire repository content to target directory (excluding .git)
-                for item in os.listdir(clone_dir):
-                    if item == '.git':
-                        continue
-
-                    source_path = os.path.join(clone_dir, item)
-                    target_path = os.path.join(model_path, item)
-
-                    if os.path.isdir(source_path):
-                        shutil.copytree(source_path, target_path, dirs_exist_ok=True)
-                    else:
-                        shutil.copy2(source_path, target_path)
+        try:
+            # Create a temporary directory for cloning
+            with tempfile.TemporaryDirectory(prefix="clarifai_model_") as clone_dir:
+                # Clone the repository with explicit branch parameter
+                if not clone_github_repo(repo_url, clone_dir, github_pat, branch):
+                    logger.error(f"Failed to clone repository from {repo_url}")
+                    github_url = None  # Fall back to template mode
 
-    except Exception as e:
-        logger.error(f"Failed to clone GitHub repository: {e}")
-        github_url = None
+                else:
+                    # Copy the entire repository content to target directory (excluding .git)
+                    for item in os.listdir(clone_dir):
+                        if item == '.git':
+                            continue
+
+                        source_path = os.path.join(clone_dir, item)
+                        target_path = os.path.join(model_path, item)
+
+                        if os.path.isdir(source_path):
+                            shutil.copytree(source_path, target_path, dirs_exist_ok=True)
+                        else:
+                            shutil.copy2(source_path, target_path)
+
+        except Exception as e:
+            logger.error(f"Failed to clone GitHub repository: {e}")
+            github_url = None
 
     if (model_name or port or context_length) and (toolkit == 'ollama'):
         customize_ollama_model(model_path, model_name, port, context_length)

{clarifai-11.8.2 → clarifai-11.8.3}/clarifai/cli/pipeline.py

@@ -26,14 +26,19 @@ def pipeline():
 
 @pipeline.command()
 @click.argument("path", type=click.Path(exists=True), required=False, default=".")
-def upload(path):
+@click.option(
+    '--no-lockfile',
+    is_flag=True,
+    help='Skip creating config-lock.yaml file.',
+)
+def upload(path, no_lockfile):
     """Upload a pipeline with associated pipeline steps to Clarifai.
 
     PATH: Path to the pipeline configuration file or directory containing config.yaml. If not specified, the current directory is used by default.
     """
     from clarifai.runners.pipelines.pipeline_builder import upload_pipeline
 
-    upload_pipeline(path)
+    upload_pipeline(path, no_lockfile=no_lockfile)
 
 
 @pipeline.command()
@@ -106,15 +111,32 @@ def run(
 
     validate_context(ctx)
 
+    # Try to load from config-lock.yaml first if no config is specified
+    lockfile_path = os.path.join(os.getcwd(), "config-lock.yaml")
+    if not config and os.path.exists(lockfile_path):
+        logger.info("Found config-lock.yaml, using it as default config source")
+        config = lockfile_path
+
     if config:
         config_data = from_yaml(config)
-        pipeline_id = config_data.get('pipeline_id', pipeline_id)
-        pipeline_version_id = config_data.get('pipeline_version_id', pipeline_version_id)
+
+        # Handle both regular config format and lockfile format
+        if 'pipeline' in config_data and isinstance(config_data['pipeline'], dict):
+            pipeline_config = config_data['pipeline']
+            pipeline_id = pipeline_config.get('id', pipeline_id)
+            pipeline_version_id = pipeline_config.get('version_id', pipeline_version_id)
+            user_id = pipeline_config.get('user_id', user_id)
+            app_id = pipeline_config.get('app_id', app_id)
+        else:
+            # Fallback to flat config structure
+            pipeline_id = config_data.get('pipeline_id', pipeline_id)
+            pipeline_version_id = config_data.get('pipeline_version_id', pipeline_version_id)
+            user_id = config_data.get('user_id', user_id)
+            app_id = config_data.get('app_id', app_id)
+
         pipeline_version_run_id = config_data.get(
             'pipeline_version_run_id', pipeline_version_run_id
         )
-        user_id = config_data.get('user_id', user_id)
-        app_id = config_data.get('app_id', app_id)
         nodepool_id = config_data.get('nodepool_id', nodepool_id)
         compute_cluster_id = config_data.get('compute_cluster_id', compute_cluster_id)
         pipeline_url = config_data.get('pipeline_url', pipeline_url)
@@ -319,6 +341,62 @@ def init(pipeline_path):
     logger.info("3. Run 'clarifai pipeline upload config.yaml' to upload your pipeline")
 
 
+@pipeline.command()
+@click.argument(
+    "lockfile_path", type=click.Path(exists=True), required=False, default="config-lock.yaml"
+)
+def validate_lock(lockfile_path):
+    """Validate a config-lock.yaml file for schema and reference consistency.
+
+    LOCKFILE_PATH: Path to the config-lock.yaml file. If not specified, looks for config-lock.yaml in current directory.
+    """
+    from clarifai.runners.utils.pipeline_validation import PipelineConfigValidator
+    from clarifai.utils.cli import from_yaml
+
+    try:
+        # Load the lockfile
+        lockfile_data = from_yaml(lockfile_path)
+
+        # Validate required fields
+        if "pipeline" not in lockfile_data:
+            raise ValueError("'pipeline' section not found in lockfile")
+
+        pipeline = lockfile_data["pipeline"]
+        required_fields = ["id", "user_id", "app_id", "version_id"]
+
+        for field in required_fields:
+            if field not in pipeline:
+                raise ValueError(f"Required field '{field}' not found in pipeline section")
+            if not pipeline[field]:
+                raise ValueError(f"Required field '{field}' cannot be empty")
+
+        # Validate orchestration spec if present
+        if "orchestration_spec" in pipeline:
+            # Create a temporary config structure for validation
+            temp_config = {
+                "pipeline": {
+                    "id": pipeline["id"],
+                    "user_id": pipeline["user_id"],
+                    "app_id": pipeline["app_id"],
+                    "orchestration_spec": pipeline["orchestration_spec"],
+                }
+            }
+
+            # Use existing validator to check orchestration spec
+            validator = PipelineConfigValidator()
+            validator._validate_orchestration_spec(temp_config)
+
+        logger.info(f"✅ Lockfile {lockfile_path} is valid")
+        logger.info(f"Pipeline: {pipeline['id']}")
+        logger.info(f"User: {pipeline['user_id']}")
+        logger.info(f"App: {pipeline['app_id']}")
+        logger.info(f"Version: {pipeline['version_id']}")
+
+    except Exception as e:
+        logger.error(f"❌ Lockfile validation failed: {e}")
+        raise click.Abort()
+
+
 @pipeline.command(['ls'])
 @click.option('--page_no', required=False, help='Page number to list.', default=1)
 @click.option('--per_page', required=False, help='Number of items per page.', default=16)

{clarifai-11.8.2 → clarifai-11.8.3}/clarifai/client/model.py

@@ -522,8 +522,9 @@ class Model(Lister, BaseClient):
                 model=self.model_info,
                 runner_selector=self._runner_selector,
             )
+            # Pass in None for async stub will create it.
             self._client = ModelClient(
-                stub=self.STUB, async_stub=self.async_stub, request_template=request_template
+                stub=self.STUB, async_stub=None, request_template=request_template
             )
         return self._client
 

{clarifai-11.8.2 → clarifai-11.8.3}/clarifai/runners/pipeline_steps/pipeline_step_builder.py

@@ -4,6 +4,7 @@ import sys
 import tarfile
 import time
 from string import Template
+from typing import List, Optional
 
 import yaml
 from clarifai_grpc.grpc.api import resources_pb2, service_pb2
@@ -11,6 +12,7 @@ from clarifai_grpc.grpc.api.status import status_code_pb2
 from google.protobuf import json_format
 
 from clarifai.client.base import BaseClient
+from clarifai.utils.hashing import hash_directory
 from clarifai.utils.logging import logger
 from clarifai.utils.misc import get_uuid
 from clarifai.versions import CLIENT_VERSION
@@ -22,12 +24,13 @@ UPLOAD_CHUNK_SIZE = 14 * 1024 * 1024
 class PipelineStepBuilder:
     """Pipeline Step Builder class for managing pipeline step upload to Clarifai."""
 
-    def __init__(self, folder: str):
+    def __init__(self, folder: str, hash_exclusions: Optional[List[str]] = None):
         """
         Initialize PipelineStepBuilder.
 
         :param folder: The folder containing the pipeline step files (config.yaml, requirements.txt,
                       dockerfile, and pipeline_step.py in 1/ subdirectory)
+        :param hash_exclusions: List of file names to exclude from hash calculation (defaults to ['config-lock.yaml'])
         """
         self._client = None
         self.folder = self._validate_folder(folder)
@@ -37,6 +40,10 @@ class PipelineStepBuilder:
         self.pipeline_step_id = self.pipeline_step_proto.id
         self.pipeline_step_version_id = None
         self.pipeline_step_compute_info = self._get_pipeline_step_compute_info()
+        # Configure files to exclude from hash calculation
+        self.hash_exclusions = (
+            hash_exclusions if hash_exclusions is not None else ['config-lock.yaml']
+        )
 
     @property
     def client(self):
@@ -490,6 +497,95 @@ COPY --link=true requirements.txt config.yaml /home/nonroot/main/
 
         raise TimeoutError("Pipeline step build did not finish in time")
 
+    def load_config_lock(self):
+        """
+        Load existing config-lock.yaml if it exists.
+
+        :return: Dictionary with config-lock data or None if file doesn't exist
+        """
+        config_lock_path = os.path.join(self.folder, "config-lock.yaml")
+        if os.path.exists(config_lock_path):
+            try:
+                with open(config_lock_path, 'r', encoding='utf-8') as f:
+                    return yaml.safe_load(f)
+            except Exception as e:
+                logger.warning(f"Failed to load config-lock.yaml: {e}")
+                return None
+        return None
+
+    def should_upload_step(self, algo="md5"):
+        """
+        Check if the pipeline step should be uploaded based on hash comparison.
+
+        :param algo: Hash algorithm to use
+        :return: True if step should be uploaded, False otherwise
+        """
+        config_lock = self.load_config_lock()
+
+        # If no config-lock.yaml exists, upload the step (first time upload)
+        if config_lock is None:
+            logger.info("No config-lock.yaml found, will upload pipeline step")
+            return True
+
+        # Compare stored hash with freshly computed one
+        current_hash = hash_directory(self.folder, algo=algo, exclude_files=self.hash_exclusions)
+        stored_hash_info = config_lock.get("hash", {})
+        stored_hash = stored_hash_info.get("value", "")
+        stored_algo = stored_hash_info.get("algo", "md5")
+
+        # If algorithm changed, re-upload to update hash
+        if stored_algo != algo:
+            logger.info(
+                f"Hash algorithm changed from {stored_algo} to {algo}, will upload pipeline step"
+            )
+            return True
+
+        # If hash changed, upload
+        if current_hash != stored_hash:
+            logger.info(
+                f"Hash changed (was: {stored_hash}, now: {current_hash}), will upload pipeline step"
+            )
+            return True
+
+        logger.info(f"Hash unchanged ({current_hash}), skipping pipeline step upload")
+        return False
+
+    def generate_config_lock(self, version_id, algo="md5"):
+        """
+        Generate config-lock.yaml content for the pipeline step.
+
+        :param version_id: Pipeline step version ID
+        :param algo: Hash algorithm used
+        :return: Dictionary with config-lock data
+        """
+        # Compute hash
+        hash_value = hash_directory(self.folder, algo=algo, exclude_files=self.hash_exclusions)
+
+        # Create config-lock structure
+        config_lock = {"id": version_id, "hash": {"algo": algo, "value": hash_value}}
+
+        # Append the original config.yaml contents
+        config_lock.update(self.config)
+
+        return config_lock
+
+    def save_config_lock(self, version_id, algo="md5"):
+        """
+        Save config-lock.yaml file with pipeline step metadata.
+
+        :param version_id: Pipeline step version ID
+        :param algo: Hash algorithm used
+        """
+        config_lock_data = self.generate_config_lock(version_id, algo)
+        config_lock_path = os.path.join(self.folder, "config-lock.yaml")
+
+        try:
+            with open(config_lock_path, 'w', encoding='utf-8') as f:
+                yaml.dump(config_lock_data, f, default_flow_style=False, allow_unicode=True)
+            logger.info(f"Generated config-lock.yaml at {config_lock_path}")
+        except Exception as e:
+            logger.error(f"Failed to save config-lock.yaml: {e}")
+
 
 def upload_pipeline_step(folder, skip_dockerfile=False):
     """