clarifai 11.8.2__py3-none-any.whl → 11.8.3__py3-none-any.whl

clarifai/__init__.py

@@ -1 +1 @@
-__version__ = "11.8.2"
+__version__ = "11.8.3"

clarifai/cli/model.py

@@ -231,6 +231,44 @@ def init(
                     repo_url = format_github_repo_url(github_url)
                     repo_url = f"https://github.com/{owner}/{repo}"
 
+                try:
+                    # Create a temporary directory for cloning
+                    with tempfile.TemporaryDirectory(prefix="clarifai_model_") as clone_dir:
+                        # Clone the repository with explicit branch parameter
+                        if not clone_github_repo(repo_url, clone_dir, github_pat, branch):
+                            logger.error(f"Failed to clone repository from {repo_url}")
+                            github_url = None  # Fall back to template mode
+
+                        else:
+                            # Copy the entire repository content to target directory (excluding .git)
+                            for item in os.listdir(clone_dir):
+                                if item == '.git':
+                                    continue
+
+                                source_path = os.path.join(clone_dir, item)
+                                target_path = os.path.join(model_path, item)
+
+                                if os.path.isdir(source_path):
+                                    shutil.copytree(source_path, target_path, dirs_exist_ok=True)
+                                else:
+                                    shutil.copy2(source_path, target_path)
+
+                            logger.info(f"Successfully cloned repository to {model_path}")
+                            logger.info(
+                                "Model initialization complete with GitHub repository clone"
+                            )
+                            logger.info("Next steps:")
+                            logger.info("1. Review the model configuration")
+                            logger.info("2. Install any required dependencies manually")
+                            logger.info(
+                                "3. Test the model locally using 'clarifai model local-test'"
+                            )
+                            return
+
+                except Exception as e:
+                    logger.error(f"Failed to clone GitHub repository: {e}")
+                    github_url = None  # Fall back to template mode
+
     if toolkit:
         logger.info(f"Initializing model from GitHub repository: {github_url}")
 
@@ -240,31 +278,31 @@ def init(
         else:
             repo_url = format_github_repo_url(github_url)
 
-    try:
-        # Create a temporary directory for cloning
-        with tempfile.TemporaryDirectory(prefix="clarifai_model_") as clone_dir:
-            # Clone the repository with explicit branch parameter
-            if not clone_github_repo(repo_url, clone_dir, github_pat, branch):
-                logger.error(f"Failed to clone repository from {repo_url}")
-                github_url = None  # Fall back to template mode
-
-            else:
-                # Copy the entire repository content to target directory (excluding .git)
-                for item in os.listdir(clone_dir):
-                    if item == '.git':
-                        continue
-
-                    source_path = os.path.join(clone_dir, item)
-                    target_path = os.path.join(model_path, item)
-
-                    if os.path.isdir(source_path):
-                        shutil.copytree(source_path, target_path, dirs_exist_ok=True)
-                    else:
-                        shutil.copy2(source_path, target_path)
+        try:
+            # Create a temporary directory for cloning
+            with tempfile.TemporaryDirectory(prefix="clarifai_model_") as clone_dir:
+                # Clone the repository with explicit branch parameter
+                if not clone_github_repo(repo_url, clone_dir, github_pat, branch):
+                    logger.error(f"Failed to clone repository from {repo_url}")
+                    github_url = None  # Fall back to template mode
 
-    except Exception as e:
-        logger.error(f"Failed to clone GitHub repository: {e}")
-        github_url = None
+                else:
+                    # Copy the entire repository content to target directory (excluding .git)
+                    for item in os.listdir(clone_dir):
+                        if item == '.git':
+                            continue
+
+                        source_path = os.path.join(clone_dir, item)
+                        target_path = os.path.join(model_path, item)
+
+                        if os.path.isdir(source_path):
+                            shutil.copytree(source_path, target_path, dirs_exist_ok=True)
+                        else:
+                            shutil.copy2(source_path, target_path)
+
+        except Exception as e:
+            logger.error(f"Failed to clone GitHub repository: {e}")
+            github_url = None
 
     if (model_name or port or context_length) and (toolkit == 'ollama'):
         customize_ollama_model(model_path, model_name, port, context_length)

clarifai/cli/pipeline.py

@@ -26,14 +26,19 @@ def pipeline():
 
 @pipeline.command()
 @click.argument("path", type=click.Path(exists=True), required=False, default=".")
-def upload(path):
+@click.option(
+    '--no-lockfile',
+    is_flag=True,
+    help='Skip creating config-lock.yaml file.',
+)
+def upload(path, no_lockfile):
     """Upload a pipeline with associated pipeline steps to Clarifai.
 
     PATH: Path to the pipeline configuration file or directory containing config.yaml. If not specified, the current directory is used by default.
     """
     from clarifai.runners.pipelines.pipeline_builder import upload_pipeline
 
-    upload_pipeline(path)
+    upload_pipeline(path, no_lockfile=no_lockfile)
 
 
 @pipeline.command()
@@ -106,15 +111,32 @@ def run(
 
     validate_context(ctx)
 
+    # Try to load from config-lock.yaml first if no config is specified
+    lockfile_path = os.path.join(os.getcwd(), "config-lock.yaml")
+    if not config and os.path.exists(lockfile_path):
+        logger.info("Found config-lock.yaml, using it as default config source")
+        config = lockfile_path
+
     if config:
         config_data = from_yaml(config)
-        pipeline_id = config_data.get('pipeline_id', pipeline_id)
-        pipeline_version_id = config_data.get('pipeline_version_id', pipeline_version_id)
+
+        # Handle both regular config format and lockfile format
+        if 'pipeline' in config_data and isinstance(config_data['pipeline'], dict):
+            pipeline_config = config_data['pipeline']
+            pipeline_id = pipeline_config.get('id', pipeline_id)
+            pipeline_version_id = pipeline_config.get('version_id', pipeline_version_id)
+            user_id = pipeline_config.get('user_id', user_id)
+            app_id = pipeline_config.get('app_id', app_id)
+        else:
+            # Fallback to flat config structure
+            pipeline_id = config_data.get('pipeline_id', pipeline_id)
+            pipeline_version_id = config_data.get('pipeline_version_id', pipeline_version_id)
+            user_id = config_data.get('user_id', user_id)
+            app_id = config_data.get('app_id', app_id)
+
         pipeline_version_run_id = config_data.get(
             'pipeline_version_run_id', pipeline_version_run_id
         )
-        user_id = config_data.get('user_id', user_id)
-        app_id = config_data.get('app_id', app_id)
         nodepool_id = config_data.get('nodepool_id', nodepool_id)
         compute_cluster_id = config_data.get('compute_cluster_id', compute_cluster_id)
         pipeline_url = config_data.get('pipeline_url', pipeline_url)
@@ -319,6 +341,62 @@ def init(pipeline_path):
     logger.info("3. Run 'clarifai pipeline upload config.yaml' to upload your pipeline")
 
 
+@pipeline.command()
+@click.argument(
+    "lockfile_path", type=click.Path(exists=True), required=False, default="config-lock.yaml"
+)
+def validate_lock(lockfile_path):
+    """Validate a config-lock.yaml file for schema and reference consistency.
+
+    LOCKFILE_PATH: Path to the config-lock.yaml file. If not specified, looks for config-lock.yaml in current directory.
+    """
+    from clarifai.runners.utils.pipeline_validation import PipelineConfigValidator
+    from clarifai.utils.cli import from_yaml
+
+    try:
+        # Load the lockfile
+        lockfile_data = from_yaml(lockfile_path)
+
+        # Validate required fields
+        if "pipeline" not in lockfile_data:
+            raise ValueError("'pipeline' section not found in lockfile")
+
+        pipeline = lockfile_data["pipeline"]
+        required_fields = ["id", "user_id", "app_id", "version_id"]
+
+        for field in required_fields:
+            if field not in pipeline:
+                raise ValueError(f"Required field '{field}' not found in pipeline section")
+            if not pipeline[field]:
+                raise ValueError(f"Required field '{field}' cannot be empty")
+
+        # Validate orchestration spec if present
+        if "orchestration_spec" in pipeline:
+            # Create a temporary config structure for validation
+            temp_config = {
+                "pipeline": {
+                    "id": pipeline["id"],
+                    "user_id": pipeline["user_id"],
+                    "app_id": pipeline["app_id"],
+                    "orchestration_spec": pipeline["orchestration_spec"],
+                }
+            }
+
+            # Use existing validator to check orchestration spec
+            validator = PipelineConfigValidator()
+            validator._validate_orchestration_spec(temp_config)
+
+        logger.info(f"✅ Lockfile {lockfile_path} is valid")
+        logger.info(f"Pipeline: {pipeline['id']}")
+        logger.info(f"User: {pipeline['user_id']}")
+        logger.info(f"App: {pipeline['app_id']}")
+        logger.info(f"Version: {pipeline['version_id']}")
+
+    except Exception as e:
+        logger.error(f"❌ Lockfile validation failed: {e}")
+        raise click.Abort()
+
+
 @pipeline.command(['ls'])
 @click.option('--page_no', required=False, help='Page number to list.', default=1)
 @click.option('--per_page', required=False, help='Number of items per page.', default=16)

clarifai/client/model.py

@@ -522,8 +522,9 @@ class Model(Lister, BaseClient):
                 model=self.model_info,
                 runner_selector=self._runner_selector,
             )
+            # Pass in None for async stub will create it.
             self._client = ModelClient(
-                stub=self.STUB, async_stub=self.async_stub, request_template=request_template
+                stub=self.STUB, async_stub=None, request_template=request_template
             )
         return self._client
 

clarifai/runners/pipeline_steps/pipeline_step_builder.py

@@ -4,6 +4,7 @@ import sys
 import tarfile
 import time
 from string import Template
+from typing import List, Optional
 
 import yaml
 from clarifai_grpc.grpc.api import resources_pb2, service_pb2
@@ -11,6 +12,7 @@ from clarifai_grpc.grpc.api.status import status_code_pb2
 from google.protobuf import json_format
 
 from clarifai.client.base import BaseClient
+from clarifai.utils.hashing import hash_directory
 from clarifai.utils.logging import logger
 from clarifai.utils.misc import get_uuid
 from clarifai.versions import CLIENT_VERSION
@@ -22,12 +24,13 @@ UPLOAD_CHUNK_SIZE = 14 * 1024 * 1024
 class PipelineStepBuilder:
     """Pipeline Step Builder class for managing pipeline step upload to Clarifai."""
 
-    def __init__(self, folder: str):
+    def __init__(self, folder: str, hash_exclusions: Optional[List[str]] = None):
         """
         Initialize PipelineStepBuilder.
 
         :param folder: The folder containing the pipeline step files (config.yaml, requirements.txt,
                       dockerfile, and pipeline_step.py in 1/ subdirectory)
+        :param hash_exclusions: List of file names to exclude from hash calculation (defaults to ['config-lock.yaml'])
         """
         self._client = None
         self.folder = self._validate_folder(folder)
@@ -37,6 +40,10 @@ class PipelineStepBuilder:
         self.pipeline_step_id = self.pipeline_step_proto.id
         self.pipeline_step_version_id = None
         self.pipeline_step_compute_info = self._get_pipeline_step_compute_info()
+        # Configure files to exclude from hash calculation
+        self.hash_exclusions = (
+            hash_exclusions if hash_exclusions is not None else ['config-lock.yaml']
+        )
 
     @property
     def client(self):
@@ -490,6 +497,95 @@ COPY --link=true requirements.txt config.yaml /home/nonroot/main/
 
         raise TimeoutError("Pipeline step build did not finish in time")
 
+    def load_config_lock(self):
+        """
+        Load existing config-lock.yaml if it exists.
+
+        :return: Dictionary with config-lock data or None if file doesn't exist
+        """
+        config_lock_path = os.path.join(self.folder, "config-lock.yaml")
+        if os.path.exists(config_lock_path):
+            try:
+                with open(config_lock_path, 'r', encoding='utf-8') as f:
+                    return yaml.safe_load(f)
+            except Exception as e:
+                logger.warning(f"Failed to load config-lock.yaml: {e}")
+                return None
+        return None
+
+    def should_upload_step(self, algo="md5"):
+        """
+        Check if the pipeline step should be uploaded based on hash comparison.
+
+        :param algo: Hash algorithm to use
+        :return: True if step should be uploaded, False otherwise
+        """
+        config_lock = self.load_config_lock()
+
+        # If no config-lock.yaml exists, upload the step (first time upload)
+        if config_lock is None:
+            logger.info("No config-lock.yaml found, will upload pipeline step")
+            return True
+
+        # Compare stored hash with freshly computed one
+        current_hash = hash_directory(self.folder, algo=algo, exclude_files=self.hash_exclusions)
+        stored_hash_info = config_lock.get("hash", {})
+        stored_hash = stored_hash_info.get("value", "")
+        stored_algo = stored_hash_info.get("algo", "md5")
+
+        # If algorithm changed, re-upload to update hash
+        if stored_algo != algo:
+            logger.info(
+                f"Hash algorithm changed from {stored_algo} to {algo}, will upload pipeline step"
+            )
+            return True
+
+        # If hash changed, upload
+        if current_hash != stored_hash:
+            logger.info(
+                f"Hash changed (was: {stored_hash}, now: {current_hash}), will upload pipeline step"
+            )
+            return True
+
+        logger.info(f"Hash unchanged ({current_hash}), skipping pipeline step upload")
+        return False
+
+    def generate_config_lock(self, version_id, algo="md5"):
+        """
+        Generate config-lock.yaml content for the pipeline step.
+
+        :param version_id: Pipeline step version ID
+        :param algo: Hash algorithm used
+        :return: Dictionary with config-lock data
+        """
+        # Compute hash
+        hash_value = hash_directory(self.folder, algo=algo, exclude_files=self.hash_exclusions)
+
+        # Create config-lock structure
+        config_lock = {"id": version_id, "hash": {"algo": algo, "value": hash_value}}
+
+        # Append the original config.yaml contents
+        config_lock.update(self.config)
+
+        return config_lock
+
+    def save_config_lock(self, version_id, algo="md5"):
+        """
+        Save config-lock.yaml file with pipeline step metadata.
+
+        :param version_id: Pipeline step version ID
+        :param algo: Hash algorithm used
+        """
+        config_lock_data = self.generate_config_lock(version_id, algo)
+        config_lock_path = os.path.join(self.folder, "config-lock.yaml")
+
+        try:
+            with open(config_lock_path, 'w', encoding='utf-8') as f:
+                yaml.dump(config_lock_data, f, default_flow_style=False, allow_unicode=True)
+            logger.info(f"Generated config-lock.yaml at {config_lock_path}")
+        except Exception as e:
+            logger.error(f"Failed to save config-lock.yaml: {e}")
+
 
 def upload_pipeline_step(folder, skip_dockerfile=False):
     """

clarifai/runners/pipelines/pipeline_builder.py

@@ -116,13 +116,33 @@ class PipelineBuilder:
         return True
 
     def _upload_pipeline_step_with_version_capture(self, step_path: str) -> tuple[bool, str]:
-        """Upload a pipeline step and capture its version ID."""
+        """
+        Upload a pipeline step and capture its version ID.
+        Implements hash-based optimization to only upload modified steps.
+        """
         try:
             # Use the existing pipeline step builder
             from clarifai.runners.pipeline_steps.pipeline_step_builder import PipelineStepBuilder
 
             builder = PipelineStepBuilder(step_path)
 
+            # Check if we should upload based on hash comparison
+            should_upload = builder.should_upload_step()
+
+            if not should_upload:
+                # Load existing version ID from config-lock.yaml
+                config_lock = builder.load_config_lock()
+                if config_lock and config_lock.get("id"):
+                    version_id = config_lock["id"]
+                    logger.info(
+                        f"Using existing pipeline step version {version_id} (no changes detected)"
+                    )
+                    return True, version_id
+                else:
+                    logger.warning(
+                        "Hash indicates no upload needed, but no version ID found in config-lock.yaml. Proceeding with upload."
+                    )
+
             # Create dockerfile if needed
             builder.create_dockerfile()
 
@@ -139,6 +159,11 @@ class PipelineBuilder:
             success = builder.upload_pipeline_step_version()
 
             if success and builder.pipeline_step_version_id:
+                # Generate config-lock.yaml with the new version ID
+                builder.save_config_lock(builder.pipeline_step_version_id)
+                logger.info(
+                    f"Generated config-lock.yaml for pipeline step with version {builder.pipeline_step_version_id}"
+                )
                 return True, builder.pipeline_step_version_id
             else:
                 logger.error("Failed to get pipeline step version ID after upload")
@@ -148,42 +173,105 @@ class PipelineBuilder:
             logger.error(f"Error uploading pipeline step: {e}")
             return False, ""
 
-    def update_config_with_versions(self) -> None:
-        """Update the config.yaml with uploaded pipeline step versions."""
+    def prepare_lockfile_with_step_versions(self) -> Dict[str, Any]:
+        """Prepare lockfile data with step versions after pipeline step upload."""
         if not self.uploaded_step_versions:
-            logger.info("No pipeline step versions to update in config")
-            return
-
-        logger.info("Updating config.yaml with pipeline step versions...")
+            logger.info("No pipeline step versions for lockfile")
 
-        # Update the orchestration spec
+        # Create a copy of the orchestration spec to modify
         pipeline_config = self.config["pipeline"]
-        orchestration_spec = pipeline_config["orchestration_spec"]
+        orchestration_spec = pipeline_config["orchestration_spec"].copy()
         argo_spec_str = orchestration_spec["argo_orchestration_spec"]
         argo_spec = yaml.safe_load(argo_spec_str)
 
         # Update templateRef names to include versions
         self._update_template_refs_with_versions(argo_spec)
 
-        # Update the config
-        orchestration_spec["argo_orchestration_spec"] = yaml.dump(
-            argo_spec, Dumper=LiteralBlockDumper, default_flow_style=False
-        )
+        # Create the partial lockfile data structure (without pipeline info)
+        lockfile_data = {
+            "pipeline": {
+                "id": self.pipeline_id,
+                "user_id": self.user_id,
+                "app_id": self.app_id,
+                "version_id": None,  # Will be filled in later
+                "orchestration_spec": {
+                    "argo_orchestration_spec": yaml.dump(
+                        argo_spec, Dumper=LiteralBlockDumper, default_flow_style=False
+                    )
+                },
+            }
+        }
+
+        return lockfile_data
+
+    def update_lockfile_with_pipeline_info(
+        self, lockfile_data: Dict[str, Any], pipeline_version_id: str
+    ) -> Dict[str, Any]:
+        """Update the prepared lockfile data with pipeline version information."""
+        lockfile_data["pipeline"]["version_id"] = pipeline_version_id
+        return lockfile_data
+
+    def generate_lockfile_data(
+        self, pipeline_id: str = None, pipeline_version_id: str = None
+    ) -> Dict[str, Any]:
+        """Generate the complete lockfile data structure without modifying config.yaml.
+
+        This method is kept for backward compatibility. The recommended approach is to use
+        prepare_lockfile_with_step_versions() followed by update_lockfile_with_pipeline_info().
+        """
+        if not self.uploaded_step_versions:
+            logger.info("No pipeline step versions for lockfile")
 
-        # Remove uploaded directories from step_directories
-        remaining_dirs = []
-        for step_dir in pipeline_config.get("step_directories", []):
-            if step_dir not in self.uploaded_step_versions:
-                remaining_dirs.append(step_dir)
+        # Create a copy of the orchestration spec to modify
+        pipeline_config = self.config["pipeline"]
+        orchestration_spec = pipeline_config["orchestration_spec"].copy()
+        argo_spec_str = orchestration_spec["argo_orchestration_spec"]
+        argo_spec = yaml.safe_load(argo_spec_str)
 
-        pipeline_config["step_directories"] = remaining_dirs
+        # Update templateRef names to include versions
+        self._update_template_refs_with_versions(argo_spec)
+
+        # Create the lockfile data structure
+        lockfile_data = {
+            "pipeline": {
+                "id": pipeline_id or self.pipeline_id,
+                "user_id": self.user_id,
+                "app_id": self.app_id,
+                "version_id": pipeline_version_id,
+                "orchestration_spec": {
+                    "argo_orchestration_spec": yaml.dump(
+                        argo_spec, Dumper=LiteralBlockDumper, default_flow_style=False
+                    )
+                },
+            }
+        }
+
+        return lockfile_data
+
+    def save_lockfile(self, lockfile_data: Dict[str, Any], lockfile_path: str = None) -> None:
+        """Save lockfile data to config-lock.yaml."""
+        if lockfile_path is None:
+            lockfile_path = os.path.join(self.config_dir, "config-lock.yaml")
 
-        # Save the updated config
-        self._save_config()
-        logger.info("Updated config.yaml with pipeline step versions")
+        try:
+            with open(lockfile_path, 'w', encoding="utf-8") as file:
+                yaml.dump(
+                    lockfile_data,
+                    file,
+                    Dumper=LiteralBlockDumper,
+                    default_flow_style=False,
+                    sort_keys=False,
+                )
+            logger.info(f"Generated lockfile: {lockfile_path}")
+        except Exception as e:
+            raise ValueError(f"Error saving lockfile {lockfile_path}: {e}")
 
     def _update_template_refs_with_versions(self, argo_spec: Dict[str, Any]) -> None:
-        """Update templateRef names in Argo spec to include version information."""
+        """
+        Update templateRef names in Argo spec to include version information.
+        The step versions should be resolved from the corresponding config-lock.yaml
+        file of each pipeline-step, located in the step_directories.
+        """
         for template in argo_spec["spec"]["templates"]:
             if "steps" in template:
                 for step_group in template["steps"]:
@@ -199,12 +287,19 @@ class PipelineBuilder:
                                 step_name = parts[-1]
                                 # The step name should match the directory name or be derivable from it
                                 version_id = self.uploaded_step_versions.get(step_name, None)
+
+                                # If not found in uploaded_step_versions, try to get from config-lock.yaml
+                                if version_id is None:
+                                    version_id = self._get_version_from_config_lock(step_name)
+
                                 if version_id is not None:
                                     # Update the templateRef to include version
                                     new_name = f"{name}/versions/{version_id}"
                                     template_ref["name"] = new_name
                                     template_ref["template"] = new_name
                                     logger.info(f"Updated templateRef from {name} to {new_name}")
+                                else:
+                                    logger.warning(f"Could not find version for step: {step_name}")
                             elif self.validator.TEMPLATE_REF_WITH_VERSION_PATTERN.match(name):
                                 # strip the /versions/{version_id} from the end of name
                                 # to get the name like above
@@ -215,6 +310,11 @@ class PipelineBuilder:
                                 # if it already has a version, make sure it matches the uploaded
                                 # version
                                 version_id = self.uploaded_step_versions.get(step_name, None)
+
+                                # If not found in uploaded_step_versions, try to get from config-lock.yaml
+                                if version_id is None:
+                                    version_id = self._get_version_from_config_lock(step_name)
+
                                 if version_id is not None:
                                     # Update the templateRef to include version
                                     new_name = f"{name}/versions/{version_id}"
@@ -223,9 +323,51 @@ class PipelineBuilder:
                                     logger.info(
                                         f"Updated templateRef from {orig_name} to {new_name}"
                                     )
+                                else:
+                                    logger.warning(f"Could not find version for step: {step_name}")
+
+    def _get_version_from_config_lock(self, step_name: str) -> str:
+        """
+        Get version ID from config-lock.yaml file in the corresponding step directory.
 
-    def create_pipeline(self) -> bool:
-        """Create the pipeline using PostPipelines RPC."""
+        :param step_name: Name of the pipeline step
+        :return: Version ID if found, None otherwise
+        """
+        pipeline_config = self.config["pipeline"]
+        step_directories = pipeline_config.get("step_directories", [])
+
+        for step_dir in step_directories:
+            # Check if step_dir matches step_name (handle both exact match and derivable cases)
+            if (
+                step_dir == step_name
+                or step_dir.endswith(f"/{step_name}")
+                or step_name in step_dir
+            ):
+                config_lock_path = os.path.join(self.config_dir, step_dir, "config-lock.yaml")
+
+                if os.path.exists(config_lock_path):
+                    try:
+                        with open(config_lock_path, 'r', encoding='utf-8') as f:
+                            config_lock = yaml.safe_load(f)
+                            version_id = config_lock.get("id")
+                            if version_id:
+                                logger.info(
+                                    f"Found version {version_id} for step {step_name} in {config_lock_path}"
+                                )
+                                return version_id
+                    except Exception as e:
+                        logger.warning(
+                            f"Failed to read config-lock.yaml at {config_lock_path}: {e}"
+                        )
+
+        return None
+
+    def create_pipeline(self) -> tuple[bool, str]:
+        """Create the pipeline using PostPipelines RPC.
+
+        Returns:
+            tuple[bool, str]: (success, pipeline_version_id)
+        """
         logger.info(f"Creating pipeline {self.pipeline_id}...")
 
         try:
@@ -243,6 +385,11 @@ class PipelineBuilder:
             argo_spec = yaml.safe_load(argo_spec_str)
             api_version = argo_spec.get("apiVersion", "argoproj.io/v1alpha1")
 
+            # Ensure that pipeline_config.argo_orchestration_spec_proto has the updated spec.templates.steps.templateRef values
+            # For each step, if the templateRef is missing a version, append the correct version at the end
+            # The step versions should be resolved from the corresponding config-lock.yaml file of each pipeline-step, located in the step_directories
+            self._update_template_refs_with_versions(argo_spec)
+
             # Create pipeline version with orchestration spec
             pipeline_version = resources_pb2.PipelineVersion()
             # Create orchestration spec proto
@@ -269,29 +416,32 @@ class PipelineBuilder:
             if response.status.code == status_code_pb2.SUCCESS:
                 logger.info(f"Successfully created pipeline {self.pipeline_id}")
 
+                pipeline_version_id = ""
                 # Log pipeline and version IDs if available in response
                 if response.pipelines:
                     created_pipeline = response.pipelines[0]
                     logger.info(f"Pipeline ID: {created_pipeline.id}")
                     if created_pipeline.pipeline_version and created_pipeline.pipeline_version.id:
-                        logger.info(f"Pipeline version ID: {created_pipeline.pipeline_version.id}")
+                        pipeline_version_id = created_pipeline.pipeline_version.id
+                        logger.info(f"Pipeline version ID: {pipeline_version_id}")
 
-                return True
+                return True, pipeline_version_id
             else:
                 logger.error(f"Failed to create pipeline: {response.status.description}")
                 logger.error(f"Details: {response.status.details}")
-                return False
+                return False, ""
 
         except Exception as e:
             logger.error(f"Error creating pipeline: {e}")
-            return False
+            return False, ""
 
 
-def upload_pipeline(path: str):
+def upload_pipeline(path: str, no_lockfile: bool = False):
     """
     Upload a pipeline with associated pipeline steps to Clarifai.
 
     :param path: Path to the pipeline configuration file or directory containing config.yaml
+    :param no_lockfile: If True, skip creating config-lock.yaml
     """
     try:
         # Determine if path is a directory or file
@@ -311,15 +461,27 @@ def upload_pipeline(path: str):
             logger.error("Failed to upload pipeline steps")
             sys.exit(1)
 
-        # Step 2: Update config with version information
-        builder.update_config_with_versions()
+        # Step 2: Generate lockfile (unless --no-lockfile is specified)
+        # This will be used to update the versions of pipeline-steps that just got uploaded in Step 1
+        lockfile_data = None
+        if not no_lockfile:
+            lockfile_data = builder.prepare_lockfile_with_step_versions()
 
         # Step 3: Create the pipeline
-        if not builder.create_pipeline():
+        success, pipeline_version_id = builder.create_pipeline()
+        if not success:
             logger.error("Failed to create pipeline")
             sys.exit(1)
 
-        logger.info("Pipeline upload completed successfully!")
+        # Step 4: Update lockfile (unless --no-lockfile is specified)
+        if not no_lockfile and lockfile_data:
+            lockfile_data = builder.update_lockfile_with_pipeline_info(
+                lockfile_data, pipeline_version_id
+            )
+            builder.save_lockfile(lockfile_data)
+            logger.info("Pipeline upload completed successfully with lockfile!")
+        else:
+            logger.info("Pipeline upload completed successfully (lockfile skipped)!")
 
     except Exception as e:
         logger.error(f"Pipeline upload failed: {e}")

clarifai/utils/hashing.py

@@ -0,0 +1,117 @@
+"""
+Hashing utilities for Clarifai Python SDK.
+
+This module provides functions for computing stable hashes of directories and files,
+commonly used for change detection in pipeline steps and other components.
+"""
+
+import hashlib
+import os
+from typing import List, Optional
+
+
+def hash_directory(
+    directory: str, algo: str = "md5", exclude_files: Optional[List[str]] = None
+) -> str:
+    """
+    Compute a stable hash of all files in a directory.
+
+    This function computes a hash that accounts for:
+    - File relative paths (to detect renames)
+    - File sizes (to detect empty files)
+    - File contents (read in chunks for large files)
+
+    :param directory: Directory to hash
+    :param algo: Hash algorithm ('md5', 'sha1', 'sha256', etc.)
+    :param exclude_files: List of file names to exclude from hash calculation.
+                         If None, defaults to ['config-lock.yaml'] for backward compatibility.
+    :return: Hash as lowercase hex digest string
+    """
+    if exclude_files is None:
+        exclude_files = ['config-lock.yaml']
+
+    # Ensure directory exists
+    if not os.path.exists(directory):
+        raise ValueError(f"Directory does not exist: {directory}")
+
+    if not os.path.isdir(directory):
+        raise ValueError(f"Path is not a directory: {directory}")
+
+    hash_func = hashlib.new(algo)
+
+    for root, _, files in os.walk(directory):
+        for name in sorted(files):
+            # Skip files in the exclusion list
+            if name in exclude_files:
+                continue
+
+            filepath = os.path.join(root, name)
+            relative_path = os.path.relpath(filepath, directory)
+
+            # Hash the relative path to detect renames
+            hash_func.update(relative_path.encode("utf-8"))
+
+            # Hash the file size to detect empties
+            file_size = os.path.getsize(filepath)
+            hash_func.update(str(file_size).encode("utf-8"))
+
+            # Hash the file contents (read in chunks for large files)
+            try:
+                with open(filepath, "rb") as f:
+                    for chunk in iter(lambda: f.read(8192), b""):
+                        hash_func.update(chunk)
+            except (IOError, OSError) as e:
+                # If we can't read the file, include the error in the hash
+                # This ensures the hash changes if file permissions change
+                hash_func.update(f"ERROR_READING_FILE: {e}".encode("utf-8"))
+
+    return hash_func.hexdigest()
+
+
+def hash_file(filepath: str, algo: str = "md5") -> str:
+    """
+    Compute a hash of a single file.
+
+    :param filepath: Path to the file to hash
+    :param algo: Hash algorithm ('md5', 'sha1', 'sha256', etc.)
+    :return: Hash as lowercase hex digest string
+    """
+    if not os.path.exists(filepath):
+        raise ValueError(f"File does not exist: {filepath}")
+
+    if not os.path.isfile(filepath):
+        raise ValueError(f"Path is not a file: {filepath}")
+
+    hash_func = hashlib.new(algo)
+
+    try:
+        with open(filepath, "rb") as f:
+            for chunk in iter(lambda: f.read(8192), b""):
+                hash_func.update(chunk)
+    except (IOError, OSError) as e:
+        raise ValueError(f"Error reading file {filepath}: {e}")
+
+    return hash_func.hexdigest()
+
+
+def verify_hash_algorithm(algo: str) -> bool:
+    """
+    Verify that a hash algorithm is supported.
+
+    :param algo: Hash algorithm name
+    :return: True if algorithm is supported, False otherwise
+    """
+    try:
+        hashlib.new(algo)
+        return True
+    except ValueError:
+        return False
+
+
+def get_available_algorithms() -> List[str]:
+    """
+    Get list of available hash algorithms.
+
+    :return: List of available algorithm names
+    """
+    return list(hashlib.algorithms_available)

{clarifai-11.8.2.dist-info → clarifai-11.8.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: clarifai
-Version: 11.8.2
+Version: 11.8.3
 Home-page: https://github.com/Clarifai/clarifai-python
 Author: Clarifai
 Author-email: support@clarifai.com

{clarifai-11.8.2.dist-info → clarifai-11.8.3.dist-info}/RECORD

@@ -1,4 +1,4 @@
-clarifai/__init__.py,sha256=wlKYds59E079Cej1oyfbQMsxCdbNX7JGHZgtuBpBG3A,23
+clarifai/__init__.py,sha256=OGoTC_KKYIrBX-5ykvzjK3rEoiEwinADzRb-TA_522M,23
 clarifai/cli.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 clarifai/errors.py,sha256=GXa6D4v_L404J83jnRNFPH7s-1V9lk7w6Ws99f1g-AY,2772
 clarifai/versions.py,sha256=ecSuEB_nOL2XSoYHDw2n23XUbm_KPOGjudMXmQrGdS8,224
@@ -8,9 +8,9 @@ clarifai/cli/__main__.py,sha256=7nPbLW7Jr2shkgMPvnxpn4xYGMvIcnqluJ69t9w4H_k,74
 clarifai/cli/base.py,sha256=FQEEmi3a9_LBOmM_-X4EYdpAmDK1UljTxrHOIIsOZbM,10696
 clarifai/cli/compute_cluster.py,sha256=8Xss0Obrp6l1XuxJe0luOqU_pf8vXGDRi6jyIe8qR6k,2282
 clarifai/cli/deployment.py,sha256=9C4I6_kyMxRkWl6h681wc79-3mAtDHtTUaxRv05OZMs,4262
-clarifai/cli/model.py,sha256=2Eeoq4Kq_kJqSTmccn4B9YZ962Wj3mz1wcPQwXEWd2M,49734
+clarifai/cli/model.py,sha256=MwNt1jaxA_8D9KGFaapwH32BFoqnq9CaH8JcafhBZR4,51911
 clarifai/cli/nodepool.py,sha256=H6OIdUW_EiyDUwZogzEDoYmVwEjLMsgoDlPyE7gjIuU,4245
-clarifai/cli/pipeline.py,sha256=MmyPaVX1XsIZlrYTrT7Ctd71ao_d3ZWkuF_fv_NJS8s,13944
+clarifai/cli/pipeline.py,sha256=bH7pnJGfjQIu_Y-f_zwKBFdPDTsBEEDtA2Oz9P6-Zj0,17129
 clarifai/cli/pipeline_step.py,sha256=dvoC2vAsDcxOCy88VV0X42PG22_7JSu9sfBVsk-Cix4,6133
 clarifai/cli/templates/__init__.py,sha256=HbMlZuYOMyVJde73ijNAevmSRUpIttGlHdwyO4W-JOs,44
 clarifai/cli/templates/model_templates.py,sha256=-xGUzadN7S-mNZ-kE4Z-kv51BlnoGHjue05Yg5OGnt0,9791
@@ -24,7 +24,7 @@ clarifai/client/dataset.py,sha256=sz5CycP3J7pG0iMREKI2JeXQuvRwlVrE4AHne8yxgtg,35
 clarifai/client/deployment.py,sha256=QBf0tzkKBEpzNgmOEmWUJMOlVWdFEFc70Y44o8y75Gs,2875
 clarifai/client/input.py,sha256=jpX47qwn7aUBBIEuSSLHF5jk70XaWEh0prD065c9b-E,51205
 clarifai/client/lister.py,sha256=1YEm2suNxPaJO4x9V5szgD_YX6N_00vgSO-7m0HagY8,2208
-clarifai/client/model.py,sha256=WJJL0fOuHe4X8aMT-gs_SnEFhhT0cVDqqHbA3VX2DE8,92520
+clarifai/client/model.py,sha256=AgZ-H0DKAVrMMzDq8m5h9E5jMEjSHDpz20FfxyLfxz0,92567
 clarifai/client/model_client.py,sha256=8N8dRqb5zfFCNxq-jc-YSL19tgS8PpevnxY69G2YzCE,38280
 clarifai/client/module.py,sha256=pTcTmR48-VQRCEj3PJK_ZT5YTsYfZDbEjxwJ43rcLMM,4753
 clarifai/client/nodepool.py,sha256=QDJOMOYrZAG962u-MZWjXOZifjWK8hDgS2zoUn59dZU,16751
@@ -86,9 +86,9 @@ clarifai/runners/models/openai_class.py,sha256=MYnL7_U4_m5FX3CoVaTd6B9Qminh-q4pY
 clarifai/runners/models/visual_classifier_class.py,sha256=1ZoLfCT2crrgRbejjTMAIwpTRgQMiH9N9yflOVpFxSg,2721
 clarifai/runners/models/visual_detector_class.py,sha256=ky4oFAkGCKPpGPdgaOso-n6D3HcmnbKee_8hBsNiV8U,2883
 clarifai/runners/pipeline_steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-clarifai/runners/pipeline_steps/pipeline_step_builder.py,sha256=jcbs3ntbeyUiAaPh00Pscni4uJopbqVNVJblUX1pYVc,21808
+clarifai/runners/pipeline_steps/pipeline_step_builder.py,sha256=0AmleVdqvPMPl6Sxhd8pZDMja7GNzckbT8AuRJLtdho,25678
 clarifai/runners/pipelines/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-clarifai/runners/pipelines/pipeline_builder.py,sha256=0FBjb8l7mWlCwBsBLkHM3znNQB9HPLEOYrrE53ntjCE,13810
+clarifai/runners/pipelines/pipeline_builder.py,sha256=bfIK7PPTspNhWjyKgPp13pkdGlHstKpsLi2Eet39Fyw,21494
 clarifai/runners/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 clarifai/runners/utils/code_script.py,sha256=-6IgNruIMTYLKJG8EqVWSaZR7lFRBoQ4ufJtuCPUCqc,14799
 clarifai/runners/utils/const.py,sha256=MK7lTzzJKbOiyiUtG_jlJXfz_xNKMn5LjkQ9vjbttXE,1538
@@ -108,6 +108,7 @@ clarifai/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 clarifai/utils/cli.py,sha256=ojPI6wBMwxpmCwWIE1nx4t_lzHqyJqby_TI6Fl3Vul4,15536
 clarifai/utils/config.py,sha256=dENYtcWW7Il5MInvIaYe0MZn0hW1fbIb0Lzk8rQ_geQ,7671
 clarifai/utils/constants.py,sha256=bHvs8L_Eai49Qm0U9YcK7Srx9FlL5iyv_pXvgSt6XDc,2497
+clarifai/utils/hashing.py,sha256=z2hHt4sDvGyqNbnOay0F2i3K_PjyX-J24IEytszyYsA,3761
 clarifai/utils/logging.py,sha256=0we53uTqUvzrulC86whu-oeWNxn1JjJL0OQ98Bwf9vo,15198
 clarifai/utils/misc.py,sha256=ATj4RR6S06GeLE0X4tMU4bmTz4Sz4j2WemTddsnSfMI,23458
 clarifai/utils/model_train.py,sha256=0XSAoTkSsrwf4f-W9yw2mkXZtkal7LBLJSoi86CFCn4,9250
@@ -121,9 +122,9 @@ clarifai/workflows/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 clarifai/workflows/export.py,sha256=HvUYG9N_-UZoRR0-_tdGbZ950_AeBqawSppgUxQebR0,1913
 clarifai/workflows/utils.py,sha256=ESL3INcouNcLKCh-nMpfXX-YbtCzX7tz7hT57_RGQ3M,2079
 clarifai/workflows/validate.py,sha256=UhmukyHkfxiMFrPPeBdUTiCOHQT5-shqivlBYEyKTlU,2931
-clarifai-11.8.2.dist-info/licenses/LICENSE,sha256=mUqF_d12-qE2n41g7C5_sq-BMLOcj6CNN-jevr15YHU,555
-clarifai-11.8.2.dist-info/METADATA,sha256=tp1H7AGojDcqwCLK9irGnVbgnsCJK6EMrCd966DAlnw,23193
-clarifai-11.8.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-clarifai-11.8.2.dist-info/entry_points.txt,sha256=X9FZ4Z-i_r2Ud1RpZ9sNIFYuu_-9fogzCMCRUD9hyX0,51
-clarifai-11.8.2.dist-info/top_level.txt,sha256=wUMdCQGjkxaynZ6nZ9FAnvBUCgp5RJUVFSy2j-KYo0s,9
-clarifai-11.8.2.dist-info/RECORD,,
+clarifai-11.8.3.dist-info/licenses/LICENSE,sha256=mUqF_d12-qE2n41g7C5_sq-BMLOcj6CNN-jevr15YHU,555
+clarifai-11.8.3.dist-info/METADATA,sha256=M9Ee27AiY9jjKzwv7zmm_qO1aMspHq0wLhO1CgbzCKc,23193
+clarifai-11.8.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+clarifai-11.8.3.dist-info/entry_points.txt,sha256=X9FZ4Z-i_r2Ud1RpZ9sNIFYuu_-9fogzCMCRUD9hyX0,51
+clarifai-11.8.3.dist-info/top_level.txt,sha256=wUMdCQGjkxaynZ6nZ9FAnvBUCgp5RJUVFSy2j-KYo0s,9
+clarifai-11.8.3.dist-info/RECORD,,