clarifai 11.6.3__py3-none-any.whl → 11.6.4rc1__py3-none-any.whl

clarifai/runners/utils/temp.py

@@ -0,0 +1,59 @@
+import json
+from clarifai.runners.utils import data_utils
+from clarifai.runners.models.model_class import ModelClass
+
+import inspect
+
+def log_stack():
+    for frame_info in inspect.stack():
+        print(f"{frame_info.filename}:{frame_info.lineno} in {frame_info.function}")
+
+
+default = 27
+
+x = json.dumps(default)
+
+print(x, type(x))
+
+x = json.loads(x)
+
+print(x, type(x))
+
+param = data_utils.Param(default)
+
+print(param, type(param))
+
+print(param.default)
+
+param_json = json.dumps(param)
+
+print(param_json, type(param_json))
+
+param_json = json.loads(param_json)
+
+print(param_json, type(param_json))
+
+# param_json = json.loads(param_json)
+
+# print(param_json, type(param_json))
+
+def test_int(default: int = data_utils.Param(default=27)):
+    print(default, type(default))
+
+test_int()
+
+
+
+
+
+class Test(ModelClass):
+    
+    @ModelClass.method
+    def test(self, max_tokens: int = data_utils.Param(default=27)) -> int:
+        print(max_tokens, type(max_tokens))
+        return max_tokens
+
+test = Test()
+
+test.test()
+

clarifai/utils/cli.py

@@ -10,6 +10,8 @@ import click
 import yaml
 from tabulate import tabulate
 
+from clarifai.utils.logging import logger
+
 
 def from_yaml(filename: str):
     try:
@@ -172,3 +174,49 @@ def validate_context(ctx):
     if ctx.obj == {}:
         logger.error("CLI config file missing. Run `clarifai login` to set up the CLI config.")
         sys.exit(1)
+
+
+def validate_context_auth(pat: str, user_id: str, api_base: str = None):
+    """
+    Validate a Personal Access Token (PAT) by making a test API call.
+
+    Args:
+        pat (str): The Personal Access Token to validate
+        user_id (str): The user ID associated with the token
+        api_base (str): The API base URL. Defaults to None (uses default).
+    """
+    try:
+        from clarifai_grpc.grpc.api.status import status_code_pb2
+
+        from clarifai.client.user import User
+
+        logger.info("Validating the Context Credentials...")
+
+        # Create user client for validation
+        if api_base:
+            user_client = User(user_id=user_id, pat=pat, base_url=api_base)
+        else:
+            user_client = User(user_id=user_id, pat=pat)
+
+        # Try to get user info as a test API call
+        response = user_client.get_user_info()
+
+        if response.status.code == status_code_pb2.SUCCESS:
+            logger.info("✅ Context is valid")
+
+    except Exception as e:
+        error_msg = str(e)
+
+        # Check for common authentication errors and provide user-friendly messages
+        if "PERMISSION_DENIED" in error_msg or "Unauthorized" in error_msg:
+            logger.error(f"Invalid PAT token or incorrect user ID '{user_id}': {error_msg}")
+        elif "UNAUTHENTICATED" in error_msg:
+            logger.error(f"Invalid PAT token or user ID: {error_msg}")
+        elif "SSL" in error_msg or "certificate" in error_msg:
+            logger.error(f"SSL/Certificate error: {error_msg}")
+        elif "Connection" in error_msg or "timeout" in error_msg:
+            logger.error(f"Network connection error: {error_msg}")
+        else:
+            logger.error(f"❌ Validation failed: \n{error_msg}")
+            logger.error("Please check your credentials and try again.")
+        raise click.Abort()  # Exit without saving the configuration

clarifai/utils/constants.py

@@ -14,20 +14,20 @@ CLARIFAI_USER_ID_ENV_VAR = "CLARIFAI_USER_ID"
 HOME_PATH = Path.home()
 DEFAULT_CONFIG = HOME_PATH / '.config/clarifai/config'
 
-# Default clusters, etc. for local dev runner easy setup
-DEFAULT_LOCAL_DEV_COMPUTE_CLUSTER_ID = "local-dev-compute-cluster"
-DEFAULT_LOCAL_DEV_NODEPOOL_ID = "local-dev-nodepool"
-DEFAULT_LOCAL_DEV_DEPLOYMENT_ID = "local-dev-deployment"
-DEFAULT_LOCAL_DEV_MODEL_ID = "local-dev-model"
-DEFAULT_LOCAL_DEV_APP_ID = "local-dev-runner-app"
+# Default clusters, etc. for local runner easy setup
+DEFAULT_LOCAL_RUNNER_COMPUTE_CLUSTER_ID = "local-runner-compute-cluster"
+DEFAULT_LOCAL_RUNNER_NODEPOOL_ID = "local-runner-nodepool"
+DEFAULT_LOCAL_RUNNER_DEPLOYMENT_ID = "local-runner-deployment"
+DEFAULT_LOCAL_RUNNER_MODEL_ID = "local-runner-model"
+DEFAULT_LOCAL_RUNNER_APP_ID = "local-runner-app"
 
 # FIXME: should have any-to-any for these cases.
-DEFAULT_LOCAL_DEV_MODEL_TYPE = "text-to-text"
+DEFAULT_LOCAL_RUNNER_MODEL_TYPE = "text-to-text"
 
-DEFAULT_LOCAL_DEV_COMPUTE_CLUSTER_CONFIG = {
+DEFAULT_LOCAL_RUNNER_COMPUTE_CLUSTER_CONFIG = {
     "compute_cluster": {
-        "id": DEFAULT_LOCAL_DEV_COMPUTE_CLUSTER_ID,
-        "description": "Default Local Dev Compute Cluster",
+        "id": DEFAULT_LOCAL_RUNNER_COMPUTE_CLUSTER_ID,
+        "description": "Default Local Runner Compute Cluster",
         "cloud_provider": {
             "id": "local",
         },
@@ -37,12 +37,12 @@ DEFAULT_LOCAL_DEV_COMPUTE_CLUSTER_CONFIG = {
     }
 }
 
-DEFAULT_LOCAL_DEV_NODEPOOL_CONFIG = {
+DEFAULT_LOCAL_RUNNER_NODEPOOL_CONFIG = {
     "nodepool": {
-        "id": DEFAULT_LOCAL_DEV_NODEPOOL_ID,
-        "description": "Default Local Dev Nodepool",
+        "id": DEFAULT_LOCAL_RUNNER_NODEPOOL_ID,
+        "description": "Default Local Runner Nodepool",
         "compute_cluster": {
-            "id": DEFAULT_LOCAL_DEV_COMPUTE_CLUSTER_ID,
+            "id": DEFAULT_LOCAL_RUNNER_COMPUTE_CLUSTER_ID,
             "user_id": None,  # This will be set when creating the compute cluster
         },
         "instance_types": [
@@ -50,8 +50,8 @@ DEFAULT_LOCAL_DEV_NODEPOOL_CONFIG = {
                 "id": "local",
                 "compute_info": {
                     "cpu_limit": str(os.cpu_count()),
-                    "cpu_memory": "16Gi",  # made up as we don't schedule based on this for local dev.
-                    "num_accelerators": 0,  # TODO if we need accelerator detection for local dev.
+                    "cpu_memory": "16Gi",  # made up as we don't schedule based on this for local runner.
+                    "num_accelerators": 0,  # TODO if we need accelerator detection for local runner.
                 },
             }
         ],
@@ -62,3 +62,5 @@ DEFAULT_LOCAL_DEV_NODEPOOL_CONFIG = {
         "max_instances": 1,
     }
 }
+DEFAULT_OLLAMA_MODEL_REPO = "https://github.com/Clarifai/runners-examples"
+DEFAULT_OLLAMA_MODEL_REPO_BRANCH = "ollama"

clarifai/utils/misc.py

@@ -2,11 +2,17 @@ import os
 import re
 import shutil
 import subprocess
+import sys
+import time
 import urllib.parse
 import uuid
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Tuple
+from urllib.parse import urlparse
 
+import requests
 from clarifai_grpc.grpc.api.status import status_code_pb2
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
 
 from clarifai.errors import UserError
 from clarifai.utils.constants import HOME_PATH
@@ -166,3 +172,377 @@ def clone_github_repo(repo_url, target_dir, github_pat=None, branch=None):
     except subprocess.CalledProcessError as e:
         logger.error(f"Failed to clone repository: {e.stderr}")
         return False
+
+
+class GitHubDownloader:
+    def __init__(
+        self, max_retries: int = 3, backoff_factor: float = 0.3, github_token: str = None
+    ):
+        self.session = requests.Session()
+        self.github_token = github_token
+
+        retry_strategy = Retry(
+            total=max_retries,
+            backoff_factor=backoff_factor,
+            status_forcelist=[429, 500, 502, 503, 504],
+            allowed_methods=["HEAD", "GET", "OPTIONS"],
+        )
+
+        adapter = HTTPAdapter(max_retries=retry_strategy)
+        self.session.mount("http://", adapter)
+        self.session.mount("https://", adapter)
+
+        self.session.headers.update({'User-Agent': 'GitHub-Folder-Downloader/1.0'})
+
+        if self.github_token:
+            self.session.headers.update({'Authorization': f'token {self.github_token}'})
+
+    def expected_folder_structure(self) -> List[Dict[str, Any]]:
+        return [
+            {"name": "1", "type": "dir", "children": [{"name": "model.py", "type": "file"}]},
+            {"name": "config.yaml", "type": "file"},
+            {"name": "requirements.txt", "type": "file"},
+        ]
+
+    def _format_expected_structure(self):
+        """Format the expected structure as a nice tree view."""
+        tree_str = ""
+        tree_str += "Expected folder structure:\n"
+        tree_str += "├── 1/\n"
+        tree_str += "│   └── model.py\n"
+        tree_str += "├── requirements.txt\n"
+        tree_str += "└── config.yaml\n"
+        return tree_str
+
+    def parse_github_url(self, url: str) -> Tuple[str, str, str, str]:
+        try:
+            parsed = urlparse(url)
+
+            if parsed.netloc not in ['github.com', 'www.github.com']:
+                raise ValueError("URL must be a GitHub repository URL")
+
+            path_parts = [p for p in parsed.path.strip('/').split('/') if p]
+
+            if len(path_parts) < 2:
+                raise ValueError("Invalid GitHub repository URL format")
+
+            owner = path_parts[0]
+            repo = path_parts[1]
+
+            if len(path_parts) >= 4 and path_parts[2] in ['tree', 'blob']:
+                branch = path_parts[3]
+                folder_path = '/'.join(path_parts[4:]) if len(path_parts) > 4 else ''
+            elif len(path_parts) >= 3:
+                branch = path_parts[2]
+                folder_path = '/'.join(path_parts[3:]) if len(path_parts) > 3 else ''
+            else:
+                branch = 'main'
+                folder_path = ''
+
+            return owner, repo, branch, folder_path
+
+        except Exception as e:
+            raise ValueError(f"Failed to parse GitHub URL: {e}")
+
+    def get_folder_contents(self, owner: str, repo: str, path: str, branch: str = 'main') -> list:
+        api_url = f"https://api.github.com/repos/{owner}/{repo}/contents/{path}"
+        params = {'ref': branch} if branch else {}
+
+        try:
+            response = self.session.get(api_url, params=params, timeout=30)
+            response.raise_for_status()
+            return response.json()
+        except requests.exceptions.Timeout:
+            raise requests.RequestException("Request timed out. Please try again.")
+        except requests.exceptions.ConnectionError:
+            raise requests.RequestException(
+                "Connection error. Please check your internet connection."
+            )
+        except requests.exceptions.HTTPError as e:
+            if e.response.status_code == 404:
+                token_msg = (
+                    ""
+                    if self.github_token
+                    else " For private repositories, use the github_token parameter."
+                )
+                raise requests.RequestException(
+                    f"Folder not found: {path}. Check if path exists or if the repository is private.{token_msg}"
+                )
+            elif e.response.status_code == 401 or e.response.status_code == 403:
+                token_msg = (
+                    " The provided GitHub token may be invalid or have insufficient permissions."
+                    if self.github_token
+                    else " For private repositories, use the github_token parameter."
+                )
+                raise requests.RequestException(f"Authentication error: {e}.{token_msg}")
+            else:
+                raise requests.RequestException(f"API request failed: {e}")
+        except requests.exceptions.RequestException as e:
+            token_msg = (
+                ""
+                if self.github_token
+                else " For private repositories, use the github_token parameter."
+            )
+            raise requests.RequestException(f"API request failed: {e}.{token_msg}")
+
+    def validate_remote_structure(
+        self,
+        owner: str,
+        repo: str,
+        path: str,
+        branch: str,
+        expected_structure: List[Dict[str, Any]],
+    ) -> Dict[str, Any]:
+        validation_result = {
+            'valid': True,
+            'missing_files': [],
+            'missing_dirs': [],
+            'warnings': [],
+            'remote_contents': [],
+        }
+
+        try:
+            remote_contents = self.get_folder_contents(owner, repo, path, branch)
+            validation_result['remote_contents'] = remote_contents
+
+            remote_items = {item['name']: item['type'] for item in remote_contents}
+
+            for item in expected_structure:
+                item_name = item['name']
+                item_type = item.get('type', 'file')
+
+                if item_name not in remote_items:
+                    if item_type == 'file':
+                        validation_result['missing_files'].append(item_name)
+                    else:
+                        validation_result['missing_dirs'].append(item_name)
+                    validation_result['valid'] = False
+                elif remote_items[item_name] != item_type:
+                    validation_result['warnings'].append(
+                        f"Item '{item_name}' exists but is a {remote_items[item_name]} instead of {item_type}"
+                    )
+                    validation_result['valid'] = False
+
+            expected_names = {item['name'] for item in expected_structure}
+            unexpected_items = [name for name in remote_items.keys() if name not in expected_names]
+            if unexpected_items:
+                validation_result['warnings'].append(
+                    f"Unexpected items found: {', '.join(unexpected_items)}"
+                )
+
+        except requests.RequestException as e:
+            validation_result['valid'] = False
+            validation_result['warnings'].append(f"Failed to access remote repository: {e}")
+
+        return validation_result
+
+    def download_file(self, download_url: str, local_path: str) -> None:
+        try:
+            response = self.session.get(download_url, stream=True, timeout=60)
+            response.raise_for_status()
+
+            os.makedirs(os.path.dirname(local_path), exist_ok=True)
+
+            total_size = int(response.headers.get('content-length', 0))
+            downloaded_size = 0
+
+            with open(local_path, 'wb') as f:
+                for chunk in response.iter_content(chunk_size=8192):
+                    if chunk:
+                        f.write(chunk)
+                        downloaded_size += len(chunk)
+
+                        if total_size > 0 and total_size > 1024 * 1024:
+                            progress = (downloaded_size / total_size) * 100
+                            logger.info(
+                                f"\rDownloading: {os.path.basename(local_path)} - {progress:.1f}%",
+                                end='',
+                                flush=True,
+                            )
+
+            if total_size > 1024 * 1024:
+                logger.info()
+
+            logger.info(f"Downloaded: {local_path}")
+
+        except requests.exceptions.Timeout:
+            logger.info(f"Timeout downloading {local_path}. Skipping...")
+        except requests.exceptions.ConnectionError:
+            logger.info(f"Connection error downloading {local_path}. Skipping...")
+        except Exception as e:
+            logger.info(f"Failed to download {local_path}: {e}")
+
+    def process_folder(
+        self, owner: str, repo: str, path: str, local_base_path: str, branch: str = 'main'
+    ) -> None:
+        try:
+            contents = self.get_folder_contents(owner, repo, path, branch)
+
+            if not contents:
+                logger.info(f"Info: Empty folder - {path}")
+                return
+
+            for item in contents:
+                item_name = item['name']
+                item_path = os.path.join(local_base_path, item_name)
+
+                if item['type'] == 'file':
+                    self.download_file(item['download_url'], item_path)
+
+                elif item['type'] == 'dir':
+                    os.makedirs(item_path, exist_ok=True)
+                    logger.info(f"Created directory: {item_path}")
+
+                    new_path = f"{path}/{item_name}" if path else item_name
+                    self.process_folder(owner, repo, new_path, item_path, branch)
+
+        except requests.exceptions.RequestException as e:
+            if "Folder not found" in str(e):
+                logger.error(f"Error: Folder not found - {path}")
+                raise
+            else:
+                logger.error(f"Error accessing folder {path}: {e}")
+                raise
+        except Exception as e:
+            logger.error(f"Unexpected error processing folder {path}: {e}")
+            raise
+
+    def validate_folder_structure(
+        self, folder_path: str, expected_structure: List[Dict[str, Any]]
+    ) -> Dict[str, Any]:
+        validation_result = {
+            'valid': True,
+            'missing_files': [],
+            'missing_dirs': [],
+            'warnings': [],
+        }
+
+        if not os.path.exists(folder_path):
+            validation_result['valid'] = False
+            validation_result['warnings'].append(f"Folder {folder_path} does not exist")
+            return validation_result
+
+        for item in expected_structure:
+            item_name = item['name']
+            item_type = item.get('type', 'file')
+            item_path = os.path.join(folder_path, item_name)
+
+            if item_type == 'file':
+                if not os.path.isfile(item_path):
+                    validation_result['missing_files'].append(item_name)
+                    validation_result['valid'] = False
+            elif item_type == 'dir':
+                if not os.path.isdir(item_path):
+                    validation_result['missing_dirs'].append(item_name)
+                    validation_result['valid'] = False
+
+        return validation_result
+
+    def download_github_folder(
+        self,
+        url: str,
+        output_dir: str,
+        github_token: str = None,
+        validate_structure: bool = False,
+        pre_validate: bool = True,
+        strict_validation: bool = False,
+    ) -> None:
+        logger.info(f"Parsing GitHub URL: {url}")
+
+        # Update token if provided as a parameter
+        if github_token:
+            self.github_token = github_token
+            self.session.headers.update({'Authorization': f'token {github_token}'})
+
+        try:
+            owner, repo, branch, folder_path = self.parse_github_url(url)
+            logger.info(f"Repository: {owner}/{repo}")
+            logger.info(f"Branch: {branch}")
+            logger.info(f"Folder: {folder_path or 'root'}")
+
+            expected_structure = self.expected_folder_structure() if pre_validate else None
+
+            if expected_structure:
+                logger.info("\nValidating remote folder structure...")
+                remote_validation = self.validate_remote_structure(
+                    owner, repo, folder_path, branch, expected_structure
+                )
+
+                if not remote_validation['valid']:
+                    logger.error("Remote structure validation failed!")
+
+                    if remote_validation['missing_files']:
+                        logger.error(
+                            f"Missing files: {', '.join(remote_validation['missing_files'])}"
+                        )
+
+                    if remote_validation['missing_dirs']:
+                        logger.error(
+                            f"Missing directories: {', '.join(remote_validation['missing_dirs'])}"
+                        )
+
+                    if remote_validation['warnings']:
+                        for warning in remote_validation['warnings']:
+                            logger.error(f"Warning: {warning}")
+
+                    # Print the expected structure in a nice format
+                    tree_view = self._format_expected_structure()
+                    logger.info("\nThe repository must have the following structure:")
+                    logger.info(tree_view)
+
+                    logger.error(
+                        "Download cancelled: Repository structure does not match the expected format."
+                    )
+                    sys.exit(1)  # Exit without proceeding with download
+                else:
+                    logger.info("Remote structure validation passed!")
+
+            os.makedirs(output_dir, exist_ok=True)
+            logger.info(f"Created output directory: {output_dir}")
+
+            logger.info("\nStarting download...")
+            start_time = time.time()
+            try:
+                self.process_folder(owner, repo, folder_path, output_dir, branch)
+
+                elapsed_time = time.time() - start_time
+                logger.info(f"\nDownload completed in {elapsed_time:.2f} seconds")
+                logger.info(f"Files saved to: {os.path.abspath(output_dir)}")
+
+                if validate_structure and expected_structure:
+                    logger.info("\nValidating downloaded folder structure...")
+                    validation_result = self.validate_folder_structure(
+                        output_dir, expected_structure
+                    )
+
+                    if validation_result['valid']:
+                        logger.info("Folder structure post validation passed!")
+                    else:
+                        logger.error("Folder structure validation failed!")
+
+                        if validation_result['missing_files']:
+                            logger.info(
+                                f"Missing files: {', '.join(validation_result['missing_files'])}"
+                            )
+
+                        if validation_result['missing_dirs']:
+                            logger.info(
+                                f"Missing directories: {', '.join(validation_result['missing_dirs'])}"
+                            )
+
+                        if validation_result['warnings']:
+                            for warning in validation_result['warnings']:
+                                logger.info(f"Warng: {warning}")
+            except requests.RequestException as e:
+                # Critical error - the main folder cannot be processed
+                logger.error(
+                    f"\nDownload failed: {e}, \n No files were downloaded to: {os.path.abspath(output_dir)}"
+                )
+                sys.exit(1)
+
+        except ValueError as e:
+            logger.error(f"Error: {e}")
+            sys.exit(1)
+        except Exception as e:
+            logger.error(f"Unexpected error: {e}")
+            sys.exit(1)

{clarifai-11.6.3.dist-info → clarifai-11.6.4rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: clarifai
-Version: 11.6.3
+Version: 11.6.4rc1
 Home-page: https://github.com/Clarifai/clarifai-python
 Author: Clarifai
 Author-email: support@clarifai.com