nexaai 1.0.12rc1__cp310-cp310-macosx_13_0_x86_64.whl → 1.0.13rc2__cp310-cp310-macosx_13_0_x86_64.whl

nexaai/_version.py

@@ -1,4 +1,4 @@
 # This file is generated by CMake from _version.py.in
 # Do not modify this file manually - it will be overwritten
 
-__version__ = "1.0.12-rc1" 
+__version__ = "1.0.13-rc2" 

nexaai/binds/__init__.py

@@ -1,3 +1,4 @@
 from .common_bind import *
 from .llm_bind import *
 from .embedder_bind import *
+from .vlm_bind import *

nexaai/utils/manifest_utils.py

@@ -11,7 +11,7 @@ This module provides utilities to:
 import os
 import json
 from datetime import datetime
-from typing import Dict, Any, List, Optional
+from typing import Dict, Any, List, Optional, Union
 
 from .quantization_utils import (
     extract_quantization_from_filename, 
@@ -101,7 +101,7 @@ def save_download_metadata(directory_path: str, metadata: Dict[str, Any]) -> Non
         pass
 
 
-def create_gguf_manifest(repo_id: str, files: List[str], directory_path: str, old_metadata: Dict[str, Any]) -> Dict[str, Any]:
+def create_gguf_manifest(repo_id: str, files: List[str], directory_path: str, old_metadata: Dict[str, Any], is_mmproj: bool = False, file_name: Optional[Union[str, List[str]]] = None) -> Dict[str, Any]:
     """Create GGUF format manifest."""
     
     # Load existing manifest to merge GGUF files if it exists
@@ -111,14 +111,27 @@ def create_gguf_manifest(repo_id: str, files: List[str], directory_path: str, ol
     if existing_manifest and "ModelFile" in existing_manifest:
         model_files = existing_manifest["ModelFile"].copy()
     
+    # Initialize MMProjFile from existing manifest or empty
+    mmproj_file = {
+        "Name": "",
+        "Downloaded": False,
+        "Size": 0
+    }
+    if existing_manifest and "MMProjFile" in existing_manifest:
+        mmproj_file = existing_manifest["MMProjFile"].copy()
+    
     # Process GGUF files
-    for file_name in files:
-        if file_name.endswith('.gguf'):
-            # Use the new enum-based quantization extraction
-            quantization_type = extract_quantization_from_filename(file_name)
-            quant_level = quantization_type.value if quantization_type else "UNKNOWN"
+    for current_file_name in files:
+        if current_file_name.endswith('.gguf'):
+            # Check if this file is an mmproj file
+            is_current_mmproj = 'mmproj' in current_file_name.lower()
+            
+            # If we're downloading specific files and this is marked as mmproj, respect that
+            if is_mmproj and file_name is not None:
+                filenames_to_check = file_name if isinstance(file_name, list) else [file_name]
+                is_current_mmproj = current_file_name in filenames_to_check
             
-            file_path = os.path.join(directory_path, file_name)
+            file_path = os.path.join(directory_path, current_file_name)
             file_size = 0
             if os.path.exists(file_path):
                 try:
@@ -126,22 +139,31 @@ def create_gguf_manifest(repo_id: str, files: List[str], directory_path: str, ol
                 except (OSError, IOError):
                     pass
             
-            model_files[quant_level] = {
-                "Name": file_name,
-                "Downloaded": True,
-                "Size": file_size
-            }
+            if is_current_mmproj:
+                # This is an mmproj file, put it in MMProjFile
+                mmproj_file = {
+                    "Name": current_file_name,
+                    "Downloaded": True,
+                    "Size": file_size
+                }
+            else:
+                # Regular model file, put in ModelFile
+                # Use the new enum-based quantization extraction
+                quantization_type = extract_quantization_from_filename(current_file_name)
+                quant_level = quantization_type.value if quantization_type else "UNKNOWN"
+                
+                model_files[quant_level] = {
+                    "Name": current_file_name,
+                    "Downloaded": True,
+                    "Size": file_size
+                }
     
     manifest = {
         "Name": repo_id,
         "ModelType": PIPELINE_TO_MODEL_TYPE.get(old_metadata.get('pipeline_tag'), "other"),
         "PluginId": "llama_cpp",
         "ModelFile": model_files,
-        "MMProjFile": {
-            "Name": "",
-            "Downloaded": False,
-            "Size": 0
-        },
+        "MMProjFile": mmproj_file,
         "TokenizerFile": {
             "Name": "",
             "Downloaded": False,
@@ -157,20 +179,27 @@ def create_gguf_manifest(repo_id: str, files: List[str], directory_path: str, ol
     return manifest
 
 
-def create_mlx_manifest(repo_id: str, files: List[str], directory_path: str, old_metadata: Dict[str, Any]) -> Dict[str, Any]:
+def create_mlx_manifest(repo_id: str, files: List[str], directory_path: str, old_metadata: Dict[str, Any], is_mmproj: bool = False, file_name: Optional[Union[str, List[str]]] = None) -> Dict[str, Any]:
     """Create MLX format manifest."""
     
     model_files = {}
     extra_files = []
     
+    # Initialize MMProjFile
+    mmproj_file = {
+        "Name": "",
+        "Downloaded": False,
+        "Size": 0
+    }
+    
     # Try different methods to extract quantization for MLX models
     quantization_type = detect_quantization_for_mlx(repo_id, directory_path)
     
     # Use the detected quantization or default to "DEFAULT"
     quant_level = quantization_type.value if quantization_type else "DEFAULT"
     
-    for file_name in files:
-        file_path = os.path.join(directory_path, file_name)
+    for current_file_name in files:
+        file_path = os.path.join(directory_path, current_file_name)
         file_size = 0
         if os.path.exists(file_path):
             try:
@@ -178,17 +207,32 @@ def create_mlx_manifest(repo_id: str, files: List[str], directory_path: str, old
             except (OSError, IOError):
                 pass
         
+        # Check if this file is an mmproj file
+        is_current_mmproj = 'mmproj' in current_file_name.lower()
+        
+        # If we're downloading specific files and this is marked as mmproj, respect that
+        if is_mmproj and file_name is not None:
+            filenames_to_check = file_name if isinstance(file_name, list) else [file_name]
+            is_current_mmproj = current_file_name in filenames_to_check
+        
+        if is_current_mmproj:
+            # This is an mmproj file, put it in MMProjFile
+            mmproj_file = {
+                "Name": current_file_name,
+                "Downloaded": True,
+                "Size": file_size
+            }
         # Check if this is a main model file (safetensors but not index files)
-        if (file_name.endswith('.safetensors') and not file_name.endswith('.index.json')):
+        elif (current_file_name.endswith('.safetensors') and not current_file_name.endswith('.index.json')):
             model_files[quant_level] = {
-                "Name": file_name,
+                "Name": current_file_name,
                 "Downloaded": True,
                 "Size": file_size
             }
         else:
             # Add to extra files
             extra_files.append({
-                "Name": file_name,
+                "Name": current_file_name,
                 "Downloaded": True,
                 "Size": file_size
             })
@@ -198,11 +242,7 @@ def create_mlx_manifest(repo_id: str, files: List[str], directory_path: str, old
         "ModelType": PIPELINE_TO_MODEL_TYPE.get(old_metadata.get('pipeline_tag'), "other"),
         "PluginId": "mlx",
         "ModelFile": model_files,
-        "MMProjFile": {
-            "Name": "",
-            "Downloaded": False,
-            "Size": 0
-        },
+        "MMProjFile": mmproj_file,
         "TokenizerFile": {
             "Name": "",
             "Downloaded": False,
@@ -232,7 +272,7 @@ def detect_model_type(files: List[str]) -> str:
         return "mlx"
 
 
-def create_manifest_from_files(repo_id: str, files: List[str], directory_path: str, old_metadata: Dict[str, Any]) -> Dict[str, Any]:
+def create_manifest_from_files(repo_id: str, files: List[str], directory_path: str, old_metadata: Dict[str, Any], is_mmproj: bool = False, file_name: Optional[Union[str, List[str]]] = None) -> Dict[str, Any]:
     """
     Create appropriate manifest format based on detected model type.
     
@@ -241,6 +281,8 @@ def create_manifest_from_files(repo_id: str, files: List[str], directory_path: s
         files: List of files in the model directory
         directory_path: Path to the model directory
         old_metadata: Existing metadata (pipeline_tag, download_time, avatar_url)
+        is_mmproj: Whether the downloaded file is an mmproj file
+        file_name: The specific file(s) that were downloaded (None if entire repo was downloaded)
         
     Returns:
         Dict containing the appropriate manifest format
@@ -248,12 +290,12 @@ def create_manifest_from_files(repo_id: str, files: List[str], directory_path: s
     model_type = detect_model_type(files)
     
     if model_type == "gguf":
-        return create_gguf_manifest(repo_id, files, directory_path, old_metadata)
+        return create_gguf_manifest(repo_id, files, directory_path, old_metadata, is_mmproj, file_name)
     else:  # mlx or other
-        return create_mlx_manifest(repo_id, files, directory_path, old_metadata)
+        return create_mlx_manifest(repo_id, files, directory_path, old_metadata, is_mmproj, file_name)
 
 
-def save_manifest_with_files_metadata(repo_id: str, local_dir: str, old_metadata: Dict[str, Any]) -> None:
+def save_manifest_with_files_metadata(repo_id: str, local_dir: str, old_metadata: Dict[str, Any], is_mmproj: bool = False, file_name: Optional[Union[str, List[str]]] = None) -> None:
     """
     Create and save manifest based on files found in the directory.
     
@@ -261,6 +303,8 @@ def save_manifest_with_files_metadata(repo_id: str, local_dir: str, old_metadata
         repo_id: Repository ID
         local_dir: Local directory containing the model files
         old_metadata: Existing metadata to preserve
+        is_mmproj: Whether the downloaded file is an mmproj file
+        file_name: The specific file(s) that were downloaded (None if entire repo was downloaded)
     """
     # Get list of files in the directory
     files = []
@@ -274,7 +318,7 @@ def save_manifest_with_files_metadata(repo_id: str, local_dir: str, old_metadata
         pass
     
     # Create appropriate manifest
-    manifest = create_manifest_from_files(repo_id, files, local_dir, old_metadata)
+    manifest = create_manifest_from_files(repo_id, files, local_dir, old_metadata, is_mmproj, file_name)
     
     # Save manifest
     save_download_metadata(local_dir, manifest)

nexaai/utils/model_manager.py

@@ -21,6 +21,13 @@ from .manifest_utils import (
 # Default path for model storage
 DEFAULT_MODEL_SAVING_PATH = "~/.cache/nexa.ai/nexa_sdk/models/"
 
+
+@dataclass
+class MMProjInfo:
+    """Data class for mmproj file information."""
+    mmproj_path: Optional[str] = None
+    size: int = 0
+
 @dataclass
 class DownloadedModel:
     """Data class representing a downloaded model with all its metadata."""
@@ -34,6 +41,7 @@ class DownloadedModel:
     pipeline_tag: Optional[str] = None  # Pipeline tag from HuggingFace model info
     download_time: Optional[str] = None  # ISO format timestamp of download
     avatar_url: Optional[str] = None  # Avatar URL for the model author
+    mmproj_info: Optional[MMProjInfo] = None  # mmproj file information
     
     def to_dict(self) -> Dict[str, Any]:
         """Convert to dictionary format for backward compatibility."""
@@ -47,7 +55,11 @@ class DownloadedModel:
             'full_repo_download_complete': self.full_repo_download_complete,
             'pipeline_tag': self.pipeline_tag,
             'download_time': self.download_time,
-            'avatar_url': self.avatar_url
+            'avatar_url': self.avatar_url,
+            'mmproj_info': {
+                'mmproj_path': self.mmproj_info.mmproj_path,
+                'size': self.mmproj_info.size
+            } if self.mmproj_info else None
         }
         return result
 
@@ -125,6 +137,46 @@ def _has_valid_metadata(directory_path: str) -> bool:
     return os.path.exists(manifest_path) or os.path.exists(old_metadata_path)
 
 
+def _extract_mmproj_info(manifest: Dict[str, Any], local_path: str) -> Optional[MMProjInfo]:
+    """
+    Extract mmproj information from manifest data.
+    
+    Args:
+        manifest: Dictionary containing manifest data
+        local_path: Local path to the model directory
+        
+    Returns:
+        MMProjInfo object if mmproj file exists, None otherwise
+    """
+    # Check if manifest has MMProjFile information
+    mmproj_file_info = manifest.get('MMProjFile')
+    if not mmproj_file_info or not mmproj_file_info.get('Downloaded') or not mmproj_file_info.get('Name'):
+        return None
+    
+    mmproj_filename = mmproj_file_info.get('Name', '')
+    if not mmproj_filename:
+        return None
+    
+    # Construct full path to mmproj file
+    mmproj_path = os.path.join(local_path, mmproj_filename)
+    
+    # Get size from manifest, but verify file exists
+    mmproj_size = mmproj_file_info.get('Size', 0)
+    if os.path.exists(mmproj_path):
+        try:
+            # Verify size matches actual file size
+            actual_size = os.path.getsize(mmproj_path)
+            mmproj_size = actual_size  # Use actual size if different
+        except (OSError, IOError):
+            # If we can't get actual size, use size from manifest
+            pass
+    else:
+        # File doesn't exist, don't include mmproj info
+        return None
+    
+    return MMProjInfo(mmproj_path=mmproj_path, size=mmproj_size)
+
+
 def _scan_for_repo_folders(base_path: str) -> List[DownloadedModel]:
     """Scan a directory for repository folders and return model information."""
     models = []
@@ -159,6 +211,10 @@ def _scan_for_repo_folders(base_path: str) -> List[DownloadedModel]:
                                 # Load metadata if it exists
                                 repo_id = f"{item}/{subitem}"
                                 metadata = load_download_metadata(subitem_path, repo_id)
+                                
+                                # Extract mmproj information
+                                mmproj_info = _extract_mmproj_info(metadata, subitem_path)
+                                
                                 models.append(DownloadedModel(
                                     repo_id=repo_id,
                                     files=files,
@@ -169,7 +225,8 @@ def _scan_for_repo_folders(base_path: str) -> List[DownloadedModel]:
                                     full_repo_download_complete=download_complete,
                                     pipeline_tag=metadata.get('pipeline_tag'),
                                     download_time=metadata.get('download_time'),
-                                    avatar_url=metadata.get('avatar_url')
+                                    avatar_url=metadata.get('avatar_url'),
+                                    mmproj_info=mmproj_info
                                 ))
                     else:
                         direct_files.append(subitem)
@@ -188,6 +245,10 @@ def _scan_for_repo_folders(base_path: str) -> List[DownloadedModel]:
                         # Load metadata if it exists
                         repo_id = item
                         metadata = load_download_metadata(item_path, repo_id)
+                        
+                        # Extract mmproj information
+                        mmproj_info = _extract_mmproj_info(metadata, item_path)
+                        
                         models.append(DownloadedModel(
                             repo_id=repo_id,
                             files=files,
@@ -198,7 +259,8 @@ def _scan_for_repo_folders(base_path: str) -> List[DownloadedModel]:
                             full_repo_download_complete=download_complete,
                             pipeline_tag=metadata.get('pipeline_tag'),
                             download_time=metadata.get('download_time'),
-                            avatar_url=metadata.get('avatar_url')
+                            avatar_url=metadata.get('avatar_url'),
+                            mmproj_info=mmproj_info
                         ))
     
     except (OSError, IOError):
@@ -236,6 +298,7 @@ def list_downloaded_models(local_dir: Optional[str] = None) -> List[DownloadedMo
             - pipeline_tag: Optional[str] - Pipeline tag from HuggingFace model info
             - download_time: Optional[str] - ISO format timestamp when the model was downloaded
             - avatar_url: Optional[str] - Avatar URL for the model author
+            - mmproj_info: Optional[MMProjInfo] - mmproj file information with mmproj_path and size
     """
     
     # Set up local directory
@@ -727,7 +790,7 @@ class HuggingFaceDownloader:
         # If no expected size, just check that file is not empty
         return os.path.getsize(file_path) > 0
     
-    def _fetch_and_save_metadata(self, repo_id: str, local_dir: str) -> None:
+    def _fetch_and_save_metadata(self, repo_id: str, local_dir: str, is_mmproj: bool = False, file_name: Optional[Union[str, List[str]]] = None) -> None:
         """Fetch model info and save metadata after successful download."""
         # Initialize metadata with defaults to ensure manifest is always created
         old_metadata = {
@@ -757,7 +820,7 @@ class HuggingFaceDownloader:
         
         # CRITICAL: Always create the manifest file, regardless of metadata fetch failures
         try:
-            save_manifest_with_files_metadata(repo_id, local_dir, old_metadata)
+            save_manifest_with_files_metadata(repo_id, local_dir, old_metadata, is_mmproj, file_name)
             print(f"[OK] Successfully created nexa.manifest for {repo_id}")
         except Exception as e:
             # This is critical - if manifest creation fails, we should know about it
@@ -819,7 +882,7 @@ class HuggingFaceDownloader:
                 progress_tracker.stop_tracking()
             
             # Save metadata after successful download
-            self._fetch_and_save_metadata(repo_id, file_local_dir)
+            self._fetch_and_save_metadata(repo_id, file_local_dir, self._current_is_mmproj, self._current_file_name)
             
             return downloaded_path
             
@@ -864,7 +927,7 @@ class HuggingFaceDownloader:
                 progress_tracker.stop_tracking()
             
             # Save metadata after successful download
-            self._fetch_and_save_metadata(repo_id, repo_local_dir)
+            self._fetch_and_save_metadata(repo_id, repo_local_dir, self._current_is_mmproj, self._current_file_name)
             
             return downloaded_path
             
@@ -926,7 +989,7 @@ class HuggingFaceDownloader:
                 progress_tracker.stop_tracking()
             
             # Save metadata after successful download
-            self._fetch_and_save_metadata(repo_id, repo_local_dir)
+            self._fetch_and_save_metadata(repo_id, repo_local_dir, self._current_is_mmproj, self._current_file_name)
             
             return repo_local_dir
             
@@ -951,7 +1014,8 @@ class HuggingFaceDownloader:
         local_dir: Optional[str] = None,
         progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
         show_progress: bool = True,
-        force_download: bool = False
+        force_download: bool = False,
+        is_mmproj: bool = False
     ) -> str:
         """
         Main download method that handles all download scenarios.
@@ -970,6 +1034,10 @@ class HuggingFaceDownloader:
         # Validate and normalize parameters
         repo_id, file_name = self._validate_and_setup_params(repo_id, file_name)
         
+        # Store parameters as instance variables for use in _fetch_and_save_metadata
+        self._current_is_mmproj = is_mmproj
+        self._current_file_name = file_name
+        
         # Set up local directory
         local_dir = self._created_dir_if_not_exists(local_dir)
         
@@ -1038,7 +1106,8 @@ def download_from_huggingface(
     show_progress: bool = True,
     token: Union[bool, str, None] = None,
     custom_endpoint: Optional[str] = None,
-    force_download: bool = False
+    force_download: bool = False,
+    is_mmproj: Optional[bool] = None
 ) -> str:
     """
     Download models or files from HuggingFace Hub or custom mirror endpoints.
@@ -1065,6 +1134,9 @@ def download_from_huggingface(
                                         The endpoint will be used to initialize HfApi for all downloads.
         force_download (bool, optional): If True, download files even if they already exist locally.
                                         Default False (skip existing files).
+        is_mmproj (bool, optional): Whether the file being downloaded is an mmproj file. Only used when
+                                   file_name is not None. If None, defaults to True if 'mmproj' is in 
+                                   the filename, False otherwise.
     
     Returns:
         str: Path to the downloaded file or directory
@@ -1101,6 +1173,15 @@ def download_from_huggingface(
             }
         }
     """
+    # Set default value for is_mmproj based on filename if not explicitly provided
+    if is_mmproj is None and file_name is not None:
+        # Check if any filename contains 'mmproj'
+        filenames_to_check = file_name if isinstance(file_name, list) else [file_name]
+        is_mmproj = any('mmproj' in filename.lower() for filename in filenames_to_check)
+    elif is_mmproj is None:
+        # Default to False if no file_name is provided
+        is_mmproj = False
+    
     # Create downloader instance with custom endpoint if provided
     downloader = HuggingFaceDownloader(
         endpoint=custom_endpoint, 
@@ -1115,7 +1196,8 @@ def download_from_huggingface(
         local_dir=local_dir,
         progress_callback=progress_callback,
         show_progress=show_progress,
-        force_download=force_download
+        force_download=force_download,
+        is_mmproj=is_mmproj
     )
 
 
@@ -1124,15 +1206,66 @@ def download_from_huggingface(
 ##########################################################################
 
 
+def _download_model_if_needed(
+    model_path: str,
+    param_name: str,
+    progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
+    token: Union[bool, str, None] = None,
+    is_mmproj: bool = False
+) -> str:
+    """
+    Helper function to download a model from HuggingFace if it doesn't exist locally.
+    
+    Args:
+        model_path: The model path that may be local or remote
+        param_name: Name of the parameter (for error messages)
+        progress_callback: Callback function for download progress updates
+        token: HuggingFace authentication token for private repositories
+        
+    Returns:
+        str: Local path to the model (either existing or downloaded)
+        
+    Raises:
+        RuntimeError: If download fails
+    """
+    # Check if model_path exists locally (file or directory)
+    if os.path.exists(model_path):
+        # Local path exists, return as-is
+        return model_path
+    
+    # Model path doesn't exist locally, try to download from HuggingFace
+    try:
+        # Parse model_path to extract repo_id and filename
+        repo_id, file_name = _parse_model_path(model_path)
+        
+        # Download the model
+        downloaded_path = download_from_huggingface(
+            repo_id=repo_id,
+            file_name=file_name,
+            local_dir=None,  # Use default cache directory
+            enable_transfer=True,
+            progress_callback=progress_callback,
+            show_progress=True,
+            token=token,
+            is_mmproj=is_mmproj
+        )
+        
+        return downloaded_path
+        
+    except Exception as e:
+        # Only handle download-related errors
+        raise RuntimeError(f"Could not load model from '{param_name}={model_path}': {e}")
+
+
 def auto_download_model(func: Callable) -> Callable:
     """
     Decorator that automatically downloads models from HuggingFace if they don't exist locally.
     
-    This decorator should be applied to __init__ methods that take a name_or_path parameter.
-    If name_or_path doesn't exist as a local file/directory, it will attempt to download
-    it from HuggingFace Hub using the download_from_huggingface function.
+    This decorator should be applied to __init__ methods that take a name_or_path parameter
+    and optionally an mmproj_path parameter. If these paths don't exist as local files/directories,
+    it will attempt to download them from HuggingFace Hub using the download_from_huggingface function.
     
-    The name_or_path can be in formats like:
+    The name_or_path and mmproj_path can be in formats like:
     - "microsoft/DialoGPT-small" (downloads entire repo)  
     - "microsoft/DialoGPT-small/pytorch_model.bin" (downloads specific file)
     - "Qwen/Qwen3-4B-GGUF/Qwen3-4B-Q4_K_M.gguf" (downloads specific file)
@@ -1149,21 +1282,6 @@ def auto_download_model(func: Callable) -> Callable:
     """
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
-        # Find name_or_path in arguments
-        # Assuming name_or_path is the first argument after self
-        if len(args) >= 2:
-            name_or_path = args[1]
-            args_list = list(args)
-            path_index = 1
-            is_positional = True
-        elif 'name_or_path' in kwargs:
-            name_or_path = kwargs['name_or_path']
-            path_index = None
-            is_positional = False
-        else:
-            # No name_or_path found, call original function
-            return func(*args, **kwargs)
-        
         # Extract progress_callback and token from arguments
         progress_callback = None
         if 'progress_callback' in kwargs:
@@ -1173,39 +1291,63 @@ def auto_download_model(func: Callable) -> Callable:
         if 'token' in kwargs:
             token = kwargs.pop('token')  # Remove from kwargs to avoid passing to original func
         
-        # Check if name_or_path exists locally (file or directory)
-        if os.path.exists(name_or_path):
-            # Local path exists, use as-is without downloading
+        # Handle name_or_path parameter
+        name_or_path = None
+        name_path_index = None
+        is_name_positional = False
+        
+        # Find name_or_path in arguments
+        # Assuming name_or_path is the first argument after self
+        if len(args) >= 2:
+            name_or_path = args[1]
+            args_list = list(args)
+            name_path_index = 1
+            is_name_positional = True
+        elif 'name_or_path' in kwargs:
+            name_or_path = kwargs['name_or_path']
+            is_name_positional = False
+        
+        # Handle mmproj_path parameter
+        mmproj_path = None
+        if 'mmproj_path' in kwargs:
+            mmproj_path = kwargs['mmproj_path']
+        
+        # If neither parameter is found, call original function
+        if name_or_path is None and mmproj_path is None:
             return func(*args, **kwargs)
         
-        # Model path doesn't exist locally, try to download from HuggingFace
-        try:
-            # Parse name_or_path to extract repo_id and filename
-            repo_id, file_name = _parse_model_path(name_or_path)
-            
-            # Download the model
-            downloaded_path = download_from_huggingface(
-                repo_id=repo_id,
-                file_name=file_name,
-                local_dir=None,  # Use default cache directory
-                enable_transfer=True,
-                progress_callback=progress_callback,  # Use the extracted callback
-                show_progress=True,
-                token=token  # Use the extracted token
-            )
-            
-            # Replace name_or_path with downloaded path
-            if is_positional:
-                args_list[path_index] = downloaded_path
-                args = tuple(args_list)
-            else:
-                kwargs['name_or_path'] = downloaded_path
-            
-        except Exception as e:
-            # Only handle download-related errors
-            raise RuntimeError(f"Could not load model from '{name_or_path}': {e}")
+        # Download name_or_path if needed
+        if name_or_path is not None:
+            try:
+                downloaded_name_path = _download_model_if_needed(
+                    name_or_path, 'name_or_path', progress_callback, token
+                )
+                
+                # Replace name_or_path with downloaded path
+                if is_name_positional:
+                    if name_path_index is not None:
+                        args_list[name_path_index] = downloaded_name_path
+                        args = tuple(args_list)
+                else:
+                    kwargs['name_or_path'] = downloaded_name_path
+                    
+            except Exception as e:
+                raise e  # Re-raise the error from _download_model_if_needed
+        
+        # Download mmproj_path if needed
+        if mmproj_path is not None:
+            try:
+                downloaded_mmproj_path = _download_model_if_needed(
+                    mmproj_path, 'mmproj_path', progress_callback, token, is_mmproj=True
+                )
+                
+                # Replace mmproj_path with downloaded path
+                kwargs['mmproj_path'] = downloaded_mmproj_path
+                
+            except Exception as e:
+                raise e  # Re-raise the error from _download_model_if_needed
         
-        # Call original function with updated path (outside try-catch to let model creation errors bubble up)
+        # Call original function with updated paths (outside try-catch to let model creation errors bubble up)
         return func(*args, **kwargs)
     
     return wrapper

nexaai/vlm_impl/pybind_vlm_impl.py

@@ -1,7 +1,6 @@
 from typing import Generator, Optional, List, Dict, Any, Union
 import queue
 import threading
-import base64
 from pathlib import Path
 
 from nexaai.common import ModelConfig, GenerationConfig, MultiModalMessage, PluginID
@@ -102,26 +101,16 @@ class PyBindVLMImpl(VLM):
                 t = c["type"]
                 if t == "text":
                     blocks.append({"type": "text", "text": c.get("text","") or ""})
+                elif t == "image":
+                    # Pass through the original structure - let vlm-bind.cpp handle field extraction
+                    blocks.append(c)
                 else:
-                    # image/audio/video
-                    src = c.get("url") or c.get("path")
-                    if not src:
-                        raise ValueError(f"No url/path for {t}")
-                    # read local file or strip data URI
-                    if Path(src).exists():
-                        raw = Path(src).read_bytes()
-                        b64 = base64.b64encode(raw).decode("ascii")
-                        blocks.append({"type": t, "text": b64})
-                    elif src.startswith("data:"):
-                        b64 = src.split(",",1)[1]
-                        blocks.append({"type": t, "text": b64})
-                    else:
-                        # remote URL
-                        blocks.append({"type": t, "text": src})
+                    raise ValueError(f"Unsupported content type: {t}. Use 'text' or 'image' to match the golden reference in vlm.cpp")
 
             payload.append({"role": role, "content": blocks})
 
-        return vlm_bind.ml_vlm_apply_chat_template(self._handle, payload, tools)
+        result = vlm_bind.ml_vlm_apply_chat_template(self._handle, payload, tools)
+        return result
 
     def generate_stream(self, prompt: str, g_cfg: GenerationConfig = GenerationConfig()) -> Generator[str, None, None]:
         """Generate text with streaming."""

{nexaai-1.0.12rc1.dist-info → nexaai-1.0.13rc2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nexaai
-Version: 1.0.12rc1
+Version: 1.0.13rc2
 Summary: Python bindings for NexaSDK C-lib backend
 Author-email: "Nexa AI, Inc." <dev@nexa.ai>
 Project-URL: Homepage, https://github.com/NexaAI/nexasdk-bridge

{nexaai-1.0.12rc1.dist-info → nexaai-1.0.13rc2.dist-info}/RECORD

@@ -1,6 +1,6 @@
 nexaai/__init__.py,sha256=jXdC4vv6DBK1fVewYTYSUhOOYfvf_Mk81UIeMGGIKUg,2029
-nexaai/_stub.cpython-310-darwin.so,sha256=9VzNDEo89QZB144PAC2Wn8s7I8iER3r0We0qTBJf1nE,49832
-nexaai/_version.py,sha256=szohTDI5dEmwOKD5OkFBXNf_BUsFGffIiuRLsNYkFes,143
+nexaai/_stub.cpython-310-darwin.so,sha256=6Rm0bf_EO8r2MetZDjJ7E65gIcteq6ZDkPSLYh2NJj4,49832
+nexaai/_version.py,sha256=phWIqzyUJzHhqe_9HXfqatPXqoSNklwhM8R24HQckh8,143
 nexaai/asr.py,sha256=NljMXDErwPNMOPaRkJZMEDka9Nk8xyur7L8i924TStY,2054
 nexaai/base.py,sha256=N8PRgDFA-XPku2vWnQIofQ7ipz3pPlO6f8YZGnuhquE,982
 nexaai/common.py,sha256=yBnIbqYaQYnfrl7IczOBh6MDibYZVxwaRJEglYcKgGs,3422
@@ -15,20 +15,19 @@ nexaai/vlm.py,sha256=3voXmAVnGlXnOiwA3wcX4p0Lvmp0X1VKkQVPObJdwBY,4649
 nexaai/asr_impl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nexaai/asr_impl/mlx_asr_impl.py,sha256=eosd8-TIWAOwV0HltmoFrLwzXHcU4jyxtncvuZE9pgA,3257
 nexaai/asr_impl/pybind_asr_impl.py,sha256=pE9Hb_hMi5yAc4MF83bLVOb8zDtreCkB3_u7XED9YpA,1516
-nexaai/binds/__init__.py,sha256=T9Ua7SzHNglSeEqXlfH5ymYXRyXhNKkC9z_y_bWCNMo,80
+nexaai/binds/__init__.py,sha256=eYuay_8DDXeOUWz2_R9HFSabohxs6hvZn391t2L0Po0,104
 nexaai/binds/common_bind.cpython-310-darwin.so,sha256=7xH5yNkK6Ot-2f9vxmultr9NIeTemlmN5tHIYp8SZHs,233960
 nexaai/binds/embedder_bind.cpython-310-darwin.so,sha256=eQwf9m58Yrl_ZZzWlstBHfHD-s3UMFi0JNPty-Pc524,202032
-nexaai/binds/libcrypto.dylib,sha256=ysW8ydmDPnnNRy3AHESjJwMTFfmGDKU9eLIaiR37ca0,5091432
-nexaai/binds/libnexa_bridge.dylib,sha256=w0XiFs2v3uoojLBFLag9mSMAydDHmaJFrT-yFaRZRyA,250592
-nexaai/binds/libssl.dylib,sha256=JHPTSbRFnImmoWDO9rFdiKb0lJMT3q78VEsx-5-S0sk,889520
+nexaai/binds/libnexa_bridge.dylib,sha256=uJ6vhHoU5Q-QE8jC2PHPqVGAKhgUig_GWZ-HODXrYqs,250376
 nexaai/binds/llm_bind.cpython-310-darwin.so,sha256=08DuSv91LIVCS5BSfSx3Nuz088h62lm3MV2yDTSIKt4,183008
+nexaai/binds/vlm_bind.cpython-310-darwin.so,sha256=x1hxuCx09P69swsVj46FXHKUH5Ln39ISBjBscyJWESM,183000
 nexaai/binds/nexa_llama_cpp/libggml-base.dylib,sha256=GyOkHOM-5uHp7NUZ4Sr9BWak6BYpcc9aqI9A-zPnQp4,629528
 nexaai/binds/nexa_llama_cpp/libggml-cpu.so,sha256=cnLUQ7WdX-5iiDaH8v45u1kX1NUmK8DanpzSMGCpXPE,1039800
 nexaai/binds/nexa_llama_cpp/libggml-metal.so,sha256=Xhhl_tLg1xmCIQVrKjqPFaLHAlx_2wUFiwDyUk0wJ-E,713680
 nexaai/binds/nexa_llama_cpp/libggml.dylib,sha256=12Q1Z98oM81hxzT_GMQsW5rlhC8DOMsX6luWVCFQHcI,58336
 nexaai/binds/nexa_llama_cpp/libllama.dylib,sha256=ORoTILXdGGX6MLMh-IWLIp4P5UEPOjE4lvWOKM18pCk,1982280
 nexaai/binds/nexa_llama_cpp/libmtmd.dylib,sha256=1plWvthTQf8IXbXthMG0MZXzWhbzV6ghdVzVFIb1FnU,701504
-nexaai/binds/nexa_llama_cpp/libnexa_plugin.dylib,sha256=zp1dcbAp9XcrIk3mUR346LAqCxzQLfSnC56pszl6jgQ,2644752
+nexaai/binds/nexa_llama_cpp/libnexa_plugin.dylib,sha256=3Dpj5aDJUcJ0CGyl7hstBBsorDwM3U0vVsCjmDB34dg,2025496
 nexaai/cv_impl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nexaai/cv_impl/mlx_cv_impl.py,sha256=gKECQOv8iaWwG3bl7xeqVy2NN_9K7tYerIFzfn4eLo4,3228
 nexaai/cv_impl/pybind_cv_impl.py,sha256=uSmwBste4cT7c8DQmXzRLmzwDf773PAbXNYWW1UzVls,1064
@@ -358,15 +357,15 @@ nexaai/tts_impl/mlx_tts_impl.py,sha256=i_uNPdvlXYtL3e01oKjDlP9jgkWCRt1bBHsExaaiJ
 nexaai/tts_impl/pybind_tts_impl.py,sha256=mpn44r6pfYLIl-NrEy2dXHjGtWtNCmM7HRyxiANxUI4,1444
 nexaai/utils/avatar_fetcher.py,sha256=bWy8ujgbOiTHFCjFxTwkn3uXbZ84PgEGUkXkR3MH4bI,3821
 nexaai/utils/decode.py,sha256=61n4Zf6c5QLyqGoctEitlI9BX3tPlP2a5aaKNHbw3T4,404
-nexaai/utils/manifest_utils.py,sha256=2waOuQErodNHhoAETQqlQgXdVes-T5A4HMb8pUIN9hg,9765
-nexaai/utils/model_manager.py,sha256=T7I4HvjMhLqkWutydzWKFOZNFWrTxVT7-WHx-XKX3KM,50269
+nexaai/utils/manifest_utils.py,sha256=sR9Nme4GbD3Cb3fMd55yLvGZpqxb71vd6b2XZTsrIGM,12328
+nexaai/utils/model_manager.py,sha256=p2kJKK63Zk-rEUucFsgY0T5PyXi_IvJY0gKewUVcAV4,56081
 nexaai/utils/model_types.py,sha256=-DER8L4lAUR_iLS99F0r57avwqWtuN21ug5pX2p24_E,1369
 nexaai/utils/progress_tracker.py,sha256=jdUqtmPqyhwC9uSKvQcJEYETwSt-OhP4oitdJ94614o,15394
 nexaai/utils/quantization_utils.py,sha256=4gvp6UQfSO9G1FYBwnFtQspTzH9sDbi1PBXw2t1N69M,7650
 nexaai/vlm_impl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nexaai/vlm_impl/mlx_vlm_impl.py,sha256=od1R1mRoIgPG3NHC7JiDlcB_YJY8aklX8Em3ZkeHNpE,10734
-nexaai/vlm_impl/pybind_vlm_impl.py,sha256=5ZMFgDATthmMzjrd-vE5KX5ZAMoWPYbF_FTLz8DBKIk,8908
-nexaai-1.0.12rc1.dist-info/METADATA,sha256=kN9eRlmv408cw-PhvdNvM4lL_GYL9UyBM4P37fWbixI,1201
-nexaai-1.0.12rc1.dist-info/WHEEL,sha256=0KYp5feZ1CMUhsfFXKpSQTbSmQbXy4mv6yPPVBXg2EM,110
-nexaai-1.0.12rc1.dist-info/top_level.txt,sha256=LRE2YERlrZk2vfuygnSzsEeqSknnZbz3Z1MHyNmBU4w,7
-nexaai-1.0.12rc1.dist-info/RECORD,,
+nexaai/vlm_impl/pybind_vlm_impl.py,sha256=n4lBTeDA9i07Fz1sq4694XkA2XQtnJnR9RaaKK0CrcM,8458
+nexaai-1.0.13rc2.dist-info/METADATA,sha256=QDTSLe7dtUuRQzCK7l0wE49_GoDsm8QN5W_3mPhdlfc,1201
+nexaai-1.0.13rc2.dist-info/WHEEL,sha256=0KYp5feZ1CMUhsfFXKpSQTbSmQbXy4mv6yPPVBXg2EM,110
+nexaai-1.0.13rc2.dist-info/top_level.txt,sha256=LRE2YERlrZk2vfuygnSzsEeqSknnZbz3Z1MHyNmBU4w,7
+nexaai-1.0.13rc2.dist-info/RECORD,,