nexaai 1.0.12rc1__cp310-cp310-win_amd64.whl → 1.0.13rc2__cp310-cp310-win_amd64.whl

nexaai/_version.py

@@ -1,4 +1,4 @@
 # This file is generated by CMake from _version.py.in
 # Do not modify this file manually - it will be overwritten
 
-__version__ = "1.0.12-rc1" 
+__version__ = "1.0.13-rc2" 

nexaai/binds/__init__.py

@@ -1,3 +1,4 @@
 from .common_bind import *
 from .llm_bind import *
 from .embedder_bind import *
+from .vlm_bind import *

nexaai/utils/manifest_utils.py

@@ -11,7 +11,7 @@ This module provides utilities to:
 import os
 import json
 from datetime import datetime
-from typing import Dict, Any, List, Optional
+from typing import Dict, Any, List, Optional, Union
 
 from .quantization_utils import (
     extract_quantization_from_filename, 
@@ -101,7 +101,7 @@ def save_download_metadata(directory_path: str, metadata: Dict[str, Any]) -> Non
         pass
 
 
-def create_gguf_manifest(repo_id: str, files: List[str], directory_path: str, old_metadata: Dict[str, Any]) -> Dict[str, Any]:
+def create_gguf_manifest(repo_id: str, files: List[str], directory_path: str, old_metadata: Dict[str, Any], is_mmproj: bool = False, file_name: Optional[Union[str, List[str]]] = None) -> Dict[str, Any]:
     """Create GGUF format manifest."""
     
     # Load existing manifest to merge GGUF files if it exists
@@ -111,14 +111,27 @@ def create_gguf_manifest(repo_id: str, files: List[str], directory_path: str, ol
     if existing_manifest and "ModelFile" in existing_manifest:
         model_files = existing_manifest["ModelFile"].copy()
     
+    # Initialize MMProjFile from existing manifest or empty
+    mmproj_file = {
+        "Name": "",
+        "Downloaded": False,
+        "Size": 0
+    }
+    if existing_manifest and "MMProjFile" in existing_manifest:
+        mmproj_file = existing_manifest["MMProjFile"].copy()
+    
     # Process GGUF files
-    for file_name in files:
-        if file_name.endswith('.gguf'):
-            # Use the new enum-based quantization extraction
-            quantization_type = extract_quantization_from_filename(file_name)
-            quant_level = quantization_type.value if quantization_type else "UNKNOWN"
+    for current_file_name in files:
+        if current_file_name.endswith('.gguf'):
+            # Check if this file is an mmproj file
+            is_current_mmproj = 'mmproj' in current_file_name.lower()
+            
+            # If we're downloading specific files and this is marked as mmproj, respect that
+            if is_mmproj and file_name is not None:
+                filenames_to_check = file_name if isinstance(file_name, list) else [file_name]
+                is_current_mmproj = current_file_name in filenames_to_check
             
-            file_path = os.path.join(directory_path, file_name)
+            file_path = os.path.join(directory_path, current_file_name)
             file_size = 0
             if os.path.exists(file_path):
                 try:
@@ -126,22 +139,31 @@ def create_gguf_manifest(repo_id: str, files: List[str], directory_path: str, ol
                 except (OSError, IOError):
                     pass
             
-            model_files[quant_level] = {
-                "Name": file_name,
-                "Downloaded": True,
-                "Size": file_size
-            }
+            if is_current_mmproj:
+                # This is an mmproj file, put it in MMProjFile
+                mmproj_file = {
+                    "Name": current_file_name,
+                    "Downloaded": True,
+                    "Size": file_size
+                }
+            else:
+                # Regular model file, put in ModelFile
+                # Use the new enum-based quantization extraction
+                quantization_type = extract_quantization_from_filename(current_file_name)
+                quant_level = quantization_type.value if quantization_type else "UNKNOWN"
+                
+                model_files[quant_level] = {
+                    "Name": current_file_name,
+                    "Downloaded": True,
+                    "Size": file_size
+                }
     
     manifest = {
         "Name": repo_id,
         "ModelType": PIPELINE_TO_MODEL_TYPE.get(old_metadata.get('pipeline_tag'), "other"),
         "PluginId": "llama_cpp",
         "ModelFile": model_files,
-        "MMProjFile": {
-            "Name": "",
-            "Downloaded": False,
-            "Size": 0
-        },
+        "MMProjFile": mmproj_file,
         "TokenizerFile": {
             "Name": "",
             "Downloaded": False,
@@ -157,20 +179,27 @@ def create_gguf_manifest(repo_id: str, files: List[str], directory_path: str, ol
     return manifest
 
 
-def create_mlx_manifest(repo_id: str, files: List[str], directory_path: str, old_metadata: Dict[str, Any]) -> Dict[str, Any]:
+def create_mlx_manifest(repo_id: str, files: List[str], directory_path: str, old_metadata: Dict[str, Any], is_mmproj: bool = False, file_name: Optional[Union[str, List[str]]] = None) -> Dict[str, Any]:
     """Create MLX format manifest."""
     
     model_files = {}
     extra_files = []
     
+    # Initialize MMProjFile
+    mmproj_file = {
+        "Name": "",
+        "Downloaded": False,
+        "Size": 0
+    }
+    
     # Try different methods to extract quantization for MLX models
     quantization_type = detect_quantization_for_mlx(repo_id, directory_path)
     
     # Use the detected quantization or default to "DEFAULT"
     quant_level = quantization_type.value if quantization_type else "DEFAULT"
     
-    for file_name in files:
-        file_path = os.path.join(directory_path, file_name)
+    for current_file_name in files:
+        file_path = os.path.join(directory_path, current_file_name)
         file_size = 0
         if os.path.exists(file_path):
             try:
@@ -178,17 +207,32 @@ def create_mlx_manifest(repo_id: str, files: List[str], directory_path: str, old
             except (OSError, IOError):
                 pass
         
+        # Check if this file is an mmproj file
+        is_current_mmproj = 'mmproj' in current_file_name.lower()
+        
+        # If we're downloading specific files and this is marked as mmproj, respect that
+        if is_mmproj and file_name is not None:
+            filenames_to_check = file_name if isinstance(file_name, list) else [file_name]
+            is_current_mmproj = current_file_name in filenames_to_check
+        
+        if is_current_mmproj:
+            # This is an mmproj file, put it in MMProjFile
+            mmproj_file = {
+                "Name": current_file_name,
+                "Downloaded": True,
+                "Size": file_size
+            }
         # Check if this is a main model file (safetensors but not index files)
-        if (file_name.endswith('.safetensors') and not file_name.endswith('.index.json')):
+        elif (current_file_name.endswith('.safetensors') and not current_file_name.endswith('.index.json')):
             model_files[quant_level] = {
-                "Name": file_name,
+                "Name": current_file_name,
                 "Downloaded": True,
                 "Size": file_size
             }
         else:
             # Add to extra files
             extra_files.append({
-                "Name": file_name,
+                "Name": current_file_name,
                 "Downloaded": True,
                 "Size": file_size
             })
@@ -198,11 +242,7 @@ def create_mlx_manifest(repo_id: str, files: List[str], directory_path: str, old
         "ModelType": PIPELINE_TO_MODEL_TYPE.get(old_metadata.get('pipeline_tag'), "other"),
         "PluginId": "mlx",
         "ModelFile": model_files,
-        "MMProjFile": {
-            "Name": "",
-            "Downloaded": False,
-            "Size": 0
-        },
+        "MMProjFile": mmproj_file,
         "TokenizerFile": {
             "Name": "",
             "Downloaded": False,
@@ -232,7 +272,7 @@ def detect_model_type(files: List[str]) -> str:
         return "mlx"
 
 
-def create_manifest_from_files(repo_id: str, files: List[str], directory_path: str, old_metadata: Dict[str, Any]) -> Dict[str, Any]:
+def create_manifest_from_files(repo_id: str, files: List[str], directory_path: str, old_metadata: Dict[str, Any], is_mmproj: bool = False, file_name: Optional[Union[str, List[str]]] = None) -> Dict[str, Any]:
     """
     Create appropriate manifest format based on detected model type.
     
@@ -241,6 +281,8 @@ def create_manifest_from_files(repo_id: str, files: List[str], directory_path: s
         files: List of files in the model directory
         directory_path: Path to the model directory
         old_metadata: Existing metadata (pipeline_tag, download_time, avatar_url)
+        is_mmproj: Whether the downloaded file is an mmproj file
+        file_name: The specific file(s) that were downloaded (None if entire repo was downloaded)
         
     Returns:
         Dict containing the appropriate manifest format
@@ -248,12 +290,12 @@ def create_manifest_from_files(repo_id: str, files: List[str], directory_path: s
     model_type = detect_model_type(files)
     
     if model_type == "gguf":
-        return create_gguf_manifest(repo_id, files, directory_path, old_metadata)
+        return create_gguf_manifest(repo_id, files, directory_path, old_metadata, is_mmproj, file_name)
     else:  # mlx or other
-        return create_mlx_manifest(repo_id, files, directory_path, old_metadata)
+        return create_mlx_manifest(repo_id, files, directory_path, old_metadata, is_mmproj, file_name)
 
 
-def save_manifest_with_files_metadata(repo_id: str, local_dir: str, old_metadata: Dict[str, Any]) -> None:
+def save_manifest_with_files_metadata(repo_id: str, local_dir: str, old_metadata: Dict[str, Any], is_mmproj: bool = False, file_name: Optional[Union[str, List[str]]] = None) -> None:
     """
     Create and save manifest based on files found in the directory.
     
@@ -261,6 +303,8 @@ def save_manifest_with_files_metadata(repo_id: str, local_dir: str, old_metadata
         repo_id: Repository ID
         local_dir: Local directory containing the model files
         old_metadata: Existing metadata to preserve
+        is_mmproj: Whether the downloaded file is an mmproj file
+        file_name: The specific file(s) that were downloaded (None if entire repo was downloaded)
     """
     # Get list of files in the directory
     files = []
@@ -274,7 +318,7 @@ def save_manifest_with_files_metadata(repo_id: str, local_dir: str, old_metadata
         pass
     
     # Create appropriate manifest
-    manifest = create_manifest_from_files(repo_id, files, local_dir, old_metadata)
+    manifest = create_manifest_from_files(repo_id, files, local_dir, old_metadata, is_mmproj, file_name)
     
     # Save manifest
     save_download_metadata(local_dir, manifest)

nexaai/utils/model_manager.py

@@ -21,6 +21,13 @@ from .manifest_utils import (
 # Default path for model storage
 DEFAULT_MODEL_SAVING_PATH = "~/.cache/nexa.ai/nexa_sdk/models/"
 
+
+@dataclass
+class MMProjInfo:
+    """Data class for mmproj file information."""
+    mmproj_path: Optional[str] = None
+    size: int = 0
+
 @dataclass
 class DownloadedModel:
     """Data class representing a downloaded model with all its metadata."""
@@ -34,6 +41,7 @@ class DownloadedModel:
     pipeline_tag: Optional[str] = None  # Pipeline tag from HuggingFace model info
     download_time: Optional[str] = None  # ISO format timestamp of download
     avatar_url: Optional[str] = None  # Avatar URL for the model author
+    mmproj_info: Optional[MMProjInfo] = None  # mmproj file information
     
     def to_dict(self) -> Dict[str, Any]:
         """Convert to dictionary format for backward compatibility."""
@@ -47,7 +55,11 @@ class DownloadedModel:
             'full_repo_download_complete': self.full_repo_download_complete,
             'pipeline_tag': self.pipeline_tag,
             'download_time': self.download_time,
-            'avatar_url': self.avatar_url
+            'avatar_url': self.avatar_url,
+            'mmproj_info': {
+                'mmproj_path': self.mmproj_info.mmproj_path,
+                'size': self.mmproj_info.size
+            } if self.mmproj_info else None
         }
         return result
 
@@ -125,6 +137,46 @@ def _has_valid_metadata(directory_path: str) -> bool:
     return os.path.exists(manifest_path) or os.path.exists(old_metadata_path)
 
 
+def _extract_mmproj_info(manifest: Dict[str, Any], local_path: str) -> Optional[MMProjInfo]:
+    """
+    Extract mmproj information from manifest data.
+    
+    Args:
+        manifest: Dictionary containing manifest data
+        local_path: Local path to the model directory
+        
+    Returns:
+        MMProjInfo object if mmproj file exists, None otherwise
+    """
+    # Check if manifest has MMProjFile information
+    mmproj_file_info = manifest.get('MMProjFile')
+    if not mmproj_file_info or not mmproj_file_info.get('Downloaded') or not mmproj_file_info.get('Name'):
+        return None
+    
+    mmproj_filename = mmproj_file_info.get('Name', '')
+    if not mmproj_filename:
+        return None
+    
+    # Construct full path to mmproj file
+    mmproj_path = os.path.join(local_path, mmproj_filename)
+    
+    # Get size from manifest, but verify file exists
+    mmproj_size = mmproj_file_info.get('Size', 0)
+    if os.path.exists(mmproj_path):
+        try:
+            # Verify size matches actual file size
+            actual_size = os.path.getsize(mmproj_path)
+            mmproj_size = actual_size  # Use actual size if different
+        except (OSError, IOError):
+            # If we can't get actual size, use size from manifest
+            pass
+    else:
+        # File doesn't exist, don't include mmproj info
+        return None
+    
+    return MMProjInfo(mmproj_path=mmproj_path, size=mmproj_size)
+
+
 def _scan_for_repo_folders(base_path: str) -> List[DownloadedModel]:
     """Scan a directory for repository folders and return model information."""
     models = []
@@ -159,6 +211,10 @@ def _scan_for_repo_folders(base_path: str) -> List[DownloadedModel]:
                                 # Load metadata if it exists
                                 repo_id = f"{item}/{subitem}"
                                 metadata = load_download_metadata(subitem_path, repo_id)
+                                
+                                # Extract mmproj information
+                                mmproj_info = _extract_mmproj_info(metadata, subitem_path)
+                                
                                 models.append(DownloadedModel(
                                     repo_id=repo_id,
                                     files=files,
@@ -169,7 +225,8 @@ def _scan_for_repo_folders(base_path: str) -> List[DownloadedModel]:
                                     full_repo_download_complete=download_complete,
                                     pipeline_tag=metadata.get('pipeline_tag'),
                                     download_time=metadata.get('download_time'),
-                                    avatar_url=metadata.get('avatar_url')
+                                    avatar_url=metadata.get('avatar_url'),
+                                    mmproj_info=mmproj_info
                                 ))
                     else:
                         direct_files.append(subitem)
@@ -188,6 +245,10 @@ def _scan_for_repo_folders(base_path: str) -> List[DownloadedModel]:
                         # Load metadata if it exists
                         repo_id = item
                         metadata = load_download_metadata(item_path, repo_id)
+                        
+                        # Extract mmproj information
+                        mmproj_info = _extract_mmproj_info(metadata, item_path)
+                        
                         models.append(DownloadedModel(
                             repo_id=repo_id,
                             files=files,
@@ -198,7 +259,8 @@ def _scan_for_repo_folders(base_path: str) -> List[DownloadedModel]:
                             full_repo_download_complete=download_complete,
                             pipeline_tag=metadata.get('pipeline_tag'),
                             download_time=metadata.get('download_time'),
-                            avatar_url=metadata.get('avatar_url')
+                            avatar_url=metadata.get('avatar_url'),
+                            mmproj_info=mmproj_info
                         ))
     
     except (OSError, IOError):
@@ -236,6 +298,7 @@ def list_downloaded_models(local_dir: Optional[str] = None) -> List[DownloadedMo
             - pipeline_tag: Optional[str] - Pipeline tag from HuggingFace model info
             - download_time: Optional[str] - ISO format timestamp when the model was downloaded
             - avatar_url: Optional[str] - Avatar URL for the model author
+            - mmproj_info: Optional[MMProjInfo] - mmproj file information with mmproj_path and size
     """
     
     # Set up local directory
@@ -727,7 +790,7 @@ class HuggingFaceDownloader:
         # If no expected size, just check that file is not empty
         return os.path.getsize(file_path) > 0
     
-    def _fetch_and_save_metadata(self, repo_id: str, local_dir: str) -> None:
+    def _fetch_and_save_metadata(self, repo_id: str, local_dir: str, is_mmproj: bool = False, file_name: Optional[Union[str, List[str]]] = None) -> None:
         """Fetch model info and save metadata after successful download."""
         # Initialize metadata with defaults to ensure manifest is always created
         old_metadata = {
@@ -757,7 +820,7 @@ class HuggingFaceDownloader:
         
         # CRITICAL: Always create the manifest file, regardless of metadata fetch failures
         try:
-            save_manifest_with_files_metadata(repo_id, local_dir, old_metadata)
+            save_manifest_with_files_metadata(repo_id, local_dir, old_metadata, is_mmproj, file_name)
             print(f"[OK] Successfully created nexa.manifest for {repo_id}")
         except Exception as e:
             # This is critical - if manifest creation fails, we should know about it
@@ -819,7 +882,7 @@ class HuggingFaceDownloader:
                 progress_tracker.stop_tracking()
             
             # Save metadata after successful download
-            self._fetch_and_save_metadata(repo_id, file_local_dir)
+            self._fetch_and_save_metadata(repo_id, file_local_dir, self._current_is_mmproj, self._current_file_name)
             
             return downloaded_path
             
@@ -864,7 +927,7 @@ class HuggingFaceDownloader:
                 progress_tracker.stop_tracking()
             
             # Save metadata after successful download
-            self._fetch_and_save_metadata(repo_id, repo_local_dir)
+            self._fetch_and_save_metadata(repo_id, repo_local_dir, self._current_is_mmproj, self._current_file_name)
             
             return downloaded_path
             
@@ -926,7 +989,7 @@ class HuggingFaceDownloader:
                 progress_tracker.stop_tracking()
             
             # Save metadata after successful download
-            self._fetch_and_save_metadata(repo_id, repo_local_dir)
+            self._fetch_and_save_metadata(repo_id, repo_local_dir, self._current_is_mmproj, self._current_file_name)
             
             return repo_local_dir
             
@@ -951,7 +1014,8 @@ class HuggingFaceDownloader:
         local_dir: Optional[str] = None,
         progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
         show_progress: bool = True,
-        force_download: bool = False
+        force_download: bool = False,
+        is_mmproj: bool = False
     ) -> str:
         """
         Main download method that handles all download scenarios.
@@ -970,6 +1034,10 @@ class HuggingFaceDownloader:
         # Validate and normalize parameters
         repo_id, file_name = self._validate_and_setup_params(repo_id, file_name)
         
+        # Store parameters as instance variables for use in _fetch_and_save_metadata
+        self._current_is_mmproj = is_mmproj
+        self._current_file_name = file_name
+        
         # Set up local directory
         local_dir = self._created_dir_if_not_exists(local_dir)
         
@@ -1038,7 +1106,8 @@ def download_from_huggingface(
     show_progress: bool = True,
     token: Union[bool, str, None] = None,
     custom_endpoint: Optional[str] = None,
-    force_download: bool = False
+    force_download: bool = False,
+    is_mmproj: Optional[bool] = None
 ) -> str:
     """
     Download models or files from HuggingFace Hub or custom mirror endpoints.
@@ -1065,6 +1134,9 @@ def download_from_huggingface(
                                         The endpoint will be used to initialize HfApi for all downloads.
         force_download (bool, optional): If True, download files even if they already exist locally.
                                         Default False (skip existing files).
+        is_mmproj (bool, optional): Whether the file being downloaded is an mmproj file. Only used when
+                                   file_name is not None. If None, defaults to True if 'mmproj' is in 
+                                   the filename, False otherwise.
     
     Returns:
         str: Path to the downloaded file or directory
@@ -1101,6 +1173,15 @@ def download_from_huggingface(
             }
         }
     """
+    # Set default value for is_mmproj based on filename if not explicitly provided
+    if is_mmproj is None and file_name is not None:
+        # Check if any filename contains 'mmproj'
+        filenames_to_check = file_name if isinstance(file_name, list) else [file_name]
+        is_mmproj = any('mmproj' in filename.lower() for filename in filenames_to_check)
+    elif is_mmproj is None:
+        # Default to False if no file_name is provided
+        is_mmproj = False
+    
     # Create downloader instance with custom endpoint if provided
     downloader = HuggingFaceDownloader(
         endpoint=custom_endpoint, 
@@ -1115,7 +1196,8 @@ def download_from_huggingface(
         local_dir=local_dir,
         progress_callback=progress_callback,
         show_progress=show_progress,
-        force_download=force_download
+        force_download=force_download,
+        is_mmproj=is_mmproj
     )
 
 
@@ -1124,15 +1206,66 @@ def download_from_huggingface(
 ##########################################################################
 
 
+def _download_model_if_needed(
+    model_path: str,
+    param_name: str,
+    progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
+    token: Union[bool, str, None] = None,
+    is_mmproj: bool = False
+) -> str:
+    """
+    Helper function to download a model from HuggingFace if it doesn't exist locally.
+    
+    Args:
+        model_path: The model path that may be local or remote
+        param_name: Name of the parameter (for error messages)
+        progress_callback: Callback function for download progress updates
+        token: HuggingFace authentication token for private repositories
+        
+    Returns:
+        str: Local path to the model (either existing or downloaded)
+        
+    Raises:
+        RuntimeError: If download fails
+    """
+    # Check if model_path exists locally (file or directory)
+    if os.path.exists(model_path):
+        # Local path exists, return as-is
+        return model_path
+    
+    # Model path doesn't exist locally, try to download from HuggingFace
+    try:
+        # Parse model_path to extract repo_id and filename
+        repo_id, file_name = _parse_model_path(model_path)
+        
+        # Download the model
+        downloaded_path = download_from_huggingface(
+            repo_id=repo_id,
+            file_name=file_name,
+            local_dir=None,  # Use default cache directory
+            enable_transfer=True,
+            progress_callback=progress_callback,
+            show_progress=True,
+            token=token,
+            is_mmproj=is_mmproj
+        )
+        
+        return downloaded_path
+        
+    except Exception as e:
+        # Only handle download-related errors
+        raise RuntimeError(f"Could not load model from '{param_name}={model_path}': {e}")
+
+
 def auto_download_model(func: Callable) -> Callable:
     """
     Decorator that automatically downloads models from HuggingFace if they don't exist locally.
     
-    This decorator should be applied to __init__ methods that take a name_or_path parameter.
-    If name_or_path doesn't exist as a local file/directory, it will attempt to download
-    it from HuggingFace Hub using the download_from_huggingface function.
+    This decorator should be applied to __init__ methods that take a name_or_path parameter
+    and optionally an mmproj_path parameter. If these paths don't exist as local files/directories,
+    it will attempt to download them from HuggingFace Hub using the download_from_huggingface function.
     
-    The name_or_path can be in formats like:
+    The name_or_path and mmproj_path can be in formats like:
     - "microsoft/DialoGPT-small" (downloads entire repo)  
     - "microsoft/DialoGPT-small/pytorch_model.bin" (downloads specific file)
     - "Qwen/Qwen3-4B-GGUF/Qwen3-4B-Q4_K_M.gguf" (downloads specific file)
@@ -1149,21 +1282,6 @@ def auto_download_model(func: Callable) -> Callable:
     """
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
-        # Find name_or_path in arguments
-        # Assuming name_or_path is the first argument after self
-        if len(args) >= 2:
-            name_or_path = args[1]
-            args_list = list(args)
-            path_index = 1
-            is_positional = True
-        elif 'name_or_path' in kwargs:
-            name_or_path = kwargs['name_or_path']
-            path_index = None
-            is_positional = False
-        else:
-            # No name_or_path found, call original function
-            return func(*args, **kwargs)
-        
         # Extract progress_callback and token from arguments
         progress_callback = None
         if 'progress_callback' in kwargs:
@@ -1173,39 +1291,63 @@ def auto_download_model(func: Callable) -> Callable:
         if 'token' in kwargs:
             token = kwargs.pop('token')  # Remove from kwargs to avoid passing to original func
         
-        # Check if name_or_path exists locally (file or directory)
-        if os.path.exists(name_or_path):
-            # Local path exists, use as-is without downloading
+        # Handle name_or_path parameter
+        name_or_path = None
+        name_path_index = None
+        is_name_positional = False
+        
+        # Find name_or_path in arguments
+        # Assuming name_or_path is the first argument after self
+        if len(args) >= 2:
+            name_or_path = args[1]
+            args_list = list(args)
+            name_path_index = 1
+            is_name_positional = True
+        elif 'name_or_path' in kwargs:
+            name_or_path = kwargs['name_or_path']
+            is_name_positional = False
+        
+        # Handle mmproj_path parameter
+        mmproj_path = None
+        if 'mmproj_path' in kwargs:
+            mmproj_path = kwargs['mmproj_path']
+        
+        # If neither parameter is found, call original function
+        if name_or_path is None and mmproj_path is None:
             return func(*args, **kwargs)
         
-        # Model path doesn't exist locally, try to download from HuggingFace
-        try:
-            # Parse name_or_path to extract repo_id and filename
-            repo_id, file_name = _parse_model_path(name_or_path)
-            
-            # Download the model
-            downloaded_path = download_from_huggingface(
-                repo_id=repo_id,
-                file_name=file_name,
-                local_dir=None,  # Use default cache directory
-                enable_transfer=True,
-                progress_callback=progress_callback,  # Use the extracted callback
-                show_progress=True,
-                token=token  # Use the extracted token
-            )
-            
-            # Replace name_or_path with downloaded path
-            if is_positional:
-                args_list[path_index] = downloaded_path
-                args = tuple(args_list)
-            else:
-                kwargs['name_or_path'] = downloaded_path
-            
-        except Exception as e:
-            # Only handle download-related errors
-            raise RuntimeError(f"Could not load model from '{name_or_path}': {e}")
+        # Download name_or_path if needed
+        if name_or_path is not None:
+            try:
+                downloaded_name_path = _download_model_if_needed(
+                    name_or_path, 'name_or_path', progress_callback, token
+                )
+                
+                # Replace name_or_path with downloaded path
+                if is_name_positional:
+                    if name_path_index is not None:
+                        args_list[name_path_index] = downloaded_name_path
+                        args = tuple(args_list)
+                else:
+                    kwargs['name_or_path'] = downloaded_name_path
+                    
+            except Exception as e:
+                raise e  # Re-raise the error from _download_model_if_needed
+        
+        # Download mmproj_path if needed
+        if mmproj_path is not None:
+            try:
+                downloaded_mmproj_path = _download_model_if_needed(
+                    mmproj_path, 'mmproj_path', progress_callback, token, is_mmproj=True
+                )
+                
+                # Replace mmproj_path with downloaded path
+                kwargs['mmproj_path'] = downloaded_mmproj_path
+                
+            except Exception as e:
+                raise e  # Re-raise the error from _download_model_if_needed
         
-        # Call original function with updated path (outside try-catch to let model creation errors bubble up)
+        # Call original function with updated paths (outside try-catch to let model creation errors bubble up)
         return func(*args, **kwargs)
     
     return wrapper

nexaai/vlm_impl/pybind_vlm_impl.py

@@ -1,7 +1,6 @@
 from typing import Generator, Optional, List, Dict, Any, Union
 import queue
 import threading
-import base64
 from pathlib import Path
 
 from nexaai.common import ModelConfig, GenerationConfig, MultiModalMessage, PluginID
@@ -102,26 +101,16 @@ class PyBindVLMImpl(VLM):
                 t = c["type"]
                 if t == "text":
                     blocks.append({"type": "text", "text": c.get("text","") or ""})
+                elif t == "image":
+                    # Pass through the original structure - let vlm-bind.cpp handle field extraction
+                    blocks.append(c)
                 else:
-                    # image/audio/video
-                    src = c.get("url") or c.get("path")
-                    if not src:
-                        raise ValueError(f"No url/path for {t}")
-                    # read local file or strip data URI
-                    if Path(src).exists():
-                        raw = Path(src).read_bytes()
-                        b64 = base64.b64encode(raw).decode("ascii")
-                        blocks.append({"type": t, "text": b64})
-                    elif src.startswith("data:"):
-                        b64 = src.split(",",1)[1]
-                        blocks.append({"type": t, "text": b64})
-                    else:
-                        # remote URL
-                        blocks.append({"type": t, "text": src})
+                    raise ValueError(f"Unsupported content type: {t}. Use 'text' or 'image' to match the golden reference in vlm.cpp")
 
             payload.append({"role": role, "content": blocks})
 
-        return vlm_bind.ml_vlm_apply_chat_template(self._handle, payload, tools)
+        result = vlm_bind.ml_vlm_apply_chat_template(self._handle, payload, tools)
+        return result
 
     def generate_stream(self, prompt: str, g_cfg: GenerationConfig = GenerationConfig()) -> Generator[str, None, None]:
         """Generate text with streaming."""

{nexaai-1.0.12rc1.dist-info → nexaai-1.0.13rc2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nexaai
-Version: 1.0.12rc1
+Version: 1.0.13rc2
 Summary: Python bindings for NexaSDK C-lib backend
 Author-email: "Nexa AI, Inc." <dev@nexa.ai>
 Project-URL: Homepage, https://github.com/NexaAI/nexasdk-bridge

{nexaai-1.0.12rc1.dist-info → nexaai-1.0.13rc2.dist-info}/RECORD

@@ -1,6 +1,6 @@
 nexaai/__init__.py,sha256=Lt8NU57eTMtWrDYzpFeYR9XtGAPXqizynP83TPU0UW0,2105
-nexaai/_stub.cp310-win_amd64.pyd,sha256=l_sOvn0sGZaAzxUURKum8xng89EtFH_aE1lcWv_ZpKA,10752
-nexaai/_version.py,sha256=56eZbYiAoFM_PEDgdfRtf4Lkj9WNLYjfs6jyjl979Rs,147
+nexaai/_stub.cp310-win_amd64.pyd,sha256=yh_bkjT4c0dWKl1ClfZ5hTHcaNN9FHKZd2PUPimpN7g,10752
+nexaai/_version.py,sha256=9VhLiWBFeo9dM94_u0HoZj6ukbol57nkGewSXnnRmpE,147
 nexaai/asr.py,sha256=_fsGaxpiU137bUtO5ujtFSYCI1RLsyeEm3Gf4GhHVRk,2118
 nexaai/base.py,sha256=qQBCiQVNzgpkQjZX9aiFDEdbAAe56TROKC3WnWra2Zg,1021
 nexaai/common.py,sha256=6keIpdX5XS5us4z79EMoa6RSkVze9SbbXax13IJ9yvs,3525
@@ -15,21 +15,22 @@ nexaai/vlm.py,sha256=STjXCw67ABrHrEll8A2NGiwmfo7MotfYgBh1k1aNxkk,4775
 nexaai/asr_impl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nexaai/asr_impl/mlx_asr_impl.py,sha256=XwMX3LYMeulp8cDS0TCCYcjvttFHAyDWQ_oMvABwQmI,3349
 nexaai/asr_impl/pybind_asr_impl.py,sha256=20o5SOPzhF9x41ra8L_qIM7YxCkYeLb5csSrNde-dds,1560
-nexaai/binds/__init__.py,sha256=tYvy0pFhoY29GstDT5r-oRiPRarPLECvJAkcamJItOg,83
-nexaai/binds/common_bind.cp310-win_amd64.pyd,sha256=rWuxRVTVPuakItIeyiXeouD25e4Ekl-niQix_GOriIs,201216
-nexaai/binds/embedder_bind.cp310-win_amd64.pyd,sha256=Bz5tKsR7W_ohrsshGq314-ZkfG39H5fs6dH9kJh584A,182784
-nexaai/binds/libcrypto-3-x64.dll,sha256=-Lau6pL5DpDXzpg9MED63gCeL8oRrSLI_e2LeaxIHqk,7314432
-nexaai/binds/libssl-3-x64.dll,sha256=Tzzyu5jRpUugFxr_65hbFlAtFpjxIDpOYMU1E0ijkJw,1313792
-nexaai/binds/llm_bind.cp310-win_amd64.pyd,sha256=Wpm1mEohC6DWlwYuK8BU_PJQoW5tjsarcjXhLFhdSgY,162816
-nexaai/binds/nexa_bridge.dll,sha256=UbbvGrUkwFk_tMJeALO5HM7O9rMfJ7xt8YXpB5LheEQ,168448
-nexaai/binds/nexa_llama_cpp/ggml-base.dll,sha256=qpJM5qmcne4UqEbsYEBeoDHOe0KyWjqPk-DrKd8TD_k,532480
-nexaai/binds/nexa_llama_cpp/ggml-cpu.dll,sha256=lbNh62Q7bW0hB_Dbps8HqUkLMNwh_VOQhT3gOTrpOt4,672768
-nexaai/binds/nexa_llama_cpp/ggml-cuda.dll,sha256=Hs_WgTlv9d5MkS2S-hENjwMVX953bhUZtM5959Q7jFs,313528832
-nexaai/binds/nexa_llama_cpp/ggml-vulkan.dll,sha256=siZ1qWTIg4LWcOBlwVsdf7mryOIUrhhiMfdITMdH-TM,36627456
-nexaai/binds/nexa_llama_cpp/ggml.dll,sha256=sASNHyXx2Kor4O5Ao5X2AhmLZwMg3hdJe2jMSxdJmi0,66560
-nexaai/binds/nexa_llama_cpp/llama.dll,sha256=S_SaHxVswdt8LlLazOsLhA72pOoD4d2iba9JKjWGTlg,1611776
-nexaai/binds/nexa_llama_cpp/mtmd.dll,sha256=5bAv5F32eaZdlJ_lbwO53Ph3b3c62t14tmZiflkVL8U,561152
-nexaai/binds/nexa_llama_cpp/nexa_plugin.dll,sha256=1AruxKWsmmuvod95SwAQaffwbrtm38EZSJT9M2-ngM0,1405440
+nexaai/binds/__init__.py,sha256=ENl-uoIF9-3XGIXitVgZ2QmJ6p7Yet4h1-X7nUDZ0Hk,108
+nexaai/binds/common_bind.cp310-win_amd64.pyd,sha256=67kvH9ytidgKpTSPSBQEIWXng4iyIKsGvLgm5qZAtYM,201216
+nexaai/binds/embedder_bind.cp310-win_amd64.pyd,sha256=4m1HZAS3eqXeHph6NRdg0mCZLzRCfVaFe0yV5kCll7g,182784
+nexaai/binds/libcrypto-3-x64.dll,sha256=PYwJJtM4xpo2IQfxm-of6VR6rtVcHCyodwcvBHmfzig,7315968
+nexaai/binds/libssl-3-x64.dll,sha256=mp_RoRjYcCeAuqMy3QDYvxQ-cjAMUNCuwA320oXVVpg,1313792
+nexaai/binds/llm_bind.cp310-win_amd64.pyd,sha256=6tLXslo9CQl77mozzHq0mJhcpQU2bG8jFn--J5ATYoo,162816
+nexaai/binds/nexa_bridge.dll,sha256=3CeitLeaXfzk_kwz8znPc2fQdJCzvey2a5EHcn41bjs,168448
+nexaai/binds/vlm_bind.cp310-win_amd64.pyd,sha256=_up-xlK3LCeKvMMFe4IbQ4gfkuYy2I3tX6GdXJV0mF8,168960
+nexaai/binds/nexa_llama_cpp/ggml-base.dll,sha256=4eKg9swCCKJ4NB5HFH1m4_UurkzIlbUm3gmQcC4ygpQ,532480
+nexaai/binds/nexa_llama_cpp/ggml-cpu.dll,sha256=P_e6-nkKKNsyhlIKBHKZJZ7158ZCj0uFlBrCQywd7ag,672768
+nexaai/binds/nexa_llama_cpp/ggml-cuda.dll,sha256=HnlhW9PufnU_x5Mr4hka5hz5pVk6VLXD9KYxxsy9ksc,313528832
+nexaai/binds/nexa_llama_cpp/ggml-vulkan.dll,sha256=47PEdfb2RzZyydi7vPReIpQKB4osrwmFkTYvE-oLap4,36627456
+nexaai/binds/nexa_llama_cpp/ggml.dll,sha256=yr5qn5GRhfw-1iZcf35hzO2unBC_qPYi5ei6Ea6NGQM,66560
+nexaai/binds/nexa_llama_cpp/llama.dll,sha256=mP71izs2Zeqw1z-p_gHRxYMWIbXI4JDXHsImeMKmmHo,1611776
+nexaai/binds/nexa_llama_cpp/mtmd.dll,sha256=Ya3J-d-b3ENtSPCrJkl9VdA3Onep77rSngM9Sycq0Ww,561152
+nexaai/binds/nexa_llama_cpp/nexa_plugin.dll,sha256=Ne7ObJB5aC2A_7LYzOsy2sNBGV1UgcieopoAw13tEW8,1405440
 nexaai/cv_impl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nexaai/cv_impl/mlx_cv_impl.py,sha256=QLd_8w90gtxH8kmssaDYatCTRvQNIJuUGKZNnYrmx6E,3317
 nexaai/cv_impl/pybind_cv_impl.py,sha256=aSOCAxmHrwJbEkSN6VX3Cykqlj_9RIpVrZXILul04GA,1096
@@ -50,15 +51,15 @@ nexaai/tts_impl/mlx_tts_impl.py,sha256=LcH9bVdIl3Q6lOzSUB_X2s-_nWFmlCl1yL7XSUK0f
 nexaai/tts_impl/pybind_tts_impl.py,sha256=n3z4zmPQayQJgAwcvETw0IBUCp8IYROuYFSg0tAy_8Y,1487
 nexaai/utils/avatar_fetcher.py,sha256=D01f8je-37Nd68zGw8MYK2m7y3fvGlC6h0KR-aN9kdU,3925
 nexaai/utils/decode.py,sha256=0Z9jDH4ICzw4YXj8nD4L-sMouDaev-TISGRQ4KzidWE,421
-nexaai/utils/manifest_utils.py,sha256=zMgQpf5dAgF2RjGhk73zBggxRDGMRKDGxh2a8m8kmYg,10045
-nexaai/utils/model_manager.py,sha256=TmaT1fFculHgfAdutpNXP4d07HIhEMPVTfPvFKE-zR0,51480
+nexaai/utils/manifest_utils.py,sha256=nPqK24srLX41x3WU4R-OGQR9u0XA7ZTHY1MQXBKXdrM,12652
+nexaai/utils/model_manager.py,sha256=3rDODGClgnM24LFWIoM-TDpXkpV1vyJSdlv7Qk4ZaPE,57434
 nexaai/utils/model_types.py,sha256=arIyb9q-1uG0nyUGdWZaxxDJAxv0cfnJEpjCzyELL5Q,1416
 nexaai/utils/progress_tracker.py,sha256=BztrFqtjwNUmeREwZ5m7H6ZcrVzQEbpZfsxndWh4z0A,15778
 nexaai/utils/quantization_utils.py,sha256=jjQaz7K4qH6TdP8Tnv5Ktb2viz8BaVBSOrb_jm3ns28,7889
 nexaai/vlm_impl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nexaai/vlm_impl/mlx_vlm_impl.py,sha256=oY_qb9z_iF0zArBuY5CCYIvZcA3R0i_NKXrr_r-QSgg,10989
-nexaai/vlm_impl/pybind_vlm_impl.py,sha256=Hu8g8OXyPn8OzLQOpRSE5lfGmhjChiKj7fMRB8mC_cI,9147
-nexaai-1.0.12rc1.dist-info/METADATA,sha256=6sAd6d9nS11AE5aIHCxvqT5MeuRxoYeezwFsQGjsGkg,1233
-nexaai-1.0.12rc1.dist-info/WHEEL,sha256=KUuBC6lxAbHCKilKua8R9W_TM71_-9Sg5uEP3uDWcoU,101
-nexaai-1.0.12rc1.dist-info/top_level.txt,sha256=LRE2YERlrZk2vfuygnSzsEeqSknnZbz3Z1MHyNmBU4w,7
-nexaai-1.0.12rc1.dist-info/RECORD,,
+nexaai/vlm_impl/pybind_vlm_impl.py,sha256=7Bo0kpSrmOdr--bWSpQBvcaexkPPRt3x1yt9e_jIyDs,8686
+nexaai-1.0.13rc2.dist-info/METADATA,sha256=PP51i9S_euhcIZSaDxFexcZU-3g3qvNOyY3biwZVtgs,1233
+nexaai-1.0.13rc2.dist-info/WHEEL,sha256=KUuBC6lxAbHCKilKua8R9W_TM71_-9Sg5uEP3uDWcoU,101
+nexaai-1.0.13rc2.dist-info/top_level.txt,sha256=LRE2YERlrZk2vfuygnSzsEeqSknnZbz3Z1MHyNmBU4w,7
+nexaai-1.0.13rc2.dist-info/RECORD,,