nexaai 1.0.4rc13__cp310-cp310-win_amd64.whl

nexaai/__init__.py

@@ -0,0 +1,71 @@
+"""
+NexaAI Python bindings for NexaSDK C-lib backend.
+"""
+
+import sys
+import os
+
+# Add mlx_backend to Python path as individual module (only if it exists)
+_current_dir = os.path.dirname(os.path.abspath(__file__))
+_mlx_backend_path = os.path.join(_current_dir, "mlx_backend")
+# Only add to path if the directory exists (it won't on Windows)
+if os.path.exists(_mlx_backend_path) and _mlx_backend_path not in sys.path:
+    sys.path.insert(0, _mlx_backend_path)
+
+try:
+    from ._version import __version__
+except ImportError:
+    # Fallback for development or when version file hasn't been generated yet
+    __version__ = "0.0.1"
+
+# Import common configuration classes first (no external dependencies)
+from .common import ModelConfig, GenerationConfig, ChatMessage, SamplerConfig
+
+# Import new feature classes (no external dependencies in base classes)
+from .llm import LLM
+from .embedder import Embedder, EmbeddingConfig
+from .vlm import VLM
+from .asr import ASR, ASRConfig, ASRResult
+from .cv import CVModel, CVModelConfig, CVResult, CVResults, CVCapabilities, BoundingBox
+from .rerank import Reranker, RerankConfig
+from .image_gen import ImageGen, ImageGenerationConfig, ImageSamplerConfig, SchedulerConfig, Image
+from .tts import TTS, TTSConfig, TTSSamplerConfig, TTSResult
+
+# Build __all__ list dynamically
+__all__ = [
+    "__version__",
+    # Common configurations (always available)
+    "ModelConfig",
+    "GenerationConfig", 
+    "ChatMessage",
+    "SamplerConfig",
+    "EmbeddingConfig",
+
+    "LLM",
+    "Embedder",
+    "VLM",
+    "ASR",
+    "CVModel",
+    "Reranker",
+    "ImageGen",
+    "TTS",
+
+    "ASRConfig",
+    "ASRResult",
+    "CVModelConfig",
+    "CVResult",
+    "CVResults", 
+    "CVCapabilities",
+    "BoundingBox",
+    "RerankConfig",
+    "ImageGenerationConfig",
+    "ImageSamplerConfig",
+    "SchedulerConfig",
+    "Image",
+    "TTSConfig",
+    "TTSSamplerConfig",
+    "TTSResult",
+]
+
+
+

nexaai/_version.py

@@ -0,0 +1,4 @@
+# This file is generated by CMake from _version.py.in
+# Do not modify this file manually - it will be overwritten
+
+__version__ = "1.0.4-rc13" 

nexaai/asr.py

@@ -0,0 +1,60 @@
+from typing import List, Optional, Sequence, Tuple
+from abc import abstractmethod
+from dataclasses import dataclass
+
+from nexaai.base import BaseModel
+
+
+@dataclass
+class ASRConfig:
+    """Configuration for ASR."""
+    timestamps: str = "none"  # "none" | "segment" | "word"
+    beam_size: int = 5
+    stream: bool = False
+
+
+@dataclass
+class ASRResult:
+    """Result from ASR processing."""
+    transcript: str
+    confidence_scores: Sequence[float]
+    timestamps: Sequence[Tuple[float, float]]
+
+
+class ASR(BaseModel):
+    """Abstract base class for Automatic Speech Recognition models."""
+
+    def __init__(self):
+        """Initialize base ASR class."""
+        pass
+
+    @classmethod
+    def _load_from(cls,
+                   model_path: str,
+                   tokenizer_path: Optional[str] = None,
+                   language: Optional[str] = None,
+                   plugin_id: str = "llama_cpp",
+                   device_id: Optional[str] = None
+        ) -> 'ASR':
+        """Load ASR model from local path, routing to appropriate implementation."""
+        if plugin_id == "mlx":
+            from nexaai.asr_impl.mlx_asr_impl import MLXASRImpl
+            return MLXASRImpl._load_from(model_path, tokenizer_path, language, plugin_id, device_id)
+        else:
+            from nexaai.asr_impl.pybind_asr_impl import PyBindASRImpl
+            return PyBindASRImpl._load_from(model_path, tokenizer_path, language, plugin_id, device_id)
+
+    @abstractmethod
+    def transcribe(
+        self,
+        audio_path: str,
+        language: Optional[str] = None,
+        config: Optional[ASRConfig] = None,
+    ) -> ASRResult:
+        """Transcribe audio file to text."""
+        pass
+
+    @abstractmethod
+    def list_supported_languages(self) -> List[str]:
+        """List supported languages."""
+        pass

nexaai/asr_impl/mlx_asr_impl.py

@@ -0,0 +1,91 @@
+# Note: This code is generated by Cursor, not tested yet.
+
+from typing import List, Optional
+
+from nexaai.asr import ASR, ASRConfig, ASRResult
+from nexaai.mlx_backend.asr.interface import MlxAsr as MLXASRInterface
+from nexaai.mlx_backend.ml import ModelConfig as MLXModelConfig, SamplerConfig as MLXSamplerConfig, GenerationConfig as MLXGenerationConfig, EmbeddingConfig
+
+
+class MLXASRImpl(ASR):
+    def __init__(self):
+        """Initialize MLX ASR implementation."""
+        super().__init__()
+        self._mlx_asr = None
+
+    @classmethod
+    def _load_from(cls,
+                   model_path: str,
+                   tokenizer_path: Optional[str] = None,
+                   language: Optional[str] = None,
+                   plugin_id: str = "mlx",
+                   device_id: Optional[str] = None
+        ) -> 'MLXASRImpl':
+        """Load ASR model from local path using MLX backend."""
+        try:
+            # MLX ASR interface is already imported
+            
+            # Create instance and load MLX ASR
+            instance = cls()
+            instance._mlx_asr = MLXASRInterface(
+                model_path=model_path,
+                tokenizer_path=tokenizer_path,
+                language=language,
+                device=device_id
+            )
+            
+            return instance
+        except Exception as e:
+            raise RuntimeError(f"Failed to load MLX ASR: {str(e)}")
+
+    def eject(self):
+        """Destroy the model and free resources."""
+        if self._mlx_asr:
+            self._mlx_asr.destroy()
+            self._mlx_asr = None
+            
+    def transcribe(
+        self,
+        audio_path: str,
+        language: Optional[str] = None,
+        config: Optional[ASRConfig] = None,
+    ) -> ASRResult:
+        """Transcribe audio file to text."""
+        if not self._mlx_asr:
+            raise RuntimeError("MLX ASR not loaded")
+        
+        try:
+            # Convert our config to MLX format if provided
+            mlx_config = None
+            if config:
+                from nexaai.mlx_backend.ml import ASRConfig as MLXASRConfig
+                
+                mlx_config = MLXASRConfig()
+                mlx_config.timestamps = config.timestamps
+                mlx_config.beam_size = config.beam_size
+                mlx_config.stream = config.stream
+            
+            # Use MLX ASR transcription
+            result = self._mlx_asr.transcribe(audio_path, language, mlx_config)
+            
+            # Convert MLX result to our format
+            return ASRResult(
+                transcript=result.transcript,
+                confidence_scores=result.confidence_scores,
+                timestamps=result.timestamps
+            )
+            
+        except Exception as e:
+            raise RuntimeError(f"Failed to transcribe audio: {str(e)}")
+
+    def list_supported_languages(self) -> List[str]:
+        """List supported languages."""
+        if not self._mlx_asr:
+            raise RuntimeError("MLX ASR not loaded")
+        
+        try:
+            return self._mlx_asr.list_supported_languages()
+        except Exception as e:
+            raise RuntimeError(f"Failed to list supported languages: {str(e)}")
+
+

nexaai/asr_impl/pybind_asr_impl.py

@@ -0,0 +1,43 @@
+from typing import List, Optional
+
+from nexaai.asr import ASR, ASRConfig, ASRResult
+
+
+class PyBindASRImpl(ASR):
+    def __init__(self):
+        """Initialize PyBind ASR implementation."""
+        super().__init__()
+        # TODO: Add PyBind-specific initialization
+
+    @classmethod
+    def _load_from(cls,
+                   model_path: str,
+                   tokenizer_path: Optional[str] = None,
+                   language: Optional[str] = None,
+                   plugin_id: str = "llama_cpp",
+                   device_id: Optional[str] = None
+        ) -> 'PyBindASRImpl':
+        """Load ASR model from local path using PyBind backend."""
+        # TODO: Implement PyBind ASR loading
+        instance = cls()
+        return instance
+
+    def eject(self):
+        """Destroy the model and free resources."""
+        # TODO: Implement PyBind ASR cleanup
+        pass
+
+    def transcribe(
+        self,
+        audio_path: str,
+        language: Optional[str] = None,
+        config: Optional[ASRConfig] = None,
+    ) -> ASRResult:
+        """Transcribe audio file to text."""
+        # TODO: Implement PyBind ASR transcription
+        raise NotImplementedError("PyBind ASR transcription not yet implemented")
+
+    def list_supported_languages(self) -> List[str]:
+        """List supported languages."""
+        # TODO: Implement PyBind ASR language listing
+        raise NotImplementedError("PyBind ASR language listing not yet implemented")

nexaai/base.py

@@ -0,0 +1,39 @@
+from abc import ABC, abstractmethod
+from nexaai.common import ProfilingData
+from nexaai.utils.model_manager import auto_download_model
+
+class BaseModel(ABC):
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.eject()
+
+    def __del__(self):
+        self.eject()
+
+    @classmethod
+    @auto_download_model
+    def from_(cls, name_or_path: str, **kwargs) -> "BaseModel":
+        """
+        initialize model from (1) HF (2) if not found, then from local path
+        """
+
+        return cls._load_from(name_or_path, **kwargs)
+
+    @classmethod
+    @abstractmethod
+    def _load_from(cls, name_or_path: str, **kwargs) -> "BaseModel":
+        """
+        Model-specific loading logic. Must be implemented by each model type.
+        Called after model is available locally.
+        """  
+        pass
+
+    @abstractmethod
+    def eject(self):
+        pass
+
+    def get_profiling_data(self) -> ProfilingData:
+        pass

nexaai/binds/__init__.py

@@ -0,0 +1,3 @@
+from .common_bind import *
+from .llm_bind import *
+from .embedder_bind import *

nexaai/common.py

@@ -0,0 +1,61 @@
+from dataclasses import dataclass
+from typing import TypedDict, Literal, Optional, List
+
+
+class ChatMessage(TypedDict):
+    role: Literal["user", "assistant", "system"]
+    content: str
+
+class MultiModalMessageContent(TypedDict):
+    type: Literal["text", "image", "audio", "video"]
+    text: Optional[str]
+    url: Optional[str]
+    path: Optional[str]
+
+class MultiModalMessage(TypedDict):
+    role: Literal["user", "assistant", "system"]
+    content: List[MultiModalMessageContent] 
+
+
+@dataclass
+class SamplerConfig:
+    temperature: float = 0.8
+    top_p: float = 0.95
+    top_k: int = 40
+    repetition_penalty: float = 1.0
+    presence_penalty: float = 0.0
+    frequency_penalty: float = 0.0
+    seed: int = -1
+    grammar_path: str = None
+    grammar_string: str = None
+
+@dataclass
+class GenerationConfig:
+    max_tokens: int = 1024
+    stop_words: list[str] = None
+    sampler_config: SamplerConfig = None
+    image_paths: list[str] = None
+    audio_paths: list[str] = None
+
+@dataclass
+class ModelConfig:
+    n_ctx: int = 4096
+    n_threads: int = None
+    n_threads_batch: int = None
+    n_batch: int = 512
+    n_ubatch: int = 512
+    n_seq_max: int = 1
+    n_gpu_layers: int = 999
+    chat_template_path: str = None
+    chat_template_content: str = None
+
+
+@dataclass(frozen=True) # Read-only
+class ProfilingData:
+    start_time: int
+    end_time: int
+    prompt_start_time: int = None
+    prompt_end_time: int = None
+    decode_start_time: int = None
+    decode_ent_time: int = None
+    first_token_time: int = None

nexaai/cv.py

@@ -0,0 +1,87 @@
+from typing import List, Optional
+from abc import abstractmethod
+from dataclasses import dataclass
+
+from nexaai.base import BaseModel
+
+
+@dataclass
+class BoundingBox:
+    """Generic bounding box structure."""
+    x: float  # X coordinate (normalized or pixel, depends on model)
+    y: float  # Y coordinate (normalized or pixel, depends on model) 
+    width: float  # Width
+    height: float  # Height
+
+
+@dataclass
+class CVResult:
+    """Generic detection/classification result."""
+    image_paths: Optional[List[str]] = None  # Output image paths
+    image_count: int = 0  # Number of output images
+    class_id: int = 0  # Class ID (example: ConvNext)
+    confidence: float = 0.0  # Confidence score [0.0-1.0]
+    bbox: Optional[BoundingBox] = None  # Bounding box (example: YOLO)
+    text: Optional[str] = None  # Text result (example: OCR)
+    embedding: Optional[List[float]] = None  # Feature embedding (example: CLIP embedding)
+    embedding_dim: int = 0  # Embedding dimension
+
+
+@dataclass
+class CVResults:
+    """Generic CV inference result."""
+    results: List[CVResult]  # Array of CV results
+    result_count: int  # Number of CV results
+
+
+class CVCapabilities:
+    """CV capabilities enum."""
+    OCR = 0  # OCR
+    CLASSIFICATION = 1  # Classification
+    SEGMENTATION = 2  # Segmentation
+    CUSTOM = 3  # Custom task
+
+
+@dataclass
+class CVModelConfig:
+    """CV model preprocessing configuration."""
+    capabilities: int  # CVCapabilities
+
+    # MLX-OCR
+    det_model_path: Optional[str] = None  # Detection model path
+    rec_model_path: Optional[str] = None  # Recognition model path
+
+    # QNN
+    model_path: Optional[str] = None  # Model path
+    system_library_path: Optional[str] = None  # System library path
+    backend_library_path: Optional[str] = None  # Backend library path
+    extension_library_path: Optional[str] = None  # Extension library path
+    config_file_path: Optional[str] = None  # Config file path
+    char_dict_path: Optional[str] = None  # Character dictionary path
+
+
+class CVModel(BaseModel):
+    """Abstract base class for generic computer vision models."""
+
+    def __init__(self):
+        """Initialize base CV model class."""
+        pass
+
+    @classmethod
+    def _load_from(cls,
+                   config: CVModelConfig,
+                   plugin_id: str = "llama_cpp",
+                   device_id: Optional[str] = None
+        ) -> 'CVModel':
+        """Load CV model from configuration, routing to appropriate implementation."""
+        if plugin_id == "mlx":
+            from nexaai.cv_impl.mlx_cv_impl import MLXCVImpl
+            return MLXCVImpl._load_from(config, plugin_id, device_id)
+        else:
+            from nexaai.cv_impl.pybind_cv_impl import PyBindCVImpl
+            return PyBindCVImpl._load_from(config, plugin_id, device_id)
+
+    @abstractmethod
+    def infer(self, input_image_path: str) -> CVResults:
+        """Perform inference on image."""
+        pass

nexaai/cv_impl/mlx_cv_impl.py

@@ -0,0 +1,88 @@
+# Note: This code is generated by Cursor, not tested yet.
+
+from typing import Optional
+import os
+
+from nexaai.cv import CVModel, CVModelConfig, CVResults
+from nexaai.mlx_backend.cv.interface import CVModel as MLXCVInterface, create_cv_model
+
+
+class MLXCVImpl(CVModel):
+    def __init__(self):
+        """Initialize MLX CV implementation."""
+        super().__init__()
+        self._mlx_cv = None
+
+    @classmethod
+    def _load_from(cls,
+                   config: CVModelConfig,
+                   plugin_id: str = "mlx",
+                   device_id: Optional[str] = None
+        ) -> 'MLXCVImpl':
+        """Load CV model from configuration using MLX backend."""
+        try:
+            # Get MLX config class
+            from nexaai.mlx_backend.ml import CVModelConfig as MLXCVModelConfig
+            
+            # Convert our config to MLX format
+            mlx_config = MLXCVModelConfig(
+                capabilities=config.capabilities,
+                det_model_path=config.det_model_path,
+                rec_model_path=config.rec_model_path,
+                model_path=config.model_path,
+                system_library_path=config.system_library_path,
+                backend_library_path=config.backend_library_path,
+                extension_library_path=config.extension_library_path,
+                config_file_path=config.config_file_path,
+                char_dict_path=config.char_dict_path
+            )
+            
+            # Create instance and load MLX CV model
+            instance = cls()
+            instance._mlx_cv = create_cv_model(mlx_config, device_id)
+            
+            return instance
+        except Exception as e:
+            raise RuntimeError(f"Failed to load MLX CV: {str(e)}")
+
+    def eject(self):
+        """Destroy the model and free resources."""
+        if self._mlx_cv:
+            self._mlx_cv.destroy()
+            self._mlx_cv = None
+
+    def infer(self, input_image_path: str) -> CVResults:
+        """Perform inference on image."""
+        if not self._mlx_cv:
+            raise RuntimeError("MLX CV not loaded")
+        
+        try:
+            # Use MLX CV inference
+            result = self._mlx_cv.infer(input_image_path)
+            
+            # Convert MLX result to our format
+            from nexaai.cv import CVResult
+            
+            our_results = []
+            for mlx_result in result.results:
+                our_result = CVResult(
+                    image_paths=mlx_result.image_paths,
+                    image_count=mlx_result.image_count,
+                    class_id=mlx_result.class_id,
+                    confidence=mlx_result.confidence,
+                    bbox=mlx_result.bbox,
+                    text=mlx_result.text,
+                    embedding=mlx_result.embedding,
+                    embedding_dim=mlx_result.embedding_dim
+                )
+                our_results.append(our_result)
+            
+            return CVResults(
+                results=our_results,
+                result_count=result.result_count
+            )
+            
+        except Exception as e:
+            raise RuntimeError(f"Failed to perform CV inference: {str(e)}")
+
+

nexaai/cv_impl/pybind_cv_impl.py

@@ -0,0 +1,31 @@
+from typing import Optional
+
+from nexaai.cv import CVModel, CVModelConfig, CVResults
+
+
+class PyBindCVImpl(CVModel):
+    def __init__(self):
+        """Initialize PyBind CV implementation."""
+        super().__init__()
+        # TODO: Add PyBind-specific initialization
+
+    @classmethod
+    def _load_from(cls,
+                   config: CVModelConfig,
+                   plugin_id: str = "llama_cpp",
+                   device_id: Optional[str] = None
+        ) -> 'PyBindCVImpl':
+        """Load CV model from configuration using PyBind backend."""
+        # TODO: Implement PyBind CV loading
+        instance = cls()
+        return instance
+
+    def eject(self):
+        """Destroy the model and free resources."""
+        # TODO: Implement PyBind CV cleanup
+        pass
+
+    def infer(self, input_image_path: str) -> CVResults:
+        """Perform inference on image."""
+        # TODO: Implement PyBind CV inference
+        raise NotImplementedError("PyBind CV inference not yet implemented")