nexaai 1.0.4rc16__cp310-cp310-win_amd64.whl → 1.0.6rc1__cp310-cp310-win_amd64.whl

nexaai/_version.py

@@ -1,4 +1,4 @@
 # This file is generated by CMake from _version.py.in
 # Do not modify this file manually - it will be overwritten
 
-__version__ = "1.0.4-rc16" 
+__version__ = "1.0.6-rc1" 

nexaai/common.py

@@ -59,10 +59,45 @@ class ModelConfig:
 
 @dataclass(frozen=True) # Read-only
 class ProfilingData:
-    start_time: int
-    end_time: int
-    prompt_start_time: int = None
-    prompt_end_time: int = None
-    decode_start_time: int = None
-    decode_ent_time: int = None
-    first_token_time: int = None
+    """Profiling data structure for LLM/VLM performance metrics."""
+    ttft: int = 0             # Time to first token (us)
+    prompt_time: int = 0      # Prompt processing time (us)
+    decode_time: int = 0      # Token generation time (us)
+    prompt_tokens: int = 0    # Number of prompt tokens
+    generated_tokens: int = 0  # Number of generated tokens
+    audio_duration: int = 0   # Audio duration (us)
+    prefill_speed: float = 0.0  # Prefill speed (tokens/sec)
+    decoding_speed: float = 0.0 # Decoding speed (tokens/sec)
+    real_time_factor: float = 0.0 # Real-Time Factor (RTF)
+    stop_reason: str = ""     # Stop reason: "eos", "length", "user", "stop_sequence"
+    
+    @classmethod
+    def from_dict(cls, data: dict) -> "ProfilingData":
+        """Create ProfilingData from dictionary."""
+        return cls(
+            ttft=data.get("ttft", 0),
+            prompt_time=data.get("prompt_time", 0),
+            decode_time=data.get("decode_time", 0),
+            prompt_tokens=data.get("prompt_tokens", 0),
+            generated_tokens=data.get("generated_tokens", 0),
+            audio_duration=data.get("audio_duration", 0),
+            prefill_speed=data.get("prefill_speed", 0.0),
+            decoding_speed=data.get("decoding_speed", 0.0),
+            real_time_factor=data.get("real_time_factor", 0.0),
+            stop_reason=data.get("stop_reason", "")
+        )
+    
+    def to_dict(self) -> dict:
+        """Convert to dictionary."""
+        return {
+            "ttft": self.ttft,
+            "prompt_time": self.prompt_time,
+            "decode_time": self.decode_time,
+            "prompt_tokens": self.prompt_tokens,
+            "generated_tokens": self.generated_tokens,
+            "audio_duration": self.audio_duration,
+            "prefill_speed": self.prefill_speed,
+            "decoding_speed": self.decoding_speed,
+            "real_time_factor": self.real_time_factor,
+            "stop_reason": self.stop_reason
+        }

nexaai/llm.py

@@ -4,7 +4,7 @@ import queue
 import threading
 
 from nexaai.common import ModelConfig, GenerationConfig, ChatMessage, PluginID
-from nexaai.base import BaseModel
+from nexaai.base import BaseModel, ProfilingData
 
 class LLM(BaseModel):
     def __init__(self, m_cfg: ModelConfig = ModelConfig()):
@@ -63,6 +63,10 @@ class LLM(BaseModel):
         """
         pass
 
+    def get_profiling_data(self) -> Optional[ProfilingData]:
+        """Get profiling data from the last generation."""  
+        pass
+
     @abstractmethod
     def save_kv_cache(self, path: str):
         """

nexaai/llm_impl/mlx_llm_impl.py

@@ -1,5 +1,6 @@
 from typing import Generator, Optional, Any, Sequence, Union
 
+from nexaai.base import ProfilingData
 from nexaai.common import ModelConfig, GenerationConfig, ChatMessage, PluginID
 from nexaai.llm import LLM
 from nexaai.mlx_backend.llm.interface import LLM as MLXLLMInterface
@@ -215,6 +216,12 @@ class MLXLLMImpl(LLM):
         except Exception as e:
             raise RuntimeError(f"Failed to generate text: {str(e)}")
 
+    def get_profiling_data(self) -> Optional[ProfilingData]:
+        """Get profiling data from the last generation."""
+        if not self._mlx_llm:
+            raise RuntimeError("MLX LLM not loaded")
+        return self._mlx_llm.get_profiling_data()
+
     def save_kv_cache(self, path: str):
         """
         Save the key-value cache to the file.

nexaai/llm_impl/pybind_llm_impl.py

@@ -2,6 +2,7 @@ from typing import Generator, Optional, Union
 import queue
 import threading
 
+from nexaai.base import ProfilingData
 from nexaai.common import ModelConfig, GenerationConfig, ChatMessage, PluginID
 from nexaai.binds import llm_bind, common_bind
 from nexaai.runtime import _ensure_runtime
@@ -13,6 +14,7 @@ class PyBindLLMImpl(LLM):
         """Private constructor, should not be called directly."""
         super().__init__(m_cfg)
         self._handle = handle  # This is a py::capsule
+        self._profiling_data = None
 
     @classmethod
     def _load_from(cls,
@@ -97,13 +99,14 @@ class PyBindLLMImpl(LLM):
         # Run generation in thread
         def generate():
             try:
-                llm_bind.ml_llm_generate(
+                result = llm_bind.ml_llm_generate(
                     handle=self._handle,
                     prompt=prompt,
                     config=config,
                     on_token=on_token,
                     user_data=None
                 )
+                self._profiling_data = ProfilingData.from_dict(result.get("profile_data", {}))
             except Exception as e:
                 exception_container[0] = e
             finally:
@@ -145,8 +148,14 @@ class PyBindLLMImpl(LLM):
             on_token=None,  # No callback for non-streaming
             user_data=None
         )
+
+        self._profiling_data = ProfilingData.from_dict(result.get("profile_data", {}))
         return result.get("text", "")
 
+    def get_profiling_data(self) -> Optional[ProfilingData]:
+        """Get profiling data."""
+        return self._profiling_data
+
     def save_kv_cache(self, path: str):
         """
         Save the key-value cache to the file.

nexaai/vlm.py

@@ -6,7 +6,7 @@ import base64
 from pathlib import Path
 
 from nexaai.common import ModelConfig, GenerationConfig, MultiModalMessage, PluginID
-from nexaai.base import BaseModel
+from nexaai.base import BaseModel, ProfilingData
 
 
 class VLM(BaseModel):
@@ -120,4 +120,8 @@ class VLM(BaseModel):
         Returns:
             str: The generated text.
         """
+        pass
+
+    def get_profiling_data(self) -> Optional[ProfilingData]:
+        """Get profiling data from the last generation."""
         pass

nexaai/vlm_impl/mlx_vlm_impl.py

@@ -1,5 +1,6 @@
 from typing import Generator, Optional, List, Dict, Any, Union
 
+from nexaai.base import ProfilingData
 from nexaai.common import ModelConfig, GenerationConfig, MultiModalMessage, PluginID
 from nexaai.vlm import VLM
 from nexaai.mlx_backend.vlm.interface import VLM as MLXVLMInterface
@@ -247,3 +248,9 @@ class MlxVlmImpl(VLM):
             
         except Exception as e:
             raise RuntimeError(f"Failed to generate text: {str(e)}")
+
+    def get_profiling_data(self) -> Optional[ProfilingData]:
+        """Get profiling data from the last generation."""
+        if not self._mlx_vlm:
+            raise RuntimeError("MLX VLM not loaded")
+        return self._mlx_vlm.get_profiling_data()

nexaai/vlm_impl/pybind_vlm_impl.py

@@ -8,6 +8,7 @@ from nexaai.common import ModelConfig, GenerationConfig, MultiModalMessage, Plug
 from nexaai.binds import vlm_bind, common_bind
 from nexaai.runtime import _ensure_runtime
 from nexaai.vlm import VLM
+from nexaai.base import ProfilingData
 
 
 class PyBindVLMImpl(VLM):
@@ -15,6 +16,7 @@ class PyBindVLMImpl(VLM):
         """Private constructor, should not be called directly."""
         super().__init__(m_cfg)
         self._handle = handle  # This is a py::capsule
+        self._profiling_data = None
 
     @classmethod
     def _load_from(cls,
@@ -143,13 +145,14 @@ class PyBindVLMImpl(VLM):
         # Run generation in thread
         def generate():
             try:
-                vlm_bind.ml_vlm_generate(
+                result = vlm_bind.ml_vlm_generate(
                     handle=self._handle,
                     prompt=prompt,
                     config=config,
                     on_token=on_token,
                     user_data=None
                 )
+                self._profiling_data = ProfilingData.from_dict(result.get("profile_data", {}))
             except Exception as e:
                 exception_container[0] = e
             finally:
@@ -191,8 +194,14 @@ class PyBindVLMImpl(VLM):
             on_token=None,  # No callback for non-streaming
             user_data=None
         )
+
+        self._profiling_data = ProfilingData.from_dict(result.get("profile_data", {}))
         return result.get("text", "")
 
+    def get_profiling_data(self) -> Optional[ProfilingData]:
+        """Get profiling data."""
+        return self._profiling_data
+
     def _convert_generation_config(self, g_cfg: GenerationConfig):
         """Convert GenerationConfig to binding format."""
         config = common_bind.GenerationConfig()

{nexaai-1.0.4rc16.dist-info → nexaai-1.0.6rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nexaai
-Version: 1.0.4rc16
+Version: 1.0.6rc1
 Summary: Python bindings for NexaSDK C-lib backend
 Author-email: "Nexa AI, Inc." <dev@nexa.ai>
 Project-URL: Homepage, https://github.com/NexaAI/nexasdk-bridge
@@ -17,11 +17,15 @@ Requires-Dist: tqdm
 Requires-Dist: hf_xet
 Requires-Dist: numpy
 Requires-Dist: httpx
-Requires-Dist: mlx
-Requires-Dist: mlx-lm
-Requires-Dist: scipy
-Requires-Dist: soundfile
-Requires-Dist: Pillow
-Requires-Dist: opencv-python
-Requires-Dist: shapely
-Requires-Dist: pyclipper
+Provides-Extra: mlx
+Requires-Dist: mlx; extra == "mlx"
+Requires-Dist: mlx-lm; extra == "mlx"
+Requires-Dist: mlx-vlm; extra == "mlx"
+Requires-Dist: tokenizers; extra == "mlx"
+Requires-Dist: safetensors; extra == "mlx"
+Requires-Dist: Pillow; extra == "mlx"
+Requires-Dist: scipy; extra == "mlx"
+Requires-Dist: soundfile; extra == "mlx"
+Requires-Dist: opencv-python; extra == "mlx"
+Requires-Dist: shapely; extra == "mlx"
+Requires-Dist: pyclipper; extra == "mlx"

{nexaai-1.0.4rc16.dist-info → nexaai-1.0.6rc1.dist-info}/RECORD

@@ -1,35 +1,35 @@
 nexaai/__init__.py,sha256=Lt8NU57eTMtWrDYzpFeYR9XtGAPXqizynP83TPU0UW0,2105
-nexaai/_stub.cp310-win_amd64.pyd,sha256=TFrp5SGUtUxU6tmAQGdW08-lZQSC1cT0ubsv-3nL9zM,10752
-nexaai/_version.py,sha256=gt1sBWA4vOkqP20VQTuFnSh43Ss73r_dHBVTNMJs5bs,147
+nexaai/_stub.cp310-win_amd64.pyd,sha256=HST07_JzYhB8bAd2oVWehmVKlFDRWUz5Df58YhKAdQY,10752
+nexaai/_version.py,sha256=dIn7jmZ9zkHQxe3amjreYeA8-Nny4TsTFC6Ib2kqTEQ,146
 nexaai/asr.py,sha256=_fsGaxpiU137bUtO5ujtFSYCI1RLsyeEm3Gf4GhHVRk,2118
 nexaai/base.py,sha256=qQBCiQVNzgpkQjZX9aiFDEdbAAe56TROKC3WnWra2Zg,1021
-nexaai/common.py,sha256=8eWPG-kBGaV3Lb2K85fpmePWK4Ex2KvFnNsoFwnfSZo,1727
+nexaai/common.py,sha256=6keIpdX5XS5us4z79EMoa6RSkVze9SbbXax13IJ9yvs,3525
 nexaai/cv.py,sha256=a6-csgYNDzPziJ0EojE9-BeM_xCny4UvWWbpnJ7GL-A,3365
 nexaai/embedder.py,sha256=3a81s7JapvYxCRbWPFKp_9EWBKW7WYqF03gk87YuGKU,2524
 nexaai/image_gen.py,sha256=4iASOKxJosMznLarTvOxJQDNaas251O81bfUWJTUBfE,4496
-nexaai/llm.py,sha256=bm01Ki8-towFLj6sBoAoZ2m6pWURKoxQWq2Vx_VQE2I,3508
+nexaai/llm.py,sha256=Qwm1q_NStLfD-JYZQIvxniWnAmwNl1V6LUON3Me7w_I,3663
 nexaai/rerank.py,sha256=_zGWmX6eDigY2kViMKCtNssp4JMEeVycZZfJH9eAZOY,1927
 nexaai/runtime.py,sha256=LxAUejH9uaci8IGz9_h0l-MMeYcwTlBjVKN_0u4Q4Qo,2021
 nexaai/tts.py,sha256=afs6sx0w0Tvs_aJlyZRPm62qQpTrs-fW_jDHrMkc4AA,2241
-nexaai/vlm.py,sha256=Yaor0xjQXOeonQw8sBGMxpzGH_9gTD8BUwvPtKX7UlA,4622
+nexaai/vlm.py,sha256=STjXCw67ABrHrEll8A2NGiwmfo7MotfYgBh1k1aNxkk,4775
 nexaai/asr_impl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nexaai/asr_impl/mlx_asr_impl.py,sha256=XwMX3LYMeulp8cDS0TCCYcjvttFHAyDWQ_oMvABwQmI,3349
 nexaai/asr_impl/pybind_asr_impl.py,sha256=20o5SOPzhF9x41ra8L_qIM7YxCkYeLb5csSrNde-dds,1560
 nexaai/binds/__init__.py,sha256=tYvy0pFhoY29GstDT5r-oRiPRarPLECvJAkcamJItOg,83
-nexaai/binds/common_bind.cp310-win_amd64.pyd,sha256=v3SBHao_kOT8t0wYpS3lSU2sVIsxhGHGRl3pUJ31uNk,201216
-nexaai/binds/embedder_bind.cp310-win_amd64.pyd,sha256=ib43HMgpcxcK1Xhv0JkrAqWnX63DTugex0PUYFeDRIg,182784
+nexaai/binds/common_bind.cp310-win_amd64.pyd,sha256=7tDfRwUhyQd1NBROT2o_AprFhI55tLIL4j3ZcZN490M,201216
+nexaai/binds/embedder_bind.cp310-win_amd64.pyd,sha256=RODHqhlNdLp-dHuI9jGak3GyxIyEtcDvVvQ4fp56ovg,182784
 nexaai/binds/libcrypto-3-x64.dll,sha256=-Lau6pL5DpDXzpg9MED63gCeL8oRrSLI_e2LeaxIHqk,7314432
 nexaai/binds/libssl-3-x64.dll,sha256=Tzzyu5jRpUugFxr_65hbFlAtFpjxIDpOYMU1E0ijkJw,1313792
-nexaai/binds/llm_bind.cp310-win_amd64.pyd,sha256=pSFXKkpuR0d70rsmsn6e5c2ZEDcs8fdlZLeM2used_U,160256
-nexaai/binds/nexa_bridge.dll,sha256=kx3-fR10aORocb8P_8i9mhJURvRfzgzPKto44OYj-R4,178688
-nexaai/binds/nexa_llama_cpp/ggml-base.dll,sha256=8_iLmK7Z03f4wv1GaujaxHpyc1Hskq4TmcrMXz2mMEY,514560
-nexaai/binds/nexa_llama_cpp/ggml-cpu.dll,sha256=wnHnU906jYBTs0Vm7mmP2ySzlQHvdRAP-uC-TAm3ay8,663552
-nexaai/binds/nexa_llama_cpp/ggml-cuda.dll,sha256=qtGxMiTtyV-gFm8r_W2I7-4WDKpNXXNx2kQjZcLEYlA,315100160
-nexaai/binds/nexa_llama_cpp/ggml-vulkan.dll,sha256=sM7EgHPprNFy0gzu6FBUY_M1vBMybqxXwjkmtCUVYYU,26204160
-nexaai/binds/nexa_llama_cpp/ggml.dll,sha256=TrxL7VobooAEB5SEnwxqlX0zL0oWP201-CteFmmp6Z4,66560
-nexaai/binds/nexa_llama_cpp/llama.dll,sha256=29-yLnMIuHgFmX29oOo5zgaOl3GXcoTJSMpaYrbavOQ,1587712
-nexaai/binds/nexa_llama_cpp/mtmd.dll,sha256=MCrTsXkpIwQajIsXEtWcO3Lycuoui1IymNSq1QeLzrA,560128
-nexaai/binds/nexa_llama_cpp/nexa_plugin.dll,sha256=tdMdLNJyo5ioxowfoQmLlnaAPfZo-fzYhZ7yvGq5uCk,1380352
+nexaai/binds/llm_bind.cp310-win_amd64.pyd,sha256=XYnaOZKM7tR6-J3cvWVe2I2RD3n-dumj9Wm82t-ghwE,162816
+nexaai/binds/nexa_bridge.dll,sha256=i6RngDhL1Jct9B3ynxQuQP4UuiDr_aFcNiD3CyG_KhM,178688
+nexaai/binds/nexa_llama_cpp/ggml-base.dll,sha256=jpgjj2HeA_oz56pF0H16JjT3FEp8dQ6t-elHV9AVOw0,532480
+nexaai/binds/nexa_llama_cpp/ggml-cpu.dll,sha256=DOaGib8JcUx6k1z1F7l2BhQfa73RREiLGtRyBUqt5sk,672768
+nexaai/binds/nexa_llama_cpp/ggml-cuda.dll,sha256=tMcjjLJlqvqxYG0torddnNUV18X13C_tA_CMg_Gkxlw,313528832
+nexaai/binds/nexa_llama_cpp/ggml-vulkan.dll,sha256=HmlESg2KXG_nz_xx-myRKby0tvShiIPQ4Rz_UYoWyhA,36627456
+nexaai/binds/nexa_llama_cpp/ggml.dll,sha256=CrrvtnALzptI10FaGD6Vg30JW64TAMWqQ8FGbYuGG38,66560
+nexaai/binds/nexa_llama_cpp/llama.dll,sha256=LAV2ZOiZ7WjJ9KtKUruX43EjOJL1wYWWGnJqY1nqWVs,1611776
+nexaai/binds/nexa_llama_cpp/mtmd.dll,sha256=aD00aXCHd5ZQbVNwsM9-QpgJAFAWIdEn3HlYDLNNRdY,561152
+nexaai/binds/nexa_llama_cpp/nexa_plugin.dll,sha256=QUf0eVX2WrRlC-IuvBfErb8Jt0LifHo4j3vIDA6CYi0,1405440
 nexaai/cv_impl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nexaai/cv_impl/mlx_cv_impl.py,sha256=QLd_8w90gtxH8kmssaDYatCTRvQNIJuUGKZNnYrmx6E,3317
 nexaai/cv_impl/pybind_cv_impl.py,sha256=aSOCAxmHrwJbEkSN6VX3Cykqlj_9RIpVrZXILul04GA,1096
@@ -40,8 +40,8 @@ nexaai/image_gen_impl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3h
 nexaai/image_gen_impl/mlx_image_gen_impl.py,sha256=peUE9ue9ApaPlZVOICBWiHtd13sY40OWQbE8EjfIUMU,11511
 nexaai/image_gen_impl/pybind_image_gen_impl.py,sha256=514RFQMSO0Rhacq0IYzlEhEr6QfaprnGew0Rjz8HZI4,3777
 nexaai/llm_impl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-nexaai/llm_impl/mlx_llm_impl.py,sha256=cHy0-xa1hSh74W0ux-V7X4RHGOtTNEyE6aQ89YKntqk,11127
-nexaai/llm_impl/pybind_llm_impl.py,sha256=VvfIqLPy9tDfq7-HkSw-wACSKo6KV6Aw3K9_cQ6exqM,7862
+nexaai/llm_impl/mlx_llm_impl.py,sha256=r9Qa1ZuduRAcRjmqo2J_zlqhprVPe3ioZtUbi7yIvYY,11426
+nexaai/llm_impl/pybind_llm_impl.py,sha256=s2Cb035xDbh1ZhGNys0LRtAR6b6n4-YVKSC5voD4_Tk,8269
 nexaai/rerank_impl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nexaai/rerank_impl/mlx_rerank_impl.py,sha256=x8L6zCccV2I4bq4V5zAOcrWFe5Ckle7z5J_00tao8bI,3335
 nexaai/rerank_impl/pybind_rerank_impl.py,sha256=1IW925bYv4FRwZDNzf9epEzdDqR6T3OONgTLBd-cOn8,1556
@@ -53,9 +53,9 @@ nexaai/utils/decode.py,sha256=0Z9jDH4ICzw4YXj8nD4L-sMouDaev-TISGRQ4KzidWE,421
 nexaai/utils/model_manager.py,sha256=Ksl-tKq-a3miTUxEn6-SSOC_KVdn6RPjcUdkWmDDwCk,49767
 nexaai/utils/progress_tracker.py,sha256=FmJBoOlzfQdc-TmccEav0cBR_iSNrrcskG3Fm1OrEJA,15482
 nexaai/vlm_impl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-nexaai/vlm_impl/mlx_vlm_impl.py,sha256=c-XGr_N270ykU3Kq0UZoUsNGz6uqGVYUW0j2lZrnXYk,10692
-nexaai/vlm_impl/pybind_vlm_impl.py,sha256=7BTgWAj7KAlZ3Js5CHc2-JdsMqIPQAAaEye_PiDyP84,8740
-nexaai-1.0.4rc16.dist-info/METADATA,sha256=cBWe2RGndeB27msPc6BGyVh-59TH0EcVHojzXF9LJ88,910
-nexaai-1.0.4rc16.dist-info/WHEEL,sha256=KUuBC6lxAbHCKilKua8R9W_TM71_-9Sg5uEP3uDWcoU,101
-nexaai-1.0.4rc16.dist-info/top_level.txt,sha256=LRE2YERlrZk2vfuygnSzsEeqSknnZbz3Z1MHyNmBU4w,7
-nexaai-1.0.4rc16.dist-info/RECORD,,
+nexaai/vlm_impl/mlx_vlm_impl.py,sha256=oY_qb9z_iF0zArBuY5CCYIvZcA3R0i_NKXrr_r-QSgg,10989
+nexaai/vlm_impl/pybind_vlm_impl.py,sha256=Hu8g8OXyPn8OzLQOpRSE5lfGmhjChiKj7fMRB8mC_cI,9147
+nexaai-1.0.6rc1.dist-info/METADATA,sha256=Wmc-f6LOyTwcWtuZXHXyvyXyK9Lk4FzhCMznFleFedg,1185
+nexaai-1.0.6rc1.dist-info/WHEEL,sha256=KUuBC6lxAbHCKilKua8R9W_TM71_-9Sg5uEP3uDWcoU,101
+nexaai-1.0.6rc1.dist-info/top_level.txt,sha256=LRE2YERlrZk2vfuygnSzsEeqSknnZbz3Z1MHyNmBU4w,7
+nexaai-1.0.6rc1.dist-info/RECORD,,