nexaai 1.0.19rc7__cp310-cp310-macosx_14_0_universal2.whl → 1.0.19rc8__cp310-cp310-macosx_14_0_universal2.whl

nexaai/binds/nexa_mlx/py-lib/vlm/modeling/models/cache.py

@@ -1,230 +0,0 @@
-from typing import Any, List, Optional, Tuple
-
-import mlx.core as mx
-import mlx.nn as nn
-from mlx_lm.models.cache import ChunkedKVCache, KVCache, RotatingKVCache, _BaseCache
-
-
-def make_prompt_cache(
-    model: nn.Module,
-    max_kv_size: Optional[int] = None,
-) -> List[Any]:
-    """
-    Construct the model's cache for use in generation.
-
-    This function will defer the cache construction to the model if it has a
-    ``make_cache`` method, otherwise it will make a default KV cache.
-
-    Args:
-        model (nn.Module): The language model.
-        max_kv_size (Optional[int]): If provided and the model does not have a
-            ``make_cache`` method, a ``RotatingKVCache`` is used with a maximum
-            size of ``max_kv_size``
-    """
-    if hasattr(model, "make_cache"):
-        return model.make_cache()
-
-    num_layers = len(model.layers)
-
-    if max_kv_size is not None:
-        return [
-            RotatingKVCache(max_size=max_kv_size, keep=4) for _ in range(num_layers)
-        ]
-    else:
-        return [KVCache() for _ in range(num_layers)]
-
-
-class SimpleKVCache:
-    """A simple key-value cache for transformer attention layers.
-
-    Stores and concatenates key/value tensors along sequence dimension.
-    """
-
-    def __init__(self):
-        self.keys = None
-        self.values = None
-        self.cache_length = 0
-
-    def update_and_fetch(self, keys, values):
-        """Update cache with new key/value tensors and return full cache.
-
-        Args:
-            keys: New key tensor to add [batch, heads, seq_len, head_dim]
-            values: New value tensor to add [batch, heads, seq_len, head_dim]
-
-        Returns:
-            Tuple of (cached_keys, cached_values) containing full cache history
-        """
-        if self.cache_length == 0:
-            # First update - just store tensors
-            self.keys = keys
-            self.values = values
-        else:
-            # Concatenate with existing cache along sequence dimension
-            self.keys = mx.concatenate([self.keys, keys], axis=2)
-            self.values = mx.concatenate([self.values, values], axis=2)
-
-        self.cache_length += keys.shape[2]
-        return self.keys, self.values
-
-    def fetch(self):
-        return self.keys, self.values
-
-    def update(self, keys, values):
-        """Update cache with new key/value tensors without returning.
-
-        Args:
-            keys: New key tensor to store
-            values: New value tensor to store
-        """
-        self.keys = keys
-        self.values = values
-        self.cache_length += keys.shape[2]
-
-
-class SlidingWindowCache(_BaseCache):
-    """A sliding window cache for local attention layers."""
-
-    def __init__(self, max_size: int, step: int = 256):
-        self.max_size = max_size
-        self.step = step
-        self.keys = None
-        self.values = None
-        self.offset = 0
-
-    def update_and_fetch(
-        self, keys: mx.array, values: mx.array
-    ) -> Tuple[mx.array, mx.array]:
-        B, n_kv_heads, seq_len, k_head_dim = keys.shape
-        v_head_dim = values.shape[-1]
-
-        if self.keys is None:
-            # Initialize cache
-            k_shape = (B, n_kv_heads, self.max_size, k_head_dim)
-            v_shape = (B, n_kv_heads, self.max_size, v_head_dim)
-            self.keys = mx.zeros(k_shape, dtype=keys.dtype)
-            self.values = mx.zeros(v_shape, dtype=values.dtype)
-
-        # Simple sliding window: keep only the last max_size tokens
-        if self.offset + seq_len <= self.max_size:
-            # Fits within current window
-            start_idx = self.offset
-            end_idx = self.offset + seq_len
-            self.keys[:, :, start_idx:end_idx, :] = keys
-            self.values[:, :, start_idx:end_idx, :] = values
-            self.offset += seq_len
-        else:
-            # Need to slide the window
-            if seq_len < self.max_size:
-                # Shift existing content left
-                shift_amount = min(seq_len, self.max_size - 1)
-                self.keys[:, :, :-shift_amount, :] = self.keys[:, :, shift_amount:, :]
-                self.values[:, :, :-shift_amount, :] = self.values[
-                    :, :, shift_amount:, :
-                ]
-                # Add new tokens at the end
-                self.keys[:, :, -shift_amount:, :] = keys[:, :, -shift_amount:, :]
-                self.values[:, :, -shift_amount:, :] = values[:, :, -shift_amount:, :]
-            else:
-                # New sequence is larger than cache, just keep the last max_size tokens
-                self.keys = keys[:, :, -self.max_size :, :]
-                self.values = values[:, :, -self.max_size :, :]
-            self.offset = self.max_size
-
-        return self.keys, self.values
-
-    @property
-    def state(self):
-        if self.keys is None:
-            return None, None
-        return self.keys, self.values
-
-    @state.setter
-    def state(self, v):
-        if v is not None and len(v) == 2:
-            self.keys, self.values = v
-            if self.keys is not None:
-                self.offset = self.max_size
-
-    def get_max_cache_shape(self):
-        return self.max_size
-
-    @property
-    def meta_state(self):
-        return tuple(map(str, (self.max_size, self.step, self.offset)))
-
-    @meta_state.setter
-    def meta_state(self, v):
-        self.max_size, self.step, self.offset = map(int, v)
-
-    def is_trimmable(self):
-        return False
-
-    def trim(self, n):
-        return 0
-
-
-class StaticKVCache(_BaseCache):
-    """A static cache that grows to accommodate all tokens."""
-
-    def __init__(self, max_size: int, step: int = 256):
-        self.max_size = max_size
-        self.step = step
-        self.keys = None
-        self.values = None
-        self.offset = 0
-
-    def update_and_fetch(
-        self, keys: mx.array, values: mx.array
-    ) -> Tuple[mx.array, mx.array]:
-        B, n_kv_heads, seq_len, k_head_dim = keys.shape
-        v_head_dim = values.shape[-1]
-
-        # Initialize cache if needed
-        if self.keys is None:
-            k_shape = (B, n_kv_heads, self.max_size, k_head_dim)
-            v_shape = (B, n_kv_heads, self.max_size, v_head_dim)
-            self.keys = mx.zeros(k_shape, dtype=keys.dtype)
-            self.values = mx.zeros(v_shape, dtype=values.dtype)
-
-        # Update cache
-        end_pos = min(self.offset + seq_len, self.max_size)
-        actual_seq_len = end_pos - self.offset
-
-        if actual_seq_len > 0:
-            self.keys[:, :, self.offset : end_pos, :] = keys[:, :, :actual_seq_len, :]
-            self.values[:, :, self.offset : end_pos, :] = values[
-                :, :, :actual_seq_len, :
-            ]
-            self.offset = end_pos
-
-        return self.keys, self.values
-
-    @property
-    def state(self):
-        if self.keys is None:
-            return None, None
-        return self.keys, self.values
-
-    @state.setter
-    def state(self, v):
-        if v is not None and len(v) == 2:
-            self.keys, self.values = v
-            if self.keys is not None:
-                self.offset = self.max_size
-
-    @property
-    def meta_state(self):
-        return tuple(map(str, (self.max_size, self.step, self.offset)))
-
-    @meta_state.setter
-    def meta_state(self, v):
-        self.max_size, self.step, self.offset = map(int, v)
-
-    def is_trimmable(self):
-        return True
-
-    def trim(self, n):
-        n = min(self.offset, n)
-        self.offset -= n
-        return n

nexaai/binds/nexa_mlx/py-lib/vlm/modeling/models/deepseek_vl_v2/__init__.py

@@ -1,10 +0,0 @@
-from .deepseek_vl_v2 import (
-    DeepseekVLV2Processor,
-    LanguageModel,
-    Model,
-    ModelConfig,
-    ProjectorConfig,
-    TextConfig,
-    VisionConfig,
-    VisionModel,
-)

nexaai/binds/nexa_mlx/py-lib/vlm/modeling/models/deepseek_vl_v2/conversation.py

@@ -1,264 +0,0 @@
-"""
-From https://github.com/lm-sys/FastChat/blob/main/fastchat/conversation.py
-"""
-
-import dataclasses
-from enum import IntEnum, auto
-from typing import Dict, List
-
-
-class SeparatorStyle(IntEnum):
-    """Separator styles."""
-
-    DeepSeek = auto()
-    DeepSeekV2 = auto()
-    PLAIN = auto()
-    ALIGNMENT = auto()
-
-
-@dataclasses.dataclass
-class Conversation:
-    """A class that manages prompt templates and keeps all conversation history."""
-
-    # The name of this template
-    name: str
-    # The template of the system prompt
-    system_template: str = "{system_message}"
-    # The system message
-    system_message: str = ""
-    # The names of two roles
-    roles: List[str] = (("USER", "ASSISTANT"),)
-    # All messages. Each item is (role, message).
-    messages: List[List[str]] = ()
-    # The number of few shot examples
-    offset: int = 0
-    # The separator style and configurations
-    sep_style: SeparatorStyle = SeparatorStyle.DeepSeek
-    sep: str = "\n"
-    sep2: str = None
-    # Stop criteria (the default one is EOS token)
-    stop_str: str = None
-    # Stops generation if meeting any token in this list
-    stop_token_ids: List[int] = None
-
-    def get_prompt(self) -> str:
-        """Get the prompt for generation."""
-        system_prompt = self.system_template.format(system_message=self.system_message)
-        if self.sep_style == SeparatorStyle.DeepSeek:
-            seps = [self.sep, self.sep2]
-            if system_prompt == "" or system_prompt is None:
-                ret = ""
-            else:
-                ret = system_prompt + seps[0]
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    ret += role + ": " + message + seps[i % 2]
-                else:
-                    ret += role + ":"
-            return ret
-        elif self.sep_style == SeparatorStyle.DeepSeekV2:
-            seps = [self.sep, self.sep2]
-            if system_prompt == "" or system_prompt is None:
-                ret = ""
-            else:
-                ret = system_prompt + seps[0]
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    if role == "User":
-                        ret += "<｜sft▁begin｜>\n" + message + self.sep
-                    else:
-                        ret += message + self.sep2
-                else:
-                    ret = ret
-            return ret
-
-        elif self.sep_style == SeparatorStyle.PLAIN:
-            seps = [self.sep, self.sep2]
-            ret = ""
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    if type(message) is tuple:
-                        message, _, _ = message
-                    if i % 2 == 0:
-                        ret += message + seps[i % 2]
-                    else:
-                        ret += message + seps[i % 2]
-                else:
-                    ret += ""
-            return ret
-        elif self.sep_style == SeparatorStyle.ALIGNMENT:
-            seps = [self.sep, self.sep2]
-            ret = ""
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    if type(message) is tuple:
-                        message, _, _ = message
-                    if i % 2 == 0:
-                        ret += "<image>\n" + seps[i % 2]
-                    else:
-                        ret += message + seps[i % 2]
-                else:
-                    ret += ""
-            return ret
-        else:
-            raise ValueError(f"Invalid style: {self.sep_style}")
-
-    def set_system_message(self, system_message: str):
-        """Set the system message."""
-        self.system_message = system_message
-
-    def append_message(self, role: str, message: str):
-        """Append a new message."""
-        self.messages.append([role, message])
-
-    def update_last_message(self, message: str):
-        """Update the last output.
-
-        The last message is typically set to be None when constructing the prompt,
-        so we need to update it in-place after getting the response from a model.
-        """
-        self.messages[-1][1] = message
-
-    def reset_message(self):
-        """Reset a new message."""
-        self.messages = []
-
-    def to_gradio_chatbot(self):
-        """Convert the conversation to gradio chatbot format."""
-        ret = []
-        for i, (role, msg) in enumerate(self.messages[self.offset :]):
-            if i % 2 == 0:
-                ret.append([msg, None])
-            else:
-                ret[-1][-1] = msg
-        return ret
-
-    def to_openai_api_messages(self):
-        """Convert the conversation to OpenAI chat completion format."""
-        system_prompt = self.system_template.format(system_message=self.system_message)
-        ret = [{"role": "system", "content": system_prompt}]
-
-        for i, (_, msg) in enumerate(self.messages[self.offset :]):
-            if i % 2 == 0:
-                ret.append({"role": "user", "content": msg})
-            else:
-                if msg is not None:
-                    ret.append({"role": "assistant", "content": msg})
-        return ret
-
-    def copy(self):
-        return Conversation(
-            name=self.name,
-            system_template=self.system_template,
-            system_message=self.system_message,
-            roles=self.roles,
-            messages=[[x, y] for x, y in self.messages],
-            offset=self.offset,
-            sep_style=self.sep_style,
-            sep=self.sep,
-            sep2=self.sep2,
-            stop_str=self.stop_str,
-            stop_token_ids=self.stop_token_ids,
-        )
-
-    def dict(self):
-        return {
-            "template_name": self.name,
-            "system_message": self.system_message,
-            "roles": self.roles,
-            "messages": self.messages,
-            "offset": self.offset,
-        }
-
-
-# A global registry for all conversation templates
-conv_templates: Dict[str, Conversation] = {}
-
-
-def register_conv_template(template: Conversation, override: bool = False):
-    """Register a new conversation template."""
-    if not override:
-        assert (
-            template.name not in conv_templates
-        ), f"{template.name} has been registered."
-
-    conv_templates[template.name] = template
-
-
-def get_conv_template(name: str) -> Conversation:
-    """Get a conversation template."""
-    return conv_templates[name].copy()
-
-
-register_conv_template(
-    Conversation(
-        name="deepseek",
-        system_template="{system_message}",
-        # system_message="You are a helpful assistant. Please answer truthfully and write out your "
-        # "thinking step by step to be sure you get the right answer.",
-        system_message="",
-        roles=("<|User|>", "<|Assistant|>"),
-        messages=(),
-        offset=0,
-        sep_style=SeparatorStyle.DeepSeek,
-        sep="\n\n",
-        sep2="<｜end▁of▁sentence｜>",
-        stop_token_ids=[100001],
-        stop_str=["User:", "<｜end▁of▁sentence｜>"],
-    )
-)
-
-register_conv_template(
-    Conversation(
-        name="deepseekv2",
-        system_template="{system_message}",
-        system_message="",
-        roles=("|<User>|", "|<Assistant>|"),
-        messages=(),
-        offset=0,
-        sep_style=SeparatorStyle.DeepSeekV2,
-        sep="\n<｜sft▁end｜>",
-        sep2="<｜end▁of▁sentence｜>",
-        stop_token_ids=[100001],
-        stop_str=["User:", "<｜end▁of▁sentence｜>"],
-    )
-)
-
-
-register_conv_template(
-    Conversation(
-        name="plain",
-        system_template="",
-        system_message="",
-        roles=("", ""),
-        messages=(),
-        offset=0,
-        sep_style=SeparatorStyle.PLAIN,
-        sep="",
-        sep2="",
-        stop_token_ids=[100001],
-        stop_str=["</s>"],
-    )
-)
-
-
-register_conv_template(
-    Conversation(
-        name="alignment",
-        system_template="",
-        system_message="",
-        roles=("", ""),
-        messages=(),
-        offset=0,
-        sep_style=SeparatorStyle.ALIGNMENT,
-        sep="",
-        sep2="",
-        stop_token_ids=[100001],
-        stop_str=["</s>"],
-    )
-)
-
-
-if __name__ == "__main__":
-    print("deepseek template:")
-    conv = get_conv_template("deepseek")