fount-vlm-nell-02 0.3.11__py3-none-any.whl

mlx_vlm/models/cache.py

@@ -0,0 +1,238 @@
+from typing import Any, List, Optional, Tuple
+
+import mlx.core as mx
+import mlx.nn as nn
+from mlx_lm.models.cache import (
+    ArraysCache,
+    BatchKVCache,
+    BatchRotatingKVCache,
+    ChunkedKVCache,
+    KVCache,
+    RotatingKVCache,
+    _BaseCache,
+)
+
+
+def make_prompt_cache(
+    model: nn.Module,
+    max_kv_size: Optional[int] = None,
+) -> List[Any]:
+    """
+    Construct the model's cache for use in generation.
+
+    This function will defer the cache construction to the model if it has a
+    ``make_cache`` method, otherwise it will make a default KV cache.
+
+    Args:
+        model (nn.Module): The language model.
+        max_kv_size (Optional[int]): If provided and the model does not have a
+            ``make_cache`` method, a ``RotatingKVCache`` is used with a maximum
+            size of ``max_kv_size``
+    """
+    if hasattr(model, "make_cache"):
+        return model.make_cache()
+
+    num_layers = len(model.layers)
+
+    if max_kv_size is not None:
+        return [
+            RotatingKVCache(max_size=max_kv_size, keep=4) for _ in range(num_layers)
+        ]
+    else:
+        return [KVCache() for _ in range(num_layers)]
+
+
+class SimpleKVCache:
+    """A simple key-value cache for transformer attention layers.
+
+    Stores and concatenates key/value tensors along sequence dimension.
+    """
+
+    def __init__(self):
+        self.keys = None
+        self.values = None
+        self.cache_length = 0
+
+    def update_and_fetch(self, keys, values):
+        """Update cache with new key/value tensors and return full cache.
+
+        Args:
+            keys: New key tensor to add [batch, heads, seq_len, head_dim]
+            values: New value tensor to add [batch, heads, seq_len, head_dim]
+
+        Returns:
+            Tuple of (cached_keys, cached_values) containing full cache history
+        """
+        if self.cache_length == 0:
+            # First update - just store tensors
+            self.keys = keys
+            self.values = values
+        else:
+            # Concatenate with existing cache along sequence dimension
+            self.keys = mx.concatenate([self.keys, keys], axis=2)
+            self.values = mx.concatenate([self.values, values], axis=2)
+
+        self.cache_length += keys.shape[2]
+        return self.keys, self.values
+
+    def fetch(self):
+        return self.keys, self.values
+
+    def update(self, keys, values):
+        """Update cache with new key/value tensors without returning.
+
+        Args:
+            keys: New key tensor to store
+            values: New value tensor to store
+        """
+        self.keys = keys
+        self.values = values
+        self.cache_length += keys.shape[2]
+
+
+class SlidingWindowCache(_BaseCache):
+    """A sliding window cache for local attention layers."""
+
+    def __init__(self, max_size: int, step: int = 256):
+        self.max_size = max_size
+        self.step = step
+        self.keys = None
+        self.values = None
+        self.offset = 0
+
+    def update_and_fetch(
+        self, keys: mx.array, values: mx.array
+    ) -> Tuple[mx.array, mx.array]:
+        B, n_kv_heads, seq_len, k_head_dim = keys.shape
+        v_head_dim = values.shape[-1]
+
+        if self.keys is None:
+            # Initialize cache
+            k_shape = (B, n_kv_heads, self.max_size, k_head_dim)
+            v_shape = (B, n_kv_heads, self.max_size, v_head_dim)
+            self.keys = mx.zeros(k_shape, dtype=keys.dtype)
+            self.values = mx.zeros(v_shape, dtype=values.dtype)
+
+        # Simple sliding window: keep only the last max_size tokens
+        if self.offset + seq_len <= self.max_size:
+            # Fits within current window
+            start_idx = self.offset
+            end_idx = self.offset + seq_len
+            self.keys[:, :, start_idx:end_idx, :] = keys
+            self.values[:, :, start_idx:end_idx, :] = values
+            self.offset += seq_len
+        else:
+            # Need to slide the window
+            if seq_len < self.max_size:
+                # Shift existing content left
+                shift_amount = min(seq_len, self.max_size - 1)
+                self.keys[:, :, :-shift_amount, :] = self.keys[:, :, shift_amount:, :]
+                self.values[:, :, :-shift_amount, :] = self.values[
+                    :, :, shift_amount:, :
+                ]
+                # Add new tokens at the end
+                self.keys[:, :, -shift_amount:, :] = keys[:, :, -shift_amount:, :]
+                self.values[:, :, -shift_amount:, :] = values[:, :, -shift_amount:, :]
+            else:
+                # New sequence is larger than cache, just keep the last max_size tokens
+                self.keys = keys[:, :, -self.max_size :, :]
+                self.values = values[:, :, -self.max_size :, :]
+            self.offset = self.max_size
+
+        return self.keys, self.values
+
+    @property
+    def state(self):
+        if self.keys is None:
+            return None, None
+        return self.keys, self.values
+
+    @state.setter
+    def state(self, v):
+        if v is not None and len(v) == 2:
+            self.keys, self.values = v
+            if self.keys is not None:
+                self.offset = self.max_size
+
+    def get_max_cache_shape(self):
+        return self.max_size
+
+    @property
+    def meta_state(self):
+        return tuple(map(str, (self.max_size, self.step, self.offset)))
+
+    @meta_state.setter
+    def meta_state(self, v):
+        self.max_size, self.step, self.offset = map(int, v)
+
+    def is_trimmable(self):
+        return False
+
+    def trim(self, n):
+        return 0
+
+
+class StaticKVCache(_BaseCache):
+    """A static cache that grows to accommodate all tokens."""
+
+    def __init__(self, max_size: int, step: int = 256):
+        self.max_size = max_size
+        self.step = step
+        self.keys = None
+        self.values = None
+        self.offset = 0
+
+    def update_and_fetch(
+        self, keys: mx.array, values: mx.array
+    ) -> Tuple[mx.array, mx.array]:
+        B, n_kv_heads, seq_len, k_head_dim = keys.shape
+        v_head_dim = values.shape[-1]
+
+        # Initialize cache if needed
+        if self.keys is None:
+            k_shape = (B, n_kv_heads, self.max_size, k_head_dim)
+            v_shape = (B, n_kv_heads, self.max_size, v_head_dim)
+            self.keys = mx.zeros(k_shape, dtype=keys.dtype)
+            self.values = mx.zeros(v_shape, dtype=values.dtype)
+
+        # Update cache
+        end_pos = min(self.offset + seq_len, self.max_size)
+        actual_seq_len = end_pos - self.offset
+
+        if actual_seq_len > 0:
+            self.keys[:, :, self.offset : end_pos, :] = keys[:, :, :actual_seq_len, :]
+            self.values[:, :, self.offset : end_pos, :] = values[
+                :, :, :actual_seq_len, :
+            ]
+            self.offset = end_pos
+
+        return self.keys, self.values
+
+    @property
+    def state(self):
+        if self.keys is None:
+            return None, None
+        return self.keys, self.values
+
+    @state.setter
+    def state(self, v):
+        if v is not None and len(v) == 2:
+            self.keys, self.values = v
+            if self.keys is not None:
+                self.offset = self.max_size
+
+    @property
+    def meta_state(self):
+        return tuple(map(str, (self.max_size, self.step, self.offset)))
+
+    @meta_state.setter
+    def meta_state(self, v):
+        self.max_size, self.step, self.offset = map(int, v)
+
+    def is_trimmable(self):
+        return True
+
+    def trim(self, n):
+        n = min(self.offset, n)
+        self.offset -= n
+        return n

mlx_vlm/models/deepseek_vl_v2/__init__.py

@@ -0,0 +1,2 @@
+from .config import MLPConfig, ModelConfig, ProjectorConfig, TextConfig, VisionConfig
+from .deepseek_vl_v2 import DeepseekVLV2Processor, LanguageModel, Model, VisionModel

mlx_vlm/models/deepseek_vl_v2/config.py

@@ -0,0 +1,159 @@
+import inspect
+from dataclasses import dataclass
+from typing import Dict, List, Optional
+
+from ..base import BaseModelConfig
+
+
+@dataclass
+class TextConfig(BaseModelConfig):
+    model_type: str = "deepseek_v2"
+    vocab_size: int = 102400
+    hidden_size: int = 1280
+    intermediate_size: int = 6848
+    moe_intermediate_size: int = 896
+    num_hidden_layers: int = 30
+    num_attention_heads: int = 32
+    num_key_value_heads: int = 32
+    n_shared_experts: Optional[int] = 2
+    n_routed_experts: Optional[int] = 64
+    routed_scaling_factor: float = 1.0
+    kv_lora_rank: int = 512
+    q_lora_rank: int = 1536
+    qk_rope_head_dim: int = 64
+    v_head_dim: int = 128
+    qk_nope_head_dim: int = 128
+    topk_method: str = "greedy"
+    n_group: Optional[int] = 1
+    topk_group: Optional[int] = 1
+    num_experts_per_tok: Optional[int] = 6
+    moe_layer_freq: int = 1
+    first_k_dense_replace: int = 0
+    max_position_embeddings: int = 2048
+    rms_norm_eps: float = 1e-6
+    rope_theta: float = 10000.0
+    rope_traditional: bool = True
+    rope_scaling: Dict = None
+    attention_bias: bool = False
+    scoring_func: str = "softmax"
+    attn_type: str = "DeepseekV2Attention"
+
+    def __post_init__(self):
+        if self.qk_nope_head_dim == 0:
+            self.attn_type = "LlamaAttention"
+
+        if self.num_key_value_heads is None:
+            self.num_key_value_heads = self.num_attention_heads
+
+
+@dataclass
+class VisionConfig(BaseModelConfig):
+    model_type: str
+    layers: int = 27
+    width: int = 1152
+    intermediate_size: int = 4304
+    num_attention_heads: int = 16
+    image_size: int = 384
+    patch_size: int = 16
+    num_channels: int = 3
+    layer_norm_eps: float = 1e-6
+    mlp_ratio: float = 3.7362
+    cls: str = None
+    params: dict = None
+
+
+@dataclass
+class MLPConfig(BaseModelConfig):
+    width: int
+    intermediate_size: int
+    hidden_act: str = "gelu"
+
+
+@dataclass
+class ProjectorConfig(BaseModelConfig):
+    projector_type: str = "downsample_mlp_gelu"
+    input_dim: int = 1152
+    n_embed: int = 2048
+    depth: int = 2
+    mlp_ratio: int = 1
+    downsample_ratio: int = 2
+    token_pooling: bool = False
+
+
+@dataclass
+class ModelConfig(BaseModelConfig):
+    text_config: TextConfig
+    vision_config: VisionConfig
+    projector_config: ProjectorConfig
+    model_type: str
+    ignore_index: int = -100
+    image_token_index: int = 100015
+    vision_feature_select_strategy: str = "default"
+    select_layer: int = -1
+    pad_id: int = 100001
+    num_image_tokens: int = 576
+    vocab_size: int = 32000
+    tile_tag: str = "2D"
+    global_view_pos: str = "head"
+    eos_token_id: Optional[List[int]] = None
+    quantization: Optional[Dict] = None
+
+    @classmethod
+    def from_dict(cls, params):
+        if "language_config" in params:
+            params["text_config"] = params["language_config"]
+            del params["language_config"]
+
+        return cls(
+            text_config=TextConfig.from_dict(params["text_config"]),
+            vision_config=VisionConfig.from_dict(params["vision_config"]),
+            projector_config=ProjectorConfig.from_dict(params["projector_config"]),
+            **{
+                k: v
+                for k, v in params.items()
+                if k in inspect.signature(cls).parameters
+                and k not in ["text_config", "vision_config", "projector_config"]
+            },
+        )
+
+
+@dataclass
+class Conversation:
+    """A class that represents a conversation."""
+
+    system: str
+    roles: List[str]
+    messages: List[List[str]]
+    offset: int
+    sep_style: int
+    sep: str
+    sep2: str
+    version: str = "Unknown"
+
+
+@dataclass
+class VLChatProcessorOutput:
+    """
+    Output of the VL chat processor.
+    """
+
+    sft_format: str
+    input_ids: List[int]
+    pixel_values: List
+    num_image_tokens: List[int]
+    image_grid_thw: List[List[int]]
+    image_sizes: Optional[List[List[int]]] = None
+    videos: Optional[List] = None
+    aspect_ratio_ids: Optional[List[int]] = None
+    aspect_ratio_mask: Optional[List[List[int]]] = None
+    cross_attention_mask: Optional[List[List[List[int]]]] = None
+    attention_mask: Optional[List[int]] = None
+    labels: Optional[List[int]] = None
+
+
+@dataclass
+class BatchCollateOutput:
+    input_ids: List
+    labels: List
+    attention_mask: List
+    pixel_values: List

mlx_vlm/models/deepseek_vl_v2/conversation.py

@@ -0,0 +1,264 @@
+"""
+From https://github.com/lm-sys/FastChat/blob/main/fastchat/conversation.py
+"""
+
+import dataclasses
+from enum import IntEnum, auto
+from typing import Dict, List
+
+
+class SeparatorStyle(IntEnum):
+    """Separator styles."""
+
+    DeepSeek = auto()
+    DeepSeekV2 = auto()
+    PLAIN = auto()
+    ALIGNMENT = auto()
+
+
+@dataclasses.dataclass
+class Conversation:
+    """A class that manages prompt templates and keeps all conversation history."""
+
+    # The name of this template
+    name: str
+    # The template of the system prompt
+    system_template: str = "{system_message}"
+    # The system message
+    system_message: str = ""
+    # The names of two roles
+    roles: List[str] = (("USER", "ASSISTANT"),)
+    # All messages. Each item is (role, message).
+    messages: List[List[str]] = ()
+    # The number of few shot examples
+    offset: int = 0
+    # The separator style and configurations
+    sep_style: SeparatorStyle = SeparatorStyle.DeepSeek
+    sep: str = "\n"
+    sep2: str = None
+    # Stop criteria (the default one is EOS token)
+    stop_str: str = None
+    # Stops generation if meeting any token in this list
+    stop_token_ids: List[int] = None
+
+    def get_prompt(self) -> str:
+        """Get the prompt for generation."""
+        system_prompt = self.system_template.format(system_message=self.system_message)
+        if self.sep_style == SeparatorStyle.DeepSeek:
+            seps = [self.sep, self.sep2]
+            if system_prompt == "" or system_prompt is None:
+                ret = ""
+            else:
+                ret = system_prompt + seps[0]
+            for i, (role, message) in enumerate(self.messages):
+                if message:
+                    ret += role + ": " + message + seps[i % 2]
+                else:
+                    ret += role + ":"
+            return ret
+        elif self.sep_style == SeparatorStyle.DeepSeekV2:
+            seps = [self.sep, self.sep2]
+            if system_prompt == "" or system_prompt is None:
+                ret = ""
+            else:
+                ret = system_prompt + seps[0]
+            for i, (role, message) in enumerate(self.messages):
+                if message:
+                    if role == "User":
+                        ret += "<｜sft▁begin｜>\n" + message + self.sep
+                    else:
+                        ret += message + self.sep2
+                else:
+                    ret = ret
+            return ret
+
+        elif self.sep_style == SeparatorStyle.PLAIN:
+            seps = [self.sep, self.sep2]
+            ret = ""
+            for i, (role, message) in enumerate(self.messages):
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    if i % 2 == 0:
+                        ret += message + seps[i % 2]
+                    else:
+                        ret += message + seps[i % 2]
+                else:
+                    ret += ""
+            return ret
+        elif self.sep_style == SeparatorStyle.ALIGNMENT:
+            seps = [self.sep, self.sep2]
+            ret = ""
+            for i, (role, message) in enumerate(self.messages):
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    if i % 2 == 0:
+                        ret += "<image>\n" + seps[i % 2]
+                    else:
+                        ret += message + seps[i % 2]
+                else:
+                    ret += ""
+            return ret
+        else:
+            raise ValueError(f"Invalid style: {self.sep_style}")
+
+    def set_system_message(self, system_message: str):
+        """Set the system message."""
+        self.system_message = system_message
+
+    def append_message(self, role: str, message: str):
+        """Append a new message."""
+        self.messages.append([role, message])
+
+    def update_last_message(self, message: str):
+        """Update the last output.
+
+        The last message is typically set to be None when constructing the prompt,
+        so we need to update it in-place after getting the response from a model.
+        """
+        self.messages[-1][1] = message
+
+    def reset_message(self):
+        """Reset a new message."""
+        self.messages = []
+
+    def to_gradio_chatbot(self):
+        """Convert the conversation to gradio chatbot format."""
+        ret = []
+        for i, (role, msg) in enumerate(self.messages[self.offset :]):
+            if i % 2 == 0:
+                ret.append([msg, None])
+            else:
+                ret[-1][-1] = msg
+        return ret
+
+    def to_openai_api_messages(self):
+        """Convert the conversation to OpenAI chat completion format."""
+        system_prompt = self.system_template.format(system_message=self.system_message)
+        ret = [{"role": "system", "content": system_prompt}]
+
+        for i, (_, msg) in enumerate(self.messages[self.offset :]):
+            if i % 2 == 0:
+                ret.append({"role": "user", "content": msg})
+            else:
+                if msg is not None:
+                    ret.append({"role": "assistant", "content": msg})
+        return ret
+
+    def copy(self):
+        return Conversation(
+            name=self.name,
+            system_template=self.system_template,
+            system_message=self.system_message,
+            roles=self.roles,
+            messages=[[x, y] for x, y in self.messages],
+            offset=self.offset,
+            sep_style=self.sep_style,
+            sep=self.sep,
+            sep2=self.sep2,
+            stop_str=self.stop_str,
+            stop_token_ids=self.stop_token_ids,
+        )
+
+    def dict(self):
+        return {
+            "template_name": self.name,
+            "system_message": self.system_message,
+            "roles": self.roles,
+            "messages": self.messages,
+            "offset": self.offset,
+        }
+
+
+# A global registry for all conversation templates
+conv_templates: Dict[str, Conversation] = {}
+
+
+def register_conv_template(template: Conversation, override: bool = False):
+    """Register a new conversation template."""
+    if not override:
+        assert (
+            template.name not in conv_templates
+        ), f"{template.name} has been registered."
+
+    conv_templates[template.name] = template
+
+
+def get_conv_template(name: str) -> Conversation:
+    """Get a conversation template."""
+    return conv_templates[name].copy()
+
+
+register_conv_template(
+    Conversation(
+        name="deepseek",
+        system_template="{system_message}",
+        # system_message="You are a helpful assistant. Please answer truthfully and write out your "
+        # "thinking step by step to be sure you get the right answer.",
+        system_message="",
+        roles=("<|User|>", "<|Assistant|>"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.DeepSeek,
+        sep="\n\n",
+        sep2="<｜end▁of▁sentence｜>",
+        stop_token_ids=[100001],
+        stop_str=["User:", "<｜end▁of▁sentence｜>"],
+    )
+)
+
+register_conv_template(
+    Conversation(
+        name="deepseekv2",
+        system_template="{system_message}",
+        system_message="",
+        roles=("|<User>|", "|<Assistant>|"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.DeepSeekV2,
+        sep="\n<｜sft▁end｜>",
+        sep2="<｜end▁of▁sentence｜>",
+        stop_token_ids=[100001],
+        stop_str=["User:", "<｜end▁of▁sentence｜>"],
+    )
+)
+
+
+register_conv_template(
+    Conversation(
+        name="plain",
+        system_template="",
+        system_message="",
+        roles=("", ""),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.PLAIN,
+        sep="",
+        sep2="",
+        stop_token_ids=[100001],
+        stop_str=["</s>"],
+    )
+)
+
+
+register_conv_template(
+    Conversation(
+        name="alignment",
+        system_template="",
+        system_message="",
+        roles=("", ""),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.ALIGNMENT,
+        sep="",
+        sep2="",
+        stop_token_ids=[100001],
+        stop_str=["</s>"],
+    )
+)
+
+
+if __name__ == "__main__":
+    print("deepseek template:")
+    conv = get_conv_template("deepseek")