fount-vlm-nell-02 0.3.11__py3-none-any.whl

mlx_vlm/models/moondream2/moondream2.py

@@ -0,0 +1,522 @@
+import re
+from functools import partial, reduce
+from typing import List, Optional, Tuple
+
+import mlx.core as mx
+import mlx.nn as nn
+import numpy as np
+from PIL import Image
+from transformers.image_transforms import (
+    convert_to_rgb,
+    resize,
+    to_channel_dimension_format,
+)
+from transformers.image_utils import PILImageResampling, to_numpy_array
+
+from ..base import BaseImageProcessor, InputEmbeddingsFeatures
+from .config import ModelConfig, VisionConfig
+from .image_crops import adaptive_avg_pool2d, overlap_crop_image, reconstruct_from_crops
+from .language import LanguageModel
+from .vision import VisionModel
+
+
+class ImageProcessor(BaseImageProcessor):
+    """Moondream image processor with multi-crop support."""
+
+    def __init__(self, max_crops: int = 12, overlap_margin: int = 4):
+        super().__init__(
+            image_mean=(0.5, 0.5, 0.5),
+            image_std=(0.5, 0.5, 0.5),
+            size=(378, 378),
+            resample=PILImageResampling.BICUBIC,
+            rescale_factor=1 / 255,
+        )
+        self.max_crops = max_crops
+        self.overlap_margin = overlap_margin
+
+    def preprocess(
+        self, images
+    ) -> Tuple[List[np.ndarray], List[int], List[Tuple[int, int]]]:
+        """
+        Preprocess images with multi-crop support.
+
+        Args:
+            images: Single PIL Image or list of PIL Images
+
+        Returns:
+            crops_list: List of [n_crops, C, H, W] arrays per image
+            crop_counts: Number of crops per image
+            tilings: (h_tiles, w_tiles) per image
+        """
+        if isinstance(images, Image.Image):
+            images = [images]
+        else:
+            assert isinstance(images, list)
+
+        crops_list = []
+        crop_counts = []
+        tilings = []
+
+        for image in images:
+            # Convert to RGB numpy array
+            image = convert_to_rgb(image)
+            image_np = to_numpy_array(image)
+
+            # Get multi-crop decomposition
+            crops, tiling = overlap_crop_image(
+                image_np,
+                max_crops=self.max_crops,
+                overlap_margin=self.overlap_margin,
+                base_size=self.size,
+                patch_size=14,
+            )
+            # crops is [n_crops, H, W, C] in range [0, 255]
+
+            # Normalize each crop: (pixel/255 - 0.5) / 0.5 = [-1, 1]
+            crops = crops.astype(np.float32) * self.rescale_factor  # [0, 1]
+            crops = (crops - 0.5) / 0.5  # [-1, 1]
+
+            # Convert to channel-first format: [n_crops, H, W, C] -> [n_crops, C, H, W]
+            crops = np.transpose(crops, (0, 3, 1, 2))
+
+            crops_list.append(crops)
+            crop_counts.append(crops.shape[0])
+            tilings.append(tiling)
+
+        return crops_list, crop_counts, tilings
+
+
+class VisionProjection(nn.Module):
+    """
+    2-layer MLP projector from vision to language space.
+
+    Projects concatenated [global, reconstructed] features (2304D) to language
+    model dimension (2048D). The input is the concatenation of:
+    - Global features: [B, 729, 1152] from full image
+    - Reconstructed features: [B, 729, 1152] pooled from local crops
+
+    Reference: moondream2/vision.py:77-89
+    """
+
+    def __init__(self, config: ModelConfig):
+        super().__init__()
+        # Input is concatenation of global and reconstructed: 1152 * 2 = 2304
+        vision_dim = config.vision_config.hidden_size * 2  # 2304
+        inner_dim = config.proj_inner_dim  # 8192
+        output_dim = config.text_config.hidden_size  # 2048
+
+        self.fc1 = nn.Linear(vision_dim, inner_dim, bias=True)
+        self.fc2 = nn.Linear(inner_dim, output_dim, bias=True)
+        self.activation = nn.GELU(approx="precise")
+
+    def __call__(
+        self, global_features: mx.array, reconstructed_features: mx.array
+    ) -> mx.array:
+        """
+        Project concatenated vision features to language model dimension.
+
+        Args:
+            global_features: [B, 729, 1152] features from global crop
+            reconstructed_features: [B, 729, 1152] features reconstructed from local crops
+
+        Returns:
+            [B, 729, 2048] projected features
+        """
+        # Concatenate along feature dimension: [B, 729, 2304]
+        x = mx.concatenate([global_features, reconstructed_features], axis=-1)
+        x = self.activation(self.fc1(x))
+        x = self.fc2(x)
+        return x
+
+
+class Model(nn.Module):
+    """Moondream 2 model for visual question answering."""
+
+    def __init__(self, config: ModelConfig):
+        super().__init__()
+        self.model_type = config.model_type
+        self.config = config
+
+        self.vision_encoder = VisionModel(config.vision_config)
+        self.vision_projection = VisionProjection(config)
+        self.language_model = LanguageModel(config.text_config)
+
+    def get_input_embeddings(
+        self,
+        input_ids: Optional[mx.array] = None,
+        pixel_values: Optional[mx.array] = None,
+        crop_counts: Optional[List[int]] = None,
+        tilings: Optional[List[Tuple[int, int]]] = None,
+        **kwargs,
+    ):
+        """
+        Get input embeddings with multi-crop image features.
+
+        Full pipeline:
+        1. Encode ALL crops through vision_encoder: [total_crops, 729, 1152]
+        2. For each image:
+           a. global_features = features[0]  # [729, 1152]
+           b. local_features = features[1:].reshape(n_local, 27, 27, 1152)
+           c. reconstructed = reconstruct_from_crops(local_features, tiling)
+           d. reconstructed = adaptive_avg_pool2d(reconstructed, (27, 27))
+           e. reconstructed = reconstructed.reshape(729, 1152)
+           f. projected = vision_projection(global, reconstructed)  # [729, 2048]
+        3. Insert projected features into embeddings
+
+        Args:
+            input_ids: Token IDs [B, seq_len]
+            pixel_values: Concatenated crops [total_crops, C, H, W]
+            crop_counts: Number of crops per image (list of ints)
+            tilings: (h_tiles, w_tiles) per image (list of tuples)
+        """
+        # #region agent log
+        import json
+        log_file = "/Users/zekieldee/Desktop/code/mlx-vlm/.cursor/debug.log"
+        def log_embed(location, message, data, hypothesis_id):
+            try:
+                with open(log_file, "a") as f:
+                    f.write(json.dumps({"sessionId": "debug-session", "runId": "inference", "hypothesisId": hypothesis_id, "location": location, "message": message, "data": data, "timestamp": __import__("time").time_ns() // 1000000}) + "\n")
+            except: pass
+        
+        log_embed("moondream2.py:get_input_embeddings_entry", "Entry to get_input_embeddings", {
+            "input_ids_shape": str(input_ids.shape) if input_ids is not None else None,
+            "pixel_values_shape": str(pixel_values.shape) if pixel_values is not None else None,
+            "crop_counts": crop_counts,
+            "tilings": tilings,
+            "input_ids_sample": input_ids[0, :20].tolist() if input_ids is not None else None
+        }, "H7,H9")
+        # #endregion
+        
+        if pixel_values is None:
+            return InputEmbeddingsFeatures(
+                inputs_embeds=self.language_model.model.embed_tokens(input_ids)
+            )
+
+        # Get text embeddings
+        inputs_embeds = self.language_model.model.embed_tokens(input_ids)
+        
+        # #region agent log
+        log_embed("moondream2.py:text_embeddings", "Text embeddings from embed_tokens", {
+            "shape": str(inputs_embeds.shape),
+            "dtype": str(inputs_embeds.dtype),
+            "mean": float(mx.mean(inputs_embeds)),
+            "std": float(mx.std(inputs_embeds)),
+            "min": float(mx.min(inputs_embeds)),
+            "max": float(mx.max(inputs_embeds))
+        }, "H9")
+        # #endregion
+
+        # Encode ALL crops through vision encoder at once
+        # pixel_values is [total_crops, C, H, W]
+        all_features = self.vision_encoder(pixel_values)  # [total_crops, 729, 1152]
+        
+        # #region agent log
+        log_embed("moondream2.py:vision_features", "Vision encoder output", {
+            "shape": str(all_features.shape),
+            "dtype": str(all_features.dtype),
+            "mean": float(mx.mean(all_features)),
+            "std": float(mx.std(all_features)),
+            "min": float(mx.min(all_features)),
+            "max": float(mx.max(all_features))
+        }, "H6,H8")
+        # #endregion
+
+        # Process each image's crops
+        batch_size = len(crop_counts) if crop_counts is not None else 1
+        projected_features_list = []
+
+        crop_offset = 0
+        for b in range(batch_size):
+            n_crops = crop_counts[b] if crop_counts is not None else all_features.shape[0]
+            tiling = tilings[b] if tilings is not None else (1, 1)
+
+            # Extract features for this image
+            img_features = all_features[crop_offset : crop_offset + n_crops]  # [n_crops, 729, 1152]
+            crop_offset += n_crops
+
+            # Global features from first crop
+            global_features = img_features[0]  # [729, 1152]
+
+            # Local crop features
+            n_local = n_crops - 1
+            if n_local > 0:
+                local_features = img_features[1:]  # [n_local, 729, 1152]
+
+                # Reshape to spatial grid: [n_local, 729, 1152] -> [n_local, 27, 27, 1152]
+                local_features = local_features.reshape(n_local, 27, 27, -1)
+
+                # Reconstruct unified feature map from local crops
+                reconstructed = reconstruct_from_crops(
+                    local_features, tiling, overlap_margin=self.config.vision_config.overlap_margin
+                )  # [H, W, 1152]
+
+                # Pool back to 27x27 to match global features
+                reconstructed = adaptive_avg_pool2d(reconstructed, (27, 27))  # [27, 27, 1152]
+
+                # Flatten to [729, 1152]
+                reconstructed_flat = reconstructed.reshape(729, -1)
+            else:
+                # No local crops, duplicate global for reconstruction
+                reconstructed_flat = global_features
+
+            # Add batch dimension and project
+            global_batch = global_features[None, :, :]  # [1, 729, 1152]
+            reconstructed_batch = reconstructed_flat[None, :, :]  # [1, 729, 1152]
+            
+            # #region agent log
+            log_embed("moondream2.py:before_projection", "Features before projection", {
+                "global_shape": str(global_batch.shape),
+                "global_mean": float(mx.mean(global_batch)),
+                "global_std": float(mx.std(global_batch)),
+                "reconstructed_shape": str(reconstructed_batch.shape),
+                "reconstructed_mean": float(mx.mean(reconstructed_batch)),
+                "reconstructed_std": float(mx.std(reconstructed_batch)),
+                "n_local_crops": n_local
+            }, "H6,H8")
+            # #endregion
+
+            # Project concatenated features: [1, 729, 2304] -> [1, 729, 2048]
+            projected = self.vision_projection(global_batch, reconstructed_batch)
+            
+            # #region agent log
+            log_embed("moondream2.py:after_projection", "Projected vision features", {
+                "shape": str(projected.shape),
+                "dtype": str(projected.dtype),
+                "mean": float(mx.mean(projected)),
+                "std": float(mx.std(projected)),
+                "min": float(mx.min(projected)),
+                "max": float(mx.max(projected))
+            }, "H6")
+            # #endregion
+            
+            projected_features_list.append(projected)
+
+        # Concatenate all projected features
+        image_features = mx.concatenate(projected_features_list, axis=0)  # [B, 729, 2048]
+        
+        # #region agent log
+        log_embed("moondream2.py:concatenated_image_features", "Concatenated image features", {
+            "shape": str(image_features.shape),
+            "dtype": str(image_features.dtype),
+            "mean": float(mx.mean(image_features)),
+            "std": float(mx.std(image_features)),
+            "min": float(mx.min(image_features)),
+            "max": float(mx.max(image_features))
+        }, "H6")
+        # #endregion
+
+        # Replace 729-token image placeholder in input_ids with vision features
+        # prepare_inputs() creates input_ids as: [BOS, <img_token>*729, <text_tokens>]
+        patch_count = image_features.shape[1]  # expected 729
+        if inputs_embeds.shape[1] >= 1 + patch_count:
+            # #region agent log
+            log_embed("moondream2.py:before_replacement", "Before replacing image tokens", {
+                "inputs_embeds_shape": str(inputs_embeds.shape),
+                "patch_count": patch_count,
+                "replacement_range": f"[1:{1 + patch_count}]",
+                "image_features_dtype": str(image_features.dtype),
+                "inputs_embeds_dtype": str(inputs_embeds.dtype),
+                "embeds_before_mean": float(mx.mean(inputs_embeds[:, 1 : 1 + patch_count, :])),
+                "embeds_before_std": float(mx.std(inputs_embeds[:, 1 : 1 + patch_count, :]))
+            }, "H7,H9")
+            # #endregion
+            
+            # Replace positions [1 : 1+patch_count] (right after BOS)
+            inputs_embeds[:, 1 : 1 + patch_count, :] = image_features.astype(
+                inputs_embeds.dtype
+            )
+            
+            # #region agent log
+            log_embed("moondream2.py:after_replacement", "After replacing image tokens", {
+                "embeds_after_mean": float(mx.mean(inputs_embeds[:, 1 : 1 + patch_count, :])),
+                "embeds_after_std": float(mx.std(inputs_embeds[:, 1 : 1 + patch_count, :])),
+                "text_tokens_mean": float(mx.mean(inputs_embeds[:, 1 + patch_count:, :])) if inputs_embeds.shape[1] > 1 + patch_count else None,
+                "text_tokens_std": float(mx.std(inputs_embeds[:, 1 + patch_count:, :])) if inputs_embeds.shape[1] > 1 + patch_count else None
+            }, "H7,H9")
+            # #endregion
+            
+            final_embeddings = inputs_embeds
+        else:
+            # Fallback: original behavior (prepend image embeddings)
+            batch_size = inputs_embeds.shape[0]
+            final_embeddings = []
+            for b in range(batch_size):
+                bos_embed = inputs_embeds[b : b + 1, :1, :]
+                text_embed = inputs_embeds[b : b + 1, 1:, :]
+                img_embed = image_features[b : b + 1]
+                combined = mx.concatenate([bos_embed, img_embed, text_embed], axis=1)
+                final_embeddings.append(combined)
+            final_embeddings = mx.concatenate(final_embeddings, axis=0)
+        
+        # #region agent log
+        log_embed("moondream2.py:final_embeddings", "Final embeddings output", {
+            "shape": str(final_embeddings.shape),
+            "dtype": str(final_embeddings.dtype),
+            "mean": float(mx.mean(final_embeddings)),
+            "std": float(mx.std(final_embeddings)),
+            "min": float(mx.min(final_embeddings)),
+            "max": float(mx.max(final_embeddings)),
+            "has_nan": bool(mx.any(mx.isnan(final_embeddings))),
+            "has_inf": bool(mx.any(mx.isinf(final_embeddings)))
+        }, "H9")
+        # #endregion
+
+        return InputEmbeddingsFeatures(inputs_embeds=final_embeddings)
+
+    @property
+    def layers(self):
+        return self.language_model.model.layers
+
+    def __call__(
+        self,
+        input_ids: mx.array,
+        pixel_values: mx.array,
+        mask: Optional[mx.array] = None,
+        cache: Optional[Tuple[mx.array, mx.array]] = None,
+        crop_counts: Optional[List[int]] = None,
+        tilings: Optional[List[Tuple[int, int]]] = None,
+        **kwargs,
+    ):
+        input_embeddings_features = self.get_input_embeddings(
+            input_ids, pixel_values, crop_counts=crop_counts, tilings=tilings, **kwargs
+        )
+
+        logits = self.language_model(
+            inputs=input_ids,
+            cache=cache,
+            inputs_embeds=input_embeddings_features.inputs_embeds,
+            mask=mask,
+        )
+        return logits
+
+    def sanitize(self, weights):
+        """
+        Map HuggingFace weights to MLX model structure.
+
+        HF Weight Structure (from moondream2/vision.py + text.py):
+        - model.vision.patch_emb.* -> vision_encoder.patch_emb.*
+        - model.vision.pos_emb -> vision_encoder.position_embedding
+        - model.vision.blocks.{i}.ln1.* -> vision_encoder.encoder.layers.{i}.ln1.*
+        - model.vision.blocks.{i}.attn.* -> vision_encoder.encoder.layers.{i}.attn.*
+        - model.vision.blocks.{i}.ln2.* -> vision_encoder.encoder.layers.{i}.ln2.*
+        - model.vision.blocks.{i}.mlp.* -> vision_encoder.encoder.layers.{i}.mlp.*
+        - model.vision.post_ln.* -> vision_encoder.post_layernorm.*
+        - model.vision.proj_mlp.* -> vision_projection.*
+        - model.text.wte -> language_model.model.embed_tokens.weight
+        - model.text.blocks.{i}.ln.* -> language_model.model.layers.{i}.input_layernorm.*
+        - model.text.blocks.{i}.attn.qkv.* -> language_model.model.layers.{i}.self_attn.qkv_proj.*
+        - model.text.blocks.{i}.attn.proj.* -> language_model.model.layers.{i}.self_attn.o_proj.*
+        - model.text.blocks.{i}.mlp.* -> language_model.model.layers.{i}.mlp.*
+        - model.text.post_ln.* -> language_model.model.norm.*
+        - model.text.lm_head.* -> language_model.lm_head.*
+        - model.region.* -> (skip, not needed for VQA)
+        """
+        # #region agent log
+        import json
+        log_file = "/Users/zekieldee/Desktop/code/mlx-vlm/.cursor/debug.log"
+        def log(location, message, data, hypothesis_id):
+            try:
+                with open(log_file, "a") as f:
+                    f.write(json.dumps({"sessionId": "debug-session", "runId": "sanitize", "hypothesisId": hypothesis_id, "location": location, "message": message, "data": data, "timestamp": __import__("time").time_ns() // 1000000}) + "\n")
+            except: pass
+        # #endregion
+        
+        new_weights = {}
+        n_skipped_region = 0
+        n_changed = 0
+        n_unchanged = 0
+        
+        # #region agent log
+        original_keys = sorted(weights.keys())
+        log("moondream2.py:sanitize_entry", "Sanitize entry - input weights", {"n_weights": len(weights), "sample_keys": original_keys[:10], "all_keys": original_keys}, "H1")
+        # #endregion
+
+        for k, v in weights.items():
+            # Skip region model weights (not needed for VQA)
+            if k.startswith("model.region."):
+                n_skipped_region += 1
+                continue
+
+            new_key = k
+
+            # Vision encoder: patch embedding
+            if k.startswith("model.vision.patch_emb."):
+                new_key = k.replace(
+                    "model.vision.patch_emb.",
+                    "vision_encoder.patch_emb.",
+                )
+
+            # Vision encoder: positional embedding
+            elif k == "model.vision.pos_emb":
+                new_key = "vision_encoder.position_embedding"
+
+            # Vision encoder: blocks
+            elif k.startswith("model.vision.blocks."):
+                # Extract block number and rest
+                match = re.match(r"model\.vision\.blocks\.(\d+)\.(.+)", k)
+                if match:
+                    block_num = match.group(1)
+                    suffix = match.group(2)
+                    # Keep the structure: ln1, attn, ln2, mlp
+                    new_key = f"vision_encoder.encoder.layers.{block_num}.{suffix}"
+
+            # Vision encoder: post layer norm
+            elif k.startswith("model.vision.post_ln."):
+                new_key = k.replace(
+                    "model.vision.post_ln.",
+                    "vision_encoder.post_layernorm.",
+                )
+
+            # Vision projection MLP
+            elif k.startswith("model.vision.proj_mlp."):
+                new_key = k.replace(
+                    "model.vision.proj_mlp.",
+                    "vision_projection.",
+                )
+
+            # Text model: embedding
+            elif k == "model.text.wte":
+                new_key = "language_model.model.embed_tokens.weight"
+
+            # Text model: transformer blocks
+            elif k.startswith("model.text.blocks."):
+                # Extract block number and rest
+                match = re.match(r"model\.text\.blocks\.(\d+)\.(.+)", k)
+                if match:
+                    block_num = match.group(1)
+                    suffix = match.group(2)
+
+                    # Map the suffixes
+                    if suffix.startswith("ln."):
+                        new_suffix = suffix.replace("ln.", "input_layernorm.")
+                    elif suffix.startswith("attn.qkv."):
+                        new_suffix = suffix.replace("attn.qkv.", "self_attn.qkv_proj.")
+                    elif suffix.startswith("attn.proj."):
+                        new_suffix = suffix.replace("attn.proj.", "self_attn.o_proj.")
+                    elif suffix.startswith("mlp."):
+                        new_suffix = suffix
+                    else:
+                        new_suffix = suffix
+
+                    new_key = f"language_model.model.layers.{block_num}.{new_suffix}"
+
+            # Text model: final layer norm
+            elif k.startswith("model.text.post_ln."):
+                new_key = k.replace("model.text.post_ln.", "language_model.model.norm.")
+
+            # Text model: lm head
+            elif k.startswith("model.text.lm_head."):
+                new_key = k.replace("model.text.lm_head.", "language_model.lm_head.")
+
+            if new_key == k:
+                n_unchanged += 1
+            else:
+                n_changed += 1
+            new_weights[new_key] = v
+
+        # #region agent log
+        sanitized_keys = sorted(new_weights.keys())
+        log("moondream2.py:sanitize_exit", "Sanitize exit - output weights", {"n_weights": len(new_weights), "n_changed": n_changed, "n_unchanged": n_unchanged, "n_skipped": n_skipped_region, "sample_keys": sanitized_keys[:10], "all_keys": sanitized_keys}, "H1")
+        # #endregion
+
+        return new_weights

mlx_vlm/models/moondream2/processing_moondream.py

@@ -0,0 +1,144 @@
+"""
+Moondream2 processor for mlx-vlm.
+"""
+
+from typing import List, Optional, Union
+
+from PIL import Image
+from transformers import AutoTokenizer
+from transformers.processing_utils import ProcessorMixin
+
+
+class MoondreamProcessor(ProcessorMixin):
+    """
+    Processor for Moondream2 model.
+
+    Wraps the tokenizer and provides compatibility with mlx-vlm's generation flow.
+    Image processing is handled separately by the model's ImageProcessor class.
+    """
+
+    tokenizer_class = "AutoTokenizer"
+    attributes = ["tokenizer"]
+
+    def __init__(self, tokenizer, chat_template: Optional[str] = None, **kwargs):
+        self.tokenizer = tokenizer
+
+        # Set up chat template for moondream
+        if chat_template is None:
+            # Moondream uses a simple format: <image>\n\nQuestion: {question}\n\nAnswer:
+            chat_template = (
+                "{% for message in messages %}"
+                "{% if message['role'] == 'user' %}"
+                "{{ message['content'] }}\n\n"
+                "{% elif message['role'] == 'assistant' %}"
+                "{{ message['content'] }}"
+                "{% endif %}"
+                "{% endfor %}"
+                "{% if add_generation_prompt %}Answer: {% endif %}"
+            )
+
+        self.chat_template = chat_template
+        tokenizer.chat_template = chat_template
+        super().__init__(tokenizer, **kwargs)
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
+        """Load processor from pretrained model path."""
+        # Convert Path to string if needed
+        if hasattr(pretrained_model_name_or_path, "__fspath__"):
+            pretrained_model_name_or_path = str(pretrained_model_name_or_path)
+
+        # Pop kwargs that are not valid for AutoTokenizer
+        trust_remote_code = kwargs.pop("trust_remote_code", True)
+
+        # Moondream2 uses a custom tokenizer (starmie-v1), not the GPT-2
+        # tokenizer files shipped in the model repo.
+        tokenizer = AutoTokenizer.from_pretrained(
+            "moondream/starmie-v1",
+            trust_remote_code=trust_remote_code,
+        )
+
+        # starmie-v1 doesn't define special token roles; set them to
+        # <|endoftext|> (ID 0) which moondream uses as BOS/EOS/PAD.
+        tokenizer.eos_token = "<|endoftext|>"
+        tokenizer.bos_token = "<|endoftext|>"
+        tokenizer.pad_token = "<|endoftext|>"
+
+        return cls(tokenizer=tokenizer)
+
+    def __call__(
+        self,
+        text: Optional[Union[str, List[str]]] = None,
+        images: Optional[Union[Image.Image, List[Image.Image]]] = None,
+        **kwargs,
+    ):
+        """
+        Process text and images for the model.
+
+        Note: Image processing is handled by the model's ImageProcessor,
+        this processor mainly handles tokenization.
+        """
+        if text is None:
+            raise ValueError("Text input is required")
+
+        # Tokenize text
+        encoding = self.tokenizer(text, **kwargs)
+
+        return encoding
+
+    def batch_decode(self, *args, **kwargs):
+        """Decode token ids to text."""
+        return self.tokenizer.batch_decode(*args, **kwargs)
+
+    def decode(self, *args, **kwargs):
+        """Decode token ids to text."""
+        return self.tokenizer.decode(*args, **kwargs)
+
+    def apply_chat_template(self, messages, add_generation_prompt=True, **kwargs):
+        """Apply chat template to messages."""
+        return self.tokenizer.apply_chat_template(
+            messages,
+            chat_template=self.chat_template,
+            add_generation_prompt=add_generation_prompt,
+            tokenize=kwargs.get("tokenize", False),
+            **{k: v for k, v in kwargs.items() if k != "tokenize"}
+        )
+
+    # Token properties delegated to tokenizer
+    @property
+    def pad_token(self):
+        return self.tokenizer.pad_token
+
+    @pad_token.setter
+    def pad_token(self, value):
+        self.tokenizer.pad_token = value
+
+    @property
+    def pad_token_id(self):
+        return self.tokenizer.pad_token_id
+
+    @pad_token_id.setter
+    def pad_token_id(self, value):
+        self.tokenizer.pad_token_id = value
+
+    @property
+    def eos_token(self):
+        return self.tokenizer.eos_token
+
+    @property
+    def eos_token_id(self):
+        return self.tokenizer.eos_token_id
+
+    @property
+    def bos_token(self):
+        return self.tokenizer.bos_token
+
+    @property
+    def bos_token_id(self):
+        return self.tokenizer.bos_token_id
+
+
+# Install the AutoProcessor patch for moondream1 model type
+from ..base import install_auto_processor_patch
+
+install_auto_processor_patch("moondream1", MoondreamProcessor)