xinference 0.11.1__py3-none-any.whl → 0.11.2.post1__py3-none-any.whl

xinference/model/llm/memory.py

@@ -0,0 +1,332 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# NOTE:
+#
+#   The algorithum is ported from https://github.com/RahulSChand/gpu_poor
+#
+#   Improvement:
+#
+#      The original js code only calculate kv_cache_dtype by float32, instead of most case we run model with float16.
+#
+#   Known Issue:
+#
+#       * On vllm, some MHA model use smaller memory than calculation (qwen1.5-7B-chat-gptq-int4,
+#       qwen1.5-14B-chat-gptq-int4 with large activation_mem).
+#
+#       * On vllm, gemma-it-7B pytorch format model use larger gpu mem than calculation
+
+import json
+import math
+from dataclasses import dataclass
+from logging import getLogger
+from math import ceil
+from typing import Any, Optional, Union
+
+from .llm_family import convert_model_size_to_float
+
+logger = getLogger(__name__)
+
+
+@dataclass
+class ModelLayersInfo:
+    vocab_size: int
+    heads: int  # num_attention_heads, num_heads or n_head
+    hidden_dim: int  # hidden_size, d_model, or n_embd
+    inter_dim: int  # intermediate_size, n_inner or d_ff
+    num_layers: int  # num_layers, num_hidden_layers or n_layer
+
+
+@dataclass
+class ModelMemInfo:
+    """Memory required by model, unit in MB"""
+
+    model_mem: int
+    kv_cache_mem: int
+    activation_mem: int
+    overhead: int
+    total: int
+
+
+QUANT_NORMALIZE = {"int4": "4-bit", "int8": "8-bit", "4-bit": "4-bit", "8-bit": "8-bit"}
+
+GGML_MULTI_FACTOR_DICT = {
+    "q4_0": 18,
+    "q4_1": 20,
+    "q5_0": 22,
+    "q5_1": 24,
+    "q8_0": 34,
+    "q8_1": 40,
+}
+
+GGML_MULTI_FACTOR_DICT_64 = {
+    "q6_K": 54.0,
+    "q3": 26.0,
+    "q4": 38.0,
+    "q5": 46.0,
+}
+
+GGML_MULTI_FACTOR_DICT_COMBINE = {
+    "q3_K_L": [38.0, 26.0],
+    "q3_K_M": [46.0, 26.0],
+    "q4_K_S": [46.0, 38.0],
+    "q4_K_M": [54.0, 38.0],
+    "q5_K_M": [54.0, 46.0],
+    "q2_K": [26.0, 22.0],
+}
+
+
+# Return gpu memory in MB
+def estimate_llm_gpu_memory(
+    model_size_in_billions: Union[str, int],
+    quantization: Optional[str],
+    context_length: int,  # input+output
+    model_format: str,
+    model_name: Optional[str] = None,
+    kv_cache_dtype: int = 16,
+) -> Optional[ModelMemInfo]:
+    """
+    model_size_in_billions: must be str like 1_8 or 46_7, to match llm.
+    """
+    info = get_model_layers_info(
+        model_size_in_billions,
+        model_name,
+        model_format,
+        quantization,
+    )
+    if info is None:
+        return None
+    size_in_billions = convert_model_size_to_float(model_size_in_billions)
+    return estimate_llm_gpu_memory_details(
+        info,
+        size_in_billions,
+        quantization,
+        context_length,
+        model_format,
+        kv_cache_dtype,
+    )
+
+
+def estimate_llm_gpu_memory_details(
+    info: ModelLayersInfo,
+    size_in_billions: float,
+    quantization: Optional[str],
+    context_length: int,  # input+output
+    model_format: str,
+    kv_cache_dtype: int = 16,
+) -> ModelMemInfo:
+    """return model_mem, kv_cache, overhead, activation_mem"""
+    if kv_cache_dtype not in [8, 16, 32]:
+        raise ValueError(f"Invalid kv_cache_dtype {kv_cache_dtype}")
+    if kv_cache_dtype == 8:
+        kv_dtype_size = 1
+    elif kv_cache_dtype == 16:
+        kv_dtype_size = 2
+    else:
+        kv_dtype_size = 4
+    overhead = 650.0
+    if model_format == "ggmlv3":
+        assert quantization is not None and quantization != "none"
+        model_size_in_mb = _compute_model_size_ggml(info, quantization)
+        inference_mem = float(
+            context_length * kv_dtype_size * info.hidden_dim * info.num_layers
+        )
+        inference_mem = inference_mem / 1024.0 / 1024.0
+        activation_mem = _compute_inference_only_activation_memory(context_length, info)
+        overhead = overhead + context_length * 0.1
+    else:
+        if quantization is not None:
+            assert isinstance(quantization, str)
+            quantization = QUANT_NORMALIZE[quantization.lower()]
+            assert quantization is not None
+
+        model_size = size_in_billions * 1000000000.0
+        model_size_in_mb = _convert_to_mb_model_size(model_size, quantization)
+        # KV cache
+        inference_mem = float(
+            context_length * 2 * kv_dtype_size * info.hidden_dim * info.num_layers
+        )
+        inference_mem = inference_mem / 1024.0 / 1024.0
+        activation_mem = _compute_inference_only_activation_memory(context_length, info)
+
+    total_mem = ceil(inference_mem + model_size_in_mb + overhead + activation_mem)
+    return ModelMemInfo(
+        model_mem=ceil(model_size_in_mb),
+        kv_cache_mem=ceil(inference_mem),
+        activation_mem=ceil(activation_mem),
+        overhead=ceil(overhead),
+        total=total_mem,
+    )
+
+
+def _load_item_from_json(config_data: Any, *keys: str) -> str:
+    assert len(keys) > 0
+    for key in keys:
+        v = config_data.get(key)
+        if v is not None:
+            return v
+    raise ValueError("load ModelLayersInfo: missing %s" % (keys[0]))
+
+
+def load_model_config_json(config_path: str) -> ModelLayersInfo:
+    with open(config_path, "r") as f:
+        config_data = json.load(f)
+        return ModelLayersInfo(
+            vocab_size=int(_load_item_from_json(config_data, "vocab_size")),
+            heads=int(
+                _load_item_from_json(
+                    config_data, "num_key_value_heads", "num_attention_heads"
+                )
+            ),
+            hidden_dim=int(
+                _load_item_from_json(config_data, "hidden_size", "d_model", "n_embd")
+            ),
+            inter_dim=int(_load_item_from_json(config_data, "intermediate_size")),
+            num_layers=int(
+                _load_item_from_json(
+                    config_data, "num_hidden_layers", "num_layers", "n_layer"
+                )
+            ),
+        )
+
+
+def get_model_layers_info(
+    model_size_in_billions: Union[str, int],
+    model_name: Optional[str],
+    model_format: Optional[str],
+    quantization: Optional[str],
+) -> Optional[ModelLayersInfo]:
+    from . import match_llm
+    from .llm_family import cache_model_config
+
+    if not model_name:
+        logger.debug("get_model_layers_info by default size=%s", model_size_in_billions)
+        size_in_billions = convert_model_size_to_float(model_size_in_billions)
+        return _get_default_layers_from_size(size_in_billions)
+    match_result = match_llm(
+        model_name=model_name,
+        model_format=model_format,
+        model_size_in_billions=model_size_in_billions,
+        quantization=quantization,
+    )
+    if not match_result:
+        return None
+    llm_family, llm_spec, _quant = match_result
+    config_path = cache_model_config(llm_family, llm_spec)
+    return load_model_config_json(config_path)
+
+
+def _get_default_layers_from_size(size_in_billion: float) -> ModelLayersInfo:
+    if size_in_billion < 5:
+        vocab_size = 32000
+        heads = 32
+        num_layers = 24
+    elif size_in_billion < 10:
+        vocab_size = 32000
+        heads = 32
+        num_layers = 32
+    elif size_in_billion < 24:
+        vocab_size = 32000
+        heads = 40
+        num_layers = 40
+    elif size_in_billion < 55:
+        vocab_size = 32000
+        heads = 60
+        num_layers = 48
+    else:
+        vocab_size = 32000
+        heads = 64
+        num_layers = 80
+
+    model_size = int(size_in_billion * 1000000000)
+    A = num_layers * 4 + 3 * 4 * num_layers
+    B = 2 * vocab_size
+    C = -1 * model_size
+    h = (-B + math.sqrt(B**2 - 4 * A * C)) / (2 * A)
+    h = math.ceil(h)
+    return ModelLayersInfo(
+        vocab_size=vocab_size,
+        heads=heads,
+        hidden_dim=h,
+        inter_dim=4 * h,
+        num_layers=num_layers,
+    )
+
+
+def _convert_to_mb_model_size(model_size: float, quantization: Optional[str]) -> float:
+    extra = 0.0
+    fB = 2.0
+    size = (model_size * fB) / (1024.0 * 1024.0)
+    # bnb_q4 == 4-bit ?
+    if quantization == "8-bit" or quantization == "4-bit":
+        extra = 0.06 * size
+    if quantization == "8-bit":
+        size = size / 2
+    if quantization == "4-bit":
+        size = size / 4
+    return size + extra
+
+
+def _compute_inference_only_activation_memory(
+    context_length: int, info: ModelLayersInfo
+) -> float:
+    hidden_dim = info.hidden_dim
+    heads = info.heads
+    ret = (
+        (context_length * hidden_dim * 5 * 2 + (context_length**2) * heads * 2)
+        / 1024
+        / 1024
+    )
+    return ret
+
+
+def _compute_model_size_ggml(info: ModelLayersInfo, quantization: str) -> float:
+    assert quantization is not None
+    vocab_size = info.vocab_size
+    num_layers = info.num_layers
+    hidden_dim = info.hidden_dim
+    inter_dim = info.inter_dim
+    total_params = int(
+        vocab_size * hidden_dim * 2
+        + num_layers * 4 * (hidden_dim**2)
+        + num_layers * 3 * inter_dim * hidden_dim
+    )
+    other_v_down_params = (
+        num_layers * (hidden_dim**2) + num_layers * hidden_dim * inter_dim
+    )
+    other_param_q2k = (
+        total_params - (hidden_dim**2) * num_layers * 2 + 2 * vocab_size * hidden_dim
+    )
+
+    total = 0.0
+    v1 = GGML_MULTI_FACTOR_DICT.get(quantization)
+    if v1 is not None:
+        total = (v1 * total_params) / (32 * 1024 * 1024)
+    v2 = GGML_MULTI_FACTOR_DICT_64.get(quantization)
+    if v2 is not None:
+        total = (v2 * total_params) / (64 * 1024 * 1024)
+    v3 = GGML_MULTI_FACTOR_DICT_COMBINE.get(quantization)
+    if v3 is not None:
+        factors = v3
+        if quantization == "q2_K":
+            total = (
+                (total_params - other_param_q2k) * factors[1]
+                + other_param_q2k * factors[0]
+            ) / (64 * 1024 * 1024)
+        else:
+            total = (
+                (total_params - other_v_down_params) * factors[1]
+                + other_v_down_params * factors[0]
+            ) / (64 * 1024 * 1024)
+    return total

xinference/model/llm/pytorch/core.py

@@ -60,6 +60,8 @@ NON_DEFAULT_MODEL_LIST: List[str] = [
     "OmniLMM",
     "yi-vl-chat",
     "deepseek-vl-chat",
+    "internvl-chat",
+    "mini-internvl-chat",
 ]
 
 

xinference/model/llm/pytorch/intern_vl.py

@@ -0,0 +1,347 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import base64
+import logging
+import time
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from io import BytesIO
+from typing import Dict, Iterator, List, Optional, Tuple, Union
+
+import requests
+import torch
+from PIL import Image
+
+from ....model.utils import select_device
+from ....types import (
+    ChatCompletion,
+    ChatCompletionChunk,
+    ChatCompletionMessage,
+    Completion,
+    CompletionChoice,
+    CompletionUsage,
+)
+from ..llm_family import LLMFamilyV1, LLMSpecV1
+from .core import PytorchChatModel, PytorchGenerateConfig
+
+logger = logging.getLogger(__name__)
+
+IMAGENET_MEAN = (0.485, 0.456, 0.406)
+IMAGENET_STD = (0.229, 0.224, 0.225)
+
+
+class InternVLChatModel(PytorchChatModel):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._tokenizer = None
+        self._model = None
+
+    @classmethod
+    def match(
+        cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
+    ) -> bool:
+        family = model_family.model_family or model_family.model_name
+        if "internvl" in family.lower():
+            return True
+        return False
+
+    def load(self, **kwargs):
+        from transformers import AutoModel, AutoTokenizer
+        from transformers.generation import GenerationConfig
+
+        device = self._pytorch_model_config.get("device", "auto")
+        device = select_device(device)
+        # for multiple GPU, set back to auto to make multiple devices work
+        device = "auto" if device == "cuda" else device
+
+        self._tokenizer = AutoTokenizer.from_pretrained(
+            self.model_path,
+            trust_remote_code=True,
+        )
+
+        kwargs = {
+            "torch_dtype": torch.bfloat16,
+            "low_cpu_mem_usage": True,
+            "trust_remote_code": True,
+            "device_map": device,
+        }
+
+        if "Int8" in self.model_spec.quantizations:
+            kwargs.update(
+                {
+                    "load_in_8bit": True,
+                    "device_map": device,
+                }
+            )
+        elif "mini" in self.model_family.model_name:
+            kwargs.pop("device_map")
+
+        self._model = AutoModel.from_pretrained(self.model_path, **kwargs).eval()
+
+        if "Int8" not in self.model_spec.quantizations:
+            self._model.cuda()
+
+        # Specify hyperparameters for generation
+        self._model.generation_config = GenerationConfig.from_pretrained(
+            self.model_path,
+            trust_remote_code=True,
+        )
+
+    def _message_content_to_intern(self, content):
+        def _load_image(_url):
+            if _url.startswith("data:"):
+                logging.info("Parse url by base64 decoder.")
+                # https://platform.openai.com/docs/guides/vision/uploading-base-64-encoded-images
+                # e.g. f"data:image/jpeg;base64,{base64_image}"
+                _type, data = _url.split(";")
+                _, ext = _type.split("/")
+                data = data[len("base64,") :]
+                data = base64.b64decode(data.encode("utf-8"))
+                return Image.open(BytesIO(data)).convert("RGB")
+            else:
+                try:
+                    response = requests.get(_url)
+                except requests.exceptions.MissingSchema:
+                    return Image.open(_url).convert("RGB")
+                else:
+                    return Image.open(BytesIO(response.content)).convert("RGB")
+
+        if not isinstance(content, str):
+            texts = []
+            image_urls = []
+            for c in content:
+                c_type = c.get("type")
+                if c_type == "text":
+                    texts.append(c["text"])
+                elif c_type == "image_url":
+                    image_urls.append(c["image_url"]["url"])
+            image_futures = []
+            with ThreadPoolExecutor() as executor:
+                for image_url in image_urls:
+                    fut = executor.submit(_load_image, image_url)
+                    image_futures.append(fut)
+            images = [fut.result() for fut in image_futures]
+            text = " ".join(texts)
+            if len(images) == 0:
+                return text, None
+            else:
+                return text, images
+        return content, None
+
+    def _history_content_to_intern(
+        self,
+        chat_history: List[ChatCompletionMessage],
+        IMG_START_TOKEN="<img>",
+        IMG_END_TOKEN="</img>",
+        IMG_CONTEXT_TOKEN="<IMG_CONTEXT>",
+    ):
+        def _image_to_piexl_values(images):
+            load_images = []
+            for image in images:
+                if image.startswith("data:"):
+                    logging.info("Parse url by base64 decoder.")
+                    # https://platform.openai.com/docs/guides/vision/uploading-base-64-encoded-images
+                    # e.g. f"data:image/jpeg;base64,{base64_image}"
+                    _type, data = image.split(";")
+                    _, ext = _type.split("/")
+                    data = data[len("base64,") :]
+                    data = base64.b64decode(data.encode("utf-8"))
+                    img = Image.open(BytesIO(data)).convert("RGB")
+                    pixel_value = (
+                        self._load_image(img, max_num=6).to(torch.bfloat16).cuda()
+                    )
+                    load_images.append(pixel_value)
+                else:
+                    try:
+                        response = requests.get(image)
+                    except requests.exceptions.MissingSchema:
+                        img = Image.open(image).convert("RGB")
+                    else:
+                        img = Image.open(BytesIO(response.content)).convert("RGB")
+                    pixel_value = (
+                        self._load_image(img, max_num=6).to(torch.bfloat16).cuda()
+                    )
+                    load_images.append(pixel_value)
+            return torch.cat(tuple(load_images), dim=0)
+
+        history: List[Tuple] = []
+        pixel_values = None
+        for i in range(0, len(chat_history), 2):
+            tmp = []
+            images: List[str] = []
+            user = chat_history[i]["content"]
+            if isinstance(user, List):
+                for content in user:
+                    c_type = content.get("type")
+                    if c_type == "text":
+                        tmp.append(content["text"])
+                    elif c_type == "image_url" and not history:
+                        images.append(content["image_url"]["url"])
+                if not history:
+                    pixel_values = _image_to_piexl_values(images)
+                    image_bs = pixel_values.shape[0]
+                    image_tokens = (
+                        IMG_START_TOKEN
+                        + IMG_CONTEXT_TOKEN * self._model.num_image_token * image_bs
+                        + IMG_END_TOKEN
+                    )
+                    tmp[0] = image_tokens + "\n" + tmp[0]
+            else:
+                tmp.append(user)
+            tmp.append(chat_history[i + 1]["content"])
+            history.append(tuple(tmp))
+        return history, pixel_values
+
+    def _find_closest_aspect_ratio(
+        self, aspect_ratio, target_ratios, width, height, image_size
+    ):
+        best_ratio_diff = float("inf")
+        best_ratio = (1, 1)
+        area = width * height
+        for ratio in target_ratios:
+            target_aspect_ratio = ratio[0] / ratio[1]
+            ratio_diff = abs(aspect_ratio - target_aspect_ratio)
+            if ratio_diff < best_ratio_diff:
+                best_ratio_diff = ratio_diff
+                best_ratio = ratio
+            elif ratio_diff == best_ratio_diff:
+                if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
+                    best_ratio = ratio
+        return best_ratio
+
+    def _dynamic_preprocess(
+        self, image, min_num=1, max_num=6, image_size=448, use_thumbnail=False
+    ):
+        orig_width, orig_height = image.size
+        aspect_ratio = orig_width / orig_height
+
+        # calculate the existing image aspect ratio
+        target_ratios = set(
+            (i, j)
+            for n in range(min_num, max_num + 1)
+            for i in range(1, n + 1)
+            for j in range(1, n + 1)
+            if i * j <= max_num and i * j >= min_num
+        )
+        target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
+
+        # find the closest aspect ratio to the target
+        target_aspect_ratio = self._find_closest_aspect_ratio(
+            aspect_ratio, target_ratios, orig_width, orig_height, image_size
+        )
+
+        # calculate the target width and height
+        target_width = image_size * target_aspect_ratio[0]
+        target_height = image_size * target_aspect_ratio[1]
+        blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
+
+        # resize the image
+        resized_img = image.resize((target_width, target_height))
+        processed_images = []
+        for i in range(blocks):
+            box = (
+                (i % (target_width // image_size)) * image_size,
+                (i // (target_width // image_size)) * image_size,
+                ((i % (target_width // image_size)) + 1) * image_size,
+                ((i // (target_width // image_size)) + 1) * image_size,
+            )
+            # split the image
+            split_img = resized_img.crop(box)
+            processed_images.append(split_img)
+        assert len(processed_images) == blocks
+        if use_thumbnail and len(processed_images) != 1:
+            thumbnail_img = image.resize((image_size, image_size))
+            processed_images.append(thumbnail_img)
+        return processed_images
+
+    def _build_transform(self, input_size):
+        import torchvision.transforms as T
+        from torchvision.transforms.functional import InterpolationMode
+
+        MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
+        transform = T.Compose(
+            [
+                T.Lambda(lambda img: img.convert("RGB") if img.mode != "RGB" else img),
+                T.Resize(
+                    (input_size, input_size), interpolation=InterpolationMode.BICUBIC
+                ),
+                T.ToTensor(),
+                T.Normalize(mean=MEAN, std=STD),
+            ]
+        )
+        return transform
+
+    def _load_image(self, image_file, input_size=448, max_num=6):
+        transform = self._build_transform(input_size=input_size)
+        images = self._dynamic_preprocess(
+            image_file, image_size=input_size, use_thumbnail=True, max_num=max_num
+        )
+        pixel_values = [transform(image) for image in images]
+        pixel_values = torch.stack(pixel_values)
+        return pixel_values
+
+    def chat(
+        self,
+        prompt: Union[str, List[Dict]],
+        system_prompt: Optional[str] = None,
+        chat_history: Optional[List[ChatCompletionMessage]] = None,
+        generate_config: Optional[PytorchGenerateConfig] = None,
+    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
+        if generate_config and generate_config.pop("stream"):
+            raise Exception(
+                f"Chat with model {self.model_family.model_name} does not support stream."
+            )
+        sanitized_config = {
+            "num_beams": 1,
+            "max_new_tokens": generate_config.get("max_tokens", 512)
+            if generate_config
+            else 512,
+            "do_sample": False,
+        }
+
+        content, image = self._message_content_to_intern(prompt)
+
+        history = None
+        if chat_history:
+            history, pixel_values = self._history_content_to_intern(chat_history)
+        else:
+            load_images = []
+            for img in image:
+                pixel_value = self._load_image(img, max_num=6).to(torch.bfloat16).cuda()
+                load_images.append(pixel_value)
+            pixel_values = torch.cat(tuple(load_images), dim=0)
+
+        response, history = self._model.chat(
+            self._tokenizer,
+            pixel_values,
+            content,
+            sanitized_config,
+            history=history,
+            return_history=True,
+        )
+        chunk = Completion(
+            id=str(uuid.uuid1()),
+            object="text_completion",
+            created=int(time.time()),
+            model=self.model_uid,
+            choices=[
+                CompletionChoice(
+                    index=0, text=response, finish_reason="stop", logprobs=None
+                )
+            ],
+            usage=CompletionUsage(
+                prompt_tokens=-1, completion_tokens=-1, total_tokens=-1
+            ),
+        )
+        return self._to_chat_completion(chunk)

xinference/model/llm/utils.py

@@ -456,6 +456,19 @@ Begin!"""
                     ret += f"<|{role}|>{prompt_style.intra_message_sep}"
             ret += "<|assistant|>\n"
             return ret
+        elif prompt_style.style_name == "c4ai-command-r":
+            ret = (
+                f"<BOS_TOKEN><|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>"
+                f"{prompt_style.system_prompt}{prompt_style.inter_message_sep}"
+            )
+            for i, message in enumerate(chat_history):
+                role = get_role(message["role"])
+                content = message["content"]
+                if content:
+                    ret += f"{role}{content}{prompt_style.inter_message_sep}"
+                else:
+                    ret += role
+            return ret
         else:
             raise ValueError(f"Invalid prompt style: {prompt_style.style_name}")