crfm-helm 0.5.4__py3-none-any.whl → 0.5.6__py3-none-any.whl

helm/clients/audio_language/diva_llama_client.py

@@ -0,0 +1,120 @@
+import threading
+from typing import Any, Dict, List, Optional, Tuple, TypedDict
+
+import numpy as np
+from transformers import AutoModel, PreTrainedModel
+
+from helm.clients.client import CachingClient
+from helm.common.cache import CacheConfig
+from helm.common.media_object import TEXT_TYPE
+from helm.common.request import (
+    GeneratedOutput,
+    Request,
+    RequestResult,
+    wrap_request_time,
+)
+from helm.common.audio_utils import get_array_from_audio_file
+from helm.proxy.retry import NonRetriableException
+
+
+_LOCK = threading.Lock()
+_PRE_TRAINED_MODEL: Optional[PreTrainedModel] = None
+
+
+def _get_pre_trained_model(model_name: str, **kwargs) -> PreTrainedModel:
+    global _LOCK
+    global _PRE_TRAINED_MODEL
+    with _LOCK:
+        if _PRE_TRAINED_MODEL is None:
+            _PRE_TRAINED_MODEL = AutoModel.from_pretrained(model_name, **kwargs)
+    return _PRE_TRAINED_MODEL
+
+
+class DivaLlamaRequest(TypedDict):
+    """Cache key for DivaLlamaClient"""
+
+    model: str
+    media_objects: List[Dict[str, Any]]
+
+
+class DivaLlamaClient(CachingClient):
+    SAMPLE_RATE = 16000
+
+    def __init__(
+        self,
+        cache_config: CacheConfig,
+        **kwargs,
+    ):
+        super().__init__(cache_config)
+        self.pre_trained_model = _get_pre_trained_model("WillHeld/DiVA-llama-3-v0-8b", trust_remote_code=True, **kwargs)
+
+    @staticmethod
+    def _get_generate_input(request: Request) -> Tuple[np.ndarray, Optional[str]]:
+        if request.prompt:
+            raise NonRetriableException("request.prompt must be empty for DivaLlamaClient")
+        if request.embedding:
+            raise NonRetriableException("request.embedding must be empty for DivaLlamaClient")
+        if request.messages:
+            raise NonRetriableException("request.messages must be empty for DivaLlamaClient")
+        if request.multimodal_prompt is None:
+            raise NonRetriableException("request.multimodal_prompt must not be None for DivaLlamaClient")
+        text_input: Optional[str] = None
+        audio_input: Optional[np.ndarray] = None
+        for media_object in request.multimodal_prompt.media_objects:
+            if media_object.is_type("audio"):
+                if audio_input is not None:
+                    raise NonRetriableException(
+                        "Only one audio object allowed in request.multimodal_prompt.media_objects"
+                    )
+                assert media_object.location
+                audio_input = get_array_from_audio_file(media_object.location, DivaLlamaClient.SAMPLE_RATE)
+            elif media_object.is_type(TEXT_TYPE):
+                if text_input is not None:
+                    raise NonRetriableException(
+                        "Only one text object allowed in request.multimodal_prompt.media_objects"
+                    )
+                assert media_object.text is not None
+                text_input = media_object.text
+            else:
+                raise NonRetriableException(f"Unsupported media content type type: {media_object.content_type}")
+        if audio_input is None:
+            raise NonRetriableException(
+                "Expected a single audio object allowed in request.multimodal_prompt.media_objects"
+            )
+        return audio_input, text_input
+
+    def make_request(self, request: Request) -> RequestResult:
+        assert request.multimodal_prompt is not None
+        raw_request: DivaLlamaRequest = {
+            "model": request.model,
+            "media_objects": [media_object.to_dict() for media_object in request.multimodal_prompt.media_objects],
+        }
+
+        try:
+
+            def do_it() -> Dict[str, Any]:
+                with _LOCK:
+                    audio_input, text_input = DivaLlamaClient._get_generate_input(request)
+                    if text_input is None:
+                        return {"completions": self.pre_trained_model.generate([audio_input])}  # type: ignore
+                    else:
+                        return {
+                            "completions": self.pre_trained_model.generate([audio_input], [text_input])  # type: ignore
+                        }
+
+            cache_key = CachingClient.make_cache_key(raw_request, request)
+            response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
+        except Exception as e:  # Do something if error is encountered.
+            error: str = f"HuggingFace error: {e}"
+            return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
+
+        generated_output = GeneratedOutput(text=response["completions"][0], logprob=0, tokens=[])
+
+        return RequestResult(
+            success=True,
+            cached=cached,
+            request_time=response["request_time"],
+            request_datetime=response.get("request_datetime"),
+            completions=[generated_output],
+            embedding=[],
+        )

helm/clients/audio_language/llama_omni_client.py

@@ -0,0 +1,198 @@
+from threading import Lock
+import torch
+from typing import Any, Dict, List, Optional
+
+from dataclasses import dataclass
+from transformers import AutoTokenizer
+import whisper
+from helm.clients.audio_language.llama_omni.model.builder import load_pretrained_model as load_llama_omni
+from helm.clients.audio_language.llama_omni.model.language_model.omni_speech2s_llama import OmniSpeech2SLlamaForCausalLM
+from helm.clients.audio_language.llama_omni.conversation import conv_templates, Conversation
+from helm.clients.audio_language.llama_omni.preprocess import tokenizer_speech_token
+
+from helm.common.cache import CacheConfig
+from helm.common.gpu_utils import get_torch_device_name
+from helm.common.hierarchical_logger import hlog, htrack_block
+from helm.common.media_object import TEXT_TYPE
+from helm.common.request import Request, RequestResult, GeneratedOutput, Token
+from helm.common.request import wrap_request_time
+from helm.clients.client import CachingClient, generate_uid_for_multimodal_prompt
+
+
+@dataclass(frozen=True)
+class LoadedLlamaOmniModelProcessor:
+    """Loaded model and processor for Qwen."""
+
+    model: OmniSpeech2SLlamaForCausalLM
+    tokenizer: AutoTokenizer
+
+
+_models_lock: Lock = Lock()
+_models: Dict[str, Optional[LoadedLlamaOmniModelProcessor]] = {
+    "ICTNLP/Llama-3.1-8B-Omni": None,
+}
+
+
+class LlamaOmniAudioLMClient(CachingClient):
+    """
+    From https://github.com/ictnlp/LLaMA-Omni,
+    LLaMA-Omni is the audio multimodal version based on the LLaMA-3.1-8B large language model,
+    developed by ICTNLP group. LLaMA-Omni accepts audio, text as inputs, and outputs text.
+
+    Paper: https://arxiv.org/abs/2409.06666
+    """
+
+    END_OF_TEXT_TOKEN: str = "<|im_end|>"
+    CONV_MODE: str = "llama_3"
+    PAD_ID: int = 128004
+    MEL_NUM: int = 128
+
+    def __init__(self, cache_config: CacheConfig):
+        super().__init__(cache_config=cache_config)
+        self._device: str = get_torch_device_name()
+
+    def _get_model(self, helm_model_name: str) -> LoadedLlamaOmniModelProcessor:
+        global _models_lock
+        global _models
+
+        model_name: str
+        if helm_model_name == "llama-3.1-8b-omni":
+            model_name = "ICTNLP/Llama-3.1-8B-Omni"
+        else:
+            raise ValueError(f"Unhandled model name: {helm_model_name}")
+
+        # Ensure that only one thread is loading the model at a time
+        with _models_lock:
+            loaded_model_processor = _models[model_name]
+            if loaded_model_processor is None:
+                hlog(f"Loading model {model_name} and caching in memory...")
+                # Follow the official LLaMA-Omni model loading pattern:
+                # https://github.com/ictnlp/LLaMA-Omni/blob/main/omni_speech/infer/run.sh
+                tokenizer, model, _ = load_llama_omni(model_name, None, s2s=True)
+                _models[model_name] = LoadedLlamaOmniModelProcessor(model, tokenizer)
+                loaded_model_processor = _models[model_name]
+
+        assert loaded_model_processor is not None
+        return loaded_model_processor
+
+    def _load_local_audio(self, media_object) -> torch.Tensor:
+        assert media_object.is_local_file, "LLaMA-Omni only supports local audio file input"
+        audio_media = whisper.load_audio(media_object.location)
+        audio_media = whisper.pad_or_trim(audio_media)
+        audio_media = whisper.log_mel_spectrogram(audio_media, n_mels=self.MEL_NUM).permute(1, 0)
+        return audio_media
+
+    def make_request(self, request: Request) -> RequestResult:
+        assert request.multimodal_prompt is not None, "Multimodal prompt is required"
+
+        loaded_model_processor: LoadedLlamaOmniModelProcessor = self._get_model(request.model_engine)
+        model = loaded_model_processor.model
+        tokenizer = loaded_model_processor.tokenizer
+
+        # The generation configs are taken from the official LLaMA-Omni repository
+        # https://github.com/ictnlp/LLaMA-Omni/blob/main/omni_speech/infer/infer.py#L116
+        generation_args = {
+            "max_new_tokens": 25,
+            "do_sample": False,
+            "use_cache": False,
+            "pad_token_id": self.PAD_ID,
+            "streaming_unit_gen": False,
+            "top_p": None,
+        }
+
+        input_text_query: Dict[str, str]
+        input_audio_query: Dict[str, Any]
+        prompt_text: str = ""
+
+        for media_object in request.multimodal_prompt.media_objects:
+            if media_object.is_type("audio") and media_object.location:
+                input_audio_query = {"audio": self._load_local_audio(media_object)}
+            elif media_object.is_type(TEXT_TYPE):
+                if media_object.text is None:
+                    raise ValueError("MediaObject of text type has missing text field value")
+                input_text_query = {"text": "<speech>\n" + media_object.text}
+                prompt_text += media_object.text
+            else:
+                raise ValueError(f"Unrecognized MediaObject type {media_object.type}")
+
+        completions: List[GeneratedOutput] = []
+        request_time: float = 0
+        request_datetime: Optional[int] = None
+        all_cached: bool = True
+
+        with htrack_block(f"Generating for prompt: {prompt_text}"):
+            for completion_index in range(request.num_completions):
+                try:
+
+                    def do_it() -> Dict[str, Any]:
+                        conv: Conversation = conv_templates[self.CONV_MODE].copy()
+                        conv.append_message(conv.roles[0], input_text_query["text"])
+                        conv.append_message(conv.roles[1], None)
+                        query: str = conv.get_prompt()
+                        # LLama-Omni requires a batch input
+                        text_inputs = (
+                            tokenizer_speech_token(query, tokenizer, return_tensors="pt").unsqueeze(0).to(self._device)
+                        )
+                        audio_inputs = (
+                            input_audio_query["audio"].to(dtype=torch.float16, device=self._device).unsqueeze(0)
+                        )
+                        speech_length = torch.LongTensor([audio_inputs.shape[1]])
+                        pred, _ = model.generate(
+                            text_inputs,
+                            audio_inputs,
+                            speech_length,
+                            None,
+                            None,
+                            None,
+                            None,
+                            None,
+                            None,
+                            None,
+                            None,
+                            False,
+                            None,
+                            None,
+                            **generation_args,
+                        )
+                        completion = tokenizer.decode(pred.cpu()[0], skip_special_tokens=True)
+                        tokens: List[str] = tokenizer.tokenize(completion)
+                        return {"output": (completion, tokens)}
+
+                    # Include the prompt and model name in the cache key
+                    cache_key = CachingClient.make_cache_key(
+                        raw_request={
+                            "completion_index": completion_index,
+                            "model": request.model,
+                            "prompt": generate_uid_for_multimodal_prompt(request.multimodal_prompt),
+                            **generation_args,
+                        },
+                        request=request,
+                    )
+                    result, cached = self.cache.get(cache_key, wrap_request_time(do_it))
+                except RuntimeError as model_error:
+                    return RequestResult(
+                        success=False, cached=False, error=str(model_error), completions=[], embedding=[]
+                    )
+
+                text, tokens = result["output"]
+
+                # Tokenize truncated text to get the list of tokens
+                completions.append(
+                    GeneratedOutput(
+                        text=text, logprob=0, tokens=[Token(text=str(token), logprob=0) for token in tokens]
+                    )
+                )
+
+                request_time += result["request_time"]
+                # Use the datetime from the first completion because that's when the request was fired
+                request_datetime = request_datetime or result.get("request_datetime")
+                all_cached = all_cached and cached
+
+        return RequestResult(
+            success=True,
+            cached=all_cached,
+            request_time=request_time,
+            request_datetime=request_datetime,
+            completions=completions,
+            embedding=[],
+        )

helm/clients/audio_language/qwen2_5_omni_client.py

@@ -0,0 +1,197 @@
+from threading import Lock
+import torch
+from typing import Any, Dict, List, Optional
+
+from dataclasses import dataclass
+from helm.clients.audio_language.qwen_omni.modeling_qwen2_5_omni import Qwen2_5OmniModel
+from helm.clients.audio_language.qwen_omni.processing_qwen2_5_omni import Qwen2_5OmniProcessor
+from helm.clients.audio_language.qwen_omni.qwen2_5_omni_utils.v2_5 import process_mm_info
+
+from helm.common.cache import CacheConfig
+from helm.common.gpu_utils import get_torch_device_name
+from helm.common.hierarchical_logger import hlog, htrack_block
+from helm.common.media_object import TEXT_TYPE
+from helm.common.request import Request, RequestResult, GeneratedOutput, Token
+from helm.common.request import wrap_request_time
+from helm.clients.client import CachingClient, generate_uid_for_multimodal_prompt
+
+
+@dataclass(frozen=True)
+class LoadedQwen2_5OmniModelProcessor:
+    """Loaded model and processor for Qwen."""
+
+    model: Qwen2_5OmniModel
+    tokenizer: Qwen2_5OmniProcessor
+
+
+_models_lock: Lock = Lock()
+_models: Dict[str, Optional[LoadedQwen2_5OmniModelProcessor]] = {
+    "Qwen/Qwen2.5-Omni-7B": None,
+}
+
+
+class Qwen2_5OmniAudioLMClient(CachingClient):
+    """
+    From https://huggingface.co/Qwen/Qwen2.5-Omni-7B,
+    Qwen2.5-Omni is an end-to-end multimodal model designed to perceive diverse modalities, including text,
+    images, audio, and video, while simultaneously generating text and natural speech responses in a streaming manner.
+
+    Paper: https://arxiv.org/abs/2503.20215
+    """
+
+    END_OF_TEXT_TOKEN: str = "<|endoftext|>>"
+
+    def __init__(self, cache_config: CacheConfig):
+        super().__init__(cache_config=cache_config)
+        self._device: str = get_torch_device_name()
+
+    def _get_model(self, helm_model_name: str) -> LoadedQwen2_5OmniModelProcessor:
+        global _models_lock
+        global _models
+
+        model_name: str
+        if helm_model_name == "qwen2.5-omni-7b":
+            model_name = "Qwen/Qwen2.5-Omni-7B"
+        else:
+            raise ValueError(f"Unhandled model name: {helm_model_name}")
+
+        # Ensure that only one thread is loading the model at a time
+        with _models_lock:
+            loaded_model_processor = _models[model_name]
+            if loaded_model_processor is None:
+                hlog(f"Loading model {model_name} and caching in memory...")
+                model = Qwen2_5OmniModel.from_pretrained(
+                    model_name,
+                    attn_implementation="flash_attention_2",
+                    torch_dtype=torch.bfloat16,
+                    device_map=self._device,
+                ).eval()
+                tokenizer = Qwen2_5OmniProcessor.from_pretrained(
+                    model_name,
+                )
+                _models[model_name] = LoadedQwen2_5OmniModelProcessor(model, tokenizer)
+                loaded_model_processor = _models[model_name]
+
+        assert loaded_model_processor is not None
+        return loaded_model_processor
+
+    def make_request(self, request: Request) -> RequestResult:
+        assert request.multimodal_prompt is not None, "Multimodal prompt is required"
+
+        loaded_model_processor: LoadedQwen2_5OmniModelProcessor = self._get_model(request.model_engine)
+        model = loaded_model_processor.model
+        tokenizer = loaded_model_processor.tokenizer
+
+        input_query: List[Dict[str, Any]] = []
+        query: List[Dict[str, str]] = []
+        prompt_text: str = ""
+
+        input_query.append(
+            {
+                "role": "system",
+                "content": (
+                    "You are Qwen, a virtual human developed by the Qwen Team,"
+                    " Alibaba Group, capable of perceiving auditory and visual inputs,"
+                    " as well as generating text and speech."
+                ),
+            }
+        )
+        # prompt_text += "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
+        for media_num, media_object in enumerate(request.multimodal_prompt.media_objects):
+            if media_object.is_type("audio") and media_object.location:
+                assert media_object.is_local_file, "Only local audio files are supported"
+                query.append({"type": "audio", "audio": media_object.location})
+
+                # prompt_text += f"<|im_start|>user\nAudio {media_num+1}: <|audio_bos|><|AUDIO|><|audio_eos|>\n"
+            elif media_object.is_type(TEXT_TYPE):
+                if media_object.text is None:
+                    raise ValueError("MediaObject of text type has missing text field value")
+                query.append({"type": "text", "text": media_object.text})
+                # prompt_text += media_object.text
+            else:
+                raise ValueError(f"Unrecognized MediaObject type {media_object.type}")
+        # prompt_text += "<|im_end|>\n<|im_start|>assistant\n"
+
+        input_query.append({"role": "user", "content": query})
+
+        completions: List[GeneratedOutput] = []
+        request_time: float = 0
+        request_datetime: Optional[int] = None
+        all_cached: bool = True
+
+        with htrack_block(f"Generating for prompt: {prompt_text}"):
+            for completion_index in range(request.num_completions):
+                try:
+
+                    def do_it() -> Dict[str, Any]:
+                        # Refer to the official Qwen2.5-Omni documentation for the format of the input query
+                        # https://huggingface.co/Qwen/Qwen2.5-Omni-7B
+                        USE_AUDIO_IN_VIDEO = True
+                        text = tokenizer.apply_chat_template(input_query, add_generation_prompt=True, tokenize=False)
+                        audios, images, videos = process_mm_info(input_query, use_audio_in_video=USE_AUDIO_IN_VIDEO)
+                        inputs = tokenizer(
+                            text=text,
+                            audios=audios,
+                            images=images,
+                            videos=videos,
+                            return_tensors="pt",
+                            padding=True,
+                            use_audio_in_video=USE_AUDIO_IN_VIDEO,
+                        )
+                        inputs = inputs.to(self._device, torch.bfloat16)
+                        input_seq_length = len(inputs.input_ids[0])
+                        # The model runs into errors when setting thinker_max_new_tokens to 1
+                        if request.max_tokens != 1:
+                            pred, _ = model.generate(**inputs, thinker_max_new_tokens=request.max_tokens)
+                            pred_decode = pred.cpu()[0][input_seq_length:]
+                        else:
+                            pred, _ = model.generate(**inputs)
+                            pred_decode = pred.cpu()[0][input_seq_length : input_seq_length + 1]
+                        completion = tokenizer.decode(
+                            pred_decode,
+                            skip_special_tokens=True,
+                            clean_up_tokenization_spaces=False,
+                        )
+                        # The processor of Qwen2-Audio-Instruct consists an AutoTokenizer and a WhisperFeatureExtractor
+                        tokens: List[str] = tokenizer.tokenizer.tokenize(completion)  # type: ignore
+                        return {"output": (completion, tokens)}
+
+                    # Include the prompt and model name in the cache key
+                    cache_key = CachingClient.make_cache_key(
+                        raw_request={
+                            "completion_index": completion_index,
+                            "model": request.model,
+                            "prompt": generate_uid_for_multimodal_prompt(request.multimodal_prompt),
+                            "max_tokens": request.max_tokens,
+                        },
+                        request=request,
+                    )
+                    result, cached = self.cache.get(cache_key, wrap_request_time(do_it))
+                except RuntimeError as model_error:
+                    return RequestResult(
+                        success=False, cached=False, error=str(model_error), completions=[], embedding=[]
+                    )
+
+                text, tokens = result["output"]
+                hlog(f"Generated: {text}")
+
+                # Tokenize truncated text to get the list of tokens
+                completions.append(
+                    GeneratedOutput(
+                        text=text, logprob=0, tokens=[Token(text=str(token), logprob=0) for token in tokens]
+                    )
+                )
+
+                request_time += result["request_time"]
+                # Use the datetime from the first completion because that's when the request was fired
+                request_datetime = request_datetime or result.get("request_datetime")
+                all_cached = all_cached and cached
+
+        return RequestResult(
+            success=True,
+            cached=all_cached,
+            request_time=request_time,
+            request_datetime=request_datetime,
+            completions=completions,
+            embedding=[],
+        )