crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/clients/audio_language/llama_omni_client.py

@@ -0,0 +1,199 @@
+from threading import Lock
+import torch
+from typing import Any, Dict, List, Optional
+
+from dataclasses import dataclass
+from transformers import AutoTokenizer
+import whisper
+from helm.clients.audio_language.llama_omni.model.builder import load_pretrained_model as load_llama_omni
+from helm.clients.audio_language.llama_omni.model.language_model.omni_speech2s_llama import OmniSpeech2SLlamaForCausalLM
+from helm.clients.audio_language.llama_omni.conversation import conv_templates, Conversation
+from helm.clients.audio_language.llama_omni.preprocess import tokenizer_speech_token
+
+from helm.common.cache import CacheConfig
+from helm.common.gpu_utils import get_torch_device_name
+from helm.common.hierarchical_logger import hexception, hlog, htrack_block
+from helm.common.media_object import TEXT_TYPE
+from helm.common.request import Request, RequestResult, GeneratedOutput, Token
+from helm.common.request import wrap_request_time
+from helm.clients.client import CachingClient, generate_uid_for_multimodal_prompt
+
+
+@dataclass(frozen=True)
+class LoadedLlamaOmniModelProcessor:
+    """Loaded model and processor for Qwen."""
+
+    model: OmniSpeech2SLlamaForCausalLM
+    tokenizer: AutoTokenizer
+
+
+_models_lock: Lock = Lock()
+_models: Dict[str, Optional[LoadedLlamaOmniModelProcessor]] = {
+    "ICTNLP/Llama-3.1-8B-Omni": None,
+}
+
+
+class LlamaOmniAudioLMClient(CachingClient):
+    """
+    From https://github.com/ictnlp/LLaMA-Omni,
+    LLaMA-Omni is the audio multimodal version based on the LLaMA-3.1-8B large language model,
+    developed by ICTNLP group. LLaMA-Omni accepts audio, text as inputs, and outputs text.
+
+    Paper: https://arxiv.org/abs/2409.06666
+    """
+
+    END_OF_TEXT_TOKEN: str = "<|im_end|>"
+    CONV_MODE: str = "llama_3"
+    PAD_ID: int = 128004
+    MEL_NUM: int = 128
+
+    def __init__(self, cache_config: CacheConfig):
+        super().__init__(cache_config=cache_config)
+        self._device: str = get_torch_device_name()
+
+    def _get_model(self, helm_model_name: str) -> LoadedLlamaOmniModelProcessor:
+        global _models_lock
+        global _models
+
+        model_name: str
+        if helm_model_name == "llama-3.1-8b-omni":
+            model_name = "ICTNLP/Llama-3.1-8B-Omni"
+        else:
+            raise ValueError(f"Unhandled model name: {helm_model_name}")
+
+        # Ensure that only one thread is loading the model at a time
+        with _models_lock:
+            loaded_model_processor = _models[model_name]
+            if loaded_model_processor is None:
+                hlog(f"Loading model {model_name} and caching in memory...")
+                # Follow the official LLaMA-Omni model loading pattern:
+                # https://github.com/ictnlp/LLaMA-Omni/blob/main/omni_speech/infer/run.sh
+                tokenizer, model, _ = load_llama_omni(model_name, None, s2s=True)
+                _models[model_name] = LoadedLlamaOmniModelProcessor(model, tokenizer)
+                loaded_model_processor = _models[model_name]
+
+        assert loaded_model_processor is not None
+        return loaded_model_processor
+
+    def _load_local_audio(self, media_object) -> torch.Tensor:
+        assert media_object.is_local_file, "LLaMA-Omni only supports local audio file input"
+        audio_media = whisper.load_audio(media_object.location)
+        audio_media = whisper.pad_or_trim(audio_media)
+        audio_media = whisper.log_mel_spectrogram(audio_media, n_mels=self.MEL_NUM).permute(1, 0)
+        return audio_media
+
+    def make_request(self, request: Request) -> RequestResult:
+        assert request.multimodal_prompt is not None, "Multimodal prompt is required"
+
+        loaded_model_processor: LoadedLlamaOmniModelProcessor = self._get_model(request.model_engine)
+        model = loaded_model_processor.model
+        tokenizer = loaded_model_processor.tokenizer
+
+        # The generation configs are taken from the official LLaMA-Omni repository
+        # https://github.com/ictnlp/LLaMA-Omni/blob/main/omni_speech/infer/infer.py#L116
+        generation_args = {
+            "max_new_tokens": 25,
+            "do_sample": False,
+            "use_cache": False,
+            "pad_token_id": self.PAD_ID,
+            "streaming_unit_gen": False,
+            "top_p": None,
+        }
+
+        input_text_query: Dict[str, str]
+        input_audio_query: Dict[str, Any]
+        prompt_text: str = ""
+
+        for media_object in request.multimodal_prompt.media_objects:
+            if media_object.is_type("audio") and media_object.location:
+                input_audio_query = {"audio": self._load_local_audio(media_object)}
+            elif media_object.is_type(TEXT_TYPE):
+                if media_object.text is None:
+                    raise ValueError("MediaObject of text type has missing text field value")
+                input_text_query = {"text": "<speech>\n" + media_object.text}
+                prompt_text += media_object.text
+            else:
+                raise ValueError(f"Unrecognized MediaObject type {media_object.type}")
+
+        completions: List[GeneratedOutput] = []
+        request_time: float = 0
+        request_datetime: Optional[int] = None
+        all_cached: bool = True
+
+        with htrack_block(f"Generating for prompt: {prompt_text}"):
+            for completion_index in range(request.num_completions):
+                try:
+
+                    def do_it() -> Dict[str, Any]:
+                        conv: Conversation = conv_templates[self.CONV_MODE].copy()
+                        conv.append_message(conv.roles[0], input_text_query["text"])
+                        conv.append_message(conv.roles[1], None)
+                        query: str = conv.get_prompt()
+                        # LLama-Omni requires a batch input
+                        text_inputs = (
+                            tokenizer_speech_token(query, tokenizer, return_tensors="pt").unsqueeze(0).to(self._device)
+                        )
+                        audio_inputs = (
+                            input_audio_query["audio"].to(dtype=torch.float16, device=self._device).unsqueeze(0)
+                        )
+                        speech_length = torch.LongTensor([audio_inputs.shape[1]])
+                        pred, _ = model.generate(
+                            text_inputs,
+                            audio_inputs,
+                            speech_length,
+                            None,
+                            None,
+                            None,
+                            None,
+                            None,
+                            None,
+                            None,
+                            None,
+                            False,
+                            None,
+                            None,
+                            **generation_args,
+                        )
+                        completion = tokenizer.decode(pred.cpu()[0], skip_special_tokens=True)
+                        tokens: List[str] = tokenizer.tokenize(completion)
+                        return {"output": (completion, tokens)}
+
+                    # Include the prompt and model name in the cache key
+                    cache_key = CachingClient.make_cache_key(
+                        raw_request={
+                            "completion_index": completion_index,
+                            "model": request.model,
+                            "prompt": generate_uid_for_multimodal_prompt(request.multimodal_prompt),
+                            **generation_args,
+                        },
+                        request=request,
+                    )
+                    result, cached = self.cache.get(cache_key, wrap_request_time(do_it))
+                except RuntimeError as model_error:
+                    hexception(model_error)
+                    return RequestResult(
+                        success=False, cached=False, error=str(model_error), completions=[], embedding=[]
+                    )
+
+                text, tokens = result["output"]
+
+                # Tokenize truncated text to get the list of tokens
+                completions.append(
+                    GeneratedOutput(
+                        text=text, logprob=0, tokens=[Token(text=str(token), logprob=0) for token in tokens]
+                    )
+                )
+
+                request_time += result["request_time"]
+                # Use the datetime from the first completion because that's when the request was fired
+                request_datetime = request_datetime or result.get("request_datetime")
+                all_cached = all_cached and cached
+
+        return RequestResult(
+            success=True,
+            cached=all_cached,
+            request_time=request_time,
+            request_datetime=request_datetime,
+            completions=completions,
+            embedding=[],
+        )

helm/clients/audio_language/qwen2_5_omni_client.py

@@ -0,0 +1,210 @@
+from threading import Lock
+import torch
+from typing import Any, Dict, List, Optional
+
+from dataclasses import dataclass
+from helm.clients.audio_language.qwen_omni.modeling_qwen2_5_omni import Qwen2_5OmniModel
+from helm.clients.audio_language.qwen_omni.processing_qwen2_5_omni import Qwen2_5OmniProcessor
+from helm.clients.audio_language.qwen_omni.qwen2_5_omni_utils.v2_5 import process_mm_info
+
+from helm.common.cache import CacheConfig
+from helm.common.gpu_utils import get_torch_device_name
+from helm.common.hierarchical_logger import hexception, hlog, htrack_block
+from helm.common.media_object import TEXT_TYPE
+from helm.common.request import Request, RequestResult, GeneratedOutput, Token
+from helm.common.request import wrap_request_time
+from helm.clients.client import CachingClient, generate_uid_for_multimodal_prompt
+
+
+@dataclass(frozen=True)
+class LoadedQwen2_5OmniModelProcessor:
+    """Loaded model and processor for Qwen."""
+
+    model: Qwen2_5OmniModel
+    tokenizer: Qwen2_5OmniProcessor
+
+
+_models_lock: Lock = Lock()
+_models: Dict[str, Optional[LoadedQwen2_5OmniModelProcessor]] = {
+    "Qwen/Qwen2.5-Omni-7B": None,
+    "Qwen/Qwen2.5-Omni-3B": None,
+}
+
+
+class Qwen2_5OmniAudioLMClient(CachingClient):
+    """
+    From https://huggingface.co/Qwen/Qwen2.5-Omni-7B,
+    Qwen2.5-Omni is an end-to-end multimodal model designed to perceive diverse modalities, including text,
+    images, audio, and video, while simultaneously generating text and natural speech responses in a streaming manner.
+
+    Paper: https://arxiv.org/abs/2503.20215
+    """
+
+    END_OF_TEXT_TOKEN: str = "<|endoftext|>"
+
+    def __init__(self, cache_config: CacheConfig):
+        super().__init__(cache_config=cache_config)
+        self._device: str = get_torch_device_name()
+
+    def _get_model(self, helm_model_name: str) -> LoadedQwen2_5OmniModelProcessor:
+        global _models_lock
+        global _models
+
+        model_name: str
+        if helm_model_name == "qwen2.5-omni-7b":
+            model_name = "Qwen/Qwen2.5-Omni-7B"
+        elif helm_model_name == "qwen2.5-omni-3b":
+            model_name = "Qwen/Qwen2.5-Omni-3B"
+        else:
+            raise ValueError(f"Unhandled model name: {helm_model_name}")
+
+        # Ensure that only one thread is loading the model at a time
+        with _models_lock:
+            loaded_model_processor = _models[model_name]
+            if loaded_model_processor is None:
+                hlog(f"Loading model {model_name} and caching in memory...")
+                if torch.cuda.get_device_capability()[0] >= 8:
+                    # Use flash attention 2 for A100 and H100 GPUs
+                    model = Qwen2_5OmniModel.from_pretrained(
+                        model_name,
+                        attn_implementation="flash_attention_2",
+                        torch_dtype=torch.bfloat16,
+                        device_map=self._device,
+                    ).eval()
+                else:
+                    # Use default attention for other GPUs
+                    model = Qwen2_5OmniModel.from_pretrained(
+                        model_name,
+                        torch_dtype=torch.float16,
+                        device_map=self._device,
+                    ).eval()
+                tokenizer = Qwen2_5OmniProcessor.from_pretrained(
+                    model_name,
+                )
+                _models[model_name] = LoadedQwen2_5OmniModelProcessor(model, tokenizer)
+                loaded_model_processor = _models[model_name]
+
+        assert loaded_model_processor is not None
+        return loaded_model_processor
+
+    def make_request(self, request: Request) -> RequestResult:
+        assert request.multimodal_prompt is not None, "Multimodal prompt is required"
+
+        loaded_model_processor: LoadedQwen2_5OmniModelProcessor = self._get_model(request.model_engine)
+        model = loaded_model_processor.model
+        tokenizer = loaded_model_processor.tokenizer
+
+        input_query: List[Dict[str, Any]] = []
+        query: List[Dict[str, str]] = []
+        prompt_text: str = ""
+
+        input_query.append(
+            {
+                "role": "system",
+                "content": (
+                    "You are Qwen, a virtual human developed by the Qwen Team,"
+                    " Alibaba Group, capable of perceiving auditory and visual inputs,"
+                    " as well as generating text and speech."
+                ),
+            }
+        )
+        # prompt_text += "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
+        for media_num, media_object in enumerate(request.multimodal_prompt.media_objects):
+            if media_object.is_type("audio") and media_object.location:
+                assert media_object.is_local_file, "Only local audio files are supported"
+                query.append({"type": "audio", "audio": media_object.location})
+
+                # prompt_text += f"<|im_start|>user\nAudio {media_num+1}: <|audio_bos|><|AUDIO|><|audio_eos|>\n"
+            elif media_object.is_type(TEXT_TYPE):
+                if media_object.text is None:
+                    raise ValueError("MediaObject of text type has missing text field value")
+                query.append({"type": "text", "text": media_object.text})
+                # prompt_text += media_object.text
+            else:
+                raise ValueError(f"Unrecognized MediaObject type {media_object.type}")
+        # prompt_text += "<|im_end|>\n<|im_start|>assistant\n"
+
+        input_query.append({"role": "user", "content": query})
+
+        completions: List[GeneratedOutput] = []
+        request_time: float = 0
+        request_datetime: Optional[int] = None
+        all_cached: bool = True
+
+        with htrack_block(f"Generating for prompt: {prompt_text}"):
+            for completion_index in range(request.num_completions):
+                try:
+
+                    def do_it() -> Dict[str, Any]:
+                        # Refer to the official Qwen2.5-Omni documentation for the format of the input query
+                        # https://huggingface.co/Qwen/Qwen2.5-Omni-7B
+                        USE_AUDIO_IN_VIDEO = True
+                        text = tokenizer.apply_chat_template(input_query, add_generation_prompt=True, tokenize=False)
+                        audios, images, videos = process_mm_info(input_query, use_audio_in_video=USE_AUDIO_IN_VIDEO)
+                        inputs = tokenizer(
+                            text=text,
+                            audios=audios,
+                            images=images,
+                            videos=videos,
+                            return_tensors="pt",
+                            padding=True,
+                            use_audio_in_video=USE_AUDIO_IN_VIDEO,
+                        )
+                        inputs = inputs.to(self._device, torch.bfloat16)
+                        input_seq_length = len(inputs.input_ids[0])
+                        # The model runs into errors when setting thinker_max_new_tokens to 1
+                        if request.max_tokens != 1:
+                            pred, _ = model.generate(**inputs, thinker_max_new_tokens=request.max_tokens)
+                            pred_decode = pred.cpu()[0][input_seq_length:]
+                        else:
+                            pred, _ = model.generate(**inputs)
+                            pred_decode = pred.cpu()[0][input_seq_length : input_seq_length + 1]
+                        completion = tokenizer.decode(
+                            pred_decode,
+                            skip_special_tokens=True,
+                            clean_up_tokenization_spaces=False,
+                        )
+                        # The processor of Qwen2-Audio-Instruct consists an AutoTokenizer and a WhisperFeatureExtractor
+                        tokens: List[str] = tokenizer.tokenizer.tokenize(completion)  # type: ignore
+                        return {"output": (completion, tokens)}
+
+                    # Include the prompt and model name in the cache key
+                    cache_key = CachingClient.make_cache_key(
+                        raw_request={
+                            "completion_index": completion_index,
+                            "model": request.model,
+                            "prompt": generate_uid_for_multimodal_prompt(request.multimodal_prompt),
+                            "max_tokens": request.max_tokens,
+                        },
+                        request=request,
+                    )
+                    result, cached = self.cache.get(cache_key, wrap_request_time(do_it))
+                except RuntimeError as model_error:
+                    hexception(model_error)
+                    return RequestResult(
+                        success=False, cached=False, error=str(model_error), completions=[], embedding=[]
+                    )
+
+                text, tokens = result["output"]
+                hlog(f"Generated: {text}")
+
+                # Tokenize truncated text to get the list of tokens
+                completions.append(
+                    GeneratedOutput(
+                        text=text, logprob=0, tokens=[Token(text=str(token), logprob=0) for token in tokens]
+                    )
+                )
+
+                request_time += result["request_time"]
+                # Use the datetime from the first completion because that's when the request was fired
+                request_datetime = request_datetime or result.get("request_datetime")
+                all_cached = all_cached and cached
+
+        return RequestResult(
+            success=True,
+            cached=all_cached,
+            request_time=request_time,
+            request_datetime=request_datetime,
+            completions=completions,
+            embedding=[],
+        )

helm/clients/audio_language/qwen2_audiolm_client.py

@@ -0,0 +1,191 @@
+from threading import Lock
+import librosa
+from typing import Any, Dict, List, Optional
+
+from dataclasses import dataclass
+from transformers import Qwen2AudioForConditionalGeneration, AutoProcessor
+
+from helm.common.cache import CacheConfig
+from helm.common.gpu_utils import get_torch_device_name
+from helm.common.hierarchical_logger import hexception, hlog, htrack_block
+from helm.common.media_object import TEXT_TYPE
+from helm.common.request import Request, RequestResult, GeneratedOutput, Token
+from helm.common.request import wrap_request_time
+from helm.clients.client import CachingClient, generate_uid_for_multimodal_prompt
+
+
+@dataclass(frozen=True)
+class LoadedQwenModelProcessor:
+    """Loaded model and processor for Qwen."""
+
+    model: Qwen2AudioForConditionalGeneration
+    tokenizer: AutoProcessor
+
+
+_models_lock: Lock = Lock()
+_models: Dict[str, Optional[LoadedQwenModelProcessor]] = {
+    "Qwen/Qwen2-Audio-7B-Instruct": None,
+}
+
+
+class Qwen2AudioLMClient(CachingClient):
+    """
+    From https://huggingface.co/Qwen/Qwen2-Audio-7B-Instruct,
+    Qwen2-Audio-Instruct (Qwen2 Large Vision Language Model) is the audito multimodal version of the large model series,
+    Qwen2 (abbr. Tongyi Qianwen), proposed by Alibaba Cloud. Qwen2-Audio-Instruct accepts audio, text as inputs,
+    outputs text.
+    Alibaba released Qwen-Audio and Qwen-Audio-Instruct, which is a instruction-following model based on Qwen-Audio.
+    We for now integrated Qwen2-Audio-Instruct for instruction-following tasks.
+
+    Paper: https://arxiv.org/abs/2407.10759
+    """
+
+    END_OF_TEXT_TOKEN: str = "<|im_end|>"
+
+    def __init__(self, cache_config: CacheConfig):
+        super().__init__(cache_config=cache_config)
+        self._device: str = get_torch_device_name()
+
+    def _get_model(self, helm_model_name: str) -> LoadedQwenModelProcessor:
+        global _models_lock
+        global _models
+
+        model_name: str
+        if helm_model_name == "qwen2-audio-7b-instruct":
+            model_name = "Qwen/Qwen2-Audio-7B-Instruct"
+        else:
+            raise ValueError(f"Unhandled model name: {helm_model_name}")
+
+        # Ensure that only one thread is loading the model at a time
+        with _models_lock:
+            loaded_model_processor = _models[model_name]
+            if loaded_model_processor is None:
+                hlog(f"Loading model {model_name} and caching in memory...")
+                model = Qwen2AudioForConditionalGeneration.from_pretrained(
+                    model_name,
+                    device_map=self._device,
+                ).eval()
+                tokenizer = AutoProcessor.from_pretrained(
+                    model_name,
+                )
+                _models[model_name] = LoadedQwenModelProcessor(model, tokenizer)
+                loaded_model_processor = _models[model_name]
+
+        assert loaded_model_processor is not None
+        return loaded_model_processor
+
+    def make_request(self, request: Request) -> RequestResult:
+        assert request.multimodal_prompt is not None, "Multimodal prompt is required"
+
+        loaded_model_processor: LoadedQwenModelProcessor = self._get_model(request.model_engine)
+        model = loaded_model_processor.model
+        tokenizer = loaded_model_processor.tokenizer
+
+        input_query: List[Dict[str, Any]] = []
+        query: List[Dict[str, str]] = []
+        prompt_text: str = ""
+
+        input_query.append({"role": "system", "content": "You are a helpful assistant."})
+        prompt_text += "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
+        for media_num, media_object in enumerate(request.multimodal_prompt.media_objects):
+            if media_object.is_type("audio") and media_object.location:
+                assert media_object.is_local_file, "Only local audio files are supported"
+                query.append({"type": "audio", "audio_url": media_object.location})
+
+                prompt_text += f"<|im_start|>user\nAudio {media_num+1}: <|audio_bos|><|AUDIO|><|audio_eos|>\n"
+            elif media_object.is_type(TEXT_TYPE):
+                if media_object.text is None:
+                    raise ValueError("MediaObject of text type has missing text field value")
+                query.append({"type": "text", "text": media_object.text})
+                prompt_text += media_object.text
+            else:
+                raise ValueError(f"Unrecognized MediaObject type {media_object.type}")
+        prompt_text += "<|im_end|>\n<|im_start|>assistant\n"
+
+        input_query.append({"role": "user", "content": query})
+        completions: List[GeneratedOutput] = []
+        request_time: float = 0
+        request_datetime: Optional[int] = None
+        all_cached: bool = True
+
+        with htrack_block(f"Generating for prompt: {prompt_text}"):
+            for completion_index in range(request.num_completions):
+                try:
+
+                    def do_it() -> Dict[str, Any]:
+                        inputs = tokenizer.apply_chat_template(  # type: ignore
+                            input_query, add_generation_prompt=True, tokenize=False
+                        )
+                        audios: List[Any] = []
+                        # Refer to the official Qwen2-Audio documentation for the format of the input query
+                        # https://huggingface.co/Qwen/Qwen2-Audio-7B-Instruct
+                        for message in input_query:
+                            if isinstance(message["content"], list):
+                                for element in message["content"]:
+                                    if element["type"] == "audio":
+                                        audios.append(
+                                            librosa.load(
+                                                element["audio_url"],
+                                                sr=tokenizer.feature_extractor.sampling_rate,  # type: ignore
+                                            )[0]
+                                        )
+                        inputs = tokenizer(  # type: ignore
+                            text=inputs,
+                            audios=audios,
+                            sampling_rate=tokenizer.feature_extractor.sampling_rate,  # type: ignore
+                            return_tensors="pt",
+                            padding=True,
+                        )
+                        input_length = inputs.input_ids.size(1)
+                        # Qwen2-Audio-Instruct counts input into the max_length,
+                        # so we need to add the length of the prompt
+                        inputs = inputs.to(self._device)
+                        pred = model.generate(**inputs, max_length=request.max_tokens + input_length)[:, input_length:]
+
+                        completion = tokenizer.decode(  # type: ignore
+                            pred.cpu()[0], skip_special_tokens=True, clean_up_tokenization_spaces=False
+                        )
+                        # The processor of Qwen2-Audio-Instruct consists an AutoTokenizer and a WhisperFeatureExtractor
+                        tokens: List[str] = tokenizer.tokenizer.tokenize(completion)  # type: ignore
+                        return {"output": (completion, tokens)}
+
+                    # Include the prompt and model name in the cache key
+                    cache_key = CachingClient.make_cache_key(
+                        raw_request={
+                            "completion_index": completion_index,
+                            "model": request.model,
+                            "prompt": generate_uid_for_multimodal_prompt(request.multimodal_prompt),
+                            "max_tokens": request.max_tokens,
+                        },
+                        request=request,
+                    )
+                    result, cached = self.cache.get(cache_key, wrap_request_time(do_it))
+                except RuntimeError as model_error:
+                    hexception(model_error)
+                    return RequestResult(
+                        success=False, cached=False, error=str(model_error), completions=[], embedding=[]
+                    )
+
+                text, tokens = result["output"]
+                hlog(f"Generated: {text}")
+
+                # Tokenize truncated text to get the list of tokens
+                completions.append(
+                    GeneratedOutput(
+                        text=text, logprob=0, tokens=[Token(text=str(token), logprob=0) for token in tokens]
+                    )
+                )
+
+                request_time += result["request_time"]
+                # Use the datetime from the first completion because that's when the request was fired
+                request_datetime = request_datetime or result.get("request_datetime")
+                all_cached = all_cached and cached
+
+        return RequestResult(
+            success=True,
+            cached=all_cached,
+            request_time=request_time,
+            request_datetime=request_datetime,
+            completions=completions,
+            embedding=[],
+        )