crfm-helm 0.5.4__py3-none-any.whl → 0.5.6__py3-none-any.whl

helm/clients/audio_language/qwen2_audiolm_client.py

@@ -0,0 +1,190 @@
+from threading import Lock
+import librosa
+from typing import Any, Dict, List, Optional
+
+from dataclasses import dataclass
+from transformers import Qwen2AudioForConditionalGeneration, AutoProcessor
+
+from helm.common.cache import CacheConfig
+from helm.common.gpu_utils import get_torch_device_name
+from helm.common.hierarchical_logger import hlog, htrack_block
+from helm.common.media_object import TEXT_TYPE
+from helm.common.request import Request, RequestResult, GeneratedOutput, Token
+from helm.common.request import wrap_request_time
+from helm.clients.client import CachingClient, generate_uid_for_multimodal_prompt
+
+
+@dataclass(frozen=True)
+class LoadedQwenModelProcessor:
+    """Loaded model and processor for Qwen."""
+
+    model: Qwen2AudioForConditionalGeneration
+    tokenizer: AutoProcessor
+
+
+_models_lock: Lock = Lock()
+_models: Dict[str, Optional[LoadedQwenModelProcessor]] = {
+    "Qwen/Qwen2-Audio-7B-Instruct": None,
+}
+
+
+class Qwen2AudioLMClient(CachingClient):
+    """
+    From https://huggingface.co/Qwen/Qwen2-Audio-7B-Instruct,
+    Qwen2-Audio-Instruct (Qwen2 Large Vision Language Model) is the audito multimodal version of the large model series,
+    Qwen2 (abbr. Tongyi Qianwen), proposed by Alibaba Cloud. Qwen2-Audio-Instruct accepts audio, text as inputs,
+    outputs text.
+    Alibaba released Qwen-Audio and Qwen-Audio-Instruct, which is a instruction-following model based on Qwen-Audio.
+    We for now integrated Qwen2-Audio-Instruct for instruction-following tasks.
+
+    Paper: https://arxiv.org/abs/2407.10759
+    """
+
+    END_OF_TEXT_TOKEN: str = "<|im_end|>"
+
+    def __init__(self, cache_config: CacheConfig):
+        super().__init__(cache_config=cache_config)
+        self._device: str = get_torch_device_name()
+
+    def _get_model(self, helm_model_name: str) -> LoadedQwenModelProcessor:
+        global _models_lock
+        global _models
+
+        model_name: str
+        if helm_model_name == "qwen2-audio-7b-instruct":
+            model_name = "Qwen/Qwen2-Audio-7B-Instruct"
+        else:
+            raise ValueError(f"Unhandled model name: {helm_model_name}")
+
+        # Ensure that only one thread is loading the model at a time
+        with _models_lock:
+            loaded_model_processor = _models[model_name]
+            if loaded_model_processor is None:
+                hlog(f"Loading model {model_name} and caching in memory...")
+                model = Qwen2AudioForConditionalGeneration.from_pretrained(
+                    model_name,
+                    device_map=self._device,
+                ).eval()
+                tokenizer = AutoProcessor.from_pretrained(
+                    model_name,
+                )
+                _models[model_name] = LoadedQwenModelProcessor(model, tokenizer)
+                loaded_model_processor = _models[model_name]
+
+        assert loaded_model_processor is not None
+        return loaded_model_processor
+
+    def make_request(self, request: Request) -> RequestResult:
+        assert request.multimodal_prompt is not None, "Multimodal prompt is required"
+
+        loaded_model_processor: LoadedQwenModelProcessor = self._get_model(request.model_engine)
+        model = loaded_model_processor.model
+        tokenizer = loaded_model_processor.tokenizer
+
+        input_query: List[Dict[str, Any]] = []
+        query: List[Dict[str, str]] = []
+        prompt_text: str = ""
+
+        input_query.append({"role": "system", "content": "You are a helpful assistant."})
+        prompt_text += "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
+        for media_num, media_object in enumerate(request.multimodal_prompt.media_objects):
+            if media_object.is_type("audio") and media_object.location:
+                assert media_object.is_local_file, "Only local audio files are supported"
+                query.append({"type": "audio", "audio_url": media_object.location})
+
+                prompt_text += f"<|im_start|>user\nAudio {media_num+1}: <|audio_bos|><|AUDIO|><|audio_eos|>\n"
+            elif media_object.is_type(TEXT_TYPE):
+                if media_object.text is None:
+                    raise ValueError("MediaObject of text type has missing text field value")
+                query.append({"type": "text", "text": media_object.text})
+                prompt_text += media_object.text
+            else:
+                raise ValueError(f"Unrecognized MediaObject type {media_object.type}")
+        prompt_text += "<|im_end|>\n<|im_start|>assistant\n"
+
+        input_query.append({"role": "user", "content": query})
+        completions: List[GeneratedOutput] = []
+        request_time: float = 0
+        request_datetime: Optional[int] = None
+        all_cached: bool = True
+
+        with htrack_block(f"Generating for prompt: {prompt_text}"):
+            for completion_index in range(request.num_completions):
+                try:
+
+                    def do_it() -> Dict[str, Any]:
+                        inputs = tokenizer.apply_chat_template(  # type: ignore
+                            input_query, add_generation_prompt=True, tokenize=False
+                        )
+                        audios: List[Any] = []
+                        # Refer to the official Qwen2-Audio documentation for the format of the input query
+                        # https://huggingface.co/Qwen/Qwen2-Audio-7B-Instruct
+                        for message in input_query:
+                            if isinstance(message["content"], list):
+                                for element in message["content"]:
+                                    if element["type"] == "audio":
+                                        audios.append(
+                                            librosa.load(
+                                                element["audio_url"],
+                                                sr=tokenizer.feature_extractor.sampling_rate,  # type: ignore
+                                            )[0]
+                                        )
+                        inputs = tokenizer(  # type: ignore
+                            text=inputs,
+                            audios=audios,
+                            sampling_rate=tokenizer.feature_extractor.sampling_rate,  # type: ignore
+                            return_tensors="pt",
+                            padding=True,
+                        )
+                        input_length = inputs.input_ids.size(1)
+                        # Qwen2-Audio-Instruct counts input into the max_length,
+                        # so we need to add the length of the prompt
+                        inputs = inputs.to(self._device)
+                        pred = model.generate(**inputs, max_length=request.max_tokens + input_length)[:, input_length:]
+
+                        completion = tokenizer.decode(  # type: ignore
+                            pred.cpu()[0], skip_special_tokens=True, clean_up_tokenization_spaces=False
+                        )
+                        # The processor of Qwen2-Audio-Instruct consists an AutoTokenizer and a WhisperFeatureExtractor
+                        tokens: List[str] = tokenizer.tokenizer.tokenize(completion)  # type: ignore
+                        return {"output": (completion, tokens)}
+
+                    # Include the prompt and model name in the cache key
+                    cache_key = CachingClient.make_cache_key(
+                        raw_request={
+                            "completion_index": completion_index,
+                            "model": request.model,
+                            "prompt": generate_uid_for_multimodal_prompt(request.multimodal_prompt),
+                            "max_tokens": request.max_tokens,
+                        },
+                        request=request,
+                    )
+                    result, cached = self.cache.get(cache_key, wrap_request_time(do_it))
+                except RuntimeError as model_error:
+                    return RequestResult(
+                        success=False, cached=False, error=str(model_error), completions=[], embedding=[]
+                    )
+
+                text, tokens = result["output"]
+                hlog(f"Generated: {text}")
+
+                # Tokenize truncated text to get the list of tokens
+                completions.append(
+                    GeneratedOutput(
+                        text=text, logprob=0, tokens=[Token(text=str(token), logprob=0) for token in tokens]
+                    )
+                )
+
+                request_time += result["request_time"]
+                # Use the datetime from the first completion because that's when the request was fired
+                request_datetime = request_datetime or result.get("request_datetime")
+                all_cached = all_cached and cached
+
+        return RequestResult(
+            success=True,
+            cached=all_cached,
+            request_time=request_time,
+            request_datetime=request_datetime,
+            completions=completions,
+            embedding=[],
+        )

helm/clients/audio_language/qwen_audiolm_client.py

@@ -0,0 +1,152 @@
+from threading import Lock
+from typing import Any, Dict, List, Optional
+
+from dataclasses import dataclass
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from helm.common.cache import CacheConfig
+from helm.common.gpu_utils import get_torch_device_name
+from helm.common.hierarchical_logger import hlog, htrack_block
+from helm.common.media_object import TEXT_TYPE
+from helm.common.request import Request, RequestResult, GeneratedOutput, Token
+from helm.common.request import wrap_request_time
+from helm.clients.client import CachingClient, generate_uid_for_multimodal_prompt
+
+
+@dataclass(frozen=True)
+class LoadedQwenModelProcessor:
+    """Loaded model and processor for Qwen."""
+
+    model: AutoModelForCausalLM
+    tokenizer: AutoTokenizer
+
+
+_models_lock: Lock = Lock()
+_models: Dict[str, Optional[LoadedQwenModelProcessor]] = {
+    "Qwen/Qwen-Audio-Chat": None,
+}
+
+
+class QwenAudioLMClient(CachingClient):
+    """
+    From https://huggingface.co/Qwen/Qwen-Audio,
+    Qwen-Audio-Chat (Qwen Large Audio Language Model) is the audio multimodal version of the large model series,
+    Qwen (abbr. Tongyi Qianwen), proposed by Alibaba Cloud. Qwen-Audio-Chat accepts audio, text as inputs, outputs text.
+    Alibaba released Qwen-Audio and Qwen-Audio-Chat, which is a chatbot model based on Qwen-Audio.
+    We for now integrated Qwen-Audio-Chat for instruction-following tasks.
+
+    Paper: https://arxiv.org/abs/2311.07919
+    """
+
+    def __init__(self, cache_config: CacheConfig):
+        super().__init__(cache_config=cache_config)
+        self._device: str = get_torch_device_name()
+
+    def _get_model(self, helm_model_name: str) -> LoadedQwenModelProcessor:
+        global _models_lock
+        global _models
+
+        model_name: str
+        if helm_model_name == "qwen-audio-chat":
+            model_name = "Qwen/Qwen-Audio-Chat"
+        else:
+            raise ValueError(f"Unhandled model name: {helm_model_name}")
+
+        # Ensure that only one thread is loading the model at a time
+        with _models_lock:
+            loaded_model_processor = _models[model_name]
+            if loaded_model_processor is None:
+                hlog(f"Loading model {model_name} and caching in memory...")
+                model = AutoModelForCausalLM.from_pretrained(
+                    model_name,
+                    device_map=self._device,
+                    trust_remote_code=True,
+                    bf16=True,
+                ).eval()
+                tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+                _models[model_name] = LoadedQwenModelProcessor(model, tokenizer)
+                loaded_model_processor = _models[model_name]
+
+        assert loaded_model_processor is not None
+        return loaded_model_processor
+
+    def make_request(self, request: Request) -> RequestResult:
+        assert request.multimodal_prompt is not None, "Multimodal prompt is required"
+
+        loaded_model_processor: LoadedQwenModelProcessor = self._get_model(request.model_engine)
+        model = loaded_model_processor.model
+        tokenizer = loaded_model_processor.tokenizer
+
+        generation_args = {
+            "max_length": request.max_tokens,
+        }
+
+        query: List[Dict[str, str]] = []
+        prompt_text: str = ""
+
+        for media_object in request.multimodal_prompt.media_objects:
+            if media_object.is_type("audio") and media_object.location:
+                query.append({"audio": media_object.location})
+            elif media_object.is_type(TEXT_TYPE):
+                if media_object.text is None:
+                    raise ValueError("MediaObject of text type has missing text field value")
+
+                query.append({"text": media_object.text})
+                prompt_text += media_object.text
+            else:
+                raise ValueError(f"Unrecognized MediaObject type {media_object.type}")
+
+        completions: List[GeneratedOutput] = []
+        request_time: float = 0
+        request_datetime: Optional[int] = None
+        all_cached: bool = True
+
+        with htrack_block(f"Generating for prompt: {prompt_text}"):
+            for completion_index in range(request.num_completions):
+                try:
+
+                    def do_it() -> Dict[str, Any]:
+                        completion, _ = model.chat(  # type: ignore
+                            tokenizer, query=tokenizer.from_list_format(query), history=None  # type: ignore
+                        )
+                        tokens: List[str] = tokenizer.tokenize(completion)  # type: ignore
+                        return {"output": (completion, tokens)}
+
+                    # Include the prompt and model name in the cache key
+                    cache_key = CachingClient.make_cache_key(
+                        raw_request={
+                            "completion_index": completion_index,
+                            "model": request.model,
+                            "prompt": generate_uid_for_multimodal_prompt(request.multimodal_prompt),
+                            **generation_args,
+                        },
+                        request=request,
+                    )
+                    result, cached = self.cache.get(cache_key, wrap_request_time(do_it))
+                except RuntimeError as model_error:
+                    return RequestResult(
+                        success=False, cached=False, error=str(model_error), completions=[], embedding=[]
+                    )
+
+                text, tokens = result["output"]
+
+                # Tokenize truncated text to get the list of tokens
+                completions.append(
+                    GeneratedOutput(
+                        text=text, logprob=0, tokens=[Token(text=str(token), logprob=0) for token in tokens]
+                    )
+                )
+
+                request_time += result["request_time"]
+                # Use the datetime from the first completion because that's when the request was fired
+                request_datetime = request_datetime or result.get("request_datetime")
+                all_cached = all_cached and cached
+
+        return RequestResult(
+            success=True,
+            cached=all_cached,
+            request_time=request_time,
+            request_datetime=request_datetime,
+            completions=completions,
+            embedding=[],
+        )

helm/clients/audio_language/test.py

@@ -0,0 +1,62 @@
+import soundfile as sf
+
+from transformers import Qwen2_5OmniModel, Qwen2_5OmniProcessor  # type: ignore
+from qwen_omni_utils import process_mm_info
+
+# default: Load the model on the available device(s)
+model = Qwen2_5OmniModel.from_pretrained("Qwen/Qwen2.5-Omni-7B", torch_dtype="auto", device_map="auto")
+
+# We recommend enabling flash_attention_2 for better acceleration and memory saving.
+# model = Qwen2_5OmniModel.from_pretrained(
+#     "Qwen/Qwen2.5-Omni-7B",
+#     torch_dtype="auto",
+#     device_map="auto",
+#     attn_implementation="flash_attention_2",
+# )
+
+processor = Qwen2_5OmniProcessor.from_pretrained("Qwen/Qwen2.5-Omni-7B")
+
+conversation = [
+    {
+        "role": "system",
+        "content": (
+            "You are Qwen, a virtual human developed by the Qwen Team,"
+            " Alibaba Group, capable of perceiving auditory and visual"
+            " inputs, as well as generating text and speech."
+        ),
+    },
+    {
+        "role": "user",
+        "content": [
+            {"type": "video", "video": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-Omni/draw.mp4"},
+        ],
+    },
+]
+
+# set use audio in video
+USE_AUDIO_IN_VIDEO = True
+
+# Preparation for inference
+text = processor.apply_chat_template(conversation, add_generation_prompt=True, tokenize=False)
+audios, images, videos = process_mm_info(conversation, use_audio_in_video=USE_AUDIO_IN_VIDEO)
+inputs = processor(
+    text=text,
+    audios=audios,
+    images=images,
+    videos=videos,
+    return_tensors="pt",
+    padding=True,
+    use_audio_in_video=USE_AUDIO_IN_VIDEO,
+)
+inputs = inputs.to(model.device).to(model.dtype)
+
+# Inference: Generation of the output text and audio
+text_ids, audio = model.generate(**inputs, use_audio_in_video=USE_AUDIO_IN_VIDEO)
+
+text = processor.batch_decode(text_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)
+print(text)
+sf.write(
+    "output.wav",
+    audio.reshape(-1).detach().cpu().numpy(),
+    samplerate=24000,
+)

helm/clients/auto_client.py

@@ -71,6 +71,7 @@ class AutoClient(Client):
                 model_deployment.client_spec,
                 constant_bindings={
                     "cache_config": cache_config,
+                    "model_name": model_deployment.model_name,
                     "tokenizer_name": model_deployment.tokenizer_name,
                 },
                 provider_bindings={
@@ -81,6 +82,7 @@ class AutoClient(Client):
                     "org_id": lambda: self.credentials.get(
                         host_organization + "OrgId", None
                     ),  # OpenAI, GooseAI, Microsoft
+                    "base_url": lambda: self.credentials.get(host_organization + "BaseUrl", None),
                     "moderation_api_client": lambda: self.get_moderation_api_client(),  # OpenAI DALL-E
                     "lock_file_path": lambda: os.path.join(
                         self.file_storage_path, f"{host_organization}.lock"
@@ -131,7 +133,7 @@ class AutoClient(Client):
             return replace(last_attempt.value, error=f"{retry_error}. Error: {last_attempt.value.error}")
 
     def get_gcs_client(self):
-        from .gcs_client import GCSClient
+        from helm.clients.gcs_client import GCSClient
 
         bucket_name: str = self.credentials["gcsBucketName"]
         cache_config: CacheConfig = self.cache_backend_config.get_cache_config("gcs")
@@ -144,7 +146,7 @@ class AutoClient(Client):
         return NudityCheckClient(cache_config)
 
     def get_clip_score_client(self):
-        from .clip_score_client import CLIPScoreClient
+        from helm.clients.clip_score_client import CLIPScoreClient
 
         cache_config: CacheConfig = self.cache_backend_config.get_cache_config("clip_score")
         return CLIPScoreClient(cache_config)

helm/clients/azure_openai_client.py

@@ -0,0 +1,55 @@
+import os
+from typing import Dict, Optional
+
+from helm.clients.openai_client import OpenAIClient
+from helm.common.cache import CacheConfig
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.proxy.retry import NonRetriableException
+from helm.tokenizers.tokenizer import Tokenizer
+
+try:
+    from openai import AzureOpenAI
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["openai"])
+
+
+class AzureOpenAIClient(OpenAIClient):
+    API_VERSION = "2024-07-01-preview"
+
+    def __init__(
+        self,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        cache_config: CacheConfig,
+        api_key: Optional[str] = None,
+        endpoint: Optional[str] = None,
+        api_version: Optional[str] = None,
+        default_headers: Optional[Dict[str, str]] = None,
+        base_url: Optional[str] = None,
+        azure_openai_deployment_name: Optional[str] = None,
+    ):
+        super().__init__(
+            tokenizer=tokenizer,
+            tokenizer_name=tokenizer_name,
+            cache_config=cache_config,
+            api_key="unused",
+            openai_model_name=azure_openai_deployment_name,
+        )
+        azure_endpoint = endpoint or os.getenv("AZURE_OPENAI_ENDPOINT")
+        assert (base_url is None) != (azure_endpoint is None), "Exactly one of endoint or base_url must be provided"
+        if base_url:
+            self.client = AzureOpenAI(
+                api_key=api_key,
+                api_version=api_version or AzureOpenAIClient.API_VERSION,
+                base_url=base_url,
+                default_headers=default_headers,
+            )
+        elif azure_endpoint:
+            self.client = AzureOpenAI(
+                api_key=api_key,
+                api_version=api_version or AzureOpenAIClient.API_VERSION,
+                azure_endpoint=azure_endpoint,
+                default_headers=default_headers,
+            )
+        else:
+            raise NonRetriableException("Must provide either endpoint or base_url")