crfm-helm 0.5.5__py3-none-any.whl → 0.5.7__py3-none-any.whl

helm/clients/audio_language/test.py

@@ -0,0 +1,62 @@
+import soundfile as sf
+
+from transformers import Qwen2_5OmniModel, Qwen2_5OmniProcessor  # type: ignore
+from qwen_omni_utils import process_mm_info
+
+# default: Load the model on the available device(s)
+model = Qwen2_5OmniModel.from_pretrained("Qwen/Qwen2.5-Omni-7B", torch_dtype="auto", device_map="auto")
+
+# We recommend enabling flash_attention_2 for better acceleration and memory saving.
+# model = Qwen2_5OmniModel.from_pretrained(
+#     "Qwen/Qwen2.5-Omni-7B",
+#     torch_dtype="auto",
+#     device_map="auto",
+#     attn_implementation="flash_attention_2",
+# )
+
+processor = Qwen2_5OmniProcessor.from_pretrained("Qwen/Qwen2.5-Omni-7B")
+
+conversation = [
+    {
+        "role": "system",
+        "content": (
+            "You are Qwen, a virtual human developed by the Qwen Team,"
+            " Alibaba Group, capable of perceiving auditory and visual"
+            " inputs, as well as generating text and speech."
+        ),
+    },
+    {
+        "role": "user",
+        "content": [
+            {"type": "video", "video": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-Omni/draw.mp4"},
+        ],
+    },
+]
+
+# set use audio in video
+USE_AUDIO_IN_VIDEO = True
+
+# Preparation for inference
+text = processor.apply_chat_template(conversation, add_generation_prompt=True, tokenize=False)
+audios, images, videos = process_mm_info(conversation, use_audio_in_video=USE_AUDIO_IN_VIDEO)
+inputs = processor(
+    text=text,
+    audios=audios,
+    images=images,
+    videos=videos,
+    return_tensors="pt",
+    padding=True,
+    use_audio_in_video=USE_AUDIO_IN_VIDEO,
+)
+inputs = inputs.to(model.device).to(model.dtype)
+
+# Inference: Generation of the output text and audio
+text_ids, audio = model.generate(**inputs, use_audio_in_video=USE_AUDIO_IN_VIDEO)
+
+text = processor.batch_decode(text_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)
+print(text)
+sf.write(
+    "output.wav",
+    audio.reshape(-1).detach().cpu().numpy(),
+    samplerate=24000,
+)

helm/clients/bedrock_client.py

@@ -117,10 +117,12 @@ class BedrockNovaClient(CachingClient):
         tokenizer_name: str,
         assumed_role: Optional[str] = None,
         region: Optional[str] = None,
+        bedrock_model_id: Optional[str] = None,
     ):
         super().__init__(cache_config=cache_config)
         self.tokenizer = tokenizer
         self.tokenizer_name = tokenizer_name
+        self.bedrock_model_id = bedrock_model_id
         self.bedrock_client = get_bedrock_client_v1(
             assumed_role=assumed_role or os.environ.get("BEDROCK_ASSUME_ROLE", None),
             region=region,
@@ -144,7 +146,7 @@ class BedrockNovaClient(CachingClient):
         messages = self._get_messages_from_request(request)
 
         return {
-            "modelId": model_id,
+            "modelId": self.bedrock_model_id or model_id,
             "inferenceConfig": {
                 "temperature": request.temperature,
                 "maxTokens": request.max_tokens,

helm/clients/client.py

@@ -2,7 +2,7 @@ import json
 from abc import ABC, abstractmethod
 from typing import List, Mapping, Optional, cast
 
-from helm.common.hierarchical_logger import hlog
+from helm.common.hierarchical_logger import hwarn
 from helm.common.media_object import MultimediaObject, TEXT_TYPE
 from helm.common.request import Request, RequestResult, GeneratedOutput, Token
 from helm.common.cache import Cache, CacheConfig
@@ -65,7 +65,7 @@ def truncate_sequence(
     # where max_tokens = 0, so there's nothing to truncate.
     if request.echo_prompt:
         if request.max_tokens != 0:
-            hlog("WARNING: don't know how to handle echo_prompt and max_tokens > 0, not truncating")
+            hwarn("don't know how to handle echo_prompt and max_tokens > 0, not truncating")
         return sequence
 
     if end_of_text_token:
@@ -90,8 +90,8 @@ def truncate_sequence(
             new_tokens.append(token)
 
         if len(new_text) < len(sequence.text) and len(new_tokens) == len(sequence.tokens):
-            hlog(
-                f"WARNING: Stripped characters from text ({len(sequence.text)} -> {len(new_text)}), "
+            hwarn(
+                f"Stripped characters from text ({len(sequence.text)} -> {len(new_text)}), "
                 f"but wasn't able to strip the tokens"
             )
 
@@ -99,14 +99,14 @@ def truncate_sequence(
         new_logprob = sum(token.logprob for token in new_tokens)
 
         if print_warning:
-            hlog(f"WARNING: truncate_sequence needs to strip {json.dumps(stop)}")
+            hwarn(f"truncate_sequence needs to strip {json.dumps(stop)}")
 
         sequence = GeneratedOutput(text=new_text, logprob=new_logprob, tokens=new_tokens)
 
     # Truncate based on the max number of tokens.
     if len(sequence.tokens) > request.max_tokens:
         if print_warning:
-            hlog(f"WARNING: truncate_sequence needs to truncate {len(sequence.tokens)} down to {request.max_tokens}")
+            hwarn(f"truncate_sequence needs to truncate {len(sequence.tokens)} down to {request.max_tokens}")
         new_tokens = sequence.tokens[: request.max_tokens]
 
         # This is imperfect stitching together of tokens, so just to make sure this is okay
@@ -114,7 +114,7 @@ def truncate_sequence(
         # Usually, in our benchmark, max_tokens is active when it's 1, so hopefully this isn't an issue.
         new_text = "".join(token.text for token in new_tokens)
         if not sequence.text.startswith(new_text):
-            hlog(f"WARNING: {json.dumps(sequence.text)} does not start with truncated text {json.dumps(new_text)}")
+            hwarn(f"{json.dumps(sequence.text)} does not start with truncated text {json.dumps(new_text)}")
 
         new_logprob = sum(token.logprob for token in new_tokens)
 

helm/clients/grok_client.py

@@ -0,0 +1,36 @@
+from typing import Any, Dict, Optional
+
+from helm.clients.openai_client import OpenAIClient
+from helm.common.cache import CacheConfig
+from helm.common.request import Request
+from helm.tokenizers.tokenizer import Tokenizer
+
+
+class GrokChatClient(OpenAIClient):
+
+    BASE_URL = "https://api.x.ai/v1"
+
+    _UNSUPPORTED_ARGUMENTS = ["presence_penalty", "frequency_penalty"]
+
+    def __init__(
+        self,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        cache_config: CacheConfig,
+        api_key: Optional[str] = None,
+    ):
+        super().__init__(
+            tokenizer=tokenizer,
+            tokenizer_name=tokenizer_name,
+            cache_config=cache_config,
+            api_key=api_key,
+            org_id=None,
+            base_url="https://api.x.ai/v1",
+        )
+
+    def _make_chat_raw_request(self, request: Request) -> Dict[str, Any]:
+        raw_request = super()._make_chat_raw_request(request)
+        for unsupported_argument in self._UNSUPPORTED_ARGUMENTS:
+            if unsupported_argument in raw_request:
+                del raw_request[unsupported_argument]
+        return raw_request

helm/clients/huggingface_client.py

@@ -8,7 +8,7 @@ from transformers.generation.stopping_criteria import (
 from typing import Any, Dict, List, Optional, TypedDict
 
 from helm.common.cache import CacheConfig
-from helm.common.hierarchical_logger import htrack_block, hlog
+from helm.common.hierarchical_logger import htrack_block, hlog, hwarn
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.common.request import (
     wrap_request_time,
@@ -18,6 +18,7 @@ from helm.common.request import (
     GeneratedOutput,
     Token,
 )
+from helm.proxy.retry import NonRetriableException
 from helm.tokenizers.tokenizer import Tokenizer
 from helm.clients.client import CachingClient, truncate_sequence
 from helm.tokenizers.huggingface_tokenizer import HuggingFaceTokenizer, WrappedPreTrainedTokenizer
@@ -256,6 +257,7 @@ class HuggingFaceClient(CachingClient):
         tokenizer: Tokenizer,
         pretrained_model_name_or_path: Optional[str] = None,
         end_of_text_token: Optional[str] = None,
+        apply_chat_template: Optional[bool] = None,
         **kwargs,
     ):
         super().__init__(cache_config=cache_config)
@@ -266,9 +268,46 @@ class HuggingFaceClient(CachingClient):
                 "but instead it is {tokenizer}"
             )
         self._wrapped_tokenizer: WrappedPreTrainedTokenizer = tokenizer.get_wrapped_tokenizer()
-        self._tokenizer = tokenizer
         self._kwargs = _process_huggingface_client_kwargs(kwargs)
         self._end_of_text_token = end_of_text_token
+        # If the user did not explicitly configure whether the model is a chat model with `apply_chat_template` arg,
+        # auto-infer if the model is a chat model based on whether the tokenizer has a chat template.
+        # Note: Auto-inference is incorrect for some non-chat models that still have chat templates
+        # e.g. Qwen2, Qwen 2.5.
+        # For these models, the `apply_chat_template` arg should be explicitly set to false.
+        if apply_chat_template is not None:
+            self._apply_chat_template = apply_chat_template
+        else:
+            with self._wrapped_tokenizer as hf_tokenizer:
+                self._apply_chat_template = bool(hf_tokenizer.chat_template)
+                hwarn(
+                    f"Automatically set `apply_chat_template` to {self._apply_chat_template} based on "
+                    "whether the tokenizer has a chat template. "
+                    "If this is incorrect, please explicitly set `apply_chat_template`."
+                )
+
+    def get_prompt(self, request: Request) -> str:
+        if request.prompt and request.messages:
+            raise NonRetriableException(f"More than one of `prompt` and `messages` was set in request: {request}")
+        # Chat model expects a list of messages as input
+        if self._apply_chat_template:
+            with self._wrapped_tokenizer as tokenizer:
+                if request.messages:
+                    prompt = tokenizer.apply_chat_template(request.messages, tokenize=False, add_generation_prompt=True)
+                    assert isinstance(prompt, str)
+                    return prompt
+                else:
+                    prompt = tokenizer.apply_chat_template(
+                        [{"role": "user", "content": request.prompt}], tokenize=False, add_generation_prompt=True
+                    )
+                    assert isinstance(prompt, str)
+                    return prompt
+        # Base non-chat model expects a string as input
+        else:
+            if request.messages:
+                raise NonRetriableException("Chat mesages not supported by non-chat model")
+            else:
+                return request.prompt
 
     def make_request(self, request: Request) -> RequestResult:
         # Embedding not supported for this model
@@ -277,7 +316,7 @@ class HuggingFaceClient(CachingClient):
 
         raw_request: HuggingFaceRequest = {
             "engine": request.model_engine,
-            "prompt": request.prompt,
+            "prompt": self.get_prompt(request),
             "temperature": 1e-7 if request.temperature == 0 else request.temperature,
             "num_return_sequences": request.num_completions,
             "max_new_tokens": request.max_tokens,

helm/clients/huggingface_pipeline_client.py

@@ -0,0 +1,138 @@
+from threading import Lock
+from typing import Any, Dict, List, Optional, Union
+
+import transformers
+
+from helm.clients.client import CachingClient
+from helm.common.cache import CacheConfig
+from helm.common.hierarchical_logger import htrack_block, hwarn
+from helm.common.request import GeneratedOutput, Request, RequestResult, wrap_request_time
+from helm.proxy.retry import NonRetriableException
+
+
+_pipelines: Dict[str, transformers.Pipeline] = {}
+_pipelines_lock: Lock = Lock()
+
+
+def _get_pipeline(
+    helm_model_name: str,
+    pipeline_kwargs: Dict[str, Any],
+) -> Any:
+    """
+    Checks if the desired HuggingFaceModel is cached. Creates the HuggingFaceModel if it's not cached.
+    Returns the HuggingFaceModel.
+    """
+    global _pipelines
+    global _pipelines_lock
+    with _pipelines_lock:
+        if helm_model_name not in _pipelines:
+            huggingface_model_name = pipeline_kwargs["model"]
+            with htrack_block(
+                f"Loading HuggingFace model {huggingface_model_name} (kwargs={pipeline_kwargs}) "
+                f"for HELM model {helm_model_name} with transformers.pipeline"
+            ):
+                _pipelines[helm_model_name] = transformers.pipeline(**pipeline_kwargs)
+
+    return _pipelines[helm_model_name]
+
+
+class HuggingFacePipelineClient(CachingClient):
+    def __init__(
+        self,
+        cache_config: CacheConfig,
+        model_name: str,
+        pretrained_model_name_or_path: Optional[str] = None,
+        apply_chat_template: Optional[bool] = None,
+        **kwargs,
+    ):
+        # Include `pretrained_model_name_or_path` parameter so that model deployments can use
+        # the `pretrained_model_name_or_path` arg to override `model_name`
+        super().__init__(cache_config=cache_config)
+        self._helm_model_name = model_name
+        self._pipeline_kwargs = {
+            "model": pretrained_model_name_or_path or self._helm_model_name,
+            "task": "text-generation",
+            **kwargs,
+        }
+        self._pipeline = _get_pipeline(self._helm_model_name, self._pipeline_kwargs)
+        if apply_chat_template is not None:
+            self._apply_chat_template = apply_chat_template
+        else:
+            # If the user did not explicitly configure whether the model is a chat model with `apply_chat_template` arg,
+            # auto-infer if the model is a chat model based on whether the tokenizer has a chat template.
+            # Note: Auto-inference is incorrect for some non-chat models that still have chat templates
+            # e.g. Qwen2, Qwen 2.5.
+            # For these models, the `apply_chat_template` arg should be explicitly set to false.
+            self._apply_chat_template = bool(self._pipeline.tokenizer.chat_template)
+            hwarn(
+                f"Automatically set `apply_chat_template` to {self._apply_chat_template} based on "
+                "whether the tokenizer has a chat template. "
+                "If this is incorrect, please explicitly set `apply_chat_template`."
+            )
+
+    def make_text_inputs(self, request: Request) -> Union[str, List[Dict[str, str]]]:
+        if request.prompt and request.messages:
+            raise NonRetriableException(f"More than one of `prompt` and `messages` was set in request: {request}")
+        # Chat model expects a list of messages as input
+        if self._apply_chat_template:
+            if request.messages:
+                return request.messages
+            else:
+                return [{"role": "user", "content": request.prompt}]
+        # Base non-chat model expects a string as input
+        else:
+            if request.messages:
+                raise NonRetriableException("Chat mesages not supported by non-chat model")
+            else:
+                return request.prompt
+
+    def make_request(self, request: Request) -> RequestResult:
+        """Make a request"""
+        if request.model != self._helm_model_name:
+            raise NonRetriableException(
+                f"This instance of HuggingFacePipelineClient has loaded model {self._helm_model_name} but the request was for model {request.model}"  # noqa: E501
+            )
+        completions: List[GeneratedOutput] = []
+        do_sample = request.temperature > 0.0
+        raw_request = {
+            "text_inputs": self.make_text_inputs(request),
+            "return_full_text": request.echo_prompt,
+            "temperature": request.temperature if do_sample else None,
+            "num_return_sequences": request.num_completions,
+            "max_new_tokens": request.max_tokens,
+            "top_p": request.top_p,
+            "top_k": request.top_k_per_token if do_sample else None,
+            "do_sample": do_sample,
+            "return_dict_in_generate": True,
+        }
+        if request.stop_sequences:
+            stop_sequence_ids = self._pipeline.tokenizer(
+                request.stop_sequences, return_token_type_ids=False, add_special_tokens=False
+            )
+            if len(stop_sequence_ids.input_ids) == 1 and len(stop_sequence_ids.input_ids[0]) == 1:
+                raw_request["eos_token_id"] = stop_sequence_ids.input_ids[0][0]
+            else:
+                raise NonRetriableException(
+                    "Multiple stop sequences and stop sequences of multiple tokens, are not yet supported by HuggingFacePipelineClient"  # noqa: E501
+                )
+
+        def do_it() -> Dict[str, Any]:
+            pipeline_outputs = self._pipeline(**raw_request)
+            return {"outputs": pipeline_outputs}
+
+        cache_key = CachingClient.make_cache_key(
+            {"pipeline_kwargs": self._pipeline_kwargs, **raw_request},
+            request,
+        )
+
+        response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
+        for raw_output in response["outputs"]:
+            completions.append(GeneratedOutput(text=raw_output["generated_text"], logprob=0, tokens=[]))
+        return RequestResult(
+            success=True,
+            cached=cached,
+            request_time=response["request_time"],
+            request_datetime=response["request_datetime"],
+            completions=completions,
+            embedding=[],
+        )

helm/clients/image_generation/dalle_mini/model/configuration.py

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-""" DalleBart model configuration """
+"""DalleBart model configuration"""
 import warnings
 
 from transformers.configuration_utils import PretrainedConfig

helm/clients/image_generation/dalle_mini/model/modeling.py

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-""" DalleBart model. """
+"""DalleBart model."""
 
 import math
 from functools import partial

helm/clients/image_generation/dalle_mini/model/processor.py

@@ -1,4 +1,4 @@
-""" DalleBart processor """
+"""DalleBart processor"""
 
 from typing import List
 

helm/clients/image_generation/dalle_mini/model/tokenizer.py

@@ -1,4 +1,4 @@
-""" DalleBart tokenizer """
+"""DalleBart tokenizer"""
 
 from transformers import BartTokenizerFast