crfm-helm 0.5.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

helm/clients/client.py

@@ -39,13 +39,17 @@ class CachingClient(Client):
         """
         if request.random is not None:
             assert "random" not in raw_request
-            cache_key: Mapping = {**raw_request, "random": request.random}
+            return {**raw_request, "random": request.random}
         else:
-            cache_key = raw_request
-        return cache_key
+            return {**raw_request}
 
 
-def truncate_sequence(sequence: GeneratedOutput, request: Request, print_warning: bool = True) -> GeneratedOutput:
+def truncate_sequence(
+    sequence: GeneratedOutput,
+    request: Request,
+    end_of_text_token: Optional[str] = None,
+    print_warning: bool = True,
+) -> GeneratedOutput:
     """
     Certain providers have bugs where they aren't respecting max_tokens,
     stop_sequences and the end of text token, so as a hack, we have to manually
@@ -64,7 +68,11 @@ def truncate_sequence(sequence: GeneratedOutput, request: Request, print_warning
             hlog("WARNING: don't know how to handle echo_prompt and max_tokens > 0, not truncating")
         return sequence
 
-    for stop in request.stop_sequences:
+    if end_of_text_token:
+        stop_sequences = request.stop_sequences + [end_of_text_token]
+    else:
+        stop_sequences = request.stop_sequences
+    for stop in stop_sequences:
         # Find `stop` in the text
         try:
             new_text = sequence.text[: sequence.text.index(stop)]
@@ -116,7 +124,12 @@ def truncate_sequence(sequence: GeneratedOutput, request: Request, print_warning
 
 
 def truncate_and_tokenize_response_text(
-    text: str, request: Request, tokenizer: Tokenizer, tokenizer_name: str, original_finish_reason: str = "endoftext"
+    text: str,
+    request: Request,
+    tokenizer: Tokenizer,
+    tokenizer_name: str,
+    end_of_text_token: Optional[str] = None,
+    original_finish_reason: str = "endoftext",
 ) -> GeneratedOutput:
     """Truncate a string-only response to respect stop_sequences and max_tokens.
 
@@ -139,7 +152,11 @@ def truncate_and_tokenize_response_text(
     if request.echo_prompt:
         raise Exception("truncate_and_tokenize_response_text() does not support requests with echo_prompt = True")
 
-    for stop_sequence in request.stop_sequences:
+    if end_of_text_token:
+        stop_sequences = request.stop_sequences + [end_of_text_token]
+    else:
+        stop_sequences = request.stop_sequences
+    for stop_sequence in stop_sequences:
         try:
             text = text[: text.index(stop_sequence)]
             finish_reason = "stop"

helm/clients/cohere_client.py

@@ -1,8 +1,9 @@
 import json
 import requests
-from typing import List
+from typing import List, Optional, Sequence, TypedDict
 
 from helm.common.cache import CacheConfig
+from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.common.request import (
     wrap_request_time,
     EMBEDDING_UNAVAILABLE_REQUEST_RESULT,
@@ -11,8 +12,13 @@ from helm.common.request import (
     GeneratedOutput,
     Token,
 )
-from .client import CachingClient, truncate_sequence
-from .cohere_utils import get_cohere_url, DEFAULT_COHERE_API_VERSION
+from helm.clients.client import CachingClient, truncate_sequence
+from helm.clients.cohere_utils import get_cohere_url, DEFAULT_COHERE_API_VERSION
+
+try:
+    import cohere
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["cohere"])
 
 
 class CohereClient(CachingClient):
@@ -152,3 +158,92 @@ class CohereClient(CachingClient):
             completions=completions,
             embedding=[],
         )
+
+
+class CohereRawChatRequest(TypedDict):
+    message: str
+    model: Optional[str]
+    preamble: Optional[str]
+    chat_history: Optional[Sequence[cohere.ChatMessage]]
+    temperature: Optional[float]
+    max_tokens: Optional[int]
+    k: Optional[int]
+    p: Optional[float]
+    seed: Optional[float]
+    stop_sequences: Optional[Sequence[str]]
+    frequency_penalty: Optional[float]
+    presence_penalty: Optional[float]
+
+
+def convert_to_raw_chat_request(request: Request) -> CohereRawChatRequest:
+    # TODO: Support chat
+    model = request.model.replace("cohere/", "")
+    return {
+        "message": request.prompt,
+        "model": model,
+        "preamble": None,
+        "chat_history": None,
+        "temperature": request.temperature,
+        "max_tokens": request.max_tokens,
+        "k": request.top_k_per_token,
+        "p": request.top_p,
+        "stop_sequences": request.stop_sequences,
+        "seed": float(request.random) if request.random is not None else None,
+        "frequency_penalty": request.frequency_penalty,
+        "presence_penalty": request.presence_penalty,
+    }
+
+
+class CohereChatClient(CachingClient):
+    """
+    Leverages the chat endpoint: https://docs.cohere.com/reference/chat
+
+    Cohere models will only support chat soon: https://docs.cohere.com/docs/migrating-from-cogenerate-to-cochat
+    """
+
+    def __init__(self, api_key: str, cache_config: CacheConfig):
+        super().__init__(cache_config=cache_config)
+        self.client = cohere.Client(api_key=api_key)
+
+    def make_request(self, request: Request) -> RequestResult:
+        if request.embedding:
+            return EMBEDDING_UNAVAILABLE_REQUEST_RESULT
+        # TODO: Support multiple completions
+        assert request.num_completions == 1, "CohereChatClient only supports num_completions=1"
+        # TODO: Support messages
+        assert not request.messages, "CohereChatClient currently does not support the messages API"
+
+        raw_request: CohereRawChatRequest = convert_to_raw_chat_request(request)
+
+        try:
+
+            def do_it():
+                """
+                Send the request to the Cohere Chat API. Responses will be structured like this:
+                cohere.Chat {
+                    message: What's up?
+                    text: Hey there! How's it going? I'm doing well, thank you for asking 😊.
+                    ...
+                }
+                """
+                raw_response = self.client.chat(**raw_request).dict()
+                assert "text" in raw_response, f"Response does not contain text: {raw_response}"
+                return raw_response
+
+            response, cached = self.cache.get(raw_request, wrap_request_time(do_it))
+        except (requests.exceptions.RequestException, AssertionError) as e:
+            error: str = f"CohereClient error: {e}"
+            return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
+
+        completions: List[GeneratedOutput] = []
+        completion: GeneratedOutput = GeneratedOutput(text=response["text"], logprob=0.0, tokens=[])
+        completions.append(completion)
+
+        return RequestResult(
+            success=True,
+            cached=cached,
+            request_time=response["request_time"],
+            request_datetime=response["request_datetime"],
+            completions=completions,
+            embedding=[],
+        )

helm/clients/huggingface_client.py

@@ -9,6 +9,7 @@ from typing import Any, Dict, List, Optional, TypedDict
 
 from helm.common.cache import CacheConfig
 from helm.common.hierarchical_logger import htrack_block, hlog
+from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.common.request import (
     wrap_request_time,
     EMBEDDING_UNAVAILABLE_REQUEST_RESULT,
@@ -17,6 +18,7 @@ from helm.common.request import (
     GeneratedOutput,
     Token,
 )
+from helm.tokenizers.tokenizer import Tokenizer
 from .client import CachingClient, truncate_sequence
 from helm.tokenizers.huggingface_tokenizer import HuggingFaceTokenizer, WrappedPreTrainedTokenizer
 from threading import Lock
@@ -53,27 +55,65 @@ class HuggingFaceRequest(TypedDict):
 class HuggingFaceServer:
     """A thin wrapper around a Hugging Face AutoModelForCausalLM for HuggingFaceClient to call."""
 
-    def __init__(self, pretrained_model_name_or_path: str, **kwargs):
-        if torch.cuda.is_available():
-            hlog("CUDA is available, initializing with a GPU...")
-            self.device: str = "cuda:0"
+    def __init__(
+        self,
+        pretrained_model_name_or_path: str,
+        wrapped_tokenizer: WrappedPreTrainedTokenizer,
+        openvino: bool = False,
+        **kwargs,
+    ):
+        self.device: Optional[str]
+        if "device_map" in kwargs:
+            try:
+                import accelerate  # noqa: F401
+            except ModuleNotFoundError as e:
+                handle_module_not_found_error(e, ["accelerate"])
+            hlog(f'Hugging Face device_map set to "{kwargs["device_map"]}".')
+            self.device = None
+        elif torch.cuda.is_available():
+            hlog('Hugging Face device set to "cuda:0" because CUDA is available.')
+            self.device = "cuda:0"
         else:
+            hlog('Hugging Face device set to "cpu" because CUDA is unavailable.')
             self.device = "cpu"
+
+        # Security issue: currently we trust remote code by default.
+        # We retain this temporarily to maintain reverse compatibility.
+        # TODO: Delete if-else and don't set trust_remote_code=True
+        if "trust_remote_code" not in kwargs:
+            kwargs["trust_remote_code"] = True
+
         with htrack_block(f"Loading Hugging Face model {pretrained_model_name_or_path}"):
             # WARNING this may fail if your GPU does not have enough memory
-            self.model = AutoModelForCausalLM.from_pretrained(
-                pretrained_model_name_or_path, trust_remote_code=True, **kwargs
-            ).to(self.device)
-        with htrack_block(f"Loading Hugging Face tokenizer for model {pretrained_model_name_or_path}"):
-            self.wrapped_tokenizer: WrappedPreTrainedTokenizer = HuggingFaceTokenizer.create_tokenizer(
-                pretrained_model_name_or_path, **kwargs
-            )
+            if openvino:
+                # Optimum Intel provides a simple interface to optimize Transformer models and convert them to \
+                # OpenVINO™ Intermediate Representation (IR) format to accelerate end-to-end pipelines on \
+                # Intel® architectures using OpenVINO™ runtime.
+                try:
+                    from optimum.intel.openvino import OVModelForCausalLM
+                except ModuleNotFoundError as e:
+                    handle_module_not_found_error(e, ["openvino"])
+
+                self.device = "cpu"
+                self.model = OVModelForCausalLM.from_pretrained(
+                    pretrained_model_name_or_path, export=True, **kwargs
+                ).to(self.device)
+            elif self.device is None:
+                # kwargs contains device_map=auto
+                # Do not call to() because accelerate will take care of model device placement.
+                self.model = AutoModelForCausalLM.from_pretrained(pretrained_model_name_or_path, **kwargs)
+            else:
+                self.model = AutoModelForCausalLM.from_pretrained(pretrained_model_name_or_path, **kwargs).to(
+                    self.device
+                )
+        self.wrapped_tokenizer = wrapped_tokenizer
 
     def serve_request(self, raw_request: HuggingFaceRequest) -> Dict:
         with self.wrapped_tokenizer as tokenizer:
             encoded_input = tokenizer(raw_request["prompt"], return_tensors="pt", return_token_type_ids=False).to(
-                self.device
+                0 if self.device is None else self.device
             )
+
         stopping_criteria: Optional[StoppingCriteriaList] = None
         optional_args = {}
         if len(raw_request["stop_sequences"]) > 0:
@@ -170,7 +210,12 @@ class HuggingFaceServerFactory:
     _servers_lock: Lock = Lock()
 
     @staticmethod
-    def get_server(helm_model_name: str, pretrained_model_name_or_path: str, **kwargs) -> Any:
+    def get_server(
+        helm_model_name: str,
+        pretrained_model_name_or_path: str,
+        wrapped_tokenizer: WrappedPreTrainedTokenizer,
+        **kwargs,
+    ) -> Any:
         """
         Checks if the desired HuggingFaceModel is cached. Creates the HuggingFaceModel if it's not cached.
         Returns the HuggingFaceModel.
@@ -182,7 +227,7 @@ class HuggingFaceServerFactory:
                     f"for HELM model {helm_model_name} with Hugging Face Transformers"
                 ):
                     HuggingFaceServerFactory._servers[helm_model_name] = HuggingFaceServer(
-                        pretrained_model_name_or_path, **kwargs
+                        pretrained_model_name_or_path, wrapped_tokenizer, **kwargs
                     )
 
         return HuggingFaceServerFactory._servers[helm_model_name]
@@ -206,18 +251,32 @@ def _process_huggingface_client_kwargs(raw_kwargs: Dict[str, Any]):
     # e.g. the string "torch.bfloat16" is converted to torch.bfloat16
     torch_dtype = processed_kwargs.get(TORCH_DTYPE_KEY)
     if torch_dtype and isinstance(torch_dtype, str):
-        if not torch_dtype.startswith(TORCH_DTYPE_VALUE_PREFIX):
-            raise ValueError(f'Unknown dtype "{torch_dtype}"; expected a string such as "torch.bfloat16"')
-        processed_kwargs[TORCH_DTYPE_KEY] = getattr(torch, torch_dtype[len(TORCH_DTYPE_VALUE_PREFIX) :])
+        if torch_dtype.startswith(TORCH_DTYPE_VALUE_PREFIX):
+            processed_kwargs[TORCH_DTYPE_KEY] = getattr(torch, torch_dtype[len(TORCH_DTYPE_VALUE_PREFIX) :])
 
     return processed_kwargs
 
 
 class HuggingFaceClient(CachingClient):
-    def __init__(self, cache_config: CacheConfig, pretrained_model_name_or_path: Optional[str] = None, **kwargs):
+    def __init__(
+        self,
+        cache_config: CacheConfig,
+        tokenizer: Tokenizer,
+        pretrained_model_name_or_path: Optional[str] = None,
+        end_of_text_token: Optional[str] = None,
+        **kwargs,
+    ):
         super().__init__(cache_config=cache_config)
         self._pretrained_model_name_or_path = pretrained_model_name_or_path
+        if not isinstance(tokenizer, HuggingFaceTokenizer):
+            raise ValueError(
+                f"Tokenizer for Hugging Face model {pretrained_model_name_or_path} must be a HuggingFaceTokenizer, "
+                "but instead it is {tokenizer}"
+            )
+        self._wrapped_tokenizer: WrappedPreTrainedTokenizer = tokenizer.get_wrapped_tokenizer()
+        self._tokenizer = tokenizer
         self._kwargs = _process_huggingface_client_kwargs(kwargs)
+        self._end_of_text_token = end_of_text_token
 
     def make_request(self, request: Request) -> RequestResult:
         # Embedding not supported for this model
@@ -242,6 +301,7 @@ class HuggingFaceClient(CachingClient):
         huggingface_model: HuggingFaceServer = HuggingFaceServerFactory.get_server(
             helm_model_name=request.model,
             pretrained_model_name_or_path=pretrained_model_name_or_path,
+            wrapped_tokenizer=self._wrapped_tokenizer,
             **self._kwargs,
         )
 
@@ -284,7 +344,7 @@ class HuggingFaceClient(CachingClient):
                 sequence_logprob += logprob
 
             completion = GeneratedOutput(text=raw_completion["text"], logprob=sequence_logprob, tokens=tokens)
-            completion = truncate_sequence(completion, request)
+            completion = truncate_sequence(completion, request, end_of_text_token=self._end_of_text_token)
             completions.append(completion)
 
         return RequestResult(

helm/clients/nvidia_nim_client.py

@@ -0,0 +1,35 @@
+from typing import Optional
+
+from helm.clients.openai_client import OpenAIClient
+from helm.common.cache import CacheConfig
+from helm.common.request import Request
+from helm.tokenizers.tokenizer import Tokenizer
+
+
+class NvidiaNimClient(OpenAIClient):
+
+    BASE_URL = "https://integrate.api.nvidia.com/v1"
+
+    def __init__(
+        self,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        cache_config: CacheConfig,
+        api_key: Optional[str] = None,
+    ):
+        self.tokenizer = tokenizer
+        self.tokenizer_name = tokenizer_name
+        super().__init__(
+            tokenizer=tokenizer,
+            tokenizer_name=tokenizer_name,
+            cache_config=cache_config,
+            api_key=api_key,
+            org_id=None,
+            base_url=NvidiaNimClient.BASE_URL,
+        )
+
+    def _get_model_for_request(self, request: Request) -> str:
+        return request.model
+
+    def _is_chat_model_engine(self, model_engine: str) -> bool:
+        return True

helm/clients/openai_client.py

@@ -12,8 +12,8 @@ from helm.common.tokenization_request import (
     TokenizationRequest,
     TokenizationRequestResult,
 )
-from helm.tokenizers.tokenizer import Tokenizer
 from .client import CachingClient, truncate_sequence, generate_uid_for_multimodal_prompt
+from helm.tokenizers.tokenizer import Tokenizer
 
 try:
     import openai
@@ -60,8 +60,7 @@ class OpenAIClient(CachingClient):
 
     def _get_cache_key(self, raw_request: Dict, request: Request):
         cache_key = CachingClient.make_cache_key(raw_request, request)
-        if is_vlm(request.model):
-            assert request.multimodal_prompt is not None
+        if request.multimodal_prompt:
             prompt_key: str = generate_uid_for_multimodal_prompt(request.multimodal_prompt)
             cache_key = {**cache_key, "multimodal_prompt": prompt_key}
             del cache_key["messages"]
@@ -103,6 +102,14 @@ class OpenAIClient(CachingClient):
 
     def _make_chat_request(self, request: Request) -> RequestResult:
         messages: Optional[List[Dict[str, Union[str, Any]]]] = request.messages
+        if (
+            (request.prompt and request.messages)
+            or (request.prompt and request.multimodal_prompt)
+            or (request.messages and request.multimodal_prompt)
+        ):
+            raise ValueError(
+                f"More than one of `prompt`, `messages` and `multimodal_prompt` was set in request: {request}"
+            )
         if request.messages is not None:
             # Checks that all messages have a role and some content
             for message in request.messages:
@@ -125,6 +132,7 @@ class OpenAIClient(CachingClient):
             content: Union[str, List[Union[str, Any]]]
             if request.multimodal_prompt is not None:
                 content = []
+                request.validate()
                 for media_object in request.multimodal_prompt.media_objects:
                     if media_object.is_type("image") and media_object.location:
                         from helm.common.images_utils import encode_base64
@@ -133,8 +141,6 @@ class OpenAIClient(CachingClient):
                         image_object: Dict[str, str] = {"url": f"data:image/jpeg;base64,{base64_image}"}
                         content.append({"type": "image_url", "image_url": image_object})
                     elif media_object.is_type(TEXT_TYPE):
-                        if media_object.text is None:
-                            raise ValueError("MediaObject of text type has missing text field value")
                         content.append({"type": media_object.type, "text": media_object.text})
                     else:
                         raise ValueError(f"Unrecognized MediaObject type {media_object.type}")

helm/clients/palmyra_client.py

@@ -3,6 +3,7 @@ import json
 import requests
 from typing import Any, Dict, List
 
+from helm.clients.openai_client import OpenAIClient
 from helm.common.cache import CacheConfig
 from helm.common.hierarchical_logger import hlog
 from helm.common.request import wrap_request_time, Request, RequestResult, GeneratedOutput, Token, ErrorFlags
@@ -142,3 +143,27 @@ class PalmyraClient(CachingClient):
             completions=completions,
             embedding=[],
         )
+
+
+class PalmyraChatClient(OpenAIClient):
+    """Sends request to a Palmyra model using a OpenAI-compatible Chat API."""
+
+    def __init__(
+        self,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        cache_config: CacheConfig,
+        api_key: str,
+        base_url: str,
+    ):
+        super().__init__(
+            tokenizer=tokenizer,
+            tokenizer_name=tokenizer_name,
+            cache_config=cache_config,
+            api_key=api_key,
+            org_id=None,
+            base_url=base_url,
+        )
+
+    def _is_chat_model_engine(self, model_engine: str) -> bool:
+        return True

helm/clients/perspective_api_client.py

@@ -4,16 +4,21 @@ from dataclasses import asdict
 from typing import Any, List, Dict, Optional
 
 from dacite import from_dict
-from googleapiclient import discovery
-from googleapiclient.errors import BatchError, HttpError
-from googleapiclient.http import BatchHttpRequest
-from httplib2 import HttpLib2Error
+
 from helm.clients.toxicity_classifier_client import ToxicityClassifierClient
+from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.proxy.retry import NonRetriableException
-
 from helm.common.cache import Cache, CacheConfig
 from helm.common.perspective_api_request import ToxicityAttributes, PerspectiveAPIRequest, PerspectiveAPIRequestResult
-from google.auth.exceptions import DefaultCredentialsError
+
+try:
+    from googleapiclient import discovery
+    from googleapiclient.errors import BatchError, HttpError
+    from googleapiclient.http import BatchHttpRequest
+    from httplib2 import HttpLib2Error
+    from google.auth.exceptions import DefaultCredentialsError
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["metrics"])
 
 
 class PerspectiveAPIClientCredentialsError(NonRetriableException):