crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/clients/clip_score_client.py

@@ -0,0 +1,49 @@
+from typing import Dict, Optional
+from dataclasses import asdict
+
+from helm.common.cache import Cache, CacheConfig
+from helm.common.clip_score_request import DEFAULT_CLIP_SCORE_MODEL, CLIPScoreRequest, CLIPScoreResult
+from helm.clients.clip_scorers.base_clip_scorer import BaseCLIPScorer
+
+
+class CLIPScoreClientError(Exception):
+    pass
+
+
+class CLIPScoreClient:
+    def __init__(self, cache_config: CacheConfig):
+        self.cache = Cache(cache_config)
+        self._clip_scorer: Optional[BaseCLIPScorer] = None
+
+    def compute_score(self, request: CLIPScoreRequest) -> CLIPScoreResult:
+        """
+        Compute a CLIPScore for a given caption and image.
+        """
+        # TODO: support multilingual CLIPScore and other CLIP models.
+        assert request.model == DEFAULT_CLIP_SCORE_MODEL, f"Unsupported model: {request.model}"
+        assert not request.multilingual
+
+        try:
+
+            def do_it():
+                if self._clip_scorer is None:
+                    from helm.clients.clip_scorers.clip_scorer import CLIPScorer
+
+                    self._clip_scorer = CLIPScorer()
+
+                score: float = self._clip_scorer.compute_score(
+                    caption=request.caption, image_location=request.image_location
+                )
+                return {"score": score}
+
+            cache_key: Dict = asdict(request)
+            results, cached = self.cache.get(cache_key, do_it)
+
+        except Exception as e:
+            raise CLIPScoreClientError(e)
+
+        return CLIPScoreResult(
+            success=True,
+            cached=cached,
+            score=results["score"],
+        )

helm/clients/clip_scorers/base_clip_scorer.py

@@ -0,0 +1,18 @@
+from abc import abstractmethod, ABC
+from typing import List
+
+
+class BaseCLIPScorer(ABC):
+    @abstractmethod
+    def compute_score(self, caption: str, image_location: str) -> float:
+        pass
+
+    def select_best_image(self, caption: str, image_locations: List[str]) -> str:
+        """Selects the image from a list of images with the highest CLIPScore given the caption."""
+        assert len(image_locations) > 0, "Need at least one image"
+
+        if len(image_locations) == 1:
+            return image_locations[0]
+
+        scores: List[float] = [self.compute_score(caption, image_location) for image_location in image_locations]
+        return image_locations[scores.index(max(scores))]

helm/clients/clip_scorers/clip_scorer.py

@@ -0,0 +1,50 @@
+from typing import Literal
+
+from torchvision import transforms
+import torch
+
+from helm.common.gpu_utils import get_torch_device
+from helm.common.images_utils import open_image
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.clients.clip_scorers.base_clip_scorer import BaseCLIPScorer
+
+
+_ = torch.manual_seed(42)
+
+
+class CLIPScorer(BaseCLIPScorer):
+    """
+    CLIPScore is a reference free metric that can be used to evaluate the correlation between an image
+    caption and the content of the image. It has been found to be highly correlated with human judgement.
+    Paper: https://arxiv.org/abs/2104.08718
+
+    We use the TorchMetrics implementation:
+    https://torchmetrics.readthedocs.io/en/stable/multimodal/clip_score.html.
+    The score is bound between 0 and 100, where a score closer to 100 is better.
+
+    Verified implementation against the scores of image-caption pairs from
+    https://wandb.ai/dalle-mini/dalle-mini/reports/OpenAI-CLIP-Score-exploration--VmlldzoxNjMwODM1.
+    """
+
+    def __init__(
+        self,
+        model_name: Literal[
+            "openai/clip-vit-base-patch16",
+            "openai/clip-vit-base-patch32",
+            "openai/clip-vit-large-patch14-336",
+            "openai/clip-vit-large-patch14",
+        ] = "openai/clip-vit-large-patch14",
+    ):
+        try:
+            from torchmetrics.multimodal import CLIPScore
+        except ModuleNotFoundError as e:
+            handle_module_not_found_error(e, ["heim"])
+
+        self._device: torch.device = get_torch_device()
+        self._metric = CLIPScore(model_name_or_path=model_name).to(self._device)
+
+    def compute_score(self, caption: str, image_location: str) -> float:
+        image = open_image(image_location)
+        image_tensor: torch.Tensor = transforms.ToTensor()(image).to(self._device)
+        score: float = self._metric(image_tensor, caption).detach().item()
+        return score

helm/clients/clip_scorers/multilingual_clip_scorer.py

@@ -0,0 +1,50 @@
+import torch
+import transformers
+
+from helm.common.gpu_utils import get_torch_device, get_torch_device_name
+from helm.common.images_utils import open_image
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.clients.clip_scorers.base_clip_scorer import BaseCLIPScorer
+
+_ = torch.manual_seed(42)
+
+
+class MultilingualCLIPScorer(BaseCLIPScorer):
+    """
+    Multilingual-CLIP extends OpenAI's English text encoders to multiple other languages.
+    Adapted from https://huggingface.co/M-CLIP/XLM-Roberta-Large-Vit-L-14
+    """
+
+    TEXT_MODEL_NAME: str = "M-CLIP/XLM-Roberta-Large-Vit-L-14"
+    IMAGE_MODEL_NAME: str = "ViT-L/14"
+
+    def __init__(self):
+        try:
+            import clip
+            from multilingual_clip import pt_multilingual_clip
+        except ModuleNotFoundError as e:
+            handle_module_not_found_error(e, ["heim"])
+
+        super().__init__()
+        self._device: torch.device = get_torch_device()
+        self._text_model = pt_multilingual_clip.MultilingualCLIP.from_pretrained(self.TEXT_MODEL_NAME)
+        self._tokenizer = transformers.AutoTokenizer.from_pretrained(self.TEXT_MODEL_NAME)
+        self._model, self._preprocess = clip.load(self.IMAGE_MODEL_NAME, device=get_torch_device_name())
+
+    def compute_score(self, caption: str, image_location: str) -> float:
+        # Get text features
+        text_features = self._text_model.forward(caption, self._tokenizer)
+        text_features = text_features / text_features.norm(p=2, dim=-1, keepdim=True)
+        text_features = text_features.to(self._device)
+
+        image = open_image(image_location)
+        image = self._preprocess(image).unsqueeze(0).to(self._device)
+
+        # Get image features
+        with torch.no_grad():
+            image_features = self._model.encode_image(image)
+        image_features = image_features / image_features.norm(p=2, dim=-1, keepdim=True)
+
+        # Compute score using text and image features
+        score = 100 * (image_features * text_features).sum(axis=-1)
+        return score.detach().item()

helm/{proxy/clients → clients}/cohere_client.py

@@ -1,18 +1,25 @@
 import json
 import requests
-from typing import List
+from typing import List, Optional, Sequence, TypedDict
 
 from helm.common.cache import CacheConfig
+from helm.common.hierarchical_logger import hexception
+from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.common.request import (
     wrap_request_time,
     EMBEDDING_UNAVAILABLE_REQUEST_RESULT,
     Request,
     RequestResult,
-    Sequence,
+    GeneratedOutput,
     Token,
 )
-from .client import CachingClient, truncate_sequence
-from .cohere_utils import get_cohere_url, DEFAULT_COHERE_API_VERSION
+from helm.clients.client import CachingClient, truncate_sequence
+from helm.clients.cohere_utils import get_cohere_url, DEFAULT_COHERE_API_VERSION
+
+try:
+    import cohere
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["cohere"])
 
 
 class CohereClient(CachingClient):
@@ -117,10 +124,11 @@ class CohereClient(CachingClient):
 
             response, cached = self.cache.get(raw_request, wrap_request_time(do_it))
         except (requests.exceptions.RequestException, AssertionError) as e:
+            hexception(e)
             error: str = f"CohereClient error: {e}"
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
 
-        completions: List[Sequence] = []
+        completions: List[GeneratedOutput] = []
         for generation in response["generations"]:
             # From https://docs.cohere.ai/generate-reference, "the likelihood refers to the average log-likelihood
             # of the entire specified string..." What we want is the sum of the log probabilities of all tokens.
@@ -132,14 +140,7 @@ class CohereClient(CachingClient):
                 logprob: float = token_likelihood.get("likelihood", 0)
                 sequence_logprob += logprob
 
-                tokens.append(
-                    Token(
-                        text=token_likelihood["token"],
-                        logprob=logprob,
-                        # Cohere does not include the top log probs in the response
-                        top_logprobs={},
-                    )
-                )
+                tokens.append(Token(text=token_likelihood["token"], logprob=logprob))
 
             sequence_text: str = generation["text"]
             if request.echo_prompt and request.max_tokens > 0:
@@ -147,7 +148,7 @@ class CohereClient(CachingClient):
                 # `return_likelihoods` is "ALL" and `max_tokens` is greater than 0.
                 sequence_text = request.prompt + sequence_text
 
-            completion: Sequence = Sequence(text=sequence_text, logprob=sequence_logprob, tokens=tokens)
+            completion: GeneratedOutput = GeneratedOutput(text=sequence_text, logprob=sequence_logprob, tokens=tokens)
             completion = truncate_sequence(completion, request)
             completions.append(completion)
 
@@ -159,3 +160,93 @@ class CohereClient(CachingClient):
             completions=completions,
             embedding=[],
         )
+
+
+class CohereRawChatRequest(TypedDict):
+    message: str
+    model: Optional[str]
+    preamble: Optional[str]
+    chat_history: Optional[Sequence[cohere.ChatbotMessage]]
+    temperature: Optional[float]
+    max_tokens: Optional[int]
+    k: Optional[int]
+    p: Optional[float]
+    seed: Optional[int]
+    stop_sequences: Optional[Sequence[str]]
+    frequency_penalty: Optional[float]
+    presence_penalty: Optional[float]
+
+
+def convert_to_raw_chat_request(request: Request) -> CohereRawChatRequest:
+    # TODO: Support chat
+    model = request.model.replace("cohere/", "")
+    return {
+        "message": request.prompt,
+        "model": model,
+        "preamble": None,
+        "chat_history": None,
+        "temperature": request.temperature,
+        "max_tokens": request.max_tokens,
+        "k": request.top_k_per_token,
+        "p": request.top_p,
+        "stop_sequences": request.stop_sequences,
+        "seed": int(request.random) if request.random is not None else None,
+        "frequency_penalty": request.frequency_penalty,
+        "presence_penalty": request.presence_penalty,
+    }
+
+
+class CohereChatClient(CachingClient):
+    """
+    Leverages the chat endpoint: https://docs.cohere.com/reference/chat
+
+    Cohere models will only support chat soon: https://docs.cohere.com/docs/migrating-from-cogenerate-to-cochat
+    """
+
+    def __init__(self, api_key: str, cache_config: CacheConfig):
+        super().__init__(cache_config=cache_config)
+        self.client = cohere.Client(api_key=api_key)
+
+    def make_request(self, request: Request) -> RequestResult:
+        if request.embedding:
+            return EMBEDDING_UNAVAILABLE_REQUEST_RESULT
+        # TODO: Support multiple completions
+        assert request.num_completions == 1, "CohereChatClient only supports num_completions=1"
+        # TODO: Support messages
+        assert not request.messages, "CohereChatClient currently does not support the messages API"
+
+        raw_request: CohereRawChatRequest = convert_to_raw_chat_request(request)
+
+        try:
+
+            def do_it():
+                """
+                Send the request to the Cohere Chat API. Responses will be structured like this:
+                cohere.Chat {
+                    message: What's up?
+                    text: Hey there! How's it going? I'm doing well, thank you for asking 😊.
+                    ...
+                }
+                """
+                raw_response = self.client.chat(**raw_request).dict()
+                assert "text" in raw_response, f"Response does not contain text: {raw_response}"
+                return raw_response
+
+            response, cached = self.cache.get(raw_request, wrap_request_time(do_it))
+        except (requests.exceptions.RequestException, AssertionError) as e:
+            hexception(e)
+            error: str = f"CohereClient error: {e}"
+            return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
+
+        completions: List[GeneratedOutput] = []
+        completion: GeneratedOutput = GeneratedOutput(text=response["text"], logprob=0.0, tokens=[])
+        completions.append(completion)
+
+        return RequestResult(
+            success=True,
+            cached=cached,
+            request_time=response["request_time"],
+            request_datetime=response["request_datetime"],
+            completions=completions,
+            embedding=[],
+        )

helm/clients/dspy_client.py

@@ -0,0 +1,135 @@
+import hashlib
+import tempfile
+from pathlib import Path
+from typing import Optional
+
+from helm.common.optional_dependencies import handle_module_not_found_error
+
+try:
+    import dspy
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["dspy"])
+
+from helm.benchmark.runner import _get_current_run_spec_name
+from helm.clients.client import Client
+from helm.common.cache import Cache, CacheConfig
+from helm.common.general import ensure_file_downloaded
+from helm.common.hierarchical_logger import hlog
+from helm.common.request import Request, RequestResult, GeneratedOutput
+from helm.proxy.retry import NonRetriableException
+from helm.tokenizers.tokenizer import Tokenizer
+
+
+class DSPyClient(Client):
+    """
+    A HELM client that uses DSPy for inference instead of directly calling the model.
+    """
+
+    def __init__(
+        self,
+        cache_config: CacheConfig,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        model_name: Optional[str] = None,
+        dspy_agent_url: Optional[str] = None,
+        dspy_module: Optional[str] = None,
+        dspy_api_model: Optional[str] = None,
+        dspy_api_base: Optional[str] = None,
+        api_key: Optional[str] = None,
+    ):
+        """
+        Initializes the DSPyClient.
+
+        Args:
+            cache_config (CacheConfig): Configuration for caching.
+            tokenizer (Tokenizer): Tokenizer instance (unused but required by HELM interface).
+            tokenizer_name (str): Name of the tokenizer (unused but required by HELM interface).
+            model_name (str): The official model name used within HELM.
+            dspy_agent_url (str): URL for the DSPy agent JSON configuration.
+            dspy_module (str): The module to use with DSPy (Predict, ChainOfThought).
+            dspy_api_model (str): The actual model name (API) to use with DSPy.
+            dspy_api_base (str): Base URL for the model API.
+            api_key (str): API key for the DSPy model provider.
+        """
+
+        if not model_name:
+            raise NonRetriableException("Please specify the model name in model_deployments.yaml")
+        if not dspy_api_model:
+            raise NonRetriableException("Please specify dspy_api_model in model_deployments.yaml")
+
+        if ("o3-mini" in model_name) or ("deepseek-r1" in model_name):
+            self.lm = dspy.LM(
+                model=dspy_api_model, api_base=dspy_api_base, api_key=api_key, temperature=1.0, max_tokens=100000
+            )
+        else:
+            self.lm = dspy.LM(model=dspy_api_model, api_base=dspy_api_base, api_key=api_key, temperature=0.0)
+
+        run_spec_name = _get_current_run_spec_name()
+        self.scenario_name = run_spec_name.split(":")[0] if run_spec_name else "unknown"
+        self.model_name = model_name
+        self.dspy_agent_url_template = dspy_agent_url
+        self.dspy_module = dspy_module
+        self.dspy_api_model = dspy_api_model
+        self.dspy_api_base = dspy_api_base
+        self.api_key = api_key
+        self.cache_dir = Path(tempfile.gettempdir()) / "dspy_agent_cache"
+        self.cache_dir.mkdir(exist_ok=True)
+        self._current_cache_file = None
+        self._load_agent_for_scenario()
+        self.cache = Cache(cache_config) if cache_config else None
+
+    def _get_cache_file(self, url):
+        url_hash = hashlib.md5(url.encode()).hexdigest()
+        return self.cache_dir / f"agent_{url_hash}.json"
+
+    def _load_agent_for_scenario(self):
+        if self.dspy_module == "ChainOfThought":
+            self.agent = dspy.ChainOfThought("inputs -> output")
+        elif self.dspy_module == "Predict":
+            self.agent = dspy.Predict("inputs -> output")
+        else:
+            raise ValueError(f"Unknown dspy_module: {self.dspy_module}")
+
+        dspy_agent_url = (
+            self.dspy_agent_url_template.format(scenario=self.scenario_name) if self.dspy_agent_url_template else None
+        )
+        if dspy_agent_url:
+            cache_file = self._get_cache_file(dspy_agent_url)
+            if self._current_cache_file == cache_file:
+                return
+            try:
+                ensure_file_downloaded(source_url=dspy_agent_url, target_path=str(cache_file))
+                self.agent.load(str(cache_file))
+                self._current_cache_file = cache_file
+
+            except Exception as e:
+                raise NonRetriableException(f"Failed to load DSPy agent from URL {dspy_agent_url}: {str(e)}")
+        hlog(
+            f"DSPy client initialized - HELM Model: {self.model_name}, DSPy Model: {self.dspy_api_model}, API Base: {self.dspy_api_base}, API Key: {'***' if self.api_key else None}, DSPy Agent: {dspy_agent_url}, DSPy Module: {self.dspy_module}"
+        )
+
+    def make_request(self, request: Request) -> RequestResult:
+        run_spec_name = _get_current_run_spec_name()
+        current_scenario = run_spec_name.split(":")[0] if run_spec_name else "unknown"
+        if current_scenario != self.scenario_name:
+            self.scenario_name = current_scenario
+            if self.dspy_agent_url_template:
+                self._load_agent_for_scenario()
+            else:
+                hlog(
+                    f"DSPy client initialized - HELM Model: {self.model_name}, DSPy Model: {self.dspy_api_model}, API Base: {self.dspy_api_base}, API Key: {'***' if self.api_key else None}, DSPy Agent: None, DSPy Module: {self.dspy_module}"
+                )
+
+        prompt_text = request.prompt
+        if request.messages:
+            prompt_text = " ".join(msg["content"] for msg in request.messages if msg.get("role") != "system")
+
+        try:
+            with dspy.context(lm=self.lm):
+                prediction = self.agent(inputs=prompt_text)
+            output_text = prediction.output if hasattr(prediction, "output") else str(prediction)
+        except Exception as e:
+            return RequestResult(success=False, cached=False, completions=[], embedding=[], error=str(e))
+
+        output = GeneratedOutput(text=output_text, logprob=0.0, tokens=[])
+        return RequestResult(success=True, cached=False, completions=[output], embedding=[])

helm/clients/gcs_client.py

@@ -0,0 +1,82 @@
+from dataclasses import asdict
+from typing import Dict, Optional
+import requests
+
+from helm.common.cache import Cache, CacheConfig
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.common.hierarchical_logger import hlog
+from helm.common.file_upload_request import FileUploadRequest, FileUploadResult
+
+
+class GCSClientError(Exception):
+    pass
+
+
+class GCSClient:
+    """
+    Uploads files to GCS. Ensure the GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials.json
+    environment variable is set.
+    """
+
+    MAX_CHECK_ATTEMPTS: int = 10
+
+    def __init__(self, bucket_name: str, cache_config: CacheConfig):
+        try:
+            from google.cloud import storage  # type: ignore
+        except ModuleNotFoundError as e:
+            handle_module_not_found_error(e, ["heim"])
+
+        self._bucket_name: str = bucket_name
+        self._cache = Cache(cache_config)
+        self._storage_client: Optional[storage.Client] = None
+
+    def upload(self, request: FileUploadRequest) -> FileUploadResult:
+        """Uploads a file to GCS."""
+        try:
+            from google.cloud import storage  # type: ignore
+        except ModuleNotFoundError as e:
+            handle_module_not_found_error(e, ["heim"])
+
+        try:
+
+            def do_it():
+                if self._storage_client is None:
+                    self._storage_client = storage.Client()
+
+                bucket = self._storage_client.bucket(self._bucket_name)
+                file_path: str = request.path
+                blob = bucket.blob(file_path)
+
+                # Optional: set a generation-match precondition to avoid potential race conditions
+                # and data corruptions. The request to upload is aborted if the object's
+                # generation number does not match your precondition. For a destination
+                # object that does not yet exist, set the if_generation_match precondition to 0.
+                # If the destination object already exists in your bucket, set instead a
+                # generation-match precondition using its generation number.
+                generation_match_precondition: int = 0
+
+                blob.upload_from_filename(file_path, if_generation_match=generation_match_precondition)
+                url: str = self._get_url(file_path)
+
+                # Ensure the file was uploaded successfully
+                uploaded: bool = False
+                for _ in range(0, self.MAX_CHECK_ATTEMPTS):
+                    check_response = requests.head(url)
+                    if check_response.status_code == 200:
+                        uploaded = True
+                        break
+                assert uploaded, f"File {file_path} was not uploaded successfully."
+
+                hlog(f"File {file_path} uploaded and is available at {url}.")
+                return {"url": url}
+
+            cache_key: Dict = asdict(request)
+            result, cached = self._cache.get(cache_key, do_it)
+
+        except Exception as e:
+            raise GCSClientError(e)
+
+        return FileUploadResult(success=True, cached=cached, url=result["url"])
+
+    def _get_url(self, path: str) -> str:
+        return f"https://storage.googleapis.com/{self._bucket_name}/{path}"

helm/{proxy/clients → clients}/google_client.py

@@ -1,8 +1,9 @@
 from typing import List, Dict
 
 from helm.common.cache import CacheConfig
-from helm.common.request import Request, RequestResult, Sequence, Token
-from .client import CachingClient, truncate_sequence
+from helm.common.hierarchical_logger import hexception
+from helm.common.request import Request, RequestResult, GeneratedOutput, Token
+from helm.clients.client import CachingClient, truncate_sequence
 
 
 class GoogleClient(CachingClient):
@@ -32,7 +33,7 @@ class GoogleClient(CachingClient):
 
     def make_request(self, request: Request) -> RequestResult:
         raw_request = GoogleClient.convert_to_raw_request(request)
-        cache_key: Dict = CachingClient.make_cache_key(raw_request, request)
+        cache_key = CachingClient.make_cache_key(raw_request, request)
 
         try:
 
@@ -44,21 +45,22 @@ class GoogleClient(CachingClient):
             # If results are not cached for a given query, fail fast
             response, cached = self.cache.get(cache_key, fail)
         except RuntimeError as e:
+            hexception(e)
             error: str = f"GoogleClient error: {e}"
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
 
         # Expect the result to be structured the same way as a response from OpenAI API.
-        completions: List[Sequence] = []
+        completions: List[GeneratedOutput] = []
         for raw_completion in response["choices"]:
             sequence_logprob = 0
             tokens: List[Token] = []
 
             raw_data = raw_completion["logprobs"]
             for text, logprob in zip(raw_data["tokens"], raw_data["token_logprobs"]):
-                tokens.append(Token(text=text, logprob=logprob or 0, top_logprobs=dict()))
+                tokens.append(Token(text=text, logprob=logprob or 0))
                 sequence_logprob += logprob or 0
 
-            completion = Sequence(
+            completion = GeneratedOutput(
                 text=raw_completion["text"],
                 logprob=sequence_logprob,
                 tokens=tokens,

helm/clients/google_translate_client.py

@@ -0,0 +1,35 @@
+from typing import Optional
+
+from helm.common.cache import Cache, SqliteCacheConfig
+from helm.common.optional_dependencies import handle_module_not_found_error
+
+try:
+    from google.cloud import translate_v2 as translate  # type: ignore
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["heim"])
+
+
+class GoogleTranslateClient:
+    """
+    Client for Google Translate.
+    Follow the instructions at https://cloud.google.com/translate/docs/setup to use this client.
+
+    # TODO: add this as a central service
+    """
+
+    def __init__(self, cache_path: str = "prod_env/cache/google_translate.sqlite"):
+        self.translate_client: Optional[translate.Client] = None
+        self.cache = Cache(SqliteCacheConfig(cache_path))
+
+    def translate(self, text: str, target_language: str) -> str:
+        def do_it():
+            if self.translate_client is None:
+                self.translate_client = translate.Client()
+
+            result = self.translate_client.translate(text, target_language=target_language)
+            del result["input"]
+            assert "translatedText" in result, f"Invalid response: {result}"
+            return result
+
+        response, _ = self.cache.get({"text": text, "target_language": target_language}, do_it)
+        return response["translatedText"]

helm/clients/grok_client.py

@@ -0,0 +1,36 @@
+from typing import Any, Dict, Optional
+
+from helm.clients.openai_client import OpenAIClient
+from helm.common.cache import CacheConfig
+from helm.common.request import Request
+from helm.tokenizers.tokenizer import Tokenizer
+
+
+class GrokChatClient(OpenAIClient):
+
+    BASE_URL = "https://api.x.ai/v1"
+
+    _UNSUPPORTED_ARGUMENTS = ["presence_penalty", "frequency_penalty"]
+
+    def __init__(
+        self,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        cache_config: CacheConfig,
+        api_key: Optional[str] = None,
+    ):
+        super().__init__(
+            tokenizer=tokenizer,
+            tokenizer_name=tokenizer_name,
+            cache_config=cache_config,
+            api_key=api_key,
+            org_id=None,
+            base_url="https://api.x.ai/v1",
+        )
+
+    def _make_chat_raw_request(self, request: Request) -> Dict[str, Any]:
+        raw_request = super()._make_chat_raw_request(request)
+        for unsupported_argument in self._UNSUPPORTED_ARGUMENTS:
+            if unsupported_argument in raw_request:
+                del raw_request[unsupported_argument]
+        return raw_request