crfm-helm 0.5.4__py3-none-any.whl → 0.5.5__py3-none-any.whl

helm/clients/mistral_client.py

@@ -1,16 +1,17 @@
 import requests
-from typing import Any, Dict, List, Optional, TypedDict
+from typing import Any, Dict, List, Optional, TypedDict, Union
 
 from helm.proxy.retry import NonRetriableException
 from helm.common.cache import CacheConfig
+from helm.common.media_object import IMAGE_TYPE, TEXT_TYPE
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.common.request import wrap_request_time, Request, RequestResult, GeneratedOutput
 from helm.tokenizers.tokenizer import Tokenizer
-from .client import CachingClient, truncate_and_tokenize_response_text
+from helm.clients.client import CachingClient, truncate_and_tokenize_response_text
 
 try:
-    from mistralai.client import MistralClient
-    from mistralai.models.chat_completion import ChatMessage, ChatCompletionResponse
+    from mistralai import Mistral
+    from mistralai.models import ChatCompletionResponse
 except ModuleNotFoundError as e:
     handle_module_not_found_error(e, ["mistral"])
 
@@ -19,11 +20,14 @@ class MistralAIRequest(TypedDict):
     """Data passed between make_request and _send_request. Used as the cache key."""
 
     model: str
-    prompt: str
+    # The prompt can be either a string or a list of messages that can be multimodal
+    prompt: Optional[Union[str, List[Dict[str, str]]]]
+    messages: Optional[List[Dict[str, Any]]]
     max_tokens: int
     temperature: float
     top_p: float
     random_seed: Optional[int]
+    stop: Optional[List[str]]
 
 
 class MistralAIClient(CachingClient):
@@ -43,21 +47,24 @@ class MistralAIClient(CachingClient):
         self.api_key: str = api_key
         self.tokenizer = tokenizer
         self.tokenizer_name = tokenizer_name
-        self._client = MistralClient(api_key=self.api_key)
+        self._client = Mistral(api_key=self.api_key)
         self.mistral_model = mistral_model
 
     def _send_request(self, raw_request: MistralAIRequest) -> Dict[str, Any]:
-        messages = [ChatMessage(role="user", content=raw_request["prompt"])]
-
-        chat_response: ChatCompletionResponse = self._client.chat(
+        if raw_request["messages"] is not None:
+            messages = raw_request["messages"]
+        else:
+            messages = [{"role": "user", "content": raw_request["prompt"]}]
+        chat_response: Optional[ChatCompletionResponse] = self._client.chat.complete(
             model=raw_request["model"],
-            messages=messages,
+            messages=messages,  # type: ignore
             temperature=raw_request["temperature"],
             max_tokens=raw_request["max_tokens"],
             top_p=raw_request["top_p"],
             random_seed=raw_request["random_seed"],
             safe_prompt=False,  # Disable safe_prompt
         )
+        assert chat_response is not None
         # Documentation: "If mode is 'json', the output will only contain JSON serializable types."
         # Source: https://docs.pydantic.dev/latest/api/base_model/#pydantic.BaseModel.model_dump
         #
@@ -86,17 +93,54 @@ class MistralAIClient(CachingClient):
         """Make a request"""
         completions: List[GeneratedOutput] = []
 
+        prompt: Union[str, List[Dict[str, str]]] = request.prompt
+        if request.multimodal_prompt:
+            # Following https://docs.mistral.ai/capabilities/vision
+            multimodal_content: List[Dict[str, str]] = []
+            for media_object in request.multimodal_prompt.media_objects:
+                if media_object.is_type(IMAGE_TYPE) and media_object.location:
+                    assert media_object.location
+                    if media_object.is_local_file:
+                        from helm.common.images_utils import encode_base64
+
+                        base64_image: str = encode_base64(media_object.location)
+                        image_url = f"data:image/jpeg;base64,{base64_image}"
+                    else:
+                        image_url = media_object.location
+                    multimodal_content.append({"type": "image_url", "image_url": image_url})
+                elif media_object.is_type(TEXT_TYPE):
+                    assert media_object.text
+                    multimodal_content.append({"type": "text", "text": media_object.text})
+                else:
+                    raise ValueError(f"Unrecognized MediaObject type {media_object.type}")
+
+            prompt = multimodal_content
+
         # `num_completions` is not supported, so instead make `num_completions` separate requests.
         for completion_index in range(request.num_completions):
             try:
-                raw_request: MistralAIRequest = {
-                    "model": self.mistral_model or request.model_engine,
-                    "prompt": request.prompt,
-                    "max_tokens": request.max_tokens,
-                    "temperature": request.temperature,
-                    "top_p": request.top_p,
-                    "random_seed": self._get_random_seed(request, completion_index),
-                }
+                if request.messages:
+                    raw_request: MistralAIRequest = {
+                        "model": self.mistral_model or request.model_engine,
+                        "prompt": None,
+                        "messages": request.messages,
+                        "max_tokens": request.max_tokens,
+                        "temperature": request.temperature,
+                        "top_p": request.top_p,
+                        "random_seed": self._get_random_seed(request, completion_index),
+                        "stop": request.stop_sequences or None,
+                    }
+                else:
+                    raw_request = {
+                        "model": self.mistral_model or request.model_engine,
+                        "prompt": prompt,
+                        "messages": None,
+                        "max_tokens": request.max_tokens,
+                        "temperature": request.temperature,
+                        "top_p": request.top_p,
+                        "random_seed": self._get_random_seed(request, completion_index),
+                        "stop": request.stop_sequences or None,
+                    }
 
                 def do_it() -> Dict[str, Any]:
                     result: Dict[str, Any] = self._send_request(raw_request)

helm/clients/nvidia_nim_client.py

@@ -30,6 +30,3 @@ class NvidiaNimClient(OpenAIClient):
 
     def _get_model_for_request(self, request: Request) -> str:
         return request.model
-
-    def _is_chat_model_engine(self, model_engine: str) -> bool:
-        return True

helm/clients/openai_client.py

@@ -1,18 +1,20 @@
 # mypy: check_untyped_defs = False
 from dataclasses import replace
-from typing import Any, Dict, List, Optional, cast, Union
+from typing import Any, Dict, List, Optional, cast, Union, Callable
 
 from helm.benchmark.model_metadata_registry import is_vlm
+from helm.common import multimodal_request_utils
 from helm.common.cache import CacheConfig
-from helm.common.media_object import TEXT_TYPE
-from helm.common.request import wrap_request_time, Request, RequestResult, GeneratedOutput, Token
+from helm.common.media_object import TEXT_TYPE, MultimediaObject
+from helm.common.request import ErrorFlags, wrap_request_time, Request, RequestResult, GeneratedOutput, Token
 from helm.common.hierarchical_logger import hlog
+from helm.common.object_spec import get_class_by_name
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.common.tokenization_request import (
     TokenizationRequest,
     TokenizationRequestResult,
 )
-from .client import CachingClient, truncate_sequence, generate_uid_for_multimodal_prompt
+from helm.clients.client import Client, CachingClient, truncate_sequence, generate_uid_for_multimodal_prompt
 from helm.tokenizers.tokenizer import Tokenizer
 
 try:
@@ -27,6 +29,19 @@ class OpenAIClient(CachingClient):
 
     # Error OpenAI throws when the image in the prompt violates their content policy
     INAPPROPRIATE_IMAGE_ERROR: str = "Your input image may contain content that is not allowed by our safety system"
+    INAPPROPRIATE_PROMPT_ERROR: str = "Invalid prompt: your prompt was flagged"
+    INAPPROPRIATE_PROMPT_AZURE_ERROR: str = (
+        "The response was filtered due to the prompt triggering Azure OpenAI's content management policy."
+    )
+    INAPPROPRIATE_PROMPT_MICROSOFT_ERROR: str = (
+        "The response was filtered due to the prompt triggering Microsoft's content management policy."
+    )
+
+    # OpenAI server error
+    OPENAI_SERVER_ERROR: str = (
+        "The server had an error processing your request. Sorry about that! You can retry your request, "
+        "or contact us through our help center at help.openai.com if you keep seeing this error."
+    )
 
     # Set the finish reason to this if the prompt violates OpenAI's content policy
     CONTENT_POLICY_VIOLATED_FINISH_REASON: str = (
@@ -42,28 +57,31 @@ class OpenAIClient(CachingClient):
         api_key: Optional[str] = None,
         org_id: Optional[str] = None,
         base_url: Optional[str] = None,
+        reasoning_effort: Optional[str] = None,
+        openai_model_name: Optional[str] = None,
+        output_processor: Optional[str] = None,
     ):
         super().__init__(cache_config=cache_config)
         self.tokenizer = tokenizer
         self.tokenizer_name = tokenizer_name
         self.client = OpenAI(api_key=api_key, organization=org_id, base_url=base_url)
-
-    def _is_chat_model_engine(self, model_engine: str) -> bool:
-        if model_engine == "gpt-3.5-turbo-instruct":
-            return False
-        elif model_engine.startswith("gpt-3.5") or model_engine.startswith("gpt-4") or model_engine.startswith("o1"):
-            return True
-        return False
+        self.reasoning_effort = reasoning_effort
+        self.openai_model_name = openai_model_name
+        self.output_processor: Optional[Callable[[str], str]] = (
+            get_class_by_name(output_processor) if output_processor else None
+        )
 
     def _get_model_for_request(self, request: Request) -> str:
-        return request.model_engine
+        return self.openai_model_name or request.model_engine
 
     def _get_cache_key(self, raw_request: Dict, request: Request):
         cache_key = CachingClient.make_cache_key(raw_request, request)
         if request.multimodal_prompt:
             prompt_key: str = generate_uid_for_multimodal_prompt(request.multimodal_prompt)
             cache_key = {**cache_key, "multimodal_prompt": prompt_key}
-            del cache_key["messages"]
+
+            if "messages" in cache_key:
+                del cache_key["messages"]
         return cache_key
 
     def _make_embedding_request(self, request: Request) -> RequestResult:
@@ -140,6 +158,19 @@ class OpenAIClient(CachingClient):
                         base64_image: str = encode_base64(media_object.location)
                         image_object: Dict[str, str] = {"url": f"data:image/jpeg;base64,{base64_image}"}
                         content.append({"type": "image_url", "image_url": image_object})
+                    elif media_object.is_type("audio") and media_object.location:
+                        base64_audio: str = multimodal_request_utils.get_contents_as_base64(media_object.location)
+                        format: str = media_object.content_type.split("/")[1]
+                        if format == "mpeg":
+                            # OpenAI expects "mp3" for mpeg audio
+                            format = "mp3"
+
+                        content.append(
+                            {
+                                "type": "input_audio",
+                                "input_audio": {"data": base64_audio, "format": format},
+                            }
+                        )
                     elif media_object.is_type(TEXT_TYPE):
                         content.append({"type": media_object.type, "text": media_object.text})
                     else:
@@ -164,15 +195,35 @@ class OpenAIClient(CachingClient):
             "frequency_penalty": request.frequency_penalty,
         }
 
-        # OpenAI's vision API doesn't allow None values for stop.
-        # Fails with "body -> stop: none is not an allowed value" if None is passed.
-        if is_vlm(request.model) and raw_request["stop"] is None:
-            raw_request.pop("stop")
+        if request.response_format and request.response_format.json_schema:
+            # Copy and modify JSON schema to conform to OpenAI's requirements
+            json_schema = dict(request.response_format.json_schema)
+
+            # additionalProperties: false must always be set in objects
+            # https://platform.openai.com/docs/guides/structured-outputs#additionalproperties-false-must-always-be-set-in-objects
+            if "additionalProperties" not in json_schema:
+                json_schema["additionalProperties"] = False
+
+            # All fields must be required
+            # https://platform.openai.com/docs/guides/structured-outputs#all-fields-must-be-required
+            if "required" not in json_schema:
+                json_schema["required"] = list(json_schema["properties"].keys())
+
+            raw_request["response_format"] = {
+                "type": "json_schema",
+                "json_schema": {
+                    "name": "response",
+                    "description": "",
+                    "schema": json_schema,
+                    "strict": True,
+                },
+            }
 
         # Special handling for o1 models.
         # Refer to the "Reasoning models" documentation further discussion of o1 model limitations:
         # https://platform.openai.com/docs/guides/reasoning
-        if request.model_engine.startswith("o1"):
+        model_engine: str = request.model_engine
+        if model_engine.startswith("o1") or model_engine.startswith("o3"):
             # Avoid error:
             # "Unsupported parameter: 'max_tokens' is not supported with this model. Use 'max_completion_tokens' instead."  # noqa: E501
             # Note that openai>=1.45 is needed for this
@@ -184,6 +235,30 @@ class OpenAIClient(CachingClient):
             if raw_request["stop"] is None:
                 raw_request.pop("stop")
 
+            # Avoid error:
+            # "Error code: 400 - {'error': {'message': "Unsupported parameter: 'temperature' is
+            # not supported with this model.", 'type': 'invalid_request_error', 'param': 'temperature',
+            # 'code': 'unsupported_parameter'}}"
+            raw_request.pop("temperature", None)
+
+            if self.reasoning_effort:
+                raw_request["reasoning_effort"] = "self.reasoning_effort"
+        elif is_vlm(request.model):
+            # Avoid error:
+            # "Invalid type for 'stop': expected an unsupported value, but got null instead."
+            if raw_request["stop"] is None:
+                raw_request.pop("stop")
+
+        # Special handling for gpt-4o-audio-preview
+        # See: https://platform.openai.com/docs/guides/audio
+        if model_engine.startswith("gpt-4o-audio-preview") or model_engine.startswith("gpt-4o-mini-audio-preview"):
+            raw_request["modalities"] = ["text"]
+
+            # Avoid error:
+            # OpenAI error: Error code: 400 - {'error': {'message': "[{'type': 'string_type', 'loc': ('body', 'stop', 'str'), 'msg': 'Input should be a valid string', 'input': None}, {'type': 'list_type', 'loc': ('body', 'stop', 'list[str]'), 'msg': 'Input should be a valid list', 'input': None}, {'type': 'list_type', 'loc': ('body', 'stop', 'list[list[int]]'), 'msg': 'Input should be a valid list', 'input': None}]", 'type': 'invalid_request_error', 'param': None, 'code': None}}  # noqa: 3501
+            if raw_request["stop"] is None:
+                raw_request.pop("stop")
+
         def do_it() -> Dict[str, Any]:
             return self.client.chat.completions.create(**raw_request).model_dump(mode="json")
 
@@ -191,7 +266,7 @@ class OpenAIClient(CachingClient):
             cache_key = self._get_cache_key(raw_request, request)
             response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
         except openai.OpenAIError as e:
-            if self.INAPPROPRIATE_IMAGE_ERROR in str(e):
+            if self.INAPPROPRIATE_IMAGE_ERROR in str(e) or self.INAPPROPRIATE_PROMPT_ERROR in str(e):
                 hlog(f"Failed safety check: {str(request)}")
                 empty_completion = GeneratedOutput(
                     text="",
@@ -206,15 +281,54 @@ class OpenAIClient(CachingClient):
                     completions=[empty_completion] * request.num_completions,
                     embedding=[],
                 )
+            elif self.OPENAI_SERVER_ERROR in str(e):
+                # Handle these errors by returning an empty completion to unblock
+                hlog(f"OpenAI server error for request: {str(request)}")
+                empty_completion = GeneratedOutput(
+                    text="",
+                    logprob=0,
+                    tokens=[],
+                    finish_reason={"reason": self.OPENAI_SERVER_ERROR},
+                )
+                return RequestResult(
+                    success=True,
+                    cached=False,
+                    request_time=0,
+                    completions=[empty_completion] * request.num_completions,
+                    embedding=[],
+                )
+            elif self.INAPPROPRIATE_PROMPT_AZURE_ERROR in str(e) or self.INAPPROPRIATE_PROMPT_MICROSOFT_ERROR in str(e):
+                return RequestResult(
+                    success=False,
+                    cached=False,
+                    error="Content blocked by Azure's content management filter",
+                    completions=[],
+                    embedding=[],
+                    error_flags=ErrorFlags(is_retriable=False, is_fatal=False),
+                )
 
             error: str = f"OpenAI error: {e}"
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
 
         completions: List[GeneratedOutput] = []
         for raw_completion in response["choices"]:
+            # Handle Azure OpenAI content filter
+            # See: https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/content-filter
+            if raw_completion["finish_reason"] == "content_filter":
+                hlog(f"Content blocked by OpenAI filter: {str(raw_request)}")
+                return RequestResult(
+                    success=False,
+                    cached=False,
+                    error="Content blocked by OpenAI filter",
+                    completions=[],
+                    embedding=[],
+                    error_flags=ErrorFlags(is_retriable=False, is_fatal=False),
+                )
             # The OpenAI chat completion API doesn't support echo.
             # If `echo_prompt` is true, combine the prompt and completion.
             raw_completion_content = raw_completion["message"]["content"]
+            if self.output_processor:
+                raw_completion_content = self.output_processor(raw_completion_content)
             text: str = request.prompt + raw_completion_content if request.echo_prompt else raw_completion_content
             # The OpenAI chat completion API doesn't return us tokens or logprobs, so we tokenize ourselves.
             tokenization_result: TokenizationRequestResult = self.tokenizer.tokenize(
@@ -313,10 +427,115 @@ class OpenAIClient(CachingClient):
             embedding=[],
         )
 
+    def _make_transcription_request(self, request: Request) -> RequestResult:
+        assert (
+            request.multimodal_prompt is not None and request.multimodal_prompt.size == 1
+        ), "Expected just a single audio file."
+        media_object = request.multimodal_prompt.media_objects[0]
+        assert media_object.is_type("audio") and media_object.location, "Expected an audio file."
+        audio_path: str = media_object.location
+        model: str = self._get_model_for_request(request)
+
+        def do_it() -> Dict[str, Any]:
+            transcription = self.client.audio.transcriptions.create(model=model, file=open(audio_path, "rb"))
+            return {"transcription": transcription.text}
+
+        try:
+            cache_key = self._get_cache_key({"audio": audio_path, "model": model}, request)
+            response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
+        except openai.OpenAIError as e:
+            error: str = f"OpenAI error: {e}"
+            return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
+
+        return RequestResult(
+            success=True,
+            cached=cached,
+            request_time=response["request_time"],
+            request_datetime=response.get("request_datetime"),
+            completions=[GeneratedOutput(text=response["transcription"], logprob=0, tokens=[])],
+            embedding=[],
+        )
+
     def make_request(self, request: Request) -> RequestResult:
         if request.embedding:
             return self._make_embedding_request(request)
-        elif self._is_chat_model_engine(request.model_engine):
-            return self._make_chat_request(request)
+        elif "whisper" in request.model_engine:
+            return self._make_transcription_request(request)
         else:
-            return self._make_completion_request(request)
+            return self._make_chat_request(request)
+
+
+class OpenAILegacyCompletionsClient(OpenAIClient):
+    def make_request(self, request: Request) -> RequestResult:
+        return self._make_completion_request(request)
+
+
+class OpenAITranscriptionThenCompletionClient(Client):
+    """
+    Wrapper around `OpenAIClient` that transcribes audio to text with a
+    speech-to-text model (e.g., Whisper) before making a completion request.
+    """
+
+    @staticmethod
+    def wrap_transcribed_indicator(transcription: str) -> str:
+        return f"\n[TRANSCRIBED AUDIO START]\n{transcription}\n[TRANSCRIBED AUDIO END]\n"
+
+    def __init__(
+        self,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        cache_config: CacheConfig,
+        api_key: Optional[str] = None,
+        org_id: Optional[str] = None,
+    ):
+        self._openai_client = OpenAIClient(
+            tokenizer=tokenizer,
+            tokenizer_name=tokenizer_name,
+            cache_config=cache_config,
+            api_key=api_key,
+            org_id=org_id,
+        )
+
+    def make_request(self, request: Request) -> RequestResult:
+        # Ensure that there is only one _ in the model engine name as the format is
+        # `transcription-model_completion-model`
+        assert request.model_engine.count("_") == 1, f"Invalid model name: {request.model_engine}"
+        # Use `model_engine` to determine both the models for transcription and completion
+        transcription_model, completion_model = request.model_engine.split("_")
+
+        # Only multimodal prompts are supported
+        assert request.multimodal_prompt is not None, "Expected a multimodal prompt"
+
+        # Gather all the text content and transcribe any audio to text
+        text_content: List[str] = []
+        for media_object in request.multimodal_prompt.media_objects:
+            if media_object.is_type("audio") and media_object.location:
+                request = Request(
+                    model=f"openai/{transcription_model}",
+                    multimodal_prompt=MultimediaObject(media_objects=[media_object]),
+                )
+                response = self._openai_client.make_request(request)
+
+                transcribed_text: str
+                if response.success and response.completions:
+                    transcribed_text = response.completions[0].text
+                else:
+                    transcribed_text = ""
+                    hlog(f"Failed to transcribe audio: {response.error}")
+
+                text_content.append(self.wrap_transcribed_indicator(transcribed_text))
+            elif media_object.is_type(TEXT_TYPE):
+                assert media_object.text is not None, "Expected text content"
+                text_content.append(media_object.text)
+            else:
+                raise ValueError(f"Unrecognized media type: {media_object.type}")
+
+        text_prompt: str = "\n".join(text_content)
+        hlog(f"Transcribed prompt:\n{text_prompt}")
+
+        # Now make the request to the completion model with just a text-only prompt and no audio
+        # Use the same decoding parameters as the original request
+        # Ensure to set multimodal_prompt to None so the request is treated as text-only.
+        return self._openai_client.make_request(
+            replace(request, prompt=text_prompt, model=f"openai/{completion_model}", multimodal_prompt=None)
+        )

helm/clients/palmyra_client.py

@@ -12,7 +12,7 @@ from helm.common.tokenization_request import (
     TokenizationRequestResult,
 )
 from helm.tokenizers.tokenizer import Tokenizer
-from .client import CachingClient, truncate_sequence
+from helm.clients.client import CachingClient, truncate_sequence
 
 
 _CONTENT_MODERATION_KEY = "fail.content.moderation.failed"
@@ -163,6 +163,3 @@ class PalmyraChatClient(OpenAIClient):
             org_id=None,
             base_url="https://api.writer.com/v1/chat",
         )
-
-    def _is_chat_model_engine(self, model_engine: str) -> bool:
-        return True

helm/clients/reka_client.py

@@ -9,7 +9,7 @@ from helm.common.request import wrap_request_time, Request, RequestResult, Gener
 from helm.common.hierarchical_logger import hlog
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.tokenizers.tokenizer import Tokenizer
-from .client import CachingClient, truncate_and_tokenize_response_text
+from helm.clients.client import CachingClient, truncate_and_tokenize_response_text
 
 try:
     import reka

helm/clients/stanfordhealthcare_azure_openai_client.py

@@ -0,0 +1,58 @@
+from typing import Optional
+
+from helm.clients.azure_openai_client import AzureOpenAIClient
+from helm.common.cache import CacheConfig
+from helm.proxy.retry import NonRetriableException
+from helm.tokenizers.tokenizer import Tokenizer
+
+
+class StanfordHealthCareAzureOpenAIClient(AzureOpenAIClient):
+    """
+    Client for accessing OpenAI models hosted on Stanford Health Care's model API.
+
+    Configure by setting the following in prod_env/credentials.conf:
+
+    ```
+    stanfordhealthcareEndpoint: https://your-domain-name/
+    stanfordhealthcareApiKey: your-private-key
+    ```
+    """
+
+    CREDENTIAL_HEADER_NAME = "Ocp-Apim-Subscription-Key"
+
+    def __init__(
+        self,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        cache_config: CacheConfig,
+        openai_model_name: str,
+        api_version: str,
+        api_key: Optional[str] = None,
+        endpoint: Optional[str] = None,
+        base_url: Optional[str] = None,
+    ):
+        if not api_key:
+            raise NonRetriableException("Must provide API key through credentials.conf")
+        if base_url:
+            base_url = base_url.format(endpoint=endpoint)
+            super().__init__(
+                tokenizer=tokenizer,
+                tokenizer_name=tokenizer_name,
+                cache_config=cache_config,
+                api_key="unused",
+                base_url=base_url,
+                azure_openai_deployment_name=openai_model_name,
+                api_version=api_version,
+                default_headers={StanfordHealthCareAzureOpenAIClient.CREDENTIAL_HEADER_NAME: api_key},
+            )
+        else:
+            super().__init__(
+                tokenizer=tokenizer,
+                tokenizer_name=tokenizer_name,
+                cache_config=cache_config,
+                api_key="unused",
+                endpoint=endpoint,
+                azure_openai_deployment_name=openai_model_name,
+                api_version=api_version,
+                default_headers={StanfordHealthCareAzureOpenAIClient.CREDENTIAL_HEADER_NAME: api_key},
+            )

helm/clients/stanfordhealthcare_claude_client.py

@@ -0,0 +1,31 @@
+from typing import Any, Dict, List
+
+from helm.common.request import (
+    Request,
+    GeneratedOutput,
+)
+from helm.clients.stanfordhealthcare_http_model_client import StanfordHealthCareHTTPModelClient
+
+
+class StanfordHealthCareClaudeClient(StanfordHealthCareHTTPModelClient):
+    """
+    Client for accessing Claude models hosted on Stanford Health Care's model API.
+
+    Configure by setting the following in prod_env/credentials.conf:
+
+    ```
+    stanfordhealthcareEndpoint: https://your-domain-name/
+    stanfordhealthcareApiKey: your-private-key
+    ```
+    """
+
+    def get_request(self, request: Request) -> Dict[str, Any]:
+        return {
+            "model_id": self.model,
+            "prompt_text": request.prompt,
+            "max_tokens": request.max_tokens,
+            "temperature": request.temperature,
+        }
+
+    def parse_response(self, response: Dict[str, Any]) -> List[GeneratedOutput]:
+        return [GeneratedOutput(text=item["text"], logprob=0, tokens=[]) for item in response["content"]]

helm/clients/stanfordhealthcare_google_client.py

@@ -0,0 +1,43 @@
+from typing import Any, Dict, List
+
+from helm.common.request import (
+    Request,
+    GeneratedOutput,
+)
+from helm.clients.stanfordhealthcare_http_model_client import StanfordHealthCareHTTPModelClient
+
+
+class StanfordHealthCareGoogleClient(StanfordHealthCareHTTPModelClient):
+    """
+    Client for accessing Google models hosted on Stanford Health Care's model API.
+
+    Configure by setting the following in prod_env/credentials.conf:
+
+    ```
+    stanfordhealthcareEndpoint: https://your-domain-name/
+    stanfordhealthcareApiKey: your-private-key
+    ```
+    """
+
+    def get_request(self, request: Request) -> Dict[str, Any]:
+        return {
+            "contents": {
+                "role": "user",
+                "parts": {"text": request.prompt},
+            },
+            "generation_config": {
+                "temperature": request.temperature,
+                "topP": request.top_p,
+                "topK": request.top_k_per_token,
+                "candidateCount": 1,
+                "maxOutputTokens": request.max_tokens,
+                "stopSequences": request.stop_sequences,
+            },
+        }
+
+    def parse_response(self, response: Dict[str, Any]) -> List[GeneratedOutput]:
+        completion = ""
+        for item in response["content"]:
+            if "content" in item["candidates"][0]:
+                completion += item["candidates"][0]["content"]["parts"][0]["text"]
+        return [GeneratedOutput(text=completion, logprob=0, tokens=[])]