crfm-helm 0.5.3__py3-none-any.whl → 0.5.5__py3-none-any.whl

helm/clients/stanfordhealthcare_google_client.py

@@ -0,0 +1,43 @@
+from typing import Any, Dict, List
+
+from helm.common.request import (
+    Request,
+    GeneratedOutput,
+)
+from helm.clients.stanfordhealthcare_http_model_client import StanfordHealthCareHTTPModelClient
+
+
+class StanfordHealthCareGoogleClient(StanfordHealthCareHTTPModelClient):
+    """
+    Client for accessing Google models hosted on Stanford Health Care's model API.
+
+    Configure by setting the following in prod_env/credentials.conf:
+
+    ```
+    stanfordhealthcareEndpoint: https://your-domain-name/
+    stanfordhealthcareApiKey: your-private-key
+    ```
+    """
+
+    def get_request(self, request: Request) -> Dict[str, Any]:
+        return {
+            "contents": {
+                "role": "user",
+                "parts": {"text": request.prompt},
+            },
+            "generation_config": {
+                "temperature": request.temperature,
+                "topP": request.top_p,
+                "topK": request.top_k_per_token,
+                "candidateCount": 1,
+                "maxOutputTokens": request.max_tokens,
+                "stopSequences": request.stop_sequences,
+            },
+        }
+
+    def parse_response(self, response: Dict[str, Any]) -> List[GeneratedOutput]:
+        completion = ""
+        for item in response["content"]:
+            if "content" in item["candidates"][0]:
+                completion += item["candidates"][0]["content"]["parts"][0]["text"]
+        return [GeneratedOutput(text=completion, logprob=0, tokens=[])]

helm/clients/stanfordhealthcare_http_model_client.py

@@ -0,0 +1,93 @@
+import requests
+
+from abc import ABC, abstractmethod
+from dataclasses import asdict
+from typing import Any, Dict, List, Optional
+
+from helm.common.cache import CacheConfig
+from helm.common.request import (
+    wrap_request_time,
+    Request,
+    RequestResult,
+    GeneratedOutput,
+    EMBEDDING_UNAVAILABLE_REQUEST_RESULT,
+)
+from helm.clients.client import CachingClient
+
+
+class StanfordHealthCareHTTPModelClient(CachingClient, ABC):
+    """
+    Client for accessing Stanford Health Care models via HTTP requests.
+
+    Configure by setting the following in prod_env/credentials.conf:
+
+    ```
+    stanfordhealthcareEndpoint: https://your-domain-name/
+    stanfordhealthcareApiKey: your-private-key
+    ```
+    """
+
+    CREDENTIAL_HEADER_NAME = "Ocp-Apim-Subscription-Key"
+
+    def __init__(
+        self,
+        cache_config: CacheConfig,
+        deployment: str,
+        endpoint: str = "http://localhost:8080",
+        do_cache: bool = False,
+        timeout: int = 3000,
+        api_key: Optional[str] = None,
+        model: Optional[str] = None,
+    ):
+        super().__init__(cache_config=cache_config)
+        assert api_key, "API key must be provided"
+        self.endpoint = endpoint
+        self.timeout = timeout
+        self.do_cache = do_cache
+        self.deployment = deployment
+        self.model = model
+        self.default_headers = {StanfordHealthCareHTTPModelClient.CREDENTIAL_HEADER_NAME: api_key}
+
+    def make_request(self, request: Request) -> RequestResult:
+        cache_key = asdict(request)
+        if request.embedding:
+            return EMBEDDING_UNAVAILABLE_REQUEST_RESULT
+
+        raw_request = self.get_request(request)
+
+        try:
+
+            def do_it() -> Dict[str, Any]:
+                url = f"{self.endpoint}/{self.deployment}"
+                response = requests.post(url, json=raw_request, headers=self.default_headers, timeout=self.timeout)
+                response.raise_for_status()
+                response_json = response.json()
+                if type(response_json) == list:
+                    response_json = {"content": response_json}
+                return response_json
+
+            if self.do_cache:
+                response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
+            else:
+                response, cached = wrap_request_time(do_it)(), False
+
+            completions = self.parse_response(response)
+
+            return RequestResult(
+                success=True,
+                cached=cached,
+                error=None,
+                completions=completions,
+                embedding=[],
+                request_time=response["request_time"],
+            )
+        except requests.exceptions.RequestException as e:
+            return RequestResult(success=False, cached=False, error=f"Request error: {e}", completions=[], embedding=[])
+
+    @abstractmethod
+    def get_request(self, request: Request) -> Dict[str, Any]:
+        pass
+
+    @abstractmethod
+    def parse_response(self, response: Dict[str, Any]) -> List[GeneratedOutput]:
+        pass

helm/clients/stanfordhealthcare_openai_client.py

@@ -0,0 +1,62 @@
+from typing import Optional
+
+from helm.clients.openai_client import OpenAIClient
+from helm.common.cache import CacheConfig
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.proxy.retry import NonRetriableException
+from helm.tokenizers.tokenizer import Tokenizer
+
+try:
+    from openai import OpenAI
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["openai"])
+
+
+class StanfordHealthCareOpenAIClient(OpenAIClient):
+    """
+    Client for accessing OpenAI models hosted on Stanford Health Care's model API.
+
+    Configure by setting the following in prod_env/credentials.conf:
+
+    ```
+    stanfordhealthcareEndpoint: https://your-domain-name/
+    stanfordhealthcareApiKey: your-private-key
+    ```
+    """
+
+    CREDENTIAL_HEADER_NAME = "Ocp-Apim-Subscription-Key"
+
+    def __init__(
+        self,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        cache_config: CacheConfig,
+        model_name: str,
+        api_key: Optional[str] = None,
+        endpoint: Optional[str] = None,
+        base_url: Optional[str] = None,
+        openai_model_name: Optional[str] = None,
+        output_processor: Optional[str] = None,
+    ):
+        super().__init__(
+            tokenizer=tokenizer,
+            tokenizer_name=tokenizer_name,
+            cache_config=cache_config,
+            api_key="unused",
+            openai_model_name=openai_model_name,
+            output_processor=output_processor,
+        )
+        if not endpoint:
+            raise NonRetriableException("Must provide endpoint through credentials.conf")
+        if not api_key:
+            raise NonRetriableException("Must provide API key through credentials.conf")
+        if not base_url:
+            raise NonRetriableException("Must provide base url through model_deployments")
+        # Guess the base URL part based on the model name
+        # Maybe make this configurable instead?
+        base_url = base_url.format(endpoint=endpoint)
+        self.client = OpenAI(
+            api_key="dummy",
+            base_url=base_url,
+            default_headers={StanfordHealthCareOpenAIClient.CREDENTIAL_HEADER_NAME: api_key},
+        )

helm/clients/stanfordhealthcare_shc_openai_client.py

@@ -0,0 +1,42 @@
+from typing import Dict, Optional
+
+from helm.clients.azure_openai_client import AzureOpenAIClient
+from helm.common.cache import CacheConfig
+from helm.proxy.retry import NonRetriableException
+from helm.tokenizers.tokenizer import Tokenizer
+
+
+class StanfordHealthCareSHCOpenAIClient(AzureOpenAIClient):
+    """
+    Client for accessing OpenAI models hosted on Stanford Health Care's model API.
+
+    Configure by setting the following in prod_env/credentials.conf:
+
+    ```
+    stanfordhealthcareEndpoint: https://your-domain-name/
+    stanfordhealthcareApiKey: your-private-key
+    ```
+    """
+
+    API_VERSION = "2024-08-01-preview"
+
+    def __init__(
+        self,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        cache_config: CacheConfig,
+        api_key: Optional[str] = None,
+        endpoint: Optional[str] = None,
+        default_headers: Optional[Dict[str, str]] = None,
+    ):
+        if not api_key:
+            raise NonRetriableException("Must provide API key through credentials.conf")
+        super().__init__(
+            tokenizer=tokenizer,
+            tokenizer_name=tokenizer_name,
+            cache_config=cache_config,
+            api_key=api_key,
+            endpoint=endpoint,
+            api_version=StanfordHealthCareSHCOpenAIClient.API_VERSION,
+            default_headers=default_headers,
+        )

helm/clients/test_client.py

@@ -1,6 +1,6 @@
 from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from helm.tokenizers.auto_tokenizer import AutoTokenizer
-from .client import truncate_sequence, truncate_and_tokenize_response_text
+from helm.clients.client import truncate_sequence, truncate_and_tokenize_response_text
 from typing import List
 from helm.common.request import Request, GeneratedOutput, Token
 

helm/clients/test_together_client.py

@@ -5,7 +5,12 @@ import tempfile
 from helm.common.cache import BlackHoleCacheConfig, SqliteCacheConfig
 from helm.common.request import Request
 
-from .together_client import TogetherClient, TogetherChatClient, TogetherCompletionClient, TogetherClientError
+from helm.clients.together_client import (
+    TogetherClient,
+    TogetherChatClient,
+    TogetherCompletionClient,
+    TogetherClientError,
+)
 
 
 class TestTogetherClient:

helm/clients/together_client.py

@@ -1,12 +1,15 @@
 from copy import deepcopy
 from itertools import zip_longest
 import threading
-from typing import List, Dict, Any, Mapping, Optional, TypedDict, Union
+from typing import Callable, List, Dict, Any, Mapping, Optional, TypedDict, Union
+from typing_extensions import NotRequired
 
 import requests
 from retrying import retry
 
 from helm.common.cache import CacheConfig
+from helm.common.media_object import IMAGE_TYPE, TEXT_TYPE
+from helm.common.object_spec import get_class_by_name
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.common.request import wrap_request_time, Request, RequestResult, GeneratedOutput, Token
 from helm.clients.client import CachingClient, truncate_sequence, cleanup_str
@@ -312,26 +315,65 @@ class TogetherRawChatRequest(TypedDict):
     logprobs: int
     echo: bool
     n: int
+    response_format: NotRequired[Dict[str, Any]]
 
 
 class TogetherChatClient(CachingClient):
     """Client that uses the Python Together library for chat models."""
 
-    def __init__(self, cache_config: CacheConfig, api_key: Optional[str], together_model: Optional[str] = None):
+    def __init__(
+        self,
+        cache_config: CacheConfig,
+        api_key: Optional[str],
+        together_model: Optional[str] = None,
+        disable_logprobs: Optional[bool] = None,
+        output_processor: Optional[str] = None,
+    ):
         super().__init__(cache_config=cache_config)
         self._client = Together(api_key=api_key)
         self._together_model = together_model
+        self._disable_logprobs = bool(disable_logprobs)
+        # self.output_processor is actually a function, not a class
+
+        self.output_processor: Optional[Callable[[str], str]] = (
+            get_class_by_name(output_processor) if output_processor else None
+        )
 
     def convert_to_raw_chat_request(self, request: Request) -> TogetherRawChatRequest:
+        request.validate()
+        messages: List[Dict[str, Any]]
         if request.messages:
             messages = request.messages
+        elif request.multimodal_prompt:
+            message_contents = []
+            for media_object in request.multimodal_prompt.media_objects:
+                if media_object.is_type(IMAGE_TYPE) and media_object.location:
+                    assert media_object.location
+                    if media_object.is_local_file:
+                        from helm.common.images_utils import encode_base64
+
+                        base64_image: str = encode_base64(media_object.location)
+                        image_url = f"data:image/jpeg;base64,{base64_image}"
+                    else:
+                        image_url = media_object.location
+                    message_contents.append({"type": "image_url", "image_url": {"url": image_url}})
+                elif media_object.is_type(TEXT_TYPE):
+                    assert media_object.text
+                    message_contents.append({"type": "text", "text": media_object.text})
+                else:
+                    raise ValueError(f"Unrecognized MediaObject type {media_object.type}")
+            messages = [{"role": "user", "content": message_contents}]
         else:
             messages = [{"role": "user", "content": request.prompt}]
         if self._together_model is not None:
             model = self._together_model
         else:
             model = request.model
-        return {
+        if self._disable_logprobs:
+            logprobs = 0
+        else:
+            logprobs = min(request.top_k_per_token, 1)
+        raw_chat_request: TogetherRawChatRequest = {
             "messages": messages,
             "model": model,
             "max_tokens": request.max_tokens,
@@ -339,10 +381,16 @@ class TogetherChatClient(CachingClient):
             "temperature": request.temperature,
             "top_p": request.top_p,
             "top_k": request.top_k_per_token,
-            "logprobs": min(request.top_k_per_token, 1),
+            "logprobs": logprobs,
             "echo": request.echo_prompt,
             "n": request.num_completions,
         }
+        if request.response_format and request.response_format.json_schema:
+            raw_chat_request["response_format"] = {
+                "type": "json_object",
+                "schema": request.response_format.json_schema,
+            }
+        return raw_chat_request
 
     def make_request(self, request: Request) -> RequestResult:
         raw_request = self.convert_to_raw_chat_request(request)
@@ -377,7 +425,10 @@ class TogetherChatClient(CachingClient):
                         break
                     tokens.append(Token(text=token_text, logprob=token_logprob or 0.0))
             assert choice.message.role == "assistant"
-            generated_outputs.append(GeneratedOutput(text=choice.message.content, logprob=0.0, tokens=tokens))
+            output_text = choice.message.content
+            if self.output_processor:
+                output_text = self.output_processor(output_text)
+            generated_outputs.append(GeneratedOutput(text=output_text, logprob=0.0, tokens=tokens))
         return RequestResult(
             success=True,
             cached=cached,
@@ -404,16 +455,27 @@ class TogetherRawCompletionRequest(TypedDict):
 class TogetherCompletionClient(CachingClient):
     """Client that uses the Python Together library for text completion models."""
 
-    def __init__(self, cache_config: CacheConfig, api_key: Optional[str], together_model: Optional[str] = None):
+    def __init__(
+        self,
+        cache_config: CacheConfig,
+        api_key: Optional[str],
+        together_model: Optional[str] = None,
+        disable_logprobs: Optional[bool] = None,
+    ):
         super().__init__(cache_config=cache_config)
         self._client = Together(api_key=api_key)
         self._together_model = together_model
+        self._disable_logprobs = bool(disable_logprobs)
 
     def convert_to_raw_completion_request(self, request: Request) -> TogetherRawCompletionRequest:
         if self._together_model is not None:
             model = self._together_model
         else:
             model = request.model
+        if self._disable_logprobs:
+            logprobs = 0
+        else:
+            logprobs = min(request.top_k_per_token, 1)
         return {
             "prompt": request.prompt,
             "model": model,
@@ -422,7 +484,7 @@ class TogetherCompletionClient(CachingClient):
             "temperature": request.temperature,
             "top_p": request.top_p,
             "top_k": request.top_k_per_token,
-            "logprobs": min(request.top_k_per_token, 1),
+            "logprobs": logprobs,
             "echo": request.echo_prompt,
             "n": request.num_completions,
         }

helm/clients/upstage_client.py

@@ -0,0 +1,23 @@
+from helm.clients.openai_client import OpenAIClient
+from helm.common.cache import CacheConfig
+from helm.tokenizers.tokenizer import Tokenizer
+
+
+class UpstageChatClient(OpenAIClient):
+    """Sends request to a Upstage model using a OpenAI-compatible Chat API."""
+
+    def __init__(
+        self,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        cache_config: CacheConfig,
+        api_key: str,
+    ):
+        super().__init__(
+            tokenizer=tokenizer,
+            tokenizer_name=tokenizer_name,
+            cache_config=cache_config,
+            api_key=api_key,
+            org_id=None,
+            base_url="https://api.upstage.ai/v1/solar",
+        )

helm/clients/vertexai_client.py

@@ -4,6 +4,7 @@ from threading import Lock
 from typing import Any, Dict, Mapping, Optional, List, Union
 
 from helm.common.cache import CacheConfig
+from helm.common.multimodal_request_utils import get_contents_as_bytes
 from helm.common.media_object import TEXT_TYPE
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.common.request import wrap_request_time, Request, RequestResult, GeneratedOutput, ErrorFlags
@@ -12,7 +13,14 @@ from helm.clients.client import CachingClient, truncate_sequence, generate_uid_f
 try:
     import vertexai
     from vertexai.language_models import TextGenerationModel, TextGenerationResponse  # PaLM2
-    from vertexai.preview.generative_models import GenerativeModel, GenerationResponse, Candidate, Part, Image  # Gemini
+    from vertexai.preview.generative_models import (
+        GenerativeModel,
+        GenerationResponse,
+        Candidate,
+        Content,
+        Part,
+        Image,
+    )  # Gemini
     from google.cloud.aiplatform_v1beta1.types import SafetySetting, HarmCategory
 except ModuleNotFoundError as e:
     handle_module_not_found_error(e, ["google"])
@@ -48,17 +56,16 @@ def _get_safety_settings_for_preset(
         raise ValueError(f"Unknown safety_settings_preset: {safety_settings_preset}")
 
 
-def _get_model_name_for_request(request: Request) -> str:
-    # We have to strip "-safety-" suffixes from model names because they are not part of the Vertex AI model name
-    # TODO: Clean up this hack
-    return request.model_engine.split("-safety-")[0]
-
-
 class VertexAIClient(CachingClient, ABC):
     """Client for Vertex AI models"""
 
     def __init__(
-        self, cache_config: CacheConfig, project_id: str, location: str, safety_settings_preset: Optional[str] = None
+        self,
+        cache_config: CacheConfig,
+        project_id: str,
+        location: str,
+        safety_settings_preset: Optional[str] = None,
+        vertexai_model: Optional[str] = None,
     ) -> None:
         super().__init__(cache_config=cache_config)
         self.project_id = project_id
@@ -67,8 +74,15 @@ class VertexAIClient(CachingClient, ABC):
         self.safety_settings_preset = safety_settings_preset
         self.safety_settings = _get_safety_settings_for_preset(safety_settings_preset)
 
+        self.vertexai_model = vertexai_model
+
         vertexai.init(project=self.project_id, location=self.location)
 
+    def _get_model_name_for_request(self, request: Request) -> str:
+        if self.vertexai_model is not None:
+            return self.vertexai_model
+        return request.model_engine
+
     def make_cache_key_with_safety_settings_preset(self, raw_request: Mapping, request: Request) -> Mapping:
         """Construct the key for the cache using the raw request.
 
@@ -111,7 +125,7 @@ class VertexAITextClient(VertexAIClient):
         }
 
         completions: List[GeneratedOutput] = []
-        model_name: str = _get_model_name_for_request(request)
+        model_name: str = self._get_model_name_for_request(request)
 
         try:
 
@@ -193,12 +207,20 @@ class VertexAIChatClient(VertexAIClient):
 
     def make_request(self, request: Request) -> RequestResult:
         """Make a request"""
-        contents: str = request.prompt
+        contents = [request.prompt]
 
         # For the multimodal case, build up the content with the media objects of `request.multimodal_prompt`
         if request.multimodal_prompt is not None:
             return self._make_multimodal_request(request)
 
+        if request.messages is not None:
+            contents = []
+            role_mapping = {"user": "user", "assistant": "model"}
+            for msg in request.messages:
+                contents.append(
+                    Content(role=role_mapping.get(msg["role"], "user"), parts=[Part.from_text(msg["content"])])
+                )
+
         parameters = {
             "temperature": request.temperature,
             "max_output_tokens": request.max_tokens,
@@ -217,7 +239,7 @@ class VertexAIChatClient(VertexAIClient):
         }
 
         completions: List[GeneratedOutput] = []
-        model_name: str = _get_model_name_for_request(request)
+        model_name: str = self._get_model_name_for_request(request)
         model = self.get_model(model_name)
 
         try:
@@ -263,7 +285,7 @@ class VertexAIChatClient(VertexAIClient):
             cache_key = self.make_cache_key_with_safety_settings_preset(
                 {
                     "model_name": model_name,
-                    "prompt": request.prompt,
+                    "prompt": request.messages or request.prompt,
                     **parameters,
                 },
                 request,
@@ -338,6 +360,10 @@ class VertexAIChatClient(VertexAIClient):
         for media_object in request.multimodal_prompt.media_objects:
             if media_object.is_type("image") and media_object.location:
                 contents.append(Part.from_image(Image.load_from_file(media_object.location)))
+            elif media_object.is_type("audio") and media_object.location:
+                contents.append(
+                    Part.from_data(get_contents_as_bytes(media_object.location), mime_type=media_object.content_type)
+                )
             elif media_object.is_type(TEXT_TYPE):
                 if media_object.text is None:
                     raise ValueError("MediaObject of text type has missing text field value")
@@ -355,7 +381,7 @@ class VertexAIChatClient(VertexAIClient):
         }
 
         completions: List[GeneratedOutput] = []
-        model_name: str = _get_model_name_for_request(request)
+        model_name: str = self._get_model_name_for_request(request)
         model = self.get_model(model_name)
 
         request_time = 0

helm/clients/vision_language/open_flamingo/__init__.py

@@ -1,2 +1,2 @@
-from .src.flamingo import Flamingo
-from .src.factory import create_model_and_transforms
+from helm.clients.vision_language.open_flamingo.src.flamingo import Flamingo
+from helm.clients.vision_language.open_flamingo.src.factory import create_model_and_transforms

helm/clients/vision_language/open_flamingo/src/factory.py

@@ -7,9 +7,9 @@ from typing import Optional
 from transformers import AutoModelForCausalLM, AutoTokenizer
 
 from helm.common.general import handle_module_not_found_error
-from .flamingo import Flamingo
-from .flamingo_lm import FlamingoLMMixin
-from .utils import extend_instance
+from helm.clients.vision_language.open_flamingo.src.flamingo import Flamingo
+from helm.clients.vision_language.open_flamingo.src.flamingo_lm import FlamingoLMMixin
+from helm.clients.vision_language.open_flamingo.src.utils import extend_instance
 
 
 def create_model_and_transforms(

helm/clients/vision_language/open_flamingo/src/flamingo.py

@@ -5,7 +5,7 @@ Source: https://github.com/mlfoundations/open_flamingo
 import torch
 from einops import rearrange
 from torch import nn
-from .helpers import PerceiverResampler
+from helm.clients.vision_language.open_flamingo.src.helpers import PerceiverResampler
 from torch.distributed.fsdp.wrap import (
     enable_wrap,
     wrap,
@@ -15,7 +15,7 @@ from torch.distributed.fsdp import (
     FullyShardedDataParallel as FSDP,
 )
 
-from .utils import apply_with_stopping_condition
+from helm.clients.vision_language.open_flamingo.src.utils import apply_with_stopping_condition
 
 
 class Flamingo(nn.Module):

helm/clients/vision_language/open_flamingo/src/flamingo_lm.py

@@ -3,8 +3,8 @@ Source: https://github.com/mlfoundations/open_flamingo
 """
 
 import torch.nn as nn
-from .helpers import GatedCrossAttentionBlock
-from .utils import getattr_recursive, setattr_recursive
+from helm.clients.vision_language.open_flamingo.src.helpers import GatedCrossAttentionBlock
+from helm.clients.vision_language.open_flamingo.src.utils import getattr_recursive, setattr_recursive
 
 
 class FlamingoLayer(nn.Module):