crfm-helm 0.5.4__py3-none-any.whl → 0.5.6__py3-none-any.whl

helm/clients/palmyra_client.py

@@ -5,14 +5,14 @@ from typing import Any, Dict, List
 
 from helm.clients.openai_client import OpenAIClient
 from helm.common.cache import CacheConfig
-from helm.common.hierarchical_logger import hlog
+from helm.common.hierarchical_logger import hwarn
 from helm.common.request import wrap_request_time, Request, RequestResult, GeneratedOutput, Token, ErrorFlags
 from helm.common.tokenization_request import (
     TokenizationRequest,
     TokenizationRequestResult,
 )
 from helm.tokenizers.tokenizer import Tokenizer
-from .client import CachingClient, truncate_sequence
+from helm.clients.client import CachingClient, truncate_sequence
 
 
 _CONTENT_MODERATION_KEY = "fail.content.moderation.failed"
@@ -103,10 +103,7 @@ class PalmyraClient(CachingClient):
                 return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
 
             if _is_content_moderation_failure(response):
-                hlog(
-                    f"WARNING: Returning empty request for {request.model_deployment} "
-                    "due to content moderation filter"
-                )
+                hwarn(f"Returning empty request for {request.model_deployment} " "due to content moderation filter")
                 return RequestResult(
                     success=False,
                     cached=False,
@@ -163,6 +160,3 @@ class PalmyraChatClient(OpenAIClient):
             org_id=None,
             base_url="https://api.writer.com/v1/chat",
         )
-
-    def _is_chat_model_engine(self, model_engine: str) -> bool:
-        return True

helm/clients/reka_client.py

@@ -6,10 +6,10 @@ from helm.proxy.retry import NonRetriableException
 from helm.common.cache import CacheConfig
 from helm.common.media_object import TEXT_TYPE
 from helm.common.request import wrap_request_time, Request, RequestResult, GeneratedOutput
-from helm.common.hierarchical_logger import hlog
+from helm.common.hierarchical_logger import hwarn
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.tokenizers.tokenizer import Tokenizer
-from .client import CachingClient, truncate_and_tokenize_response_text
+from helm.clients.client import CachingClient, truncate_and_tokenize_response_text
 
 try:
     import reka
@@ -121,7 +121,7 @@ class RekaClient(CachingClient):
             if messages[-1]["role"] != "user":
                 raise ValueError("Last message must have role 'user'")
             if request.prompt != "":
-                hlog("WARNING: Since message is set, prompt will be ignored")
+                hwarn("Since message is set, prompt will be ignored")
             reka_chat_history = self._convert_messages_to_reka_chat_history(messages)
         else:
             current_chat_history: Dict[str, Any] = {

helm/clients/stanfordhealthcare_azure_openai_client.py

@@ -0,0 +1,58 @@
+from typing import Optional
+
+from helm.clients.azure_openai_client import AzureOpenAIClient
+from helm.common.cache import CacheConfig
+from helm.proxy.retry import NonRetriableException
+from helm.tokenizers.tokenizer import Tokenizer
+
+
+class StanfordHealthCareAzureOpenAIClient(AzureOpenAIClient):
+    """
+    Client for accessing OpenAI models hosted on Stanford Health Care's model API.
+
+    Configure by setting the following in prod_env/credentials.conf:
+
+    ```
+    stanfordhealthcareEndpoint: https://your-domain-name/
+    stanfordhealthcareApiKey: your-private-key
+    ```
+    """
+
+    CREDENTIAL_HEADER_NAME = "Ocp-Apim-Subscription-Key"
+
+    def __init__(
+        self,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        cache_config: CacheConfig,
+        openai_model_name: str,
+        api_version: str,
+        api_key: Optional[str] = None,
+        endpoint: Optional[str] = None,
+        base_url: Optional[str] = None,
+    ):
+        if not api_key:
+            raise NonRetriableException("Must provide API key through credentials.conf")
+        if base_url:
+            base_url = base_url.format(endpoint=endpoint)
+            super().__init__(
+                tokenizer=tokenizer,
+                tokenizer_name=tokenizer_name,
+                cache_config=cache_config,
+                api_key="unused",
+                base_url=base_url,
+                azure_openai_deployment_name=openai_model_name,
+                api_version=api_version,
+                default_headers={StanfordHealthCareAzureOpenAIClient.CREDENTIAL_HEADER_NAME: api_key},
+            )
+        else:
+            super().__init__(
+                tokenizer=tokenizer,
+                tokenizer_name=tokenizer_name,
+                cache_config=cache_config,
+                api_key="unused",
+                endpoint=endpoint,
+                azure_openai_deployment_name=openai_model_name,
+                api_version=api_version,
+                default_headers={StanfordHealthCareAzureOpenAIClient.CREDENTIAL_HEADER_NAME: api_key},
+            )

helm/clients/stanfordhealthcare_claude_client.py

@@ -0,0 +1,31 @@
+from typing import Any, Dict, List
+
+from helm.common.request import (
+    Request,
+    GeneratedOutput,
+)
+from helm.clients.stanfordhealthcare_http_model_client import StanfordHealthCareHTTPModelClient
+
+
+class StanfordHealthCareClaudeClient(StanfordHealthCareHTTPModelClient):
+    """
+    Client for accessing Claude models hosted on Stanford Health Care's model API.
+
+    Configure by setting the following in prod_env/credentials.conf:
+
+    ```
+    stanfordhealthcareEndpoint: https://your-domain-name/
+    stanfordhealthcareApiKey: your-private-key
+    ```
+    """
+
+    def get_request(self, request: Request) -> Dict[str, Any]:
+        return {
+            "model_id": self.model,
+            "prompt_text": request.prompt,
+            "max_tokens": request.max_tokens,
+            "temperature": request.temperature,
+        }
+
+    def parse_response(self, response: Dict[str, Any]) -> List[GeneratedOutput]:
+        return [GeneratedOutput(text=item["text"], logprob=0, tokens=[]) for item in response["content"]]

helm/clients/stanfordhealthcare_google_client.py

@@ -0,0 +1,43 @@
+from typing import Any, Dict, List
+
+from helm.common.request import (
+    Request,
+    GeneratedOutput,
+)
+from helm.clients.stanfordhealthcare_http_model_client import StanfordHealthCareHTTPModelClient
+
+
+class StanfordHealthCareGoogleClient(StanfordHealthCareHTTPModelClient):
+    """
+    Client for accessing Google models hosted on Stanford Health Care's model API.
+
+    Configure by setting the following in prod_env/credentials.conf:
+
+    ```
+    stanfordhealthcareEndpoint: https://your-domain-name/
+    stanfordhealthcareApiKey: your-private-key
+    ```
+    """
+
+    def get_request(self, request: Request) -> Dict[str, Any]:
+        return {
+            "contents": {
+                "role": "user",
+                "parts": {"text": request.prompt},
+            },
+            "generation_config": {
+                "temperature": request.temperature,
+                "topP": request.top_p,
+                "topK": request.top_k_per_token,
+                "candidateCount": 1,
+                "maxOutputTokens": request.max_tokens,
+                "stopSequences": request.stop_sequences,
+            },
+        }
+
+    def parse_response(self, response: Dict[str, Any]) -> List[GeneratedOutput]:
+        completion = ""
+        for item in response["content"]:
+            if "content" in item["candidates"][0]:
+                completion += item["candidates"][0]["content"]["parts"][0]["text"]
+        return [GeneratedOutput(text=completion, logprob=0, tokens=[])]

helm/clients/stanfordhealthcare_http_model_client.py

@@ -0,0 +1,93 @@
+import requests
+
+from abc import ABC, abstractmethod
+from dataclasses import asdict
+from typing import Any, Dict, List, Optional
+
+from helm.common.cache import CacheConfig
+from helm.common.request import (
+    wrap_request_time,
+    Request,
+    RequestResult,
+    GeneratedOutput,
+    EMBEDDING_UNAVAILABLE_REQUEST_RESULT,
+)
+from helm.clients.client import CachingClient
+
+
+class StanfordHealthCareHTTPModelClient(CachingClient, ABC):
+    """
+    Client for accessing Stanford Health Care models via HTTP requests.
+
+    Configure by setting the following in prod_env/credentials.conf:
+
+    ```
+    stanfordhealthcareEndpoint: https://your-domain-name/
+    stanfordhealthcareApiKey: your-private-key
+    ```
+    """
+
+    CREDENTIAL_HEADER_NAME = "Ocp-Apim-Subscription-Key"
+
+    def __init__(
+        self,
+        cache_config: CacheConfig,
+        deployment: str,
+        endpoint: str = "http://localhost:8080",
+        do_cache: bool = False,
+        timeout: int = 3000,
+        api_key: Optional[str] = None,
+        model: Optional[str] = None,
+    ):
+        super().__init__(cache_config=cache_config)
+        assert api_key, "API key must be provided"
+        self.endpoint = endpoint
+        self.timeout = timeout
+        self.do_cache = do_cache
+        self.deployment = deployment
+        self.model = model
+        self.default_headers = {StanfordHealthCareHTTPModelClient.CREDENTIAL_HEADER_NAME: api_key}
+
+    def make_request(self, request: Request) -> RequestResult:
+        cache_key = asdict(request)
+        if request.embedding:
+            return EMBEDDING_UNAVAILABLE_REQUEST_RESULT
+
+        raw_request = self.get_request(request)
+
+        try:
+
+            def do_it() -> Dict[str, Any]:
+                url = f"{self.endpoint}/{self.deployment}"
+                response = requests.post(url, json=raw_request, headers=self.default_headers, timeout=self.timeout)
+                response.raise_for_status()
+                response_json = response.json()
+                if type(response_json) == list:
+                    response_json = {"content": response_json}
+                return response_json
+
+            if self.do_cache:
+                response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
+            else:
+                response, cached = wrap_request_time(do_it)(), False
+
+            completions = self.parse_response(response)
+
+            return RequestResult(
+                success=True,
+                cached=cached,
+                error=None,
+                completions=completions,
+                embedding=[],
+                request_time=response["request_time"],
+            )
+        except requests.exceptions.RequestException as e:
+            return RequestResult(success=False, cached=False, error=f"Request error: {e}", completions=[], embedding=[])
+
+    @abstractmethod
+    def get_request(self, request: Request) -> Dict[str, Any]:
+        pass
+
+    @abstractmethod
+    def parse_response(self, response: Dict[str, Any]) -> List[GeneratedOutput]:
+        pass

helm/clients/stanfordhealthcare_openai_client.py

@@ -0,0 +1,62 @@
+from typing import Optional
+
+from helm.clients.openai_client import OpenAIClient
+from helm.common.cache import CacheConfig
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.proxy.retry import NonRetriableException
+from helm.tokenizers.tokenizer import Tokenizer
+
+try:
+    from openai import OpenAI
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["openai"])
+
+
+class StanfordHealthCareOpenAIClient(OpenAIClient):
+    """
+    Client for accessing OpenAI models hosted on Stanford Health Care's model API.
+
+    Configure by setting the following in prod_env/credentials.conf:
+
+    ```
+    stanfordhealthcareEndpoint: https://your-domain-name/
+    stanfordhealthcareApiKey: your-private-key
+    ```
+    """
+
+    CREDENTIAL_HEADER_NAME = "Ocp-Apim-Subscription-Key"
+
+    def __init__(
+        self,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        cache_config: CacheConfig,
+        model_name: str,
+        api_key: Optional[str] = None,
+        endpoint: Optional[str] = None,
+        base_url: Optional[str] = None,
+        openai_model_name: Optional[str] = None,
+        output_processor: Optional[str] = None,
+    ):
+        super().__init__(
+            tokenizer=tokenizer,
+            tokenizer_name=tokenizer_name,
+            cache_config=cache_config,
+            api_key="unused",
+            openai_model_name=openai_model_name,
+            output_processor=output_processor,
+        )
+        if not endpoint:
+            raise NonRetriableException("Must provide endpoint through credentials.conf")
+        if not api_key:
+            raise NonRetriableException("Must provide API key through credentials.conf")
+        if not base_url:
+            raise NonRetriableException("Must provide base url through model_deployments")
+        # Guess the base URL part based on the model name
+        # Maybe make this configurable instead?
+        base_url = base_url.format(endpoint=endpoint)
+        self.client = OpenAI(
+            api_key="dummy",
+            base_url=base_url,
+            default_headers={StanfordHealthCareOpenAIClient.CREDENTIAL_HEADER_NAME: api_key},
+        )

helm/clients/stanfordhealthcare_shc_openai_client.py

@@ -0,0 +1,42 @@
+from typing import Dict, Optional
+
+from helm.clients.azure_openai_client import AzureOpenAIClient
+from helm.common.cache import CacheConfig
+from helm.proxy.retry import NonRetriableException
+from helm.tokenizers.tokenizer import Tokenizer
+
+
+class StanfordHealthCareSHCOpenAIClient(AzureOpenAIClient):
+    """
+    Client for accessing OpenAI models hosted on Stanford Health Care's model API.
+
+    Configure by setting the following in prod_env/credentials.conf:
+
+    ```
+    stanfordhealthcareEndpoint: https://your-domain-name/
+    stanfordhealthcareApiKey: your-private-key
+    ```
+    """
+
+    API_VERSION = "2024-08-01-preview"
+
+    def __init__(
+        self,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        cache_config: CacheConfig,
+        api_key: Optional[str] = None,
+        endpoint: Optional[str] = None,
+        default_headers: Optional[Dict[str, str]] = None,
+    ):
+        if not api_key:
+            raise NonRetriableException("Must provide API key through credentials.conf")
+        super().__init__(
+            tokenizer=tokenizer,
+            tokenizer_name=tokenizer_name,
+            cache_config=cache_config,
+            api_key=api_key,
+            endpoint=endpoint,
+            api_version=StanfordHealthCareSHCOpenAIClient.API_VERSION,
+            default_headers=default_headers,
+        )

helm/clients/test_client.py

@@ -1,6 +1,6 @@
 from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from helm.tokenizers.auto_tokenizer import AutoTokenizer
-from .client import truncate_sequence, truncate_and_tokenize_response_text
+from helm.clients.client import truncate_sequence, truncate_and_tokenize_response_text
 from typing import List
 from helm.common.request import Request, GeneratedOutput, Token
 

helm/clients/test_together_client.py

@@ -5,7 +5,12 @@ import tempfile
 from helm.common.cache import BlackHoleCacheConfig, SqliteCacheConfig
 from helm.common.request import Request
 
-from .together_client import TogetherClient, TogetherChatClient, TogetherCompletionClient, TogetherClientError
+from helm.clients.together_client import (
+    TogetherClient,
+    TogetherChatClient,
+    TogetherCompletionClient,
+    TogetherClientError,
+)
 
 
 class TestTogetherClient:

helm/clients/together_client.py

@@ -1,15 +1,18 @@
 from copy import deepcopy
 from itertools import zip_longest
+import re
 import threading
-from typing import List, Dict, Any, Mapping, Optional, TypedDict, Union
+from typing import Callable, List, Dict, Any, Mapping, Optional, Tuple, TypedDict, Union
+from typing_extensions import NotRequired
 
 import requests
 from retrying import retry
 
 from helm.common.cache import CacheConfig
 from helm.common.media_object import IMAGE_TYPE, TEXT_TYPE
+from helm.common.object_spec import get_class_by_name
 from helm.common.optional_dependencies import handle_module_not_found_error
-from helm.common.request import wrap_request_time, Request, RequestResult, GeneratedOutput, Token
+from helm.common.request import Thinking, wrap_request_time, Request, RequestResult, GeneratedOutput, Token
 from helm.clients.client import CachingClient, truncate_sequence, cleanup_str
 
 try:
@@ -98,6 +101,19 @@ class JobNotFinishedError(TogetherClientError):
     pass
 
 
+def _parse_thinking(input: str) -> Tuple[str, str]:
+    """Return a tuple of thinking text and output text."""
+    match = re.match(r"<think>\n(.*)\n</think>\n{0,2}(.*)", input, re.DOTALL)
+    if match:
+        return (match.group(1), match.group(2))
+
+    match = re.match(r"<think>\n?(.*)", input, re.DOTALL)
+    if match:
+        return (match.group(1), "")
+
+    return (input, "")
+
+
 class TogetherClient(CachingClient):
     """
     Client for the models where we evaluate offline. Since the queries are handled offline, the `TogetherClient` just
@@ -313,15 +329,31 @@ class TogetherRawChatRequest(TypedDict):
     logprobs: int
     echo: bool
     n: int
+    response_format: NotRequired[Dict[str, Any]]
 
 
 class TogetherChatClient(CachingClient):
     """Client that uses the Python Together library for chat models."""
 
-    def __init__(self, cache_config: CacheConfig, api_key: Optional[str], together_model: Optional[str] = None):
+    def __init__(
+        self,
+        cache_config: CacheConfig,
+        api_key: Optional[str],
+        together_model: Optional[str] = None,
+        disable_logprobs: Optional[bool] = None,
+        output_processor: Optional[str] = None,
+        parse_thinking: Optional[bool] = None,
+    ):
         super().__init__(cache_config=cache_config)
         self._client = Together(api_key=api_key)
         self._together_model = together_model
+        self._disable_logprobs = bool(disable_logprobs)
+        # self.output_processor is actually a function, not a class
+        self._parse_thinking = bool(parse_thinking)
+
+        self.output_processor: Optional[Callable[[str], str]] = (
+            get_class_by_name(output_processor) if output_processor else None
+        )
 
     def convert_to_raw_chat_request(self, request: Request) -> TogetherRawChatRequest:
         request.validate()
@@ -353,7 +385,11 @@ class TogetherChatClient(CachingClient):
             model = self._together_model
         else:
             model = request.model
-        return {
+        if self._disable_logprobs:
+            logprobs = 0
+        else:
+            logprobs = min(request.top_k_per_token, 1)
+        raw_chat_request: TogetherRawChatRequest = {
             "messages": messages,
             "model": model,
             "max_tokens": request.max_tokens,
@@ -361,10 +397,16 @@ class TogetherChatClient(CachingClient):
             "temperature": request.temperature,
             "top_p": request.top_p,
             "top_k": request.top_k_per_token,
-            "logprobs": min(request.top_k_per_token, 1),
+            "logprobs": logprobs,
             "echo": request.echo_prompt,
             "n": request.num_completions,
         }
+        if request.response_format and request.response_format.json_schema:
+            raw_chat_request["response_format"] = {
+                "type": "json_object",
+                "schema": request.response_format.json_schema,
+            }
+        return raw_chat_request
 
     def make_request(self, request: Request) -> RequestResult:
         raw_request = self.convert_to_raw_chat_request(request)
@@ -398,8 +440,21 @@ class TogetherChatClient(CachingClient):
                     if token_text is None:
                         break
                     tokens.append(Token(text=token_text, logprob=token_logprob or 0.0))
+            logprob = sum([token.logprob for token in tokens]) if tokens else 0.0
             assert choice.message.role == "assistant"
-            generated_outputs.append(GeneratedOutput(text=choice.message.content, logprob=0.0, tokens=tokens))
+            output_text = choice.message.content
+            if self.output_processor:
+                output_text = self.output_processor(output_text)
+
+            if self._parse_thinking:
+                thinking_text, output_text = _parse_thinking(output_text)
+                generated_outputs.append(
+                    GeneratedOutput(
+                        text=output_text, logprob=logprob, tokens=tokens, thinking=Thinking(text=thinking_text)
+                    )
+                )
+            else:
+                generated_outputs.append(GeneratedOutput(text=output_text, logprob=logprob, tokens=tokens))
         return RequestResult(
             success=True,
             cached=cached,
@@ -426,16 +481,27 @@ class TogetherRawCompletionRequest(TypedDict):
 class TogetherCompletionClient(CachingClient):
     """Client that uses the Python Together library for text completion models."""
 
-    def __init__(self, cache_config: CacheConfig, api_key: Optional[str], together_model: Optional[str] = None):
+    def __init__(
+        self,
+        cache_config: CacheConfig,
+        api_key: Optional[str],
+        together_model: Optional[str] = None,
+        disable_logprobs: Optional[bool] = None,
+    ):
         super().__init__(cache_config=cache_config)
         self._client = Together(api_key=api_key)
         self._together_model = together_model
+        self._disable_logprobs = bool(disable_logprobs)
 
     def convert_to_raw_completion_request(self, request: Request) -> TogetherRawCompletionRequest:
         if self._together_model is not None:
             model = self._together_model
         else:
             model = request.model
+        if self._disable_logprobs:
+            logprobs = 0
+        else:
+            logprobs = min(request.top_k_per_token, 1)
         return {
             "prompt": request.prompt,
             "model": model,
@@ -444,7 +510,7 @@ class TogetherCompletionClient(CachingClient):
             "temperature": request.temperature,
             "top_p": request.top_p,
             "top_k": request.top_k_per_token,
-            "logprobs": min(request.top_k_per_token, 1),
+            "logprobs": logprobs,
             "echo": request.echo_prompt,
             "n": request.num_completions,
         }
@@ -481,8 +547,9 @@ class TogetherCompletionClient(CachingClient):
                     if token_text is None:
                         break
                     tokens.append(Token(text=token_text, logprob=token_logprob or 0.0))
+            logprob = sum([token.logprob for token in tokens]) if tokens else 0.0
             assert choice.text
-            generated_outputs.append(GeneratedOutput(text=choice.text, logprob=0.0, tokens=tokens))
+            generated_outputs.append(GeneratedOutput(text=choice.text, logprob=logprob, tokens=tokens))
         return RequestResult(
             success=True,
             cached=cached,

helm/clients/upstage_client.py

@@ -0,0 +1,23 @@
+from helm.clients.openai_client import OpenAIClient
+from helm.common.cache import CacheConfig
+from helm.tokenizers.tokenizer import Tokenizer
+
+
+class UpstageChatClient(OpenAIClient):
+    """Sends request to a Upstage model using a OpenAI-compatible Chat API."""
+
+    def __init__(
+        self,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        cache_config: CacheConfig,
+        api_key: str,
+    ):
+        super().__init__(
+            tokenizer=tokenizer,
+            tokenizer_name=tokenizer_name,
+            cache_config=cache_config,
+            api_key=api_key,
+            org_id=None,
+            base_url="https://api.upstage.ai/v1/solar",
+        )