crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/clients/openai_responses_client.py

@@ -0,0 +1,200 @@
+# mypy: check_untyped_defs = False
+import dataclasses
+from typing import Any, Dict, List, Optional, Union
+
+
+from helm.clients.openai_client import OpenAIClientUtils
+from helm.common.cache import CacheConfig
+from helm.common.hierarchical_logger import hwarn
+from helm.common.media_object import TEXT_TYPE
+from helm.common.request import (
+    Thinking,
+    wrap_request_time,
+    Request,
+    RequestResult,
+    GeneratedOutput,
+)
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.clients.client import (
+    CachingClient,
+    truncate_and_tokenize_response_text,
+    generate_uid_for_multimodal_prompt,
+)
+from helm.tokenizers.tokenizer import Tokenizer
+
+try:
+    import openai
+    from openai import OpenAI
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["openai"])
+
+
+class OpenAIResponseClient(CachingClient):
+    def __init__(
+        self,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        cache_config: CacheConfig,
+        api_key: Optional[str] = None,
+        org_id: Optional[str] = None,
+        base_url: Optional[str] = None,
+        reasoning_effort: Optional[str] = None,
+        openai_model_name: Optional[str] = None,
+    ):
+        super().__init__(cache_config=cache_config)
+        self.tokenizer = tokenizer
+        self.tokenizer_name = tokenizer_name
+        self.client = OpenAI(
+            api_key=api_key,
+            organization=org_id,
+            base_url=base_url,
+        )
+        self.reasoning_effort = reasoning_effort
+        self.openai_model_name = openai_model_name
+
+    def _get_cache_key(self, raw_request: Dict, request: Request):
+        cache_key = CachingClient.make_cache_key(raw_request, request)
+        if request.multimodal_prompt:
+            prompt_key: str = generate_uid_for_multimodal_prompt(request.multimodal_prompt)
+            cache_key = {**cache_key, "multimodal_prompt": prompt_key}
+        return cache_key
+
+    def _make_raw_request(self, request: Request) -> dict[str, Any]:
+        input: Union[str, List[Dict[str, Any]]]
+
+        if (
+            (request.prompt and request.messages)
+            or (request.prompt and request.multimodal_prompt)
+            or (request.messages and request.multimodal_prompt)
+        ):
+            raise ValueError(
+                f"More than one of `prompt`, `messages` and `multimodal_prompt` was set in request: {request}"
+            )
+
+        if request.messages is not None:
+            # Checks that all messages have a role and some content
+            for message in request.messages:
+                if not message.get("role") or not message.get("content"):
+                    raise ValueError("All messages must have a role and content")
+            # Checks that the last role is "user"
+            if request.messages[-1]["role"] != "user":
+                raise ValueError("Last message must have role 'user'")
+            if request.prompt != "":
+                hwarn("Since message is set, prompt will be ignored")
+            input = request.messages
+        elif request.multimodal_prompt is not None:
+            content = []
+            request.validate()
+            for media_object in request.multimodal_prompt.media_objects:
+                if media_object.is_type("image") and media_object.location:
+                    from helm.common.images_utils import encode_base64
+
+                    base64_image: str = encode_base64(media_object.location)
+                    content.append(
+                        {
+                            "type": "input_image",
+                            "image_url": f"data:image/jpeg;base64,{base64_image}",
+                        }
+                    )
+                elif media_object.is_type(TEXT_TYPE):
+                    assert media_object.text is not None
+                    content.append({"type": "input_text", "text": media_object.text})
+                else:
+                    raise ValueError(f"Unrecognized MediaObject type {media_object.type}")
+            input = [{"role": "user", "content": content}]
+        else:
+            input = request.prompt
+
+        raw_request: Dict[str, Any] = {
+            "model": self._get_model_for_request(request),
+            "input": input,
+            "top_p": request.top_p,
+            # API errors if max_output_tokens is less than 16
+            # (Error you get: "Invalid 'max_output_tokens': integer below minimum value.
+            #    Expected a value >= 16, but got 5 instead.")
+            "max_output_tokens": max(16, request.max_tokens),
+            "temperature": request.temperature,
+            # Don't store responses for later retrieval
+            "store": False,
+        }
+        if self.reasoning_effort:
+            raw_request["reasoning"] = {"effort": self.reasoning_effort}
+        # If o-series model, get reasoning summaries
+        # Plus other changes
+        model_engine: str = request.model_engine
+        if OpenAIClientUtils.is_reasoning_model(model_engine):
+            if "reasoning" not in raw_request:
+                raw_request["reasoning"] = {}
+            raw_request["reasoning"]["summary"] = "detailed"
+            # Avoid error:
+            # "Error code: 400 - {'error': {'message': "Unsupported parameter: 'temperature' is
+            # not supported with this model.", 'type': 'invalid_request_error', 'param': 'temperature',
+            # 'code': 'unsupported_parameter'}}"
+            raw_request.pop("temperature", None)
+
+            # The following parameters also happen to be unsupported by the o-series (code unsupported_parameter)
+            raw_request.pop("top_p", None)
+
+        return raw_request
+
+    def _get_model_for_request(self, request: Request) -> str:
+        return self.openai_model_name or request.model_engine
+
+    def make_request(self, request: Request) -> RequestResult:
+        # Content can either be text or a list of multimodal content made up of text and images:
+        # https://platform.openai.com/docs/api-reference/responses/create
+        raw_request = self._make_raw_request(request)
+
+        # The responses API does not support a "num_completions" parameter,
+        # so we need to handle it ourselves with a simple loop
+        completions: list[GeneratedOutput] = []
+        for _ in range(request.num_completions):
+
+            def do_it() -> Dict[str, Any]:
+                raw_response = self.client.responses.create(**raw_request).model_dump(mode="json")
+                assert not raw_response.get("error", None), f"Error in response: {raw_response}"
+                return raw_response
+
+            try:
+                cache_key = self._get_cache_key(raw_request, request)
+                response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
+            except openai.OpenAIError as e:
+                return OpenAIClientUtils.handle_openai_error(e, request)
+
+            # We can only return one completition really,
+            # but we get an array of messages back, so we need to contact them
+            reasoning_output = ""
+            text_output = ""
+
+            if request.echo_prompt:
+                text_output += request.prompt
+            for output in response["output"]:
+                output_type = output[
+                    "type"
+                ]  # one of "message" or "reasoning" from API observation, but can also include tool calls
+
+                if output_type == "reasoning":
+                    reasoning_output += "\n\n".join([raw_output["text"] for raw_output in output["summary"]])
+                elif output_type == "message":
+                    text_output += "\n\n".join([raw_output["text"] for raw_output in output["content"]])
+                # (Other output types are ignored)
+
+            completion = truncate_and_tokenize_response_text(
+                text_output,
+                request,
+                self.tokenizer,
+                self.tokenizer_name,
+                original_finish_reason="",
+            )
+            if reasoning_output:
+                completion = dataclasses.replace(completion, thinking=Thinking(text=reasoning_output))
+            completions.append(completion)
+
+        return RequestResult(
+            success=True,
+            cached=cached,
+            request_time=response["request_time"],
+            request_datetime=response.get("request_datetime"),
+            completions=completions,
+            embedding=[],
+        )

helm/clients/openrouter_client.py

@@ -0,0 +1,31 @@
+import os
+from typing import Optional
+from helm.clients.openai_client import OpenAIClient
+from helm.common.cache import CacheConfig
+from helm.tokenizers.tokenizer import Tokenizer
+
+
+class OpenRouterClient(OpenAIClient):
+    def __init__(
+        self,
+        tokenizer_name: str,
+        tokenizer: Tokenizer,
+        cache_config: CacheConfig,
+        api_key: Optional[str] = None,
+        model_name: Optional[str] = None,
+        output_processor: Optional[str] = None,
+    ):
+        self.api_key = api_key or os.getenv("OPENROUTER_API_KEY")
+        self.base_url = "https://openrouter.ai/api/v1/"
+        super().__init__(
+            tokenizer,
+            tokenizer_name,
+            cache_config=cache_config,
+            output_processor=output_processor,
+            base_url=self.base_url,
+            api_key=self.api_key,
+        )
+        self.model_name = model_name
+
+    def _get_model_for_request(self, request):
+        return self.model_name or request.model

helm/{proxy/clients → clients}/palmyra_client.py

@@ -3,15 +3,16 @@ import json
 import requests
 from typing import Any, Dict, List
 
+from helm.clients.openai_client import OpenAIClient
 from helm.common.cache import CacheConfig
-from helm.common.hierarchical_logger import hlog
-from helm.common.request import wrap_request_time, Request, RequestResult, Sequence, Token, ErrorFlags
+from helm.common.hierarchical_logger import hexception, hwarn
+from helm.common.request import wrap_request_time, Request, RequestResult, GeneratedOutput, Token, ErrorFlags
 from helm.common.tokenization_request import (
     TokenizationRequest,
     TokenizationRequestResult,
 )
-from helm.proxy.tokenizers.tokenizer import Tokenizer
-from .client import CachingClient, truncate_sequence
+from helm.tokenizers.tokenizer import Tokenizer
+from helm.clients.client import CachingClient, truncate_sequence
 
 
 _CONTENT_MODERATION_KEY = "fail.content.moderation.failed"
@@ -67,14 +68,14 @@ class PalmyraClient(CachingClient):
             # "random_seed": request.random,
         }
 
-        completions: List[Sequence] = []
+        completions: List[GeneratedOutput] = []
         model_name: str = request.model_engine
 
         # `num_completions` is not supported, so instead make `num_completions` separate requests.
         for completion_index in range(request.num_completions):
             try:
 
-                def do_it():
+                def do_it() -> Dict[str, Any]:
                     # Add an argument timeout to raw_request to avoid waiting getting timeout of 60s
                     # which happens for long prompts.
                     request_with_timeout = {"timeout": 300, **raw_request}
@@ -98,14 +99,12 @@ class PalmyraClient(CachingClient):
 
                 response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
             except (requests.exceptions.RequestException, AssertionError) as e:
+                hexception(e)
                 error: str = f"PalmyraClient error: {e}"
                 return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
 
             if _is_content_moderation_failure(response):
-                hlog(
-                    f"WARNING: Returning empty request for {request.model_deployment} "
-                    "due to content moderation filter"
-                )
+                hwarn(f"Returning empty request for {request.model_deployment} " "due to content moderation filter")
                 return RequestResult(
                     success=False,
                     cached=False,
@@ -128,11 +127,9 @@ class PalmyraClient(CachingClient):
             )
 
             # Log probs are not currently not supported by the Writer, so set to 0 for now.
-            tokens: List[Token] = [
-                Token(text=str(text), logprob=0, top_logprobs={}) for text in tokenization_result.raw_tokens
-            ]
+            tokens: List[Token] = [Token(text=str(text), logprob=0) for text in tokenization_result.raw_tokens]
 
-            completion = Sequence(text=response_text, logprob=0, tokens=tokens)
+            completion = GeneratedOutput(text=response_text, logprob=0, tokens=tokens)
             sequence = truncate_sequence(completion, request, print_warning=True)
             completions.append(sequence)
 
@@ -144,3 +141,23 @@ class PalmyraClient(CachingClient):
             completions=completions,
             embedding=[],
         )
+
+
+class PalmyraChatClient(OpenAIClient):
+    """Sends request to a Palmyra model using a OpenAI-compatible Chat API."""
+
+    def __init__(
+        self,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        cache_config: CacheConfig,
+        api_key: str,
+    ):
+        super().__init__(
+            tokenizer=tokenizer,
+            tokenizer_name=tokenizer_name,
+            cache_config=cache_config,
+            api_key=api_key,
+            org_id=None,
+            base_url="https://api.writer.com/v1/chat",
+        )

helm/{proxy/clients → clients}/perspective_api_client.py

@@ -1,19 +1,24 @@
 # mypy: check_untyped_defs = False
 import threading
 from dataclasses import asdict
-from typing import List, Dict, Optional
+from typing import Any, List, Dict, Optional
 
 from dacite import from_dict
-from googleapiclient import discovery
-from googleapiclient.errors import BatchError, HttpError
-from googleapiclient.http import BatchHttpRequest
-from httplib2 import HttpLib2Error
-from helm.proxy.clients.toxicity_classifier_client import ToxicityClassifierClient
-from helm.proxy.retry import NonRetriableException
 
+from helm.clients.toxicity_classifier_client import ToxicityClassifierClient
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.proxy.retry import NonRetriableException
 from helm.common.cache import Cache, CacheConfig
 from helm.common.perspective_api_request import ToxicityAttributes, PerspectiveAPIRequest, PerspectiveAPIRequestResult
-from google.auth.exceptions import DefaultCredentialsError
+
+try:
+    from googleapiclient import discovery
+    from googleapiclient.errors import BatchError, HttpError
+    from googleapiclient.http import BatchHttpRequest
+    from httplib2 import HttpLib2Error
+    from google.auth.exceptions import DefaultCredentialsError
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["metrics"])
 
 
 class PerspectiveAPIClientCredentialsError(NonRetriableException):
@@ -91,14 +96,9 @@ class PerspectiveAPIClient(ToxicityClassifierClient):
         Batch several requests into a single API request and get the toxicity attributes and scores.
         For more information, see https://googleapis.github.io/google-api-python-client/docs/batch.html.
         """
-
-        with self._client_lock:
-            if not self._client:
-                self._client = self._create_client()
-
         try:
 
-            def do_it():
+            def do_it() -> Dict[str, Any]:
                 text_to_response: Dict[str, Dict] = dict()
 
                 def callback(request_id: str, response: Dict, error: HttpError):
@@ -106,6 +106,10 @@ class PerspectiveAPIClient(ToxicityClassifierClient):
                         raise error
                     text_to_response[request_id] = response
 
+                with self._client_lock:
+                    if not self._client:
+                        self._client = self._create_client()
+
                 # Create a batch request. We will add a request to the batch request for each text string
                 batch_request: BatchHttpRequest = self._client.new_batch_http_request()
 

helm/clients/reka_client.py

@@ -0,0 +1,190 @@
+# mypy: check_untyped_defs = False
+import requests
+from typing import Any, Dict, List, Optional, TypedDict
+
+from helm.proxy.retry import NonRetriableException
+from helm.common.cache import CacheConfig
+from helm.common.media_object import TEXT_TYPE
+from helm.common.request import wrap_request_time, Request, RequestResult, GeneratedOutput
+from helm.common.hierarchical_logger import hexception, hwarn
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.tokenizers.tokenizer import Tokenizer
+from helm.clients.client import CachingClient, truncate_and_tokenize_response_text
+
+try:
+    import reka
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["reka-api"])
+
+
+class RekaAIRequest(TypedDict):
+    """Data passed between make_request and _send_request. Used as the cache key."""
+
+    model_name: str
+    conversation_history: List[Dict[str, str]]
+    request_output_len: int
+    temperature: float
+    runtime_top_p: float
+    random_seed: Optional[int]
+    stop_words: Optional[List[str]]
+    presence_penalty: float
+    frequency_penalty: float
+
+
+class RekaClient(CachingClient):
+    REKA_CHAT_ROLE_MAPPING: Dict[str, str] = {
+        "user": "human",
+        "assistant": "model",
+    }
+
+    def __init__(
+        self,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        cache_config: CacheConfig,
+        api_key: Optional[str] = None,
+    ):
+        super().__init__(cache_config=cache_config)
+        self.tokenizer = tokenizer
+        self.tokenizer_name = tokenizer_name
+        self.client = reka
+        self.client.API_KEY = api_key
+
+    def _is_reka_model_engine(self, model_engine: str) -> bool:
+        if (
+            model_engine.startswith("reka-edge")
+            or model_engine.startswith("reka-flash")
+            or model_engine.startswith("reka-core")
+        ):
+            return True
+        else:
+            return False
+
+    def _get_model_for_request(self, request: Request) -> str:
+        return request.model_engine
+
+    def _get_random_seed(self, request: Request, completion_index: int) -> Optional[int]:
+        if request.random is None and completion_index == 0:
+            return None
+
+        # Treat the user's request.random as an integer for the random seed.
+        try:
+            request_random_seed = int(request.random) if request.random is not None else 0
+        except ValueError:
+            raise NonRetriableException("RekaAIClient only supports integer values for request.random")
+
+        # A large prime is used so that the resulting values are unlikely to collide
+        # with request.random values chosen by the user.
+        fixed_large_prime = 1911011
+        completion_index_random_seed = completion_index * fixed_large_prime
+
+        return request_random_seed + completion_index_random_seed
+
+    def _convert_messages_to_reka_chat_history(self, messages: List[Dict[str, Any]]):
+        chat_history = []
+        num_images: int = 0
+        for chat_turn, message in enumerate(messages):
+            role = message["role"]
+            content = message["content"]
+            current_chat_history: Dict[str, Any] = {
+                "type": self.REKA_CHAT_ROLE_MAPPING[role],
+                "text": "",  # text placeholder
+                "media_url": None,
+            }
+            for item in content:
+                if item["type"] == "image_url":
+                    if chat_turn == 0 and num_images == 0:
+                        current_chat_history["media_url"] = item["image_url"]["url"]
+                        num_images += 1
+                    else:
+                        raise ValueError(
+                            f"Only the first message can contain one image. Found image input "
+                            f"in message {chat_turn + 1}"
+                        )
+                elif item["type"] == "text":
+                    current_chat_history["text"] = item["text"]
+                else:
+                    raise ValueError(f"Unrecognized message type {item['type']}")
+            chat_history.append(current_chat_history)
+        return chat_history
+
+    def make_request(self, request: Request) -> RequestResult:
+        completions: List[GeneratedOutput] = []
+        messages: Optional[List[Dict[str, Any]]] = request.messages
+        reka_chat_history: List[Dict[str, Any]]
+        if messages is not None:
+            # Checks that all messages have a role and some content
+            for message in messages:
+                if not message.get("role") or not message.get("content"):
+                    raise ValueError("All messages must have a role and content")
+            # Checks that the last role is "user"
+            if messages[-1]["role"] != "user":
+                raise ValueError("Last message must have role 'user'")
+            if request.prompt != "":
+                hwarn("Since message is set, prompt will be ignored")
+            reka_chat_history = self._convert_messages_to_reka_chat_history(messages)
+        else:
+            current_chat_history: Dict[str, Any] = {
+                "type": "human",
+                "text": "",
+                "media_url": None,
+            }
+            if request.multimodal_prompt is not None:
+                for media_object in request.multimodal_prompt.media_objects:
+                    if media_object.is_type("image") and media_object.location:
+                        from helm.common.images_utils import encode_base64
+
+                        base64_image: str = encode_base64(media_object.location)
+                        current_chat_history["media_url"] = f"data:image/jpeg;base64,{base64_image}"
+                    elif media_object.is_type(TEXT_TYPE):
+                        if media_object.text is None:
+                            raise ValueError("MediaObject of text type has missing text field value")
+                        current_chat_history["text"] = media_object.text
+                    else:
+                        raise ValueError(f"Unrecognized MediaObject type {media_object.type}")
+
+            else:
+                current_chat_history["text"] = request.prompt
+            reka_chat_history = [current_chat_history]
+
+        # `num_completions` is not supported, so instead make `num_completions` separate requests.
+        for completion_index in range(request.num_completions):
+            try:
+                raw_request: RekaAIRequest = {
+                    "model_name": self._get_model_for_request(request),
+                    "conversation_history": reka_chat_history,  # we only use chat_history as the input
+                    "request_output_len": request.max_tokens,
+                    "temperature": request.temperature,
+                    "random_seed": self._get_random_seed(request, completion_index),
+                    "stop_words": request.stop_sequences or None,  # API doesn't like empty list
+                    "runtime_top_p": request.top_p,
+                    "presence_penalty": request.presence_penalty,
+                    "frequency_penalty": request.frequency_penalty,
+                }
+
+                def do_it() -> Dict[str, Any]:
+                    return self.client.chat(**raw_request)
+
+                response, cached = self.cache.get(raw_request, wrap_request_time(do_it))
+            except (requests.exceptions.RequestException, AssertionError) as e:
+                hexception(e)
+                error: str = f"RekaClient error: {e}"
+                return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
+
+            response_message: Dict[str, Any] = response
+            assert response_message["type"] == "model"
+            response_text: str = response_message["text"]
+
+            # The Reka API doesn't support echo. If `echo_prompt` is true, combine the prompt and completion.
+            text: str = request.prompt + response_text if request.echo_prompt else response_text
+            completion = truncate_and_tokenize_response_text(text, request, self.tokenizer, self.tokenizer_name)
+            completions.append(completion)
+
+        return RequestResult(
+            success=True,
+            cached=cached,
+            request_time=response["request_time"],
+            request_datetime=response.get("request_datetime"),
+            completions=completions,
+            embedding=[],
+        )

helm/clients/simple_client.py

@@ -0,0 +1,64 @@
+import itertools
+from typing import List, TypedDict
+from typing import Dict, Any
+
+from helm.common.cache import CacheConfig
+from helm.common.request import wrap_request_time, Request, RequestResult, GeneratedOutput, Token
+from helm.clients.client import CachingClient
+
+
+class SimpleClientRequest(TypedDict):
+    engine: str
+    prompt: str
+    num_completions: int
+
+
+class SimpleClient(CachingClient):
+    """Simple client for tutorials and for debugging."""
+
+    def __init__(self, cache_config: CacheConfig):
+        super().__init__(cache_config=cache_config)
+
+    def make_request(self, request: Request) -> RequestResult:
+        raw_request: SimpleClientRequest = {
+            "engine": request.model_engine,
+            "prompt": request.prompt,
+            "num_completions": request.num_completions,
+        }
+
+        def do_it() -> Dict[str, Any]:
+            return self.invoke_model(raw_request)
+
+        cache_key = CachingClient.make_cache_key(raw_request, request)
+        response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
+        logprob = 0
+        completions = [
+            GeneratedOutput(
+                text=text,
+                logprob=logprob,
+                tokens=[Token(text=text, logprob=logprob)],
+            )
+            for text in response["completions"]
+        ]
+
+        return RequestResult(
+            success=True,
+            cached=cached,
+            request_time=response["request_time"],
+            request_datetime=response.get("request_datetime"),
+            completions=completions,
+            embedding=[],
+        )
+
+    def invoke_model(self, raw_request: SimpleClientRequest) -> Dict[str, Any]:
+        """
+        Example:
+        Prompt: 7 2 4 6
+        Completions (num_completions = 3):
+        - 6
+        - 4
+        - 2
+        """
+        prompt_words: List[str] = raw_request["prompt"].split()
+        completions = list(itertools.islice(itertools.cycle(reversed(prompt_words)), raw_request["num_completions"]))
+        return {"completions": completions}