crfm-helm 0.5.0__py3-none-any.whl → 0.5.2__py3-none-any.whl

helm/clients/reka_client.py

@@ -0,0 +1,189 @@
+# mypy: check_untyped_defs = False
+import requests
+from typing import Any, Dict, List, Optional, TypedDict
+
+from helm.proxy.retry import NonRetriableException
+from helm.common.cache import CacheConfig
+from helm.common.media_object import TEXT_TYPE
+from helm.common.request import wrap_request_time, Request, RequestResult, GeneratedOutput
+from helm.common.hierarchical_logger import hlog
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.tokenizers.tokenizer import Tokenizer
+from .client import CachingClient, truncate_and_tokenize_response_text
+
+try:
+    import reka
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["reka-api"])
+
+
+class RekaAIRequest(TypedDict):
+    """Data passed between make_request and _send_request. Used as the cache key."""
+
+    model_name: str
+    conversation_history: List[Dict[str, str]]
+    request_output_len: int
+    temperature: float
+    runtime_top_p: float
+    random_seed: Optional[int]
+    stop_words: Optional[List[str]]
+    presence_penalty: float
+    frequency_penalty: float
+
+
+class RekaClient(CachingClient):
+    REKA_CHAT_ROLE_MAPPING: Dict[str, str] = {
+        "user": "human",
+        "assistant": "model",
+    }
+
+    def __init__(
+        self,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        cache_config: CacheConfig,
+        api_key: Optional[str] = None,
+    ):
+        super().__init__(cache_config=cache_config)
+        self.tokenizer = tokenizer
+        self.tokenizer_name = tokenizer_name
+        self.client = reka
+        self.client.API_KEY = api_key
+
+    def _is_reka_model_engine(self, model_engine: str) -> bool:
+        if (
+            model_engine.startswith("reka-edge")
+            or model_engine.startswith("reka-flash")
+            or model_engine.startswith("reka-core")
+        ):
+            return True
+        else:
+            return False
+
+    def _get_model_for_request(self, request: Request) -> str:
+        return request.model_engine
+
+    def _get_random_seed(self, request: Request, completion_index: int) -> Optional[int]:
+        if request.random is None and completion_index == 0:
+            return None
+
+        # Treat the user's request.random as an integer for the random seed.
+        try:
+            request_random_seed = int(request.random) if request.random is not None else 0
+        except ValueError:
+            raise NonRetriableException("RekaAIClient only supports integer values for request.random")
+
+        # A large prime is used so that the resulting values are unlikely to collide
+        # with request.random values chosen by the user.
+        fixed_large_prime = 1911011
+        completion_index_random_seed = completion_index * fixed_large_prime
+
+        return request_random_seed + completion_index_random_seed
+
+    def _convert_messages_to_reka_chat_history(self, messages: List[Dict[str, Any]]):
+        chat_history = []
+        num_images: int = 0
+        for chat_turn, message in enumerate(messages):
+            role = message["role"]
+            content = message["content"]
+            current_chat_history: Dict[str, Any] = {
+                "type": self.REKA_CHAT_ROLE_MAPPING[role],
+                "text": "",  # text placeholder
+                "media_url": None,
+            }
+            for item in content:
+                if item["type"] == "image_url":
+                    if chat_turn == 0 and num_images == 0:
+                        current_chat_history["media_url"] = item["image_url"]["url"]
+                        num_images += 1
+                    else:
+                        raise ValueError(
+                            f"Only the first message can contain one image. Found image input "
+                            f"in message {chat_turn + 1}"
+                        )
+                elif item["type"] == "text":
+                    current_chat_history["text"] = item["text"]
+                else:
+                    raise ValueError(f"Unrecognized message type {item['type']}")
+            chat_history.append(current_chat_history)
+        return chat_history
+
+    def make_request(self, request: Request) -> RequestResult:
+        completions: List[GeneratedOutput] = []
+        messages: Optional[List[Dict[str, Any]]] = request.messages
+        reka_chat_history: List[Dict[str, Any]]
+        if messages is not None:
+            # Checks that all messages have a role and some content
+            for message in messages:
+                if not message.get("role") or not message.get("content"):
+                    raise ValueError("All messages must have a role and content")
+            # Checks that the last role is "user"
+            if messages[-1]["role"] != "user":
+                raise ValueError("Last message must have role 'user'")
+            if request.prompt != "":
+                hlog("WARNING: Since message is set, prompt will be ignored")
+            reka_chat_history = self._convert_messages_to_reka_chat_history(messages)
+        else:
+            current_chat_history: Dict[str, Any] = {
+                "type": "human",
+                "text": "",
+                "media_url": None,
+            }
+            if request.multimodal_prompt is not None:
+                for media_object in request.multimodal_prompt.media_objects:
+                    if media_object.is_type("image") and media_object.location:
+                        from helm.common.images_utils import encode_base64
+
+                        base64_image: str = encode_base64(media_object.location)
+                        current_chat_history["media_url"] = f"data:image/jpeg;base64,{base64_image}"
+                    elif media_object.is_type(TEXT_TYPE):
+                        if media_object.text is None:
+                            raise ValueError("MediaObject of text type has missing text field value")
+                        current_chat_history["text"] = media_object.text
+                    else:
+                        raise ValueError(f"Unrecognized MediaObject type {media_object.type}")
+
+            else:
+                current_chat_history["text"] = request.prompt
+            reka_chat_history = [current_chat_history]
+
+        # `num_completions` is not supported, so instead make `num_completions` separate requests.
+        for completion_index in range(request.num_completions):
+            try:
+                raw_request: RekaAIRequest = {
+                    "model_name": self._get_model_for_request(request),
+                    "conversation_history": reka_chat_history,  # we only use chat_history as the input
+                    "request_output_len": request.max_tokens,
+                    "temperature": request.temperature,
+                    "random_seed": self._get_random_seed(request, completion_index),
+                    "stop_words": request.stop_sequences or None,  # API doesn't like empty list
+                    "runtime_top_p": request.top_p,
+                    "presence_penalty": request.presence_penalty,
+                    "frequency_penalty": request.frequency_penalty,
+                }
+
+                def do_it() -> Dict[str, Any]:
+                    return self.client.chat(**raw_request)
+
+                response, cached = self.cache.get(raw_request, wrap_request_time(do_it))
+            except (requests.exceptions.RequestException, AssertionError) as e:
+                error: str = f"RekaClient error: {e}"
+                return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
+
+            response_message: Dict[str, Any] = response
+            assert response_message["type"] == "model"
+            response_text: str = response_message["text"]
+
+            # The Reka API doesn't support echo. If `echo_prompt` is true, combine the prompt and completion.
+            text: str = request.prompt + response_text if request.echo_prompt else response_text
+            completion = truncate_and_tokenize_response_text(text, request, self.tokenizer, self.tokenizer_name)
+            completions.append(completion)
+
+        return RequestResult(
+            success=True,
+            cached=cached,
+            request_time=response["request_time"],
+            request_datetime=response.get("request_datetime"),
+            completions=completions,
+            embedding=[],
+        )

helm/clients/test_client.py

@@ -1,5 +1,5 @@
-from helm.common.cache import BlackHoleCacheConfig
-from helm.tokenizers.huggingface_tokenizer import HuggingFaceTokenizer
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
+from helm.tokenizers.auto_tokenizer import AutoTokenizer
 from .client import truncate_sequence, truncate_and_tokenize_response_text
 from typing import List
 from helm.common.request import Request, GeneratedOutput, Token
@@ -52,8 +52,8 @@ def test_truncate_sequence():
 
 
 def test_truncate_and_tokenize_response_text():
-    tokenizer = HuggingFaceTokenizer(BlackHoleCacheConfig())
     tokenizer_name = "huggingface/gpt2"
+    tokenizer = AutoTokenizer(credentials={}, cache_backend_config=BlackHoleCacheBackendConfig())
 
     # No truncation
     response = truncate_and_tokenize_response_text(

helm/clients/test_huggingface_client.py

@@ -3,12 +3,18 @@ import pytest
 from helm.common.cache import BlackHoleCacheConfig
 from helm.common.request import Request, RequestResult
 from helm.clients.huggingface_client import HuggingFaceClient
+from helm.tokenizers.huggingface_tokenizer import HuggingFaceTokenizer
 
 
 class TestHuggingFaceClient:
     def test_gpt2(self):
+        tokenizer = HuggingFaceTokenizer(
+            BlackHoleCacheConfig(), "huggingface/gpt2", pretrained_model_name_or_path="openai/gpt2"
+        )
         client = HuggingFaceClient(
-            cache_config=BlackHoleCacheConfig(), pretrained_model_name_or_path="openai-community/gpt2"
+            cache_config=BlackHoleCacheConfig(),
+            tokenizer=tokenizer,
+            pretrained_model_name_or_path="openai-community/gpt2",
         )
         prompt: str = "I am a computer scientist."
         result: RequestResult = client.make_request(
@@ -29,8 +35,13 @@ class TestHuggingFaceClient:
 
     @pytest.mark.skip(reason="GPT-J 6B is 22 GB and extremely slow without a GPU.")
     def test_gptj_6b(self):
+        tokenizer = HuggingFaceTokenizer(
+            BlackHoleCacheConfig(), "huggingface/gpt2", pretrained_model_name_or_path="openai/gpt2"
+        )
         client = HuggingFaceClient(
-            cache_config=BlackHoleCacheConfig(), pretrained_model_name_or_path="openai-community/gpt2"
+            cache_config=BlackHoleCacheConfig(),
+            tokenizer=tokenizer,
+            pretrained_model_name_or_path="openai-community/gpt2",
         )
         result: RequestResult = client.make_request(
             Request(
@@ -45,8 +56,13 @@ class TestHuggingFaceClient:
         assert len(result.completions) == 3
 
     def test_logprob(self):
+        tokenizer = HuggingFaceTokenizer(
+            BlackHoleCacheConfig(), "huggingface/gpt2", pretrained_model_name_or_path="openai/gpt2"
+        )
         client = HuggingFaceClient(
-            cache_config=BlackHoleCacheConfig(), pretrained_model_name_or_path="openai-community/gpt2"
+            cache_config=BlackHoleCacheConfig(),
+            tokenizer=tokenizer,
+            pretrained_model_name_or_path="openai-community/gpt2",
         )
         prompt: str = "I am a computer scientist."
         result: RequestResult = client.make_request(

helm/clients/test_together_client.py

@@ -2,10 +2,10 @@ import os
 import pytest
 import tempfile
 
-from helm.common.cache import SqliteCacheConfig
+from helm.common.cache import BlackHoleCacheConfig, SqliteCacheConfig
 from helm.common.request import Request
 
-from .together_client import TogetherClient, TogetherClientError
+from .together_client import TogetherClient, TogetherChatClient, TogetherCompletionClient, TogetherClientError
 
 
 class TestTogetherClient:
@@ -107,3 +107,73 @@ class TestTogetherClient:
                     model_deployment="together/redpajama-incite-base-3b-v1",
                 )
             )
+
+
+@pytest.mark.models
+def test_together_chat_client_make_request():
+    # Requires setting TOGETHER_API_KEY environment variable.
+    client = TogetherChatClient(
+        cache_config=BlackHoleCacheConfig(), api_key=None, together_model="meta-llama/Llama-3-8b-chat-hf"
+    )
+    request = Request(
+        model="meta/llama-3-8b-instruct",
+        model_deployment="together/llama-3-8b-instruct",
+        prompt="Elephants are one of the most",
+        temperature=0.0,
+        max_tokens=10,
+    )
+    result = client.make_request(request)
+    assert result.success
+    assert not result.cached
+    assert result.embedding == []
+    assert len(result.completions) == 1
+    assert result.completions[0].text == "...intelligent animals on Earth!assistant"
+    assert result.completions[0].logprob == 0.0
+    result_token_strings = [token.text for token in result.completions[0].tokens]
+    assert result_token_strings == [
+        "...",
+        "int",
+        "elligent",
+        " animals",
+        " on",
+        " Earth",
+        "!",
+        "<|eot_id|>",
+        "<|start_header_id|>",
+        "assistant",
+    ]
+
+
+@pytest.mark.models
+def test_together_completion_client_make_request():
+    # Requires setting TOGETHER_API_KEY environment variable.
+    client = TogetherCompletionClient(
+        cache_config=BlackHoleCacheConfig(), api_key=None, together_model="meta-llama/Llama-3-8b-hf"
+    )
+    request = Request(
+        model="meta/llama-3-8b",
+        model_deployment="together/llama-3-8b",
+        prompt="Elephants are one of the most",
+        temperature=0.0,
+        max_tokens=10,
+    )
+    result = client.make_request(request)
+    assert result.success
+    assert not result.cached
+    assert result.embedding == []
+    assert len(result.completions) == 1
+    assert result.completions[0].text == " popular animals in the world. They are known for"
+    assert result.completions[0].logprob == 0.0
+    result_token_strings = [token.text for token in result.completions[0].tokens]
+    assert result_token_strings == [
+        " popular",
+        " animals",
+        " in",
+        " the",
+        " world",
+        ".",
+        " They",
+        " are",
+        " known",
+        " for",
+    ]

helm/clients/together_client.py

@@ -1,12 +1,21 @@
 from copy import deepcopy
-from typing import List, Dict, Any, Optional, Union
+from itertools import zip_longest
+import threading
+from typing import List, Dict, Any, Mapping, Optional, TypedDict, Union
 
 import requests
 from retrying import retry
 
 from helm.common.cache import CacheConfig
+from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.common.request import wrap_request_time, Request, RequestResult, GeneratedOutput, Token
-from .client import CachingClient, truncate_sequence, cleanup_str
+from helm.clients.client import CachingClient, truncate_sequence, cleanup_str
+
+try:
+    from together import Together
+    from together.types import ChatCompletionResponse, CompletionResponse
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["together"])
 
 
 class _RewriteRequestTags:
@@ -272,3 +281,191 @@ class TogetherClient(CachingClient):
                 completions=completions,
                 embedding=[],
             )
+
+
+_MODEL_TO_DEFAULT_STOP_TOKENS: Optional[Mapping[str, List[str]]] = None
+_MODEL_TO_DEFAULT_STOP_TOKENS_LOCK = threading.Lock()
+
+
+def get_default_stop_tokens_for_model(together_model: str, together_client: Together) -> List[str]:
+    global _MODEL_TO_DEFAULT_STOP_TOKENS
+    global _MODEL_TO_DEFAULT_STOP_TOKENS_LOCK
+    with _MODEL_TO_DEFAULT_STOP_TOKENS_LOCK:
+        if _MODEL_TO_DEFAULT_STOP_TOKENS is None:
+            _MODEL_TO_DEFAULT_STOP_TOKENS = {}
+            for model in together_client.models.list():
+                _MODEL_TO_DEFAULT_STOP_TOKENS[model.id.lower()] = model.config["stop"]
+    stop_tokens = _MODEL_TO_DEFAULT_STOP_TOKENS.get(together_model.lower())
+    if stop_tokens is None:
+        raise ValueError(f"Unknown together_model {together_model}")
+    return stop_tokens
+
+
+class TogetherRawChatRequest(TypedDict):
+    messages: List[Dict[str, str]]
+    model: str
+    max_tokens: int
+    stop: List[str]
+    temperature: float
+    top_p: float
+    top_k: int
+    logprobs: int
+    echo: bool
+    n: int
+
+
+class TogetherChatClient(CachingClient):
+    """Client that uses the Python Together library for chat models."""
+
+    def __init__(self, cache_config: CacheConfig, api_key: Optional[str], together_model: Optional[str] = None):
+        super().__init__(cache_config=cache_config)
+        self._client = Together(api_key=api_key)
+        self._together_model = together_model
+
+    def convert_to_raw_chat_request(self, request: Request) -> TogetherRawChatRequest:
+        if request.messages:
+            messages = request.messages
+        else:
+            messages = [{"role": "user", "content": request.prompt}]
+        if self._together_model is not None:
+            model = self._together_model
+        else:
+            model = request.model
+        return {
+            "messages": messages,
+            "model": model,
+            "max_tokens": request.max_tokens,
+            "stop": request.stop_sequences + get_default_stop_tokens_for_model(model, self._client),
+            "temperature": request.temperature,
+            "top_p": request.top_p,
+            "top_k": request.top_k_per_token,
+            "logprobs": min(request.top_k_per_token, 1),
+            "echo": request.echo_prompt,
+            "n": request.num_completions,
+        }
+
+    def make_request(self, request: Request) -> RequestResult:
+        raw_request = self.convert_to_raw_chat_request(request)
+        cache_key = CachingClient.make_cache_key(raw_request, request)
+
+        def do_it() -> Dict[Any, Any]:
+            response = self._client.chat.completions.create(**raw_request)
+            return response.model_dump(mode="json")
+
+        try:
+            raw_response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
+            response = ChatCompletionResponse.model_validate(raw_response)
+        except Exception as error:
+            return RequestResult(
+                success=False,
+                cached=False,
+                error=str(error),
+                completions=[],
+                embedding=[],
+            )
+
+        generated_outputs: List[GeneratedOutput] = []
+        for choice in response.choices:
+            # NOTE: Together always returns None for choice.finish_reason
+            # NOTE: Together does not return logprobs for the whole generated output, only for individual tokens
+            tokens: List[Token] = []
+            if choice.logprobs:
+                for token_text, token_logprob in zip_longest(
+                    choice.logprobs.tokens or [], choice.logprobs.token_logprobs or []
+                ):
+                    if token_text is None:
+                        break
+                    tokens.append(Token(text=token_text, logprob=token_logprob or 0.0))
+            assert choice.message.role == "assistant"
+            generated_outputs.append(GeneratedOutput(text=choice.message.content, logprob=0.0, tokens=tokens))
+        return RequestResult(
+            success=True,
+            cached=cached,
+            request_time=raw_response["request_time"],
+            request_datetime=raw_response["request_datetime"],
+            completions=generated_outputs,
+            embedding=[],
+        )
+
+
+class TogetherRawCompletionRequest(TypedDict):
+    prompt: str
+    model: str
+    max_tokens: int
+    stop: List[str]
+    temperature: float
+    top_p: float
+    top_k: int
+    logprobs: int
+    echo: bool
+    n: int
+
+
+class TogetherCompletionClient(CachingClient):
+    """Client that uses the Python Together library for text completion models."""
+
+    def __init__(self, cache_config: CacheConfig, api_key: Optional[str], together_model: Optional[str] = None):
+        super().__init__(cache_config=cache_config)
+        self._client = Together(api_key=api_key)
+        self._together_model = together_model
+
+    def convert_to_raw_completion_request(self, request: Request) -> TogetherRawCompletionRequest:
+        if self._together_model is not None:
+            model = self._together_model
+        else:
+            model = request.model
+        return {
+            "prompt": request.prompt,
+            "model": model,
+            "max_tokens": request.max_tokens,
+            "stop": request.stop_sequences + get_default_stop_tokens_for_model(model, self._client),
+            "temperature": request.temperature,
+            "top_p": request.top_p,
+            "top_k": request.top_k_per_token,
+            "logprobs": min(request.top_k_per_token, 1),
+            "echo": request.echo_prompt,
+            "n": request.num_completions,
+        }
+
+    def make_request(self, request: Request) -> RequestResult:
+        raw_request = self.convert_to_raw_completion_request(request)
+        cache_key = CachingClient.make_cache_key(raw_request, request)
+
+        def do_it() -> Dict[Any, Any]:
+            response = self._client.completions.create(**raw_request)
+            return response.model_dump(mode="json")
+
+        try:
+            raw_response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
+            response = CompletionResponse.model_validate(raw_response)
+        except Exception as error:
+            return RequestResult(
+                success=False,
+                cached=False,
+                error=str(error),
+                completions=[],
+                embedding=[],
+            )
+
+        generated_outputs: List[GeneratedOutput] = []
+        for choice in response.choices:
+            # NOTE: Together always returns None for choice.finish_reason
+            # NOTE: Together does not return logprobs for the whole generated output, only for individual tokens
+            tokens: List[Token] = []
+            if choice.logprobs:
+                for token_text, token_logprob in zip_longest(
+                    choice.logprobs.tokens or [], choice.logprobs.token_logprobs or []
+                ):
+                    if token_text is None:
+                        break
+                    tokens.append(Token(text=token_text, logprob=token_logprob or 0.0))
+            assert choice.text
+            generated_outputs.append(GeneratedOutput(text=choice.text, logprob=0.0, tokens=tokens))
+        return RequestResult(
+            success=True,
+            cached=cached,
+            request_time=raw_response["request_time"],
+            request_datetime=raw_response["request_datetime"],
+            completions=generated_outputs,
+            embedding=[],
+        )