crfm-helm 0.5.5__py3-none-any.whl → 0.5.7__py3-none-any.whl

helm/clients/vision_language/huggingface_vision2seq_client.py

@@ -95,8 +95,8 @@ class HuggingFaceVision2SeqClient(CachingClient):
 
                 def do_it() -> Dict[str, Any]:
                     messages = [{"role": "user", "content": multimodal_prompt}]
-                    prompt = processor.apply_chat_template(messages, add_generation_prompt=True)
-                    inputs = processor(
+                    prompt = processor.apply_chat_template(messages, add_generation_prompt=True)  # type: ignore
+                    inputs = processor(  # type: ignore
                         text=[prompt] * request.num_completions,
                         images=[
                             [load_image(image_path) for image_path in image_paths]
@@ -107,8 +107,10 @@ class HuggingFaceVision2SeqClient(CachingClient):
                     inputs = {k: v.to(self._device) for k, v in inputs.items()}
 
                     # Generate
-                    generated_ids = model.generate(**inputs, **generation_args)
-                    generated_texts: List[str] = processor.batch_decode(generated_ids, skip_special_tokens=True)
+                    generated_ids = model.generate(**inputs, **generation_args)  # type: ignore
+                    generated_texts: List[str] = processor.batch_decode(  # type: ignore
+                        generated_ids, skip_special_tokens=True
+                    )
                     return {"output": generated_texts}
 
                 # Include the prompt and model name in the cache key

helm/clients/vision_language/huggingface_vlm_client.py

@@ -50,7 +50,7 @@ class HuggingFaceVLMClient(CachingClient):
         with self._models_lock:
             model_id: str = self._models_aliases.get(model_name, model_name)
             if model_id not in self._models:
-                self._models[model_id] = pipeline("image-to-text", model=model_id, device_map="auto")
+                self._models[model_id] = pipeline("image-to-text", model=model_id, device_map="auto")  # type: ignore
             return self._models[model_id]
 
     def make_request(self, request: Request) -> RequestResult:
@@ -80,7 +80,7 @@ class HuggingFaceVLMClient(CachingClient):
 
             def do_it() -> Dict[str, Any]:
                 model: ImageToTextPipeline = self._get_model(request.model_deployment)
-                outputs = model(image, prompt=prompt, generate_kwargs=generation_args)
+                outputs = model(image, prompt=prompt, generate_kwargs=generation_args)  # type: ignore
                 return outputs[0]
 
             cache_key = CachingClient.make_cache_key(

helm/clients/vision_language/idefics_client.py

@@ -89,14 +89,18 @@ class IDEFICSClient(CachingClient):
         input_args: Dict[str, Union[str, bool]] = {"return_tensors": "pt"}
         generation_args = {
             "max_new_tokens": request.max_tokens,
-            "bad_words_ids": processor.tokenizer(self.BAD_WORD_TOKENS, add_special_tokens=False).input_ids,
+            "bad_words_ids": processor.tokenizer(  # type: ignore
+                self.BAD_WORD_TOKENS, add_special_tokens=False
+            ).input_ids,
         }
 
         if self.END_OF_UTTERANCE_TOKEN in request.stop_sequences:
             # Following https://huggingface.co/HuggingFaceM4/idefics-80b-instruct,
             # specify <end_of_utterance> as an exit condition.
             input_args["add_end_of_utterance_token"] = False
-            exit_condition = processor.tokenizer(self.END_OF_UTTERANCE_TOKEN, add_special_tokens=False).input_ids
+            exit_condition = processor.tokenizer(  # type: ignore
+                self.END_OF_UTTERANCE_TOKEN, add_special_tokens=False
+            ).input_ids
             generation_args["eos_token_id"] = exit_condition
 
         multimodal_prompt: List[Union[str, Image.Image]] = []

helm/clients/vision_language/paligemma_client.py

@@ -93,7 +93,7 @@ class PaliGemmaClient(CachingClient):
             else:
                 raise ValueError(f"Unrecognized MediaObject type {media_object.type}")
         prompt_text: str = "\n".join(prompt_pieces)
-        model_inputs = processor(text=prompt_text, images=images, return_tensors="pt").to(self._device)
+        model_inputs = processor(text=prompt_text, images=images, return_tensors="pt").to(self._device)  # type: ignore
         input_len = model_inputs["input_ids"].shape[-1]
 
         completions: List[GeneratedOutput] = []
@@ -109,7 +109,7 @@ class PaliGemmaClient(CachingClient):
                             )[0]
                             if not request.echo_prompt:
                                 generation = generation[input_len:]
-                            decoded = processor.decode(generation, skip_special_tokens=True)
+                            decoded = processor.decode(generation, skip_special_tokens=True)  # type: ignore
                             return {"output": decoded}
 
                     # Include the prompt and model name in the cache key

helm/clients/vision_language/qwen2_vlm_client.py

@@ -2,7 +2,7 @@ from threading import Lock
 from typing import Any, Dict, List, Optional
 from dataclasses import dataclass
 
-from transformers import Qwen2VLForConditionalGeneration, AutoProcessor
+from transformers import AutoProcessor
 from qwen_vl_utils import process_vision_info
 import torch
 
@@ -16,15 +16,20 @@ from helm.clients.client import CachingClient, generate_uid_for_multimodal_promp
 
 
 @dataclass(frozen=True)
-class LoadedQwen2ModelProcessor:
-    model: Qwen2VLForConditionalGeneration
+class LoadedModelProcessor:
+    model: Any
     processor: AutoProcessor
 
 
+# Global cache for all models
 _models_lock: Lock = Lock()
-_models: Dict[str, Optional[LoadedQwen2ModelProcessor]] = {
+_models: Dict[str, Optional[LoadedModelProcessor]] = {
     "Qwen/Qwen2-VL-7B-Instruct": None,
     "Qwen/Qwen2-VL-72B-Instruct": None,
+    "Qwen/Qwen2.5-VL-3B-Instruct": None,
+    "Qwen/Qwen2.5-VL-7B-Instruct": None,
+    "Qwen/Qwen2.5-VL-32B-Instruct": None,
+    "Qwen/Qwen2.5-VL-72B-Instruct": None,
 }
 
 
@@ -38,50 +43,52 @@ class Qwen2VLMClient(CachingClient):
             return "Qwen/Qwen2-VL-7B-Instruct"
         elif helm_model_name == "qwen2-vl-72b-instruct":
             return "Qwen/Qwen2-VL-72B-Instruct"
+        elif helm_model_name == "qwen2.5-vl-3b-instruct":
+            return "Qwen/Qwen2.5-VL-3B-Instruct"
+        elif helm_model_name == "qwen2.5-vl-7b-instruct":
+            return "Qwen/Qwen2.5-VL-7B-Instruct"
+        elif helm_model_name == "qwen2.5-vl-32b-instruct":
+            return "Qwen/Qwen2.5-VL-32B-Instruct"
+        elif helm_model_name == "qwen2.5-vl-72b-instruct":
+            return "Qwen/Qwen2.5-VL-72B-Instruct"
         else:
             raise ValueError(f"Unhandled model name: {helm_model_name}")
 
-    def _get_model(self, helm_model_name: str) -> LoadedQwen2ModelProcessor:
-        global _models_lock
-        global _models
+    def _get_model(self, helm_model_name: str) -> LoadedModelProcessor:
+        from transformers import Qwen2VLForConditionalGeneration, Qwen2_5_VLForConditionalGeneration
 
-        model_name = self._get_model_name(helm_model_name)
+        global _models_lock, _models
 
+        model_name = self._get_model_name(helm_model_name)
         with _models_lock:
             loaded = _models[model_name]
             if loaded is None:
                 hlog(f"Loading model {model_name} and caching in memory...")
-                # https://huggingface.co/docs/transformers/model_doc/qwen2_vl#flash-attention-2-to-speed-up-generation
-                model = Qwen2VLForConditionalGeneration.from_pretrained(
-                    model_name,
-                    torch_dtype=torch.bfloat16,
-                    device_map="auto",
-                    attn_implementation="flash_attention_2",
-                ).eval()
+                # Use different loading routines depending on whether it's Qwen2.5 or Qwen2.
+                if "2.5" in model_name:
+                    # Qwen2.5: by default use torch_dtype="auto". You can enable flash_attention_2 if desired.
+                    model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
+                        model_name,
+                        torch_dtype=torch.bfloat16,
+                        device_map="auto",
+                        attn_implementation="flash_attention_2",
+                    ).eval()
+                else:
+                    model = Qwen2VLForConditionalGeneration.from_pretrained(
+                        model_name,
+                        torch_dtype=torch.bfloat16,
+                        device_map="auto",
+                        attn_implementation="flash_attention_2",
+                    ).eval()
                 processor = AutoProcessor.from_pretrained(model_name)
-                loaded = LoadedQwen2ModelProcessor(model=model, processor=processor)
+                loaded = LoadedModelProcessor(model=model, processor=processor)
                 _models[model_name] = loaded
-
         return loaded
 
     def make_request(self, request: Request) -> RequestResult:
         assert request.multimodal_prompt is not None, "Multimodal prompt is required"
-        loaded = self._get_model(request.model_engine)
-        model = loaded.model
-        processor = loaded.processor
-
-        # Build Qwen2 messages
-        # We assume all media objects go into a single "user" message:
-        # messages = [
-        #   {
-        #     "role": "user",
-        #     "content": [
-        #       {"type": "image", "image": "file:///path/to/image1.jpg"},
-        #       {"type": "image", "image": "file:///path/to/image2.jpg"},
-        #       {"type": "text", "text": "Describe these images."}
-        #     ]
-        #   }
-        # ]
+
+        # Build messages by collating all media objects into a single "user" message.
         message_content = []
         for media_object in request.multimodal_prompt.media_objects:
             if media_object.is_type("image") and media_object.location:
@@ -95,18 +102,6 @@ class Qwen2VLMClient(CachingClient):
 
         messages = [{"role": "user", "content": message_content}]
 
-        # Prepare text and vision inputs
-        text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
-        image_inputs, video_inputs = process_vision_info(messages)
-
-        inputs = processor(
-            text=[text],
-            images=image_inputs,
-            videos=video_inputs,
-            padding=True,
-            return_tensors="pt",
-        ).to(self._device)
-
         generation_args = {
             "max_new_tokens": request.max_tokens,
         }
@@ -116,23 +111,38 @@ class Qwen2VLMClient(CachingClient):
         request_datetime: Optional[int] = None
         all_cached: bool = True
 
-        with htrack_block(f"Generating for prompt: {text}"):
+        with htrack_block(f"Generating for prompt: {request.multimodal_prompt.text}"):
             for completion_index in range(request.num_completions):
                 try:
 
                     def do_it() -> Dict[str, Any]:
+                        loaded = self._get_model(request.model_engine)
+                        model = loaded.model
+                        processor = loaded.processor
+
+                        # Prepare text and vision inputs.
+                        text = processor.apply_chat_template(  # type: ignore
+                            messages, tokenize=False, add_generation_prompt=True
+                        )
+                        image_inputs, video_inputs = process_vision_info(messages)
+                        inputs = processor(  # type: ignore
+                            text=[text],
+                            images=image_inputs,
+                            videos=video_inputs,
+                            padding=True,
+                            return_tensors="pt",
+                        ).to(self._device)
+
                         generated_ids = model.generate(**inputs, **generation_args)
-                        # Remove the input prefix from outputs
+                        # Remove the input prefix from outputs.
                         generated_ids_trimmed = [
                             out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
                         ]
-                        output_text = processor.batch_decode(
+                        output_text = processor.batch_decode(  # type: ignore
                             generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
                         )
-                        # There's only one batch element
-                        completion = output_text[0]
                         # For simplicity, we split tokens by whitespace.
-                        # A more accurate tokenization would require a tokenizer for Qwen2, if desired.
+                        completion = output_text[0]
                         tokens = completion.split()
                         return {"output": (completion, tokens)}
 
@@ -148,7 +158,11 @@ class Qwen2VLMClient(CachingClient):
                     result, cached = self.cache.get(cache_key, wrap_request_time(do_it))
                 except RuntimeError as model_error:
                     return RequestResult(
-                        success=False, cached=False, error=str(model_error), completions=[], embedding=[]
+                        success=False,
+                        cached=False,
+                        error=str(model_error),
+                        completions=[],
+                        embedding=[],
                     )
 
                 text_out, tokens = result["output"]
@@ -160,7 +174,6 @@ class Qwen2VLMClient(CachingClient):
                     )
                 )
                 hlog(f"Generated: {text_out}")
-
                 request_time += result["request_time"]
                 request_datetime = request_datetime or result.get("request_datetime")
                 all_cached = all_cached and cached

helm/clients/vision_language/qwen_vlm_client.py

@@ -115,14 +115,16 @@ class QwenVLMClient(CachingClient):
 
                     def do_it() -> Dict[str, Any]:
                         if request.model_engine == "qwen-vl-chat":
-                            completion, _ = model.chat(tokenizer, query=tokenizer.from_list_format(query), history=None)
+                            completion, _ = model.chat(  # type: ignore
+                                tokenizer, query=tokenizer.from_list_format(query), history=None  # type: ignore
+                            )
                         else:
-                            inputs = tokenizer(tokenizer.from_list_format(query), return_tensors="pt")
+                            inputs = tokenizer(tokenizer.from_list_format(query), return_tensors="pt")  # type: ignore
                             inputs = inputs.to(self._device)
-                            pred = model.generate(**inputs, **generation_args)
-                            completion = tokenizer.decode(pred.cpu()[0], skip_special_tokens=False)
+                            pred = model.generate(**inputs, **generation_args)  # type: ignore
+                            completion = tokenizer.decode(pred.cpu()[0], skip_special_tokens=False)  # type: ignore
 
-                        tokens: List[str] = tokenizer.tokenize(completion)
+                        tokens: List[str] = tokenizer.tokenize(completion)  # type: ignore
                         return {"output": (completion, tokens)}
 
                     # Include the prompt and model name in the cache key

helm/clients/vllm_client.py

@@ -2,7 +2,7 @@ from typing import Any, Dict, Optional
 
 from helm.common.cache import CacheConfig
 from helm.common.request import Request
-from helm.clients.openai_client import OpenAILegacyCompletionsClient
+from helm.clients.openai_client import OpenAIClient, OpenAILegacyCompletionsClient
 from helm.tokenizers.tokenizer import Tokenizer
 
 
@@ -19,6 +19,8 @@ class VLLMClient(OpenAILegacyCompletionsClient):
         tokenizer_name: str,
         cache_config: CacheConfig,
         base_url: Optional[str] = None,
+        vllm_model_name: Optional[str] = None,
+        **kwargs,
     ):
         super().__init__(
             tokenizer=tokenizer,
@@ -27,18 +29,52 @@ class VLLMClient(OpenAILegacyCompletionsClient):
             api_key="EMPTY",
             org_id=None,
             base_url=base_url,
+            openai_model_name=vllm_model_name,
+            **kwargs,
         )
         self.tokenizer = tokenizer
         self.tokenizer_name = tokenizer_name
-
-    def _get_model_for_request(self, request: Request) -> str:
-        # The `model` parameter for vLLM should be the whole model name including the creator organization,
-        # unlike OpenAI which only uses the model engine.
-        return request.model
+        self.vllm_model_name = vllm_model_name
 
     def _to_raw_completion_request(self, request: Request) -> Dict[str, Any]:
         raw_request = super()._to_raw_completion_request(request)
         # This avoids the error: best_of must be 1 when using greedy sampling
-        if "best_of" in raw_request and raw_request["best_of"] > 1:
+        if (
+            "temperature" in raw_request
+            and raw_request["temperature"] == 0.0
+            and "best_of" in raw_request
+            and raw_request["best_of"] > 1
+        ):
             raw_request["best_of"] = 1
         return raw_request
+
+
+class VLLMChatClient(OpenAIClient):
+    """Sends request to a vLLM server using the OpenAI-compatible API.
+
+    Only uses the Chat Completions API.
+
+    See: https://docs.vllm.ai/en/latest/getting_started/quickstart.html#openai-compatible-server"""
+
+    def __init__(
+        self,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        cache_config: CacheConfig,
+        base_url: Optional[str] = None,
+        vllm_model_name: Optional[str] = None,
+        **kwargs,
+    ):
+        super().__init__(
+            tokenizer=tokenizer,
+            tokenizer_name=tokenizer_name,
+            cache_config=cache_config,
+            api_key="EMPTY",
+            org_id=None,
+            base_url=base_url,
+            openai_model_name=vllm_model_name,
+            **kwargs,
+        )
+        self.tokenizer = tokenizer
+        self.tokenizer_name = tokenizer_name
+        self.vllm_model_name = vllm_model_name

helm/clients/vllm_granite_thinking_client.py

@@ -0,0 +1,56 @@
+from dataclasses import replace
+import re
+from typing import Any, Dict, List, Tuple
+
+from helm.clients.vllm_client import VLLMChatClient
+from helm.common.request import GeneratedOutput, Request, RequestResult, Thinking
+
+
+class VLLMGraniteThinkingClient(VLLMChatClient):
+    """Sends request to a Granite model on vLLM server with thinking enabled.
+
+    From vLLM documentation at
+    https://docs.vllm.ai/en/v0.9.1/features/reasoning_outputs.html
+
+    IBM Granite 3.2 reasoning is disabled by default;
+    to enable it, you must also pass thinking=True in your chat_template_kwargs.
+    """
+
+    def _make_chat_raw_request(self, request: Request) -> Dict[str, Any]:
+        raw_request = super()._make_chat_raw_request(request)
+        raw_request["extra_body"] = {"chat_template_kwargs": {"thinking": True}}
+        return raw_request
+
+    def _parse_thinking(self, input: str) -> Tuple[str, str]:
+        """Return a tuple of thinking text and output text."""
+        match = re.match(r"<think>(.*)</think>\s*<response>(.*)</response>", input, re.DOTALL)
+        if match:
+            return (match.group(1), match.group(2))
+
+        match = re.match(r"<think>(.*)</think>\s*<response>(.*)", input, re.DOTALL)
+        if match:
+            return (match.group(1), match.group(2))
+
+        match = re.match(r"<think>(.*)</think>\s*", input, re.DOTALL)
+        if match:
+            return (match.group(1), "")
+
+        match = re.match(r"<think>(.*)", input, re.DOTALL)
+        if match:
+            return (match.group(1), "")
+
+        return (input, "")
+
+    def _make_chat_request(self, request: Request) -> RequestResult:
+        request_result = super()._make_chat_request(request)
+        modified_completions: List[GeneratedOutput] = []
+        for completion in request_result.completions:
+            thinking, modified_text = self._parse_thinking(completion.text)
+            modified_completions.append(
+                replace(
+                    completion,
+                    text=modified_text,
+                    thinking=Thinking(text=thinking),
+                )
+            )
+        return replace(request_result, completions=modified_completions)

helm/clients/writer_client.py

@@ -0,0 +1,102 @@
+from typing import Any, Dict, List, Mapping, Optional
+
+from helm.clients.client import CachingClient
+from helm.common.cache import CacheConfig
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.common.request import wrap_request_time, Request, RequestResult, GeneratedOutput, Token
+
+try:
+    from writerai import Writer
+    from writerai.types.chat_completion import ChatCompletion
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["openai"])
+
+
+class WriterClient(CachingClient):
+    def __init__(self, cache_config: CacheConfig, api_key: Optional[str] = None):
+        super().__init__(cache_config=cache_config)
+        self._writer_client = Writer(api_key=api_key)
+
+    def _get_messages_from_request(self, request: Request) -> List[Dict]:
+        if request.prompt and request.messages:
+            raise ValueError(f"Only one of `prompt` and `messages` may be set in request: {request}")
+        if request.multimodal_prompt:
+            raise ValueError("`multimodal_prompt` is not supported by WriterClient")
+        if request.messages:
+            return [{"role": message["role"], "content": message["content"]} for message in request.messages]
+        else:
+            return [{"role": "user", "content": request.prompt}]
+
+    def _convert_chat_completion_to_generated_outputs(
+        self, chat_completion: ChatCompletion, request: Request
+    ) -> List[GeneratedOutput]:
+        generated_outputs: List[GeneratedOutput] = []
+        for choice in chat_completion.choices:
+            raw_completion_content = choice.message.content
+            # The Writer chat completion API doesn't support echo.
+            # If `echo_prompt` is true, combine the prompt and completion.
+            text: str = request.prompt + raw_completion_content if request.echo_prompt else raw_completion_content
+            tokens: List[Token] = []
+            if choice.logprobs and choice.logprobs.content:
+                tokens = [
+                    Token(text=choice_token.token, logprob=choice_token.logprob)
+                    for choice_token in choice.logprobs.content
+                ]
+            generated_output = GeneratedOutput(
+                text=text,
+                logprob=sum(token.logprob for token in tokens) if tokens else 0.0,
+                tokens=tokens,
+                finish_reason={"reason": choice.finish_reason},
+            )
+            generated_outputs.append(generated_output)
+        return generated_outputs
+
+    def _convert_request_to_raw_request(self, request: Request) -> Dict:
+        raw_request = {
+            "messages": self._get_messages_from_request(request),
+            "model": request.model.split("/")[-1],
+            "logprobs": bool(request.top_k_per_token),
+            "max_tokens": request.max_tokens,
+            "n": request.num_completions,
+            "stop": request.stop_sequences,
+            "temperature": request.temperature,
+            "top_p": request.top_p,
+        }
+        if request.response_format and request.response_format.json_schema:
+            raw_request["response_format"] = {
+                "type": "json_schema",
+                "json_schema": {
+                    "schema": request.response_format.json_schema,
+                },
+            }
+        return raw_request
+
+    def make_request(self, request: Request) -> RequestResult:
+        raw_request = self._convert_request_to_raw_request(request)
+        cache_key: Mapping = CachingClient.make_cache_key(raw_request, request)
+
+        def do_it() -> Dict[Any, Any]:
+            return self._writer_client.chat.chat(**raw_request).model_dump()
+
+        try:
+            raw_response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
+            chat_completion: ChatCompletion = ChatCompletion.model_validate(raw_response)
+        except Exception as error:
+            return RequestResult(
+                success=False,
+                cached=False,
+                error=str(error),
+                completions=[],
+                embedding=[],
+            )
+
+        generated_outputs = self._convert_chat_completion_to_generated_outputs(chat_completion, request)
+
+        return RequestResult(
+            success=True,
+            cached=cached,
+            request_time=raw_response["request_time"],
+            request_datetime=raw_response["request_datetime"],
+            completions=generated_outputs,
+            embedding=[],
+        )

helm/common/context.py

@@ -0,0 +1,80 @@
+from abc import ABC, abstractmethod
+
+from helm.common.critique_request import CritiqueRequest, CritiqueRequestResult
+from helm.common.clip_score_request import CLIPScoreRequest, CLIPScoreResult
+from helm.common.file_upload_request import FileUploadResult, FileUploadRequest
+from helm.common.nudity_check_request import NudityCheckRequest, NudityCheckResult
+from helm.common.perspective_api_request import PerspectiveAPIRequestResult, PerspectiveAPIRequest
+from helm.common.moderations_api_request import ModerationAPIRequest, ModerationAPIRequestResult
+from helm.common.tokenization_request import (
+    TokenizationRequest,
+    TokenizationRequestResult,
+    DecodeRequest,
+    DecodeRequestResult,
+)
+from helm.common.request import Request, RequestResult
+from helm.proxy.query import Query, QueryResult
+from helm.common.cache import CacheConfig
+from helm.proxy.services.service import GeneralInfo
+
+
+class Context(ABC):
+    @abstractmethod
+    def get_general_info(self) -> GeneralInfo:
+        """Get general info."""
+        pass
+
+    @abstractmethod
+    def expand_query(self, query: Query) -> QueryResult:
+        """Turn the `query` into requests."""
+        pass
+
+    @abstractmethod
+    def make_request(self, request: Request) -> RequestResult:
+        """Actually make a request to an API."""
+        pass
+
+    @abstractmethod
+    def tokenize(self, request: TokenizationRequest) -> TokenizationRequestResult:
+        """Tokenize via an API."""
+        pass
+
+    @abstractmethod
+    def decode(self, request: DecodeRequest) -> DecodeRequestResult:
+        """Decodes to text."""
+        pass
+
+    @abstractmethod
+    def upload(self, request: FileUploadRequest) -> FileUploadResult:
+        """Uploads a file to external storage."""
+        pass
+
+    @abstractmethod
+    def check_nudity(self, request: NudityCheckRequest) -> NudityCheckResult:
+        """Check for nudity for a batch of images."""
+        pass
+
+    @abstractmethod
+    def compute_clip_score(self, request: CLIPScoreRequest) -> CLIPScoreResult:
+        """Computes CLIPScore for a given caption and image."""
+        pass
+
+    @abstractmethod
+    def get_toxicity_scores(self, request: PerspectiveAPIRequest) -> PerspectiveAPIRequestResult:
+        """Get toxicity scores for a batch of text."""
+        pass
+
+    @abstractmethod
+    def get_moderation_results(self, request: ModerationAPIRequest) -> ModerationAPIRequestResult:
+        """Get OpenAI's moderation results for some text."""
+        pass
+
+    @abstractmethod
+    def make_critique_request(self, request: CritiqueRequest) -> CritiqueRequestResult:
+        """Get responses to a critique request."""
+        pass
+
+    @abstractmethod
+    def get_cache_config(self, shard_name: str) -> CacheConfig:
+        """Returns a CacheConfig"""
+        pass

helm/common/credentials_utils.py

@@ -2,7 +2,7 @@
 
 from typing import Any, Mapping, Optional
 
-from helm.common.hierarchical_logger import hlog
+from helm.common.hierarchical_logger import hlog, hwarn
 
 
 def provide_api_key(
@@ -13,16 +13,16 @@ def provide_api_key(
         hlog(f"Using host_organization api key defined in credentials.conf: {api_key_name}")
         return credentials[api_key_name]
     if "deployments" not in credentials:
-        hlog(
-            "WARNING: Could not find key 'deployments' in credentials.conf, "
+        hwarn(
+            "Could not find key 'deployments' in credentials.conf, "
             f"therefore the API key {api_key_name} should be specified."
         )
         return None
     deployment_api_keys = credentials["deployments"]
     if model is None:
-        hlog(f"WARNING: Could not find key '{host_organization}' in credentials.conf and no model provided")
+        hwarn(f"Could not find key '{host_organization}' in credentials.conf and no model provided")
         return None
     if model not in deployment_api_keys:
-        hlog(f"WARNING: Could not find key '{model}' under key 'deployments' in credentials.conf")
+        hwarn(f"Could not find key '{model}' under key 'deployments' in credentials.conf")
         return None
     return deployment_api_keys[model]

helm/common/critique_request.py

@@ -6,7 +6,6 @@ from helm.common.media_object import MediaObject
 class QuestionType:
     """String enum of question types."""
 
-    # TODO: Make this a StrEnum after upgrading to Python 3.11
     MULTIPLE_CHOICE: str = "multiple_choice"
     CHECKBOX: str = "checkbox"
     FREE_RESPONSE: str = "free_response"