crfm-helm 0.5.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

helm/clients/vertexai_client.py

@@ -1,7 +1,7 @@
 import requests
 from abc import ABC, abstractmethod
 from threading import Lock
-from typing import Any, Dict, Optional, List, Union
+from typing import Any, Dict, Mapping, Optional, List, Union
 
 from helm.common.cache import CacheConfig
 from helm.common.media_object import TEXT_TYPE
@@ -26,22 +26,62 @@ class VertexAIContentBlockedError(Exception):
     pass
 
 
+class SafetySettingPresets:
+    BLOCK_NONE = "block_none"  # Disable all blocking
+    DEFAULT = "default"  # Use default safety settings
+
+
+def _get_safety_settings_for_preset(
+    safety_settings_preset: Optional[str],
+) -> Optional[Dict[HarmCategory, SafetySetting.HarmBlockThreshold]]:
+    """Get the safety settings for the safety_settings_preset.
+
+    If safety_settings_preset is None, use the default value of BLOCK_NONE (*not* DEFAULT)."""
+    if safety_settings_preset is None or safety_settings_preset == SafetySettingPresets.BLOCK_NONE:
+        return {
+            harm_category: SafetySetting.HarmBlockThreshold(SafetySetting.HarmBlockThreshold.BLOCK_NONE)
+            for harm_category in iter(HarmCategory)
+        }
+    elif safety_settings_preset == SafetySettingPresets.DEFAULT:
+        return None
+    else:
+        raise ValueError(f"Unknown safety_settings_preset: {safety_settings_preset}")
+
+
+def _get_model_name_for_request(request: Request) -> str:
+    # We have to strip "-safety-" suffixes from model names because they are not part of the Vertex AI model name
+    # TODO: Clean up this hack
+    return request.model_engine.split("-safety-")[0]
+
+
 class VertexAIClient(CachingClient, ABC):
     """Client for Vertex AI models"""
 
-    def __init__(self, cache_config: CacheConfig, project_id: str, location: str) -> None:
+    def __init__(
+        self, cache_config: CacheConfig, project_id: str, location: str, safety_settings_preset: Optional[str] = None
+    ) -> None:
         super().__init__(cache_config=cache_config)
         self.project_id = project_id
         self.location = location
 
-        # VertexAI's default safety filter is overly sensitive, so we disable it.
-        self.safety_settings: Dict[HarmCategory, SafetySetting.HarmBlockThreshold] = {
-            harm_category: SafetySetting.HarmBlockThreshold(SafetySetting.HarmBlockThreshold.BLOCK_NONE)
-            for harm_category in iter(HarmCategory)
-        }
+        self.safety_settings_preset = safety_settings_preset
+        self.safety_settings = _get_safety_settings_for_preset(safety_settings_preset)
 
         vertexai.init(project=self.project_id, location=self.location)
 
+    def make_cache_key_with_safety_settings_preset(self, raw_request: Mapping, request: Request) -> Mapping:
+        """Construct the key for the cache using the raw request.
+
+        Add `self.safety_settings_preset` to the key, if not None."""
+        if self.safety_settings_preset is not None:
+            assert "safety_settings_preset" not in raw_request
+            return {
+                **CachingClient.make_cache_key(raw_request, request),
+                "safety_settings_preset": self.safety_settings_preset,
+            }
+        else:
+            return CachingClient.make_cache_key(raw_request, request)
+
     @abstractmethod
     def make_request(self, request: Request) -> RequestResult:
         raise NotImplementedError
@@ -71,7 +111,7 @@ class VertexAITextClient(VertexAIClient):
         }
 
         completions: List[GeneratedOutput] = []
-        model_name: str = request.model_engine
+        model_name: str = _get_model_name_for_request(request)
 
         try:
 
@@ -87,9 +127,9 @@ class VertexAITextClient(VertexAIClient):
             # We need to include the engine's name to differentiate among requests made for different model
             # engines since the engine name is not included in the request itself.
             # Same for the prompt.
-            cache_key = CachingClient.make_cache_key(
+            cache_key = self.make_cache_key_with_safety_settings_preset(
                 {
-                    "engine": request.model_engine,
+                    "engine": model_name,
                     "prompt": request.prompt,
                     **parameters,
                 },
@@ -177,7 +217,7 @@ class VertexAIChatClient(VertexAIClient):
         }
 
         completions: List[GeneratedOutput] = []
-        model_name: str = request.model_engine
+        model_name: str = _get_model_name_for_request(request)
         model = self.get_model(model_name)
 
         try:
@@ -197,7 +237,7 @@ class VertexAIChatClient(VertexAIClient):
 
                 # Depending on the version of the Vertex AI library and the type of prompt blocking,
                 # prompt blocking can show up in many ways, so this defensively handles most of these ways
-                if response.prompt_feedback.block_reason:
+                if response.prompt_feedback and response.prompt_feedback.block_reason:
                     raise VertexAIContentBlockedError(
                         f"Prompt blocked with reason: {response.prompt_feedback.block_reason}"
                     )
@@ -209,8 +249,10 @@ class VertexAIChatClient(VertexAIClient):
                     # content blocking can show up in many ways, so this defensively handles most of these ways
                     if candidate.finish_reason in VertexAIChatClient.CONTENT_BLOCKED_FINISH_REASONS:
                         raise VertexAIContentBlockedError(f"Content blocked with reason: {candidate.finish_reason}")
+                    if not candidate.content:
+                        raise VertexAIContentBlockedError(f"No content in candidate: {candidate}")
                     if not candidate.content.parts:
-                        raise VertexAIContentBlockedError(f"No parts in candidate: {candidate}")
+                        raise VertexAIContentBlockedError(f"No content parts in candidate: {candidate}")
                     predictions.append({"text": candidate.content.text})
                     # TODO: Extract more information from the response
                 return {"predictions": predictions}
@@ -218,7 +260,7 @@ class VertexAIChatClient(VertexAIClient):
             # We need to include the engine's name to differentiate among requests made for different model
             # engines since the engine name is not included in the request itself.
             # Same for the prompt.
-            cache_key = CachingClient.make_cache_key(
+            cache_key = self.make_cache_key_with_safety_settings_preset(
                 {
                     "model_name": model_name,
                     "prompt": request.prompt,
@@ -313,7 +355,7 @@ class VertexAIChatClient(VertexAIClient):
         }
 
         completions: List[GeneratedOutput] = []
-        model_name: str = request.model_engine
+        model_name: str = _get_model_name_for_request(request)
         model = self.get_model(model_name)
 
         request_time = 0
@@ -330,7 +372,7 @@ class VertexAIChatClient(VertexAIClient):
                     )
                     # Depending on the version of the Vertex AI library and the type of prompt blocking,
                     # prompt blocking can show up in many ways, so this defensively handles most of these ways
-                    if response.prompt_feedback.block_reason:
+                    if response.prompt_feedback and response.prompt_feedback.block_reason:
                         raise VertexAIContentBlockedError(
                             f"Prompt blocked with reason: {response.prompt_feedback.block_reason}"
                         )
@@ -345,15 +387,17 @@ class VertexAIChatClient(VertexAIClient):
                     # content blocking can show up in many ways, so this defensively handles most of these ways
                     if candidate.finish_reason in VertexAIChatClient.CONTENT_BLOCKED_FINISH_REASONS:
                         raise VertexAIContentBlockedError(f"Content blocked with reason: {candidate.finish_reason}")
+                    if not candidate.content:
+                        raise VertexAIContentBlockedError(f"No content in candidate: {candidate}")
                     if not candidate.content.parts:
-                        raise VertexAIContentBlockedError(f"No parts in candidate: {candidate}")
+                        raise VertexAIContentBlockedError(f"No content parts in candidate: {candidate}")
                     return {"predictions": [{"text": candidate.text}]}
 
                 raw_cache_key = {"model_name": model_name, "prompt": prompt_key, **parameters}
                 if completion_index > 0:
                     raw_cache_key["completion_index"] = completion_index
 
-                cache_key = CachingClient.make_cache_key(raw_cache_key, request)
+                cache_key = self.make_cache_key_with_safety_settings_preset(raw_cache_key, request)
                 response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
             except requests.exceptions.RequestException as e:
                 error: str = f"Gemini Vision error: {e}"

helm/clients/vision_language/huggingface_vlm_client.py

@@ -38,6 +38,7 @@ class HuggingFaceVLMClient(CachingClient):
         "huggingface/llava-v1.6-vicuna-13b-hf": "llava-hf/llava-v1.6-vicuna-13b-hf",
         "huggingface/llava-v1.6-mistral-7b-hf": "llava-hf/llava-v1.6-mistral-7b-hf",
         "huggingface/llava-v1.6-34b-hf": "llava-hf/llava-v1.6-34b-hf",
+        "huggingface/prometheus-vision-13b-v1.0-hf": "PahaII/prometheus-vision-13b-v1.0-hf",
     }
 
     def __init__(self, tokenizer: Tokenizer, tokenizer_name: str, cache_config: CacheConfig):

helm/clients/vision_language/open_flamingo_client.py

@@ -82,13 +82,12 @@ class OpenFlamingoClient(CachingClient):
         # Build the prompt
         prompt_text: str = ""
         images: List[Image.Image] = []
+        request.validate()
         for media_object in request.multimodal_prompt.media_objects:
             if media_object.is_type("image") and media_object.location:
                 images.append(open_image(media_object.location))
                 prompt_text += self.IMAGE_TOKEN
             elif media_object.is_type(TEXT_TYPE):
-                if media_object.text is None:
-                    raise ValueError("MediaObject of text type has missing text field value")
                 prompt_text += media_object.text
             else:
                 raise ValueError(f"Unrecognized MediaObject type {media_object.type}")

helm/clients/vision_language/paligemma_client.py

@@ -0,0 +1,146 @@
+from threading import Lock
+from typing import Any, Dict, List, Optional
+
+import torch
+from dataclasses import dataclass
+from transformers import AutoProcessor, PaliGemmaForConditionalGeneration
+
+from helm.common.cache import CacheConfig
+from helm.common.images_utils import open_image
+from helm.common.gpu_utils import get_torch_device_name
+from helm.common.hierarchical_logger import hlog, htrack_block
+from helm.common.media_object import TEXT_TYPE
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.common.request import Request, RequestResult, GeneratedOutput, Token
+from helm.common.tokenization_request import TokenizationRequest
+from helm.common.request import wrap_request_time
+from helm.clients.client import CachingClient, generate_uid_for_multimodal_prompt
+from helm.tokenizers.tokenizer import Tokenizer
+
+try:
+    from PIL import Image
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["images"])
+
+# Added to solve: cutlassF: no kernel found to launch!
+torch.backends.cuda.enable_mem_efficient_sdp(False)
+torch.backends.cuda.enable_flash_sdp(False)
+
+
+@dataclass(frozen=True)
+class LoadedPaliGemmaForConditionalGeneration:
+    """Loaded model and processor for PaliGemma."""
+
+    model: PaliGemmaForConditionalGeneration
+    processor: AutoProcessor
+
+
+_models_lock: Lock = Lock()
+_models: Dict[str, Optional[LoadedPaliGemmaForConditionalGeneration]] = {}
+
+
+class PaliGemmaClient(CachingClient):
+    """
+    PaliGemma is a versatile and lightweight vision-language model (VLM) inspired by PaLI-3
+    and based on open components such as the SigLIP vision model and the Gemma language model.
+    It takes both image and text as input and generates text as output, supporting multiple languages.
+    It is designed for class-leading fine-tune performance on a wide range of vision-language tasks
+    such as image and short video caption, visual question answering, text reading, object detection
+    and object segmentation.
+    """
+
+    def __init__(self, tokenizer: Tokenizer, tokenizer_name: str, cache_config: CacheConfig):
+        super().__init__(cache_config=cache_config)
+        self.tokenizer = tokenizer
+        self.tokenizer_name = tokenizer_name
+        self._device: str = get_torch_device_name()
+
+    def _get_model(self, checkpoint: str) -> LoadedPaliGemmaForConditionalGeneration:
+        global _models_lock
+        global _models
+
+        # Ensure that only one thread is loading the model at a time
+        with _models_lock:
+            if checkpoint not in _models or _models[checkpoint] is None:
+                hlog(f"Loading model {checkpoint} and caching in memory...")
+                model = PaliGemmaForConditionalGeneration.from_pretrained(
+                    checkpoint, torch_dtype=torch.bfloat16, device_map="auto"
+                ).eval()
+                processor = AutoProcessor.from_pretrained(checkpoint)
+                _models[checkpoint] = LoadedPaliGemmaForConditionalGeneration(model, processor)
+            loaded_model_processor = _models[checkpoint]
+
+        assert loaded_model_processor is not None
+        return loaded_model_processor
+
+    def make_request(self, request: Request) -> RequestResult:
+        assert request.multimodal_prompt is not None, "Multimodal prompt is required"
+
+        loaded_model_processor: LoadedPaliGemmaForConditionalGeneration = self._get_model(request.model_deployment)
+        model = loaded_model_processor.model
+        processor = loaded_model_processor.processor
+        generation_args = {"max_new_tokens": request.max_tokens}
+
+        images: List[Image.Image] = []
+        prompt_pieces: List[str] = []
+        for media_object in request.multimodal_prompt.media_objects:
+            if media_object.is_type("image") and media_object.location:
+                images += [open_image(media_object.location).convert("RGB")]
+            elif media_object.is_type(TEXT_TYPE):
+                if media_object.text is None:
+                    raise ValueError("MediaObject of text type has missing text field value")
+                prompt_pieces.append(media_object.text)
+            else:
+                raise ValueError(f"Unrecognized MediaObject type {media_object.type}")
+        prompt_text: str = "\n".join(prompt_pieces)
+        model_inputs = processor(text=prompt_text, images=images, return_tensors="pt").to(self._device)
+        input_len = model_inputs["input_ids"].shape[-1]
+
+        completions: List[GeneratedOutput] = []
+        with htrack_block(f"Generating for prompt: {prompt_text}"):
+            try:
+                concat_results = []
+                for i_completion in range(request.num_completions):
+
+                    def do_it() -> Dict[str, Any]:
+                        with torch.inference_mode():
+                            generation = model.generate(
+                                **model_inputs, max_new_tokens=request.max_tokens, do_sample=False
+                            )[0]
+                            if not request.echo_prompt:
+                                generation = generation[input_len:]
+                            decoded = processor.decode(generation, skip_special_tokens=True)
+                            return {"output": decoded}
+
+                    # Include the prompt and model name in the cache key
+                    cache_key = CachingClient.make_cache_key(
+                        raw_request={
+                            "n": request.num_completions,
+                            "i": i_completion,
+                            "model": request.model,
+                            "prompt": generate_uid_for_multimodal_prompt(request.multimodal_prompt),
+                            **generation_args,
+                        },
+                        request=request,
+                    )
+                    result, cached = self.cache.get(cache_key, wrap_request_time(do_it))
+                    concat_results.append(result)
+            except RuntimeError as model_error:
+                return RequestResult(success=False, cached=False, error=str(model_error), completions=[], embedding=[])
+
+            for result in concat_results:
+                text = result["output"]
+                hlog(f"Generated text: {text}")
+                tokenization_result = self.tokenizer.tokenize(
+                    TokenizationRequest(text, tokenizer=self.tokenizer_name, encode=False)
+                )
+                tokens: List[Token] = [Token(text=str(text), logprob=0) for text in tokenization_result.raw_tokens]
+                completions.append(GeneratedOutput(text=text, logprob=0, tokens=tokens))
+
+        return RequestResult(
+            success=True,
+            cached=cached,
+            request_time=result["request_time"],
+            completions=completions,
+            embedding=[],
+        )

helm/clients/vision_language/palmyra_vision_client.py

@@ -0,0 +1,99 @@
+from typing import Dict, List
+import json
+
+import requests
+
+from helm.common.cache import CacheConfig
+from helm.common.images_utils import encode_base64
+from helm.common.media_object import TEXT_TYPE
+from helm.common.request import Request, RequestResult, GeneratedOutput, ErrorFlags
+from helm.common.request import wrap_request_time
+from helm.clients.client import CachingClient, generate_uid_for_multimodal_prompt, truncate_and_tokenize_response_text
+from helm.tokenizers.tokenizer import Tokenizer
+
+
+class PalmyraVisionContentBlockedError(Exception):
+    pass
+
+
+class PalmyraVisionClient(CachingClient):
+    CONTENT_BLOCKED_ERROR: str = "fail.input.content.moderation"
+
+    def __init__(self, tokenizer: Tokenizer, tokenizer_name: str, endpoint: str, cache_config: CacheConfig):
+        super().__init__(cache_config)
+        self.tokenizer: Tokenizer = tokenizer
+        self.tokenizer_name: str = tokenizer_name
+
+        # Currently, the Palmyra Vision model does not have a public API, so we need to use a secret endpoint
+        self.endpoint: str = endpoint
+
+    def make_request(self, request: Request) -> RequestResult:
+        assert request.multimodal_prompt is not None, "Multimodal prompt is required"
+
+        # Build the prompt
+        prompt: List[Dict[str, str]] = []
+        for media_object in request.multimodal_prompt.media_objects:
+            if media_object.is_type("image") and media_object.location:
+                prompt.append(
+                    {
+                        "type": "InlineData",
+                        "value": encode_base64(media_object.location, format="JPEG"),
+                        "contentType": "image/jpeg",
+                    }
+                )
+            elif media_object.is_type(TEXT_TYPE):
+                if media_object.text is None:
+                    raise ValueError("MediaObject of text type has missing text field value")
+                prompt.append({"type": "Text", "value": media_object.text})
+            else:
+                raise ValueError(f"Unrecognized MediaObject type {media_object.type}")
+
+        # Generate
+        try:
+
+            def do_it():
+                response = requests.post(
+                    self.endpoint, headers={"Content-Type": "application/json"}, data=json.dumps({"parts": prompt})
+                )
+                json_response = json.loads(response.text)
+
+                # Check for content blocked error
+                if (
+                    "errors" in json_response
+                    and "tpe" in json_response
+                    and json_response["tpe"] == self.CONTENT_BLOCKED_ERROR
+                ):
+                    raise PalmyraVisionContentBlockedError(json_response["errors"])
+
+                # Hard fail if the `choices` is missing from the response
+                assert "choices" in json_response, f"Invalid response: {response.text}"
+
+                return json_response
+
+            cache_key = CachingClient.make_cache_key(
+                raw_request={"prompt": generate_uid_for_multimodal_prompt(request.multimodal_prompt)},
+                request=request,
+            )
+            result, cached = self.cache.get(cache_key, wrap_request_time(do_it))
+        except PalmyraVisionContentBlockedError as ex:
+            return RequestResult(
+                success=False,
+                cached=False,
+                error=f"Content blocked: {str(ex)}",
+                completions=[],
+                embedding=[],
+                error_flags=ErrorFlags(is_retriable=False, is_fatal=False),
+            )
+
+        # The internal endpoint doesn't support any other parameters, so we have to truncate ourselves
+        completions: List[GeneratedOutput] = [
+            truncate_and_tokenize_response_text(choice["text"], request, self.tokenizer, self.tokenizer_name)
+            for choice in result["choices"]
+        ]
+        return RequestResult(
+            success=True,
+            cached=cached,
+            request_time=result["request_time"],
+            completions=completions,
+            embedding=[],
+        )

helm/clients/yi_client.py

@@ -0,0 +1,31 @@
+from typing import Optional
+
+from helm.clients.openai_client import OpenAIClient
+from helm.common.cache import CacheConfig
+from helm.tokenizers.tokenizer import Tokenizer
+
+
+class YiChatClient(OpenAIClient):
+
+    BASE_URL = "http://api.01ww.xyz/v1"
+
+    def __init__(
+        self,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        cache_config: CacheConfig,
+        api_key: Optional[str] = None,
+    ):
+        self.tokenizer = tokenizer
+        self.tokenizer_name = tokenizer_name
+        super().__init__(
+            tokenizer=tokenizer,
+            tokenizer_name=tokenizer_name,
+            cache_config=cache_config,
+            api_key=api_key,
+            org_id=None,
+            base_url=YiChatClient.BASE_URL,
+        )
+
+    def _is_chat_model_engine(self, model_engine: str) -> bool:
+        return True

helm/common/critique_request.py

@@ -1,5 +1,6 @@
 from dataclasses import dataclass
-from typing import Dict, List, Union
+from typing import Dict, List, Union, Optional
+from helm.common.media_object import MediaObject
 
 
 class QuestionType:
@@ -34,6 +35,11 @@ class CritiqueQuestionTemplate:
 
     Can contain placeholders like {{placeholder}} that will be interpolated using the fields in CritiqueRequest."""
 
+    media_object: Optional[MediaObject] = None
+    """Path of image for multimodal input.
+
+    Image path or URL of the question."""
+
 
 @dataclass(frozen=True)
 class CritiqueTaskTemplate:
@@ -53,6 +59,9 @@ class CritiqueTaskTemplate:
     questions: List[CritiqueQuestionTemplate]
     """List of templates for the questions."""
 
+    max_tokens: Optional[int] = None
+    """Max token to be generated for the free-end generation."""
+
 
 @dataclass(frozen=True)
 class CritiqueRequest:

helm/common/images_utils.py

@@ -1,5 +1,7 @@
+from hashlib import md5
 import base64
 import io
+import os
 
 import requests
 import shutil
@@ -43,6 +45,11 @@ def encode_base64(image_location: str, format="JPEG") -> str:
     return base64.b64encode(image_file.getvalue()).decode("ascii")
 
 
+def generate_hash(image: Image.Image) -> str:
+    """Generates a hash for the image."""
+    return md5(image.tobytes()).hexdigest()
+
+
 def copy_image(src: str, dest: str, width: Optional[int] = None, height: Optional[int] = None) -> None:
     """
     Copies the image file from `src` path to `dest` path. If dimensions `width` and `height`
@@ -57,6 +64,24 @@ def copy_image(src: str, dest: str, width: Optional[int] = None, height: Optiona
         shutil.copy(src, dest)
 
 
+def resize_image_to_max_file_size(src: str, dest: str, max_size_in_bytes: int, step=10):
+    # Open an image file
+    with Image.open(src) as img:
+        width, height = img.size
+
+        # Reduce dimensions iteratively until the file size is under the limit
+        while True:
+            # Save the image temporarily to check the file size
+            img.save(dest, quality=95)  # Start with high quality
+            if os.path.getsize(dest) < max_size_in_bytes:
+                break
+
+            # Reduce dimensions
+            width -= step
+            height -= step
+            img = img.resize((width, height), Image.Resampling.LANCZOS)
+
+
 def is_blacked_out_image(image_location: str) -> bool:
     """Returns True if the image is all black. False otherwise."""
     try:

helm/common/mongo_key_value_store.py

@@ -85,4 +85,5 @@ class MongoKeyValueStore(KeyValueStore):
         self._collection.bulk_write(operations)
 
     def remove(self, key: Dict) -> None:
-        self._collection.delete_one(key)
+        query = {self._REQUEST_KEY: self._canonicalize_key(key)}
+        self._collection.delete_one(query)

helm/common/request.py

@@ -72,6 +72,22 @@ class Request:
     image_generation_parameters: Optional[ImageGenerationParameters] = None
     """Parameters for image generation."""
 
+    def validate(self):
+        if (
+            (self.messages and self.prompt)
+            or (self.messages and self.multimodal_prompt)
+            or (self.prompt and self.multimodal_prompt)
+        ):
+            raise ValueError("Exactly one of the messages, prompt, multimodal_prompt fields should be set")
+
+        if self.multimodal_prompt:
+            for media_object in self.multimodal_prompt.media_objects:
+                if media_object.content_type == "text" and media_object.text is None:
+                    raise ValueError("Media object with text content type must have text set")
+
+                if media_object.content_type == "image" and media_object.location is None:
+                    raise ValueError("Media object with image content type must have location set")
+
     @property
     def model_host(self) -> str:
         """Returns the model host (referring to the deployment).