crfm-helm 0.5.0__py3-none-any.whl → 0.5.2__py3-none-any.whl

helm/clients/vision_language/palmyra_vision_client.py

@@ -0,0 +1,84 @@
+from typing import Dict, List
+import json
+
+import requests
+
+from helm.common.cache import CacheConfig
+from helm.common.images_utils import encode_base64
+from helm.common.media_object import TEXT_TYPE
+from helm.common.request import Request, RequestResult, GeneratedOutput
+from helm.common.request import wrap_request_time
+from helm.clients.client import CachingClient, generate_uid_for_multimodal_prompt, truncate_and_tokenize_response_text
+from helm.tokenizers.tokenizer import Tokenizer
+
+
+class PalmyraVisionClient(CachingClient):
+    def __init__(self, tokenizer: Tokenizer, tokenizer_name: str, endpoint: str, cache_config: CacheConfig):
+        super().__init__(cache_config)
+        self.tokenizer: Tokenizer = tokenizer
+        self.tokenizer_name: str = tokenizer_name
+
+        # Currently, the Palmyra Vision model does not have a public API, so we need to use a secret endpoint
+        self.endpoint: str = endpoint
+
+    def make_request(self, request: Request) -> RequestResult:
+        assert request.multimodal_prompt is not None, "Multimodal prompt is required"
+
+        # Build the prompt
+        prompt: List[Dict[str, str]] = []
+        for media_object in request.multimodal_prompt.media_objects:
+            if media_object.is_type("image") and media_object.location:
+                prompt.append(
+                    {
+                        "type": "InlineData",
+                        "value": encode_base64(media_object.location, format="JPEG"),
+                        "contentType": "image/jpeg",
+                    }
+                )
+            elif media_object.is_type(TEXT_TYPE):
+                if media_object.text is None:
+                    raise ValueError("MediaObject of text type has missing text field value")
+                prompt.append({"type": "Text", "value": media_object.text})
+            else:
+                raise ValueError(f"Unrecognized MediaObject type {media_object.type}")
+
+        # Generate
+        try:
+
+            def do_it():
+                response = requests.post(
+                    self.endpoint, headers={"Content-Type": "application/json"}, data=json.dumps({"parts": prompt})
+                )
+                if response.status_code != 200:
+                    curl_command: str = (
+                        f"curl --location '{self.endpoint}' --header 'Content-Type: application/json' "
+                        f"--data '{json.dumps({'parts': prompt})}'"
+                    )
+                    assert False, f"Got status code {response.status_code}. Try {curl_command}"
+
+                json_response = json.loads(response.text)
+                assert (
+                    "choices" in json_response and "errors" not in json_response
+                ), f"Invalid response: {response.text}"
+                return json_response
+
+            cache_key = CachingClient.make_cache_key(
+                raw_request={"prompt": generate_uid_for_multimodal_prompt(request.multimodal_prompt)},
+                request=request,
+            )
+            result, cached = self.cache.get(cache_key, wrap_request_time(do_it))
+        except RuntimeError as ex:
+            return RequestResult(success=False, cached=False, error=str(ex), completions=[], embedding=[])
+
+        # The internal endpoint doesn't support any other parameters, so we have to truncate ourselves
+        completions: List[GeneratedOutput] = [
+            truncate_and_tokenize_response_text(choice["text"], request, self.tokenizer, self.tokenizer_name)
+            for choice in result["choices"]
+        ]
+        return RequestResult(
+            success=True,
+            cached=cached,
+            request_time=result["request_time"],
+            completions=completions,
+            embedding=[],
+        )

helm/clients/yi_client.py

@@ -0,0 +1,31 @@
+from typing import Optional
+
+from helm.clients.openai_client import OpenAIClient
+from helm.common.cache import CacheConfig
+from helm.tokenizers.tokenizer import Tokenizer
+
+
+class YiChatClient(OpenAIClient):
+
+    BASE_URL = "http://api.01ww.xyz/v1"
+
+    def __init__(
+        self,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        cache_config: CacheConfig,
+        api_key: Optional[str] = None,
+    ):
+        self.tokenizer = tokenizer
+        self.tokenizer_name = tokenizer_name
+        super().__init__(
+            tokenizer=tokenizer,
+            tokenizer_name=tokenizer_name,
+            cache_config=cache_config,
+            api_key=api_key,
+            org_id=None,
+            base_url=YiChatClient.BASE_URL,
+        )
+
+    def _is_chat_model_engine(self, model_engine: str) -> bool:
+        return True

helm/common/critique_request.py

@@ -1,5 +1,6 @@
 from dataclasses import dataclass
-from typing import Dict, List, Union
+from typing import Dict, List, Union, Optional
+from helm.common.media_object import MediaObject
 
 
 class QuestionType:
@@ -34,6 +35,11 @@ class CritiqueQuestionTemplate:
 
     Can contain placeholders like {{placeholder}} that will be interpolated using the fields in CritiqueRequest."""
 
+    media_object: Optional[MediaObject] = None
+    """Path of image for multimodal input.
+
+    Image path or URL of the question."""
+
 
 @dataclass(frozen=True)
 class CritiqueTaskTemplate:
@@ -53,6 +59,9 @@ class CritiqueTaskTemplate:
     questions: List[CritiqueQuestionTemplate]
     """List of templates for the questions."""
 
+    max_tokens: Optional[int] = None
+    """Max token to be generated for the free-end generation."""
+
 
 @dataclass(frozen=True)
 class CritiqueRequest:

helm/common/images_utils.py

@@ -1,8 +1,10 @@
 import base64
 import io
+import os
+
 import requests
 import shutil
-from typing import List, Optional
+from typing import List, Optional, Tuple
 from urllib.request import urlopen
 
 import numpy as np
@@ -28,6 +30,12 @@ def open_image(image_location: str) -> Image.Image:
     return image.convert("RGB")
 
 
+def get_dimensions(image_location: str) -> Tuple[int, int]:
+    """Returns the dimensions of the image."""
+    image: Image.Image = open_image(image_location)
+    return image.size
+
+
 def encode_base64(image_location: str, format="JPEG") -> str:
     """Returns the base64 representation of an image file."""
     image_file = io.BytesIO()
@@ -36,7 +44,7 @@ def encode_base64(image_location: str, format="JPEG") -> str:
     return base64.b64encode(image_file.getvalue()).decode("ascii")
 
 
-def copy_image(src: str, dest: str, width: Optional[int] = None, height: Optional[int] = None):
+def copy_image(src: str, dest: str, width: Optional[int] = None, height: Optional[int] = None) -> None:
     """
     Copies the image file from `src` path to `dest` path. If dimensions `width` and `height`
     are specified, resizes the image before copying. `src` can be a URL.
@@ -44,12 +52,30 @@ def copy_image(src: str, dest: str, width: Optional[int] = None, height: Optiona
     if (width is not None and height is not None) or is_url(src):
         image = open_image(src)
         if width is not None and height is not None:
-            image = image.resize((width, height), Image.ANTIALIAS)
+            image = image.resize((width, height), Image.Resampling.LANCZOS)
         image.save(dest)
     else:
         shutil.copy(src, dest)
 
 
+def resize_image_to_max_file_size(src: str, dest: str, max_size_in_bytes: int, step=10):
+    # Open an image file
+    with Image.open(src) as img:
+        width, height = img.size
+
+        # Reduce dimensions iteratively until the file size is under the limit
+        while True:
+            # Save the image temporarily to check the file size
+            img.save(dest, quality=95)  # Start with high quality
+            if os.path.getsize(dest) < max_size_in_bytes:
+                break
+
+            # Reduce dimensions
+            width -= step
+            height -= step
+            img = img.resize((width, height), Image.Resampling.LANCZOS)
+
+
 def is_blacked_out_image(image_location: str) -> bool:
     """Returns True if the image is all black. False otherwise."""
     try: