crfm-helm 0.4.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

helm/clients/vision_language/qwen_vlm_client.py

@@ -0,0 +1,171 @@
+from threading import Lock
+from typing import Any, Dict, List, Optional
+
+from dataclasses import dataclass
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers.generation import GenerationConfig
+
+from helm.common.cache import CacheConfig
+from helm.common.gpu_utils import get_torch_device_name
+from helm.common.hierarchical_logger import hlog, htrack_block
+from helm.common.media_object import TEXT_TYPE
+from helm.common.request import Request, RequestResult, GeneratedOutput, Token
+from helm.common.request import wrap_request_time
+from helm.clients.client import CachingClient, generate_uid_for_multimodal_prompt
+
+
+@dataclass(frozen=True)
+class LoadedQwenModelProcessor:
+    """Loaded model and processor for Qwen."""
+
+    model: AutoModelForCausalLM
+    tokenizer: AutoTokenizer
+
+
+_models_lock: Lock = Lock()
+_models: Dict[str, Optional[LoadedQwenModelProcessor]] = {
+    "Qwen/Qwen-VL": None,
+    "Qwen/Qwen-VL-Chat": None,
+}
+
+
+class QwenVLMClient(CachingClient):
+    """
+    From https://huggingface.co/Qwen/Qwen-VL,
+    Qwen-VL (Qwen Large Vision Language Model) is the visual multimodal version of the large model series,
+    Qwen (abbr. Tongyi Qianwen), proposed by Alibaba Cloud. Qwen-VL accepts image, text, and bounding box
+    as inputs, outputs text and bounding box.
+    Alibaba released Qwen-VL and Qwen-VL-Chat, which is a chatbot model based on Qwen-VL.
+
+    Paper: https://arxiv.org/abs/2308.12966
+    """
+
+    END_OF_TEXT_TOKEN: str = "<|endoftext|>"
+
+    def __init__(self, cache_config: CacheConfig):
+        super().__init__(cache_config=cache_config)
+        self._device: str = get_torch_device_name()
+
+    def _get_model(self, helm_model_name: str) -> LoadedQwenModelProcessor:
+        global _models_lock
+        global _models
+
+        model_name: str
+        if helm_model_name == "qwen-vl-chat":
+            model_name = "Qwen/Qwen-VL-Chat"
+        elif helm_model_name == "qwen-vl":
+            model_name = "Qwen/Qwen-VL"
+        else:
+            raise ValueError(f"Unhandled model name: {helm_model_name}")
+
+        # Ensure that only one thread is loading the model at a time
+        with _models_lock:
+            loaded_model_processor = _models[model_name]
+            if loaded_model_processor is None:
+                hlog(f"Loading model {model_name} and caching in memory...")
+                model = AutoModelForCausalLM.from_pretrained(
+                    model_name, device_map=self._device, trust_remote_code=True, bf16=True
+                ).eval()
+                if model_name == "Qwen/Qwen-VL-Chat":
+                    model.generation_config = GenerationConfig.from_pretrained(model_name, trust_remote_code=True)
+                tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+                _models[model_name] = LoadedQwenModelProcessor(model, tokenizer)
+                loaded_model_processor = _models[model_name]
+
+        assert loaded_model_processor is not None
+        return loaded_model_processor
+
+    def make_request(self, request: Request) -> RequestResult:
+        assert request.multimodal_prompt is not None, "Multimodal prompt is required"
+
+        loaded_model_processor: LoadedQwenModelProcessor = self._get_model(request.model_engine)
+        model = loaded_model_processor.model
+        tokenizer = loaded_model_processor.tokenizer
+
+        generation_args = {
+            "max_length": request.max_tokens,
+        }
+
+        query: List[Dict[str, str]] = []
+        prompt_text: str = ""
+
+        image_index: int = 1
+        for media_object in request.multimodal_prompt.media_objects:
+            if media_object.is_type("image") and media_object.location:
+                query.append({"image": media_object.location})
+                prompt_text += f"Picture {image_index}: <img>{media_object.location}</img>\n"
+                image_index += 1
+            elif media_object.is_type(TEXT_TYPE):
+                if media_object.text is None:
+                    raise ValueError("MediaObject of text type has missing text field value")
+
+                query.append({"text": media_object.text})
+                prompt_text += media_object.text
+            else:
+                raise ValueError(f"Unrecognized MediaObject type {media_object.type}")
+
+        completions: List[GeneratedOutput] = []
+        request_time: float = 0
+        request_datetime: Optional[int] = None
+        all_cached: bool = True
+
+        with htrack_block(f"Generating for prompt: {prompt_text}"):
+            for completion_index in range(request.num_completions):
+                try:
+
+                    def do_it() -> Dict[str, Any]:
+                        if request.model_engine == "qwen-vl-chat":
+                            completion, _ = model.chat(tokenizer, query=tokenizer.from_list_format(query), history=None)
+                        else:
+                            inputs = tokenizer(tokenizer.from_list_format(query), return_tensors="pt")
+                            inputs = inputs.to(self._device)
+                            pred = model.generate(**inputs, **generation_args)
+                            completion = tokenizer.decode(pred.cpu()[0], skip_special_tokens=False)
+
+                        tokens: List[str] = tokenizer.tokenize(completion)
+                        return {"output": (completion, tokens)}
+
+                    # Include the prompt and model name in the cache key
+                    cache_key = CachingClient.make_cache_key(
+                        raw_request={
+                            "completion_index": completion_index,
+                            "model": request.model,
+                            "prompt": generate_uid_for_multimodal_prompt(request.multimodal_prompt),
+                            **generation_args,
+                        },
+                        request=request,
+                    )
+                    result, cached = self.cache.get(cache_key, wrap_request_time(do_it))
+                except RuntimeError as model_error:
+                    return RequestResult(
+                        success=False, cached=False, error=str(model_error), completions=[], embedding=[]
+                    )
+
+                text, tokens = result["output"]
+
+                # Truncate the output text as the original Qwen includes the prompt in the output sequence
+                if request.model_engine == "qwen-vl":
+                    text = text[len(prompt_text) :]
+                    text = text.replace(self.END_OF_TEXT_TOKEN, "")
+                    hlog(f"Truncated: {text}")
+
+                # Tokenize truncated text to get the list of tokens
+                completions.append(
+                    GeneratedOutput(
+                        text=text, logprob=0, tokens=[Token(text=str(token), logprob=0) for token in tokens]
+                    )
+                )
+
+                request_time += result["request_time"]
+                # Use the datetime from the first completion because that's when the request was fired
+                request_datetime = request_datetime or result.get("request_datetime")
+                all_cached = all_cached and cached
+
+        return RequestResult(
+            success=True,
+            cached=all_cached,
+            request_time=request_time,
+            request_datetime=request_datetime,
+            completions=completions,
+            embedding=[],
+        )

helm/clients/vllm_client.py

@@ -0,0 +1,46 @@
+from typing import Any, Dict, Optional
+
+from helm.common.cache import CacheConfig
+from helm.common.request import Request
+from helm.clients.openai_client import OpenAIClient
+from helm.tokenizers.tokenizer import Tokenizer
+
+
+class VLLMClient(OpenAIClient):
+    """Sends request to a vLLM server using the OpenAI-compatible API.
+
+    See: https://docs.vllm.ai/en/latest/getting_started/quickstart.html#openai-compatible-server"""
+
+    def __init__(
+        self,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        cache_config: CacheConfig,
+        base_url: Optional[str] = None,
+    ):
+        super().__init__(
+            tokenizer=tokenizer,
+            tokenizer_name=tokenizer_name,
+            cache_config=cache_config,
+            api_key="EMPTY",
+            org_id=None,
+            base_url=base_url,
+        )
+        self.tokenizer = tokenizer
+        self.tokenizer_name = tokenizer_name
+
+    def _is_chat_model_engine(self, model_engine: str) -> bool:
+        # Only support vLLM completion models for now.
+        return False
+
+    def _get_model_for_request(self, request: Request) -> str:
+        # The `model` parameter for vLLM should be the whole model name including the creator organization,
+        # unlike OpenAI which only uses the model engine.
+        return request.model
+
+    def _to_raw_completion_request(self, request: Request) -> Dict[str, Any]:
+        raw_request = super()._to_raw_completion_request(request)
+        # This avoids the error: best_of must be 1 when using greedy sampling
+        if "best_of" in raw_request and raw_request["best_of"] > 1:
+            raw_request["best_of"] = 1
+        return raw_request

helm/common/cache.py

@@ -1,13 +1,13 @@
 from collections import defaultdict
 from dataclasses import dataclass
-from typing import Dict, Callable, Generator, Optional, Tuple
+from typing import Dict, Callable, Generator, Mapping, Optional, Tuple
 import json
 import threading
 
 import sqlite3
 
 from helm.common.general import hlog, htrack
-from helm.common.key_value_store import KeyValueStore, SqliteKeyValueStore
+from helm.common.key_value_store import BlackHoleKeyValueStore, KeyValueStore, SqliteKeyValueStore
 from helm.proxy.retry import get_retry_decorator
 
 try:
@@ -51,6 +51,16 @@ class SqliteCacheConfig(KeyValueStoreCacheConfig):
         return self.path
 
 
+@dataclass(frozen=True)
+class BlackHoleCacheConfig(KeyValueStoreCacheConfig):
+    """Configuration for a cache that does not save any data."""
+
+    @property
+    def cache_stats_key(self) -> str:
+        """The string key used by CacheStats to identify this cache."""
+        return "disabled_cache"
+
+
 @dataclass(frozen=True)
 class MongoCacheConfig(KeyValueStoreCacheConfig):
     """Configuration for a cache backed by a MongoDB collection."""
@@ -113,12 +123,14 @@ def create_key_value_store(config: KeyValueStoreCacheConfig) -> KeyValueStore:
         return MongoKeyValueStore(config.uri, config.collection_name)
     elif isinstance(config, SqliteCacheConfig):
         return SqliteKeyValueStore(config.path)
+    elif isinstance(config, BlackHoleCacheConfig):
+        return BlackHoleKeyValueStore()
     else:
         raise ValueError(f"KeyValueStoreCacheConfig with unknown type: {config}")
 
 
 @retry
-def write_to_key_value_store(key_value_store: KeyValueStore, key: Dict, response: Dict) -> bool:
+def write_to_key_value_store(key_value_store: KeyValueStore, key: Mapping, response: Dict) -> bool:
     """
     Write to the key value store with retry. Returns boolean indicating whether the write was successful or not.
     """
@@ -188,7 +200,7 @@ class Cache(object):
         else:
             raise ValueError(f"CacheConfig with unknown type: {config}")
 
-    def get(self, request: Dict, compute: Callable[[], Dict]) -> Tuple[Dict, bool]:
+    def get(self, request: Mapping, compute: Callable[[], Dict]) -> Tuple[Dict, bool]:
         """Get the result of `request` (by calling `compute` as needed)."""
         cache_stats.increment_query(self.config.cache_stats_key)
 

helm/common/cache_backend_config.py

@@ -0,0 +1,47 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+import os
+
+from helm.common.cache import CacheConfig, MongoCacheConfig, BlackHoleCacheConfig, SqliteCacheConfig
+
+
+class CacheBackendConfig(ABC):
+    """Config for a cache backend."""
+
+    @abstractmethod
+    def get_cache_config(self, shard_name: str) -> CacheConfig:
+        """Get a CacheConfig for the given shard."""
+        pass
+
+
+@dataclass(frozen=True)
+class MongoCacheBackendConfig(CacheBackendConfig):
+    """Config for a MongoDB cache backend."""
+
+    uri: str
+    """URL for the MongoDB database that contains the collection.
+
+    Example format: mongodb://[username:password@]host1[:port1]/[dbname]
+    For full format, see: https://www.mongodb.com/docs/manual/reference/connection-string/"""
+
+    def get_cache_config(self, shard_name: str) -> CacheConfig:
+        return MongoCacheConfig(uri=self.uri, collection_name=shard_name)
+
+
+@dataclass(frozen=True)
+class BlackHoleCacheBackendConfig(CacheBackendConfig):
+    """Config for a cache backend that does not save any data."""
+
+    def get_cache_config(self, shard_name: str) -> CacheConfig:
+        return BlackHoleCacheConfig()
+
+
+@dataclass(frozen=True)
+class SqliteCacheBackendConfig(CacheBackendConfig):
+    """Config for a Sqlite cache backend."""
+
+    path: str
+    """Path for the directory that will contain Sqlite files for caches."""
+
+    def get_cache_config(self, shard_name: str) -> CacheConfig:
+        return SqliteCacheConfig(path=os.path.join(self.path, f"{shard_name}.sqlite"))

helm/common/clip_score_request.py

@@ -0,0 +1,41 @@
+from dataclasses import dataclass
+from typing import Optional
+
+
+DEFAULT_CLIP_SCORE_MODEL = "openai/clip-vit-large-patch14"
+
+
+@dataclass(frozen=True)
+class CLIPScoreRequest:
+    """
+    Computes a CLIPScore for a given caption and image.
+    """
+
+    # Caption to compute CLIPScore for
+    caption: str
+
+    # Location of the image
+    image_location: str
+
+    # Which CLIP model to use
+    model: str = DEFAULT_CLIP_SCORE_MODEL
+
+    # Compute multilingual CLIPScore
+    multilingual: bool = False
+
+
+@dataclass(frozen=True)
+class CLIPScoreResult:
+    """Result after sending a `CLIPScoreRequest`."""
+
+    # Whether the request was successful
+    success: bool
+
+    # Whether the request was cached
+    cached: bool
+
+    # The CLIPScore
+    score: float = 0.0
+
+    # If `success` is false, what was the error?
+    error: Optional[str] = None

helm/common/file_caches/file_cache.py

@@ -0,0 +1,16 @@
+from abc import ABC, abstractmethod
+from typing import Callable
+
+
+class FileCache(ABC):
+    """
+    Cache to store files.
+    """
+
+    @abstractmethod
+    def store(self, compute: Callable[[], bytes]) -> str:
+        """
+        Stores the output of `compute` as a file at a unique location.
+        Returns the location of the file.
+        """
+        pass

helm/common/file_caches/local_file_cache.py

@@ -0,0 +1,61 @@
+import os
+from typing import Callable
+
+from helm.common.general import ensure_directory_exists, generate_unique_id
+from .file_cache import FileCache
+
+from helm.common.optional_dependencies import handle_module_not_found_error
+
+try:
+    from PIL import Image
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["images"])
+
+
+class LocalFileCache(FileCache):
+    def __init__(self, base_path: str, file_extension: str):
+        ensure_directory_exists(base_path)
+        self._location: str = base_path
+        self._file_extension: str = file_extension
+
+    def store(self, compute: Callable[[], bytes]) -> str:
+        """
+        Stores the output of `compute` as a file at a unique path.
+        Returns the file path.
+        """
+        file_path: str = self.generate_unique_new_file_path()
+        with open(file_path, "wb") as f:
+            f.write(compute())
+
+        return file_path
+
+    def generate_unique_new_file_path(self) -> str:
+        """Generate an unique file name at `base_path`"""
+
+        def generate_one() -> str:
+            file_name: str = f"{generate_unique_id()}.{self._file_extension}"
+            return os.path.join(self._location, file_name)
+
+        file_path: str
+        while True:
+            file_path = generate_one()
+            if not os.path.exists(file_path):
+                break
+        return file_path
+
+
+class LocalPILFileCache(LocalFileCache):
+    def __init__(self, base_path: str):
+        super().__init__(base_path, "png")
+
+    def store_image(self, compute: Callable[[], Image.Image]) -> str:
+        """
+        Stores the output of `compute` as a file at a unique path.
+        Returns the file path.
+        """
+        file_path: str = self.generate_unique_new_file_path()
+        compute().save(file_path)
+        return file_path
+
+    def load_image(self, file_path: str) -> Image.Image:
+        return Image.open(file_path).convert("RGB")

helm/common/file_caches/test_local_file_cache.py

@@ -0,0 +1,25 @@
+import os
+import shutil
+import tempfile
+import unittest
+
+from .local_file_cache import LocalFileCache
+
+
+class TestLocalFileCache(unittest.TestCase):
+    def setup_method(self, _):
+        self.path: str = tempfile.mkdtemp()
+
+    def teardown_method(self, _):
+        shutil.rmtree(self.path)
+
+    def test_get(self):
+        cache = LocalFileCache(self.path, file_extension="txt")
+        file_path1: str = cache.store(lambda: "hello.".encode())
+
+        # Verify the contents of the file
+        with open(file_path1, "r") as f:
+            assert f.read() == "hello."
+
+        cache.store(lambda: "bye.".encode())
+        assert len(os.listdir(self.path)) == 2

helm/common/file_upload_request.py

@@ -0,0 +1,27 @@
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass(frozen=True)
+class FileUploadRequest:
+    """Uploads a file at `path`."""
+
+    # Path of the file to upload
+    path: str
+
+
+@dataclass(frozen=True)
+class FileUploadResult:
+    """Result after sending a `FileUploadRequest`."""
+
+    # Whether the request was successful
+    success: bool
+
+    # Whether the request was cached
+    cached: bool
+
+    # URL of the uploaded file
+    url: str
+
+    # If `success` is false, what was the error?
+    error: Optional[str] = None

helm/common/general.py

@@ -63,7 +63,7 @@ def shell(args: List[str]):
     hlog(f"Executing: {cmd}")
     exit_code = subprocess.call(args)
     if exit_code != 0:
-        hlog(f"Failed with exit code {exit_code}: {cmd}")
+        raise Exception(f"Failed with exit code {exit_code}: {cmd}")
 
 
 @htrack(None)

helm/common/image_generation_parameters.py

@@ -0,0 +1,25 @@
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass(frozen=True)
+class ImageGenerationParameters:
+    """
+    Parameters for image generation.
+    """
+
+    output_image_width: Optional[int] = None
+    """Width of the generated image. The model will generate images with the model's
+    default dimensions when unspecified."""
+
+    output_image_height: Optional[int] = None
+    """Height of the generated image. The model will generate images with the model's
+    default dimensions when unspecified."""
+
+    guidance_scale: Optional[float] = None
+    """A non-negative number determining how much importance is given to the prompt
+    when generating images. Higher values will generate images that follow more
+    closely to the prompt. Currently only for diffusion models."""
+
+    diffusion_denoising_steps: Optional[int] = None
+    """The number of denoising steps for diffusion models."""

helm/common/images_utils.py

@@ -1,8 +1,12 @@
 import base64
 import io
+
 import requests
 import shutil
-from typing import Optional
+from typing import List, Optional, Tuple
+from urllib.request import urlopen
+
+import numpy as np
 
 from .general import is_url
 from helm.common.optional_dependencies import handle_module_not_found_error
@@ -25,6 +29,12 @@ def open_image(image_location: str) -> Image.Image:
     return image.convert("RGB")
 
 
+def get_dimensions(image_location: str) -> Tuple[int, int]:
+    """Returns the dimensions of the image."""
+    image: Image.Image = open_image(image_location)
+    return image.size
+
+
 def encode_base64(image_location: str, format="JPEG") -> str:
     """Returns the base64 representation of an image file."""
     image_file = io.BytesIO()
@@ -33,7 +43,7 @@ def encode_base64(image_location: str, format="JPEG") -> str:
     return base64.b64encode(image_file.getvalue()).decode("ascii")
 
 
-def copy_image(src: str, dest: str, width: Optional[int] = None, height: Optional[int] = None):
+def copy_image(src: str, dest: str, width: Optional[int] = None, height: Optional[int] = None) -> None:
     """
     Copies the image file from `src` path to `dest` path. If dimensions `width` and `height`
     are specified, resizes the image before copying. `src` can be a URL.
@@ -41,7 +51,27 @@ def copy_image(src: str, dest: str, width: Optional[int] = None, height: Optiona
     if (width is not None and height is not None) or is_url(src):
         image = open_image(src)
         if width is not None and height is not None:
-            image = image.resize((width, height), Image.ANTIALIAS)
+            image = image.resize((width, height), Image.Resampling.LANCZOS)
         image.save(dest)
     else:
         shutil.copy(src, dest)
+
+
+def is_blacked_out_image(image_location: str) -> bool:
+    """Returns True if the image is all black. False otherwise."""
+    try:
+        import cv2
+    except ModuleNotFoundError as e:
+        handle_module_not_found_error(e, ["heim"])
+
+    if is_url(image_location):
+        arr = np.asarray(bytearray(urlopen(image_location).read()), dtype=np.uint8)
+        image = cv2.imdecode(arr, -1)
+    else:
+        image = cv2.imread(image_location, 0)
+    return cv2.countNonZero(image) == 0
+
+
+def filter_blacked_out_images(image_locations: List[str]) -> List[str]:
+    """Returns a list of image locations that are not blacked out."""
+    return [image_location for image_location in image_locations if not is_blacked_out_image(image_location)]

helm/common/key_value_store.py

@@ -1,12 +1,12 @@
 from abc import abstractmethod
 import contextlib
 import json
-from typing import Dict, Generator, Iterable, Optional, Tuple
+from typing import Dict, Generator, Iterable, Mapping, Optional, Tuple
 
 from sqlitedict import SqliteDict
 
 
-def request_to_key(request: Dict) -> str:
+def request_to_key(request: Mapping) -> str:
     """Normalize a `request` into a `key` so that we can hash using it."""
     return json.dumps(request, sort_keys=True)
 
@@ -27,7 +27,7 @@ class KeyValueStore(contextlib.AbstractContextManager):
         pass
 
     @abstractmethod
-    def put(self, key: Dict, value: Dict) -> None:
+    def put(self, key: Mapping, value: Dict) -> None:
         pass
 
     @abstractmethod
@@ -68,7 +68,7 @@ class SqliteKeyValueStore(KeyValueStore):
         for key, value in self._sqlite_dict.items():
             yield (key, value)
 
-    def put(self, key: Dict, value: Dict) -> None:
+    def put(self, key: Mapping, value: Dict) -> None:
         key_string = request_to_key(key)
         self._sqlite_dict[key_string] = value
         self._sqlite_dict.commit()
@@ -80,3 +80,34 @@ class SqliteKeyValueStore(KeyValueStore):
     def remove(self, key: Dict) -> None:
         del self._sqlite_dict[key]
         self._sqlite_dict.commit()
+
+
+class BlackHoleKeyValueStore(KeyValueStore):
+    """Key value store that discards all data."""
+
+    def __enter__(self) -> "BlackHoleKeyValueStore":
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback) -> None:
+        pass
+
+    def contains(self, key: Dict) -> bool:
+        return False
+
+    def get(self, key: Dict) -> Optional[Dict]:
+        return None
+
+    def get_all(self) -> Generator[Tuple[Dict, Dict], None, None]:
+        # Return an empty generator.
+        # See: https://stackoverflow.com/a/13243870
+        return
+        yield
+
+    def put(self, key: Mapping, value: Dict) -> None:
+        return None
+
+    def multi_put(self, pairs: Iterable[Tuple[Dict, Dict]]) -> None:
+        return None
+
+    def remove(self, key: Dict) -> None:
+        return None