crfm-helm 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

helm/clients/gcs_client.py

@@ -0,0 +1,82 @@
+from dataclasses import asdict
+from typing import Dict, Optional
+import requests
+
+from helm.common.cache import Cache, CacheConfig
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.common.hierarchical_logger import hlog
+from helm.common.file_upload_request import FileUploadRequest, FileUploadResult
+
+
+class GCSClientError(Exception):
+    pass
+
+
+class GCSClient:
+    """
+    Uploads files to GCS. Ensure the GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials.json
+    environment variable is set.
+    """
+
+    MAX_CHECK_ATTEMPTS: int = 10
+
+    def __init__(self, bucket_name: str, cache_config: CacheConfig):
+        try:
+            from google.cloud import storage  # type: ignore
+        except ModuleNotFoundError as e:
+            handle_module_not_found_error(e, ["heim"])
+
+        self._bucket_name: str = bucket_name
+        self._cache = Cache(cache_config)
+        self._storage_client: Optional[storage.Client] = None
+
+    def upload(self, request: FileUploadRequest) -> FileUploadResult:
+        """Uploads a file to GCS."""
+        try:
+            from google.cloud import storage  # type: ignore
+        except ModuleNotFoundError as e:
+            handle_module_not_found_error(e, ["heim"])
+
+        try:
+
+            def do_it():
+                if self._storage_client is None:
+                    self._storage_client = storage.Client()
+
+                bucket = self._storage_client.bucket(self._bucket_name)
+                file_path: str = request.path
+                blob = bucket.blob(file_path)
+
+                # Optional: set a generation-match precondition to avoid potential race conditions
+                # and data corruptions. The request to upload is aborted if the object's
+                # generation number does not match your precondition. For a destination
+                # object that does not yet exist, set the if_generation_match precondition to 0.
+                # If the destination object already exists in your bucket, set instead a
+                # generation-match precondition using its generation number.
+                generation_match_precondition: int = 0
+
+                blob.upload_from_filename(file_path, if_generation_match=generation_match_precondition)
+                url: str = self._get_url(file_path)
+
+                # Ensure the file was uploaded successfully
+                uploaded: bool = False
+                for _ in range(0, self.MAX_CHECK_ATTEMPTS):
+                    check_response = requests.head(url)
+                    if check_response.status_code == 200:
+                        uploaded = True
+                        break
+                assert uploaded, f"File {file_path} was not uploaded successfully."
+
+                hlog(f"File {file_path} uploaded and is available at {url}.")
+                return {"url": url}
+
+            cache_key: Dict = asdict(request)
+            result, cached = self._cache.get(cache_key, do_it)
+
+        except Exception as e:
+            raise GCSClientError(e)
+
+        return FileUploadResult(success=True, cached=cached, url=result["url"])
+
+    def _get_url(self, path: str) -> str:
+        return f"https://storage.googleapis.com/{self._bucket_name}/{path}"

helm/{proxy/clients → clients}/google_client.py

@@ -1,8 +1,7 @@
 from typing import List, Dict
 
 from helm.common.cache import CacheConfig
-from helm.common.request import Request, RequestResult, Sequence, Token
-from helm.proxy.tokenizers.tokenizer import Tokenizer
+from helm.common.request import Request, RequestResult, GeneratedOutput, Token
 from .client import CachingClient, truncate_sequence
 
 
@@ -28,12 +27,12 @@ class GoogleClient(CachingClient):
             "top_p": request.top_p,
         }
 
-    def __init__(self, tokenizer: Tokenizer, cache_config: CacheConfig):
-        super().__init__(cache_config=cache_config, tokenizer=tokenizer)
+    def __init__(self, cache_config: CacheConfig):
+        super().__init__(cache_config=cache_config)
 
     def make_request(self, request: Request) -> RequestResult:
         raw_request = GoogleClient.convert_to_raw_request(request)
-        cache_key: Dict = CachingClient.make_cache_key(raw_request, request)
+        cache_key = CachingClient.make_cache_key(raw_request, request)
 
         try:
 
@@ -49,17 +48,17 @@ class GoogleClient(CachingClient):
             return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
 
         # Expect the result to be structured the same way as a response from OpenAI API.
-        completions: List[Sequence] = []
+        completions: List[GeneratedOutput] = []
         for raw_completion in response["choices"]:
             sequence_logprob = 0
             tokens: List[Token] = []
 
             raw_data = raw_completion["logprobs"]
             for text, logprob in zip(raw_data["tokens"], raw_data["token_logprobs"]):
-                tokens.append(Token(text=text, logprob=logprob or 0, top_logprobs=dict()))
+                tokens.append(Token(text=text, logprob=logprob or 0))
                 sequence_logprob += logprob or 0
 
-            completion = Sequence(
+            completion = GeneratedOutput(
                 text=raw_completion["text"],
                 logprob=sequence_logprob,
                 tokens=tokens,

helm/clients/google_translate_client.py

@@ -0,0 +1,35 @@
+from typing import Optional
+
+from helm.common.cache import Cache, SqliteCacheConfig
+from helm.common.optional_dependencies import handle_module_not_found_error
+
+try:
+    from google.cloud import translate_v2 as translate  # type: ignore
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["heim"])
+
+
+class GoogleTranslateClient:
+    """
+    Client for Google Translate.
+    Follow the instructions at https://cloud.google.com/translate/docs/setup to use this client.
+
+    # TODO: add this as a central service
+    """
+
+    def __init__(self, cache_path: str = "prod_env/cache/google_translate.sqlite"):
+        self.translate_client: Optional[translate.Client] = None
+        self.cache = Cache(SqliteCacheConfig(cache_path))
+
+    def translate(self, text: str, target_language: str) -> str:
+        def do_it():
+            if self.translate_client is None:
+                self.translate_client = translate.Client()
+
+            result = self.translate_client.translate(text, target_language=target_language)
+            del result["input"]
+            assert "translatedText" in result, f"Invalid response: {result}"
+            return result
+
+        response, _ = self.cache.get({"text": text, "target_language": target_language}, do_it)
+        return response["translatedText"]

helm/{proxy/clients → clients}/http_model_client.py

@@ -1,16 +1,16 @@
 import os
 from dataclasses import asdict
+from typing import Any, Dict
 
 from helm.common.cache import CacheConfig
 from helm.common.request import (
     wrap_request_time,
     Request,
     RequestResult,
-    Sequence,
+    GeneratedOutput,
     Token,
     EMBEDDING_UNAVAILABLE_REQUEST_RESULT,
 )
-from helm.proxy.tokenizers.tokenizer import Tokenizer
 from .client import CachingClient
 
 import requests
@@ -21,13 +21,12 @@ class HTTPModelClient(CachingClient):
 
     def __init__(
         self,
-        tokenizer: Tokenizer,
         cache_config: CacheConfig,
         base_url: str = "http://localhost:8080",
         timeout: int = 3000,
         do_cache: bool = False,
     ):
-        super().__init__(cache_config=cache_config, tokenizer=tokenizer)
+        super().__init__(cache_config=cache_config)
         self.base_url = (
             base_url if not os.environ.get("HELM_HTTP_MODEL_BASE_URL") else os.environ["HELM_HTTP_MODEL_BASE_URL"]
         )
@@ -53,7 +52,7 @@ class HTTPModelClient(CachingClient):
 
         try:
 
-            def do_it():
+            def do_it() -> Dict[str, Any]:
                 url = f"{self.base_url}/process"
                 response = requests.post(url, json=raw_request, timeout=self.timeout)
                 response.raise_for_status()
@@ -65,11 +64,8 @@ class HTTPModelClient(CachingClient):
             else:
                 response, cached = do_it(), False
 
-            tokens = [
-                Token(text=token["text"], logprob=token["logprob"], top_logprobs=token["top_logprob"])
-                for token in response["tokens"]
-            ]
-            completions = [Sequence(text=response["text"], logprob=response["logprob"], tokens=tokens)]
+            tokens = [Token(text=token["text"], logprob=token["logprob"]) for token in response["tokens"]]
+            completions = [GeneratedOutput(text=response["text"], logprob=response["logprob"], tokens=tokens)]
 
             return RequestResult(
                 success=True,

helm/{proxy/clients → clients}/huggingface_client.py

@@ -1,11 +1,11 @@
 from copy import deepcopy
 import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoModelForCausalLM
 from transformers.generation.stopping_criteria import (
     StoppingCriteria,
     StoppingCriteriaList,
 )
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, TypedDict
 
 from helm.common.cache import CacheConfig
 from helm.common.hierarchical_logger import htrack_block, hlog
@@ -14,12 +14,11 @@ from helm.common.request import (
     EMBEDDING_UNAVAILABLE_REQUEST_RESULT,
     Request,
     RequestResult,
-    Sequence,
+    GeneratedOutput,
     Token,
 )
 from .client import CachingClient, truncate_sequence
-from helm.proxy.tokenizers.huggingface_tokenizer import HuggingFaceTokenizer, resolve_alias
-from helm.proxy.tokenizers.tokenizer import Tokenizer
+from helm.tokenizers.huggingface_tokenizer import HuggingFaceTokenizer, WrappedPreTrainedTokenizer
 from threading import Lock
 
 
@@ -37,108 +36,127 @@ class StopAtSpecificTokenCriteria(StoppingCriteria):
         return bool(torch.all(current_sequence == stop_sequence_tensor).item())
 
 
+class HuggingFaceRequest(TypedDict):
+    """Data passed between make_request and serve_request. Used as the cache key."""
+
+    engine: str
+    prompt: str
+    temperature: float
+    num_return_sequences: int
+    max_new_tokens: int
+    top_p: float
+    echo_prompt: bool
+    top_k_per_token: int
+    stop_sequences: List
+
+
 class HuggingFaceServer:
     """A thin wrapper around a Hugging Face AutoModelForCausalLM for HuggingFaceClient to call."""
 
-    def __init__(self, pretrained_model_name_or_path: str, revision: Optional[str] = None):
+    def __init__(self, pretrained_model_name_or_path: str, **kwargs):
         if torch.cuda.is_available():
             hlog("CUDA is available, initializing with a GPU...")
             self.device: str = "cuda:0"
         else:
             self.device = "cpu"
-        model_kwargs = {}
-        if revision:
-            model_kwargs["revision"] = revision
         with htrack_block(f"Loading Hugging Face model {pretrained_model_name_or_path}"):
             # WARNING this may fail if your GPU does not have enough memory
             self.model = AutoModelForCausalLM.from_pretrained(
-                pretrained_model_name_or_path, trust_remote_code=True, **model_kwargs
+                pretrained_model_name_or_path, trust_remote_code=True, **kwargs
             ).to(self.device)
         with htrack_block(f"Loading Hugging Face tokenizer for model {pretrained_model_name_or_path}"):
-            self.tokenizer: AutoTokenizer = HuggingFaceTokenizer.create_tokenizer(
-                pretrained_model_name_or_path, revision
+            self.wrapped_tokenizer: WrappedPreTrainedTokenizer = HuggingFaceTokenizer.create_tokenizer(
+                pretrained_model_name_or_path, **kwargs
             )
 
-    def serve_request(self, raw_request: Dict[str, Any]):
-        encoded_input = self.tokenizer(raw_request["prompt"], return_tensors="pt", return_token_type_ids=False).to(
-            self.device
-        )
-        raw_request = deepcopy(raw_request)
-        raw_request["do_sample"] = True
-        raw_request["return_dict_in_generate"] = True
-        raw_request["output_scores"] = True
-        top_k_per_token: int = raw_request["top_k_per_token"]
-        del raw_request["top_k_per_token"]
+    def serve_request(self, raw_request: HuggingFaceRequest) -> Dict:
+        with self.wrapped_tokenizer as tokenizer:
+            encoded_input = tokenizer(raw_request["prompt"], return_tensors="pt", return_token_type_ids=False).to(
+                self.device
+            )
         stopping_criteria: Optional[StoppingCriteriaList] = None
+        optional_args = {}
         if len(raw_request["stop_sequences"]) > 0:
-            stop_sequence_ids = self.tokenizer(
-                raw_request["stop_sequences"], return_token_type_ids=False, add_special_tokens=False
-            )
+            with self.wrapped_tokenizer as tokenizer:
+                stop_sequence_ids = tokenizer(
+                    raw_request["stop_sequences"], return_token_type_ids=False, add_special_tokens=False
+                )
             if len(stop_sequence_ids.input_ids) == 1 and len(stop_sequence_ids.input_ids[0]) == 1:
-                raw_request["eos_token_id"] = stop_sequence_ids.input_ids[0][0]
+                optional_args["eos_token_id"] = stop_sequence_ids.input_ids[0][0]
             else:
                 stopping_criteria = StoppingCriteriaList()
                 for stop_sequence_input_ids in stop_sequence_ids.input_ids:
                     stopping_criteria.append(StopAtSpecificTokenCriteria(stop_sequence=stop_sequence_input_ids))
-            del raw_request["stop_sequences"]
 
-        # Strip out irrelevant parameters
-        relevant_raw_request = {
-            key: raw_request[key]
-            for key in raw_request
-            if key not in ["engine", "prompt", "echo_prompt", "stop_sequences"]
-        }
-
-        # Use HuggingFace's `generate` method.
-        output = self.model.generate(
-            **encoded_input,
-            **relevant_raw_request,
-            stopping_criteria=stopping_criteria,
+        # Check if we need to compute the perplexity of the prompt (#1497)
+        compute_logprobs_only = (
+            raw_request["max_new_tokens"] == 0
+            and raw_request["num_return_sequences"] == 1
+            and raw_request["echo_prompt"]
         )
-        sequences = output.sequences
-        scores = output.scores
 
-        # Compute logprobs for each completed sequence.
-        all_logprobs_of_chosen_tokens = []
-        all_top_logprobs_dicts = []
+        # Use HuggingFace's `generate` method.
+        if compute_logprobs_only:
+            with torch.no_grad():
+                output = self.model(encoded_input["input_ids"])
+            sequences = encoded_input["input_ids"]
+            scores = output.logits
+        else:
+            output = self.model.generate(
+                **encoded_input,
+                temperature=raw_request["temperature"],
+                num_return_sequences=raw_request["num_return_sequences"],
+                max_new_tokens=raw_request["max_new_tokens"],
+                top_p=raw_request["top_p"],
+                do_sample=True,
+                return_dict_in_generate=True,
+                output_scores=True,
+                **optional_args,
+                stopping_criteria=stopping_criteria,
+            )
+            sequences = output.sequences
+            scores = output.scores
+
+        prompt_tokens_logprobs = []
+        if compute_logprobs_only:
+            # Append the logprob of the first token of the prompt.
+            prompt_tokens_logprobs.append(0.0)
+
+            # Compute logprobs of prompt tokens.
+            for completion_id in range(raw_request["num_return_sequences"]):
+                for i in range(len(sequences[completion_id]) - 1):
+                    logprobs = torch.nn.functional.log_softmax(scores[completion_id][i], dim=0)
+                    prompt_tokens_logprobs.append(logprobs[sequences[completion_id][i + 1]].item())
+
+        # Compute logprobs of generated tokens for each completed sequence.
+        all_generated_tokens_logprobs = []
         for completion_id in range(raw_request["num_return_sequences"]):
-            logprobs_of_chosen_tokens = []
-            top_logprobs_dicts = []
+            generated_tokens_logprobs = []
             for i in range(len(sequences[completion_id]) - len(encoded_input.input_ids[0])):
                 logprobs = torch.nn.functional.log_softmax(scores[i][completion_id], dim=0)
-
-                # Get top tokens in terms of log probability.
-                topk_logprobs = torch.topk(logprobs, k=top_k_per_token)
-                top_logprobs_dicts.append(
-                    {
-                        self.tokenizer.convert_ids_to_tokens(k.item()): v.item()
-                        for (k, v) in zip(topk_logprobs.indices, topk_logprobs.values)
-                    }
-                )
-
                 # Get log probability of chosen token.
                 j = i + len(encoded_input.input_ids[0])
-                logprobs_of_chosen_tokens.append(logprobs[sequences[completion_id][j]].item())
-            all_logprobs_of_chosen_tokens.append(logprobs_of_chosen_tokens)
-            all_top_logprobs_dicts.append(top_logprobs_dicts)
+                generated_tokens_logprobs.append(logprobs[sequences[completion_id][j]].item())
+            all_generated_tokens_logprobs.append(generated_tokens_logprobs)
 
         # Remove prompt from the start of each sequence if echo_prompt is False.
         if not raw_request["echo_prompt"]:
             sequences = [sequence[len(encoded_input.input_ids[0]) :] for sequence in sequences]
 
-        all_tokens = [[self.tokenizer.decode(token) for token in sequence_tokens] for sequence_tokens in sequences]
-        all_decoded_text = self.tokenizer.batch_decode(sequences)
+        with self.wrapped_tokenizer as tokenizer:
+            all_tokens = [[tokenizer.decode(token) for token in sequence_tokens] for sequence_tokens in sequences]
+            all_decoded_text = tokenizer.batch_decode(sequences)
 
         completions = []
-        for decoded_text, tokens, logprobs_of_chosen_tokens, top_logprobs_dicts in zip(
-            all_decoded_text, all_tokens, all_logprobs_of_chosen_tokens, all_top_logprobs_dicts
+        for decoded_text, tokens, generated_tokens_logprobs in zip(
+            all_decoded_text, all_tokens, all_generated_tokens_logprobs
         ):
             completions.append(
                 {
                     "text": decoded_text,
                     "tokens": tokens,
-                    "logprobs": logprobs_of_chosen_tokens,
-                    "top_logprobs_dicts": top_logprobs_dicts,
+                    "logprobs": generated_tokens_logprobs,
+                    "prompt_logprobs": prompt_tokens_logprobs,
                 }
             )
 
@@ -152,7 +170,7 @@ class HuggingFaceServerFactory:
     _servers_lock: Lock = Lock()
 
     @staticmethod
-    def get_server(helm_model_name: str, pretrained_model_name_or_path: str, revision: Optional[str] = None) -> Any:
+    def get_server(helm_model_name: str, pretrained_model_name_or_path: str, **kwargs) -> Any:
         """
         Checks if the desired HuggingFaceModel is cached. Creates the HuggingFaceModel if it's not cached.
         Returns the HuggingFaceModel.
@@ -160,34 +178,53 @@ class HuggingFaceServerFactory:
         with HuggingFaceServerFactory._servers_lock:
             if helm_model_name not in HuggingFaceServerFactory._servers:
                 with htrack_block(
-                    f"Loading {pretrained_model_name_or_path} (revision={revision}) "
+                    f"Loading {pretrained_model_name_or_path} (kwargs={kwargs}) "
                     f"for HELM model {helm_model_name} with Hugging Face Transformers"
                 ):
                     HuggingFaceServerFactory._servers[helm_model_name] = HuggingFaceServer(
-                        pretrained_model_name_or_path, revision
+                        pretrained_model_name_or_path, **kwargs
                     )
 
         return HuggingFaceServerFactory._servers[helm_model_name]
 
 
+TORCH_DTYPE_KEY = "torch_dtype"
+TORCH_DTYPE_VALUE_PREFIX = "torch."
+
+
+def _process_huggingface_client_kwargs(raw_kwargs: Dict[str, Any]):
+    """Process the kwargs for HuggingFaceClient.
+
+    The kwargs passed to HuggingFaceClient will eventually be passed to AutoModel.from_pretrained().
+    Since the kwargs from HuggingFaceClient may be derived from configuration YAML,
+    they may contain primitive types instead of the unserializable types that
+    AutoModel.from_pretrained() expects (e.g. torch_dtype). This function converts values of
+    primitive types to values of the unserializable types."""
+    processed_kwargs = deepcopy(raw_kwargs)
+
+    # Convert torch_dtype string value to actual dtypes
+    # e.g. the string "torch.bfloat16" is converted to torch.bfloat16
+    torch_dtype = processed_kwargs.get(TORCH_DTYPE_KEY)
+    if torch_dtype and isinstance(torch_dtype, str):
+        if not torch_dtype.startswith(TORCH_DTYPE_VALUE_PREFIX):
+            raise ValueError(f'Unknown dtype "{torch_dtype}"; expected a string such as "torch.bfloat16"')
+        processed_kwargs[TORCH_DTYPE_KEY] = getattr(torch, torch_dtype[len(TORCH_DTYPE_VALUE_PREFIX) :])
+
+    return processed_kwargs
+
+
 class HuggingFaceClient(CachingClient):
-    def __init__(
-        self,
-        tokenizer: Tokenizer,
-        cache_config: CacheConfig,
-        pretrained_model_name_or_path: Optional[str] = None,
-        revision: Optional[str] = None,
-    ):
-        super().__init__(cache_config=cache_config, tokenizer=tokenizer)
+    def __init__(self, cache_config: CacheConfig, pretrained_model_name_or_path: Optional[str] = None, **kwargs):
+        super().__init__(cache_config=cache_config)
         self._pretrained_model_name_or_path = pretrained_model_name_or_path
-        self._revision = revision
+        self._kwargs = _process_huggingface_client_kwargs(kwargs)
 
     def make_request(self, request: Request) -> RequestResult:
         # Embedding not supported for this model
         if request.embedding:
             return EMBEDDING_UNAVAILABLE_REQUEST_RESULT
 
-        raw_request = {
+        raw_request: HuggingFaceRequest = {
             "engine": request.model_engine,
             "prompt": request.prompt,
             "temperature": 1e-7 if request.temperature == 0 else request.temperature,
@@ -199,20 +236,18 @@ class HuggingFaceClient(CachingClient):
             "stop_sequences": request.stop_sequences,
         }
 
-        pretrained_model_name_or_path: str
-        if self._pretrained_model_name_or_path:
-            pretrained_model_name_or_path = self._pretrained_model_name_or_path
-        else:
-            pretrained_model_name_or_path = resolve_alias(request.model)
+        pretrained_model_name_or_path = (
+            self._pretrained_model_name_or_path if self._pretrained_model_name_or_path else request.model
+        )
         huggingface_model: HuggingFaceServer = HuggingFaceServerFactory.get_server(
             helm_model_name=request.model,
             pretrained_model_name_or_path=pretrained_model_name_or_path,
-            revision=self._revision,
+            **self._kwargs,
         )
 
         try:
 
-            def do_it():
+            def do_it() -> Dict[str, Any]:
                 return huggingface_model.serve_request(raw_request)
 
             cache_key = CachingClient.make_cache_key(raw_request, request)
@@ -229,19 +264,26 @@ class HuggingFaceClient(CachingClient):
             if request.echo_prompt:
                 # Add prompt to list of generated tokens.
                 generated_tokens = raw_completion["tokens"][response["input_length"] :]
-                for token_text in raw_completion["tokens"][: response["input_length"]]:
-                    tokens.append(Token(text=token_text, logprob=0.0, top_logprobs={}))
+                if raw_completion.get("prompt_logprobs"):
+                    for token_text, logprob in zip(
+                        raw_completion["tokens"][: response["input_length"]],
+                        raw_completion["prompt_logprobs"][: response["input_length"]],
+                    ):
+                        tokens.append(Token(text=token_text, logprob=logprob))
+                        sequence_logprob += logprob
+                else:
+                    for token_text in raw_completion["tokens"][: response["input_length"]]:
+                        tokens.append(Token(text=token_text, logprob=0.0))
+
             else:
                 generated_tokens = raw_completion["tokens"]
 
             # Compute logprob for the entire sequence.
-            for token_text, logprob, top_logprobs_dict in zip(
-                generated_tokens, raw_completion["logprobs"], raw_completion["top_logprobs_dicts"]
-            ):
-                tokens.append(Token(text=token_text, logprob=logprob, top_logprobs=top_logprobs_dict))
+            for token_text, logprob in zip(generated_tokens, raw_completion["logprobs"]):
+                tokens.append(Token(text=token_text, logprob=logprob))
                 sequence_logprob += logprob
 
-            completion = Sequence(text=raw_completion["text"], logprob=sequence_logprob, tokens=tokens)
+            completion = GeneratedOutput(text=raw_completion["text"], logprob=sequence_logprob, tokens=tokens)
             completion = truncate_sequence(completion, request)
             completions.append(completion)
 

helm/clients/image_generation/adobe_vision_client.py

@@ -0,0 +1,78 @@
+from typing import List, Dict
+
+from helm.common.cache import Cache, CacheConfig
+from helm.common.request import Request, RequestResult, GeneratedOutput
+from helm.common.tokenization_request import (
+    TokenizationRequest,
+    TokenizationRequestResult,
+    DecodeRequest,
+    DecodeRequestResult,
+)
+from helm.clients.client import Client, CachingClient
+from .image_generation_client_utils import get_single_image_multimedia_object
+
+
+class AdobeVisionClient(Client):
+    """
+    Client for Adobe vision models. Offline eval only.
+    """
+
+    SUPPORTED_MODELS: List[str] = ["giga-gan", "firefly"]
+
+    @staticmethod
+    def convert_to_raw_request(request: Request) -> Dict:
+        # Use default hyperparameters for everything else
+        raw_request: Dict = {
+            "request_type": "image-model-inference",
+            "model": request.model_engine,
+            "prompt": request.prompt,
+            "n": request.num_completions,
+        }
+        if request.random is not None:
+            raw_request["random"] = request.random
+        return raw_request
+
+    def __init__(self, cache_config: CacheConfig):
+        self._cache = Cache(cache_config)
+        self._promptist_model = None
+        self._promptist_tokenizer = None
+
+    def make_request(self, request: Request) -> RequestResult:
+        if request.model_engine not in self.SUPPORTED_MODELS:
+            raise ValueError(f"Unsupported model: {request.model_engine}")
+
+        raw_request = AdobeVisionClient.convert_to_raw_request(request)
+        raw_request.pop("random", None)
+        cache_key = CachingClient.make_cache_key(raw_request, request)
+
+        try:
+
+            def fail():
+                raise RuntimeError(
+                    f"The result has not been uploaded to the cache for the following request: {cache_key}"
+                )
+
+            response, cached = self._cache.get(cache_key, fail)
+        except RuntimeError as e:
+            error: str = f"Adobe Vision Client error: {e}"
+            return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
+
+        completions: List[GeneratedOutput] = [
+            GeneratedOutput(
+                text="", logprob=0, tokens=[], multimodal_content=get_single_image_multimedia_object(file_path)
+            )
+            for file_path in response["images"]
+        ]
+        return RequestResult(
+            success=True,
+            cached=cached,
+            request_time=response["request_time"],
+            completions=completions,
+            embedding=[],
+        )
+
+    def tokenize(self, request: TokenizationRequest) -> TokenizationRequestResult:
+        raise NotImplementedError("This client does not support tokenizing.")
+
+    def decode(self, request: DecodeRequest) -> DecodeRequestResult:
+        raise NotImplementedError("This client does not support decoding.")