crfm-helm 0.5.4__py3-none-any.whl → 0.5.5__py3-none-any.whl

helm/clients/audio_language/qwen_audiolm_client.py

@@ -0,0 +1,150 @@
+from threading import Lock
+from typing import Any, Dict, List, Optional
+
+from dataclasses import dataclass
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from helm.common.cache import CacheConfig
+from helm.common.gpu_utils import get_torch_device_name
+from helm.common.hierarchical_logger import hlog, htrack_block
+from helm.common.media_object import TEXT_TYPE
+from helm.common.request import Request, RequestResult, GeneratedOutput, Token
+from helm.common.request import wrap_request_time
+from helm.clients.client import CachingClient, generate_uid_for_multimodal_prompt
+
+
+@dataclass(frozen=True)
+class LoadedQwenModelProcessor:
+    """Loaded model and processor for Qwen."""
+
+    model: AutoModelForCausalLM
+    tokenizer: AutoTokenizer
+
+
+_models_lock: Lock = Lock()
+_models: Dict[str, Optional[LoadedQwenModelProcessor]] = {
+    "Qwen/Qwen-Audio-Chat": None,
+}
+
+
+class QwenAudioLMClient(CachingClient):
+    """
+    From https://huggingface.co/Qwen/Qwen-Audio,
+    Qwen-Audio-Chat (Qwen Large Audio Language Model) is the audio multimodal version of the large model series,
+    Qwen (abbr. Tongyi Qianwen), proposed by Alibaba Cloud. Qwen-Audio-Chat accepts audio, text as inputs, outputs text.
+    Alibaba released Qwen-Audio and Qwen-Audio-Chat, which is a chatbot model based on Qwen-Audio.
+    We for now integrated Qwen-Audio-Chat for instruction-following tasks.
+
+    Paper: https://arxiv.org/abs/2311.07919
+    """
+
+    def __init__(self, cache_config: CacheConfig):
+        super().__init__(cache_config=cache_config)
+        self._device: str = get_torch_device_name()
+
+    def _get_model(self, helm_model_name: str) -> LoadedQwenModelProcessor:
+        global _models_lock
+        global _models
+
+        model_name: str
+        if helm_model_name == "qwen-audio-chat":
+            model_name = "Qwen/Qwen-Audio-Chat"
+        else:
+            raise ValueError(f"Unhandled model name: {helm_model_name}")
+
+        # Ensure that only one thread is loading the model at a time
+        with _models_lock:
+            loaded_model_processor = _models[model_name]
+            if loaded_model_processor is None:
+                hlog(f"Loading model {model_name} and caching in memory...")
+                model = AutoModelForCausalLM.from_pretrained(
+                    model_name,
+                    device_map=self._device,
+                    trust_remote_code=True,
+                    bf16=True,
+                ).eval()
+                tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+                _models[model_name] = LoadedQwenModelProcessor(model, tokenizer)
+                loaded_model_processor = _models[model_name]
+
+        assert loaded_model_processor is not None
+        return loaded_model_processor
+
+    def make_request(self, request: Request) -> RequestResult:
+        assert request.multimodal_prompt is not None, "Multimodal prompt is required"
+
+        loaded_model_processor: LoadedQwenModelProcessor = self._get_model(request.model_engine)
+        model = loaded_model_processor.model
+        tokenizer = loaded_model_processor.tokenizer
+
+        generation_args = {
+            "max_length": request.max_tokens,
+        }
+
+        query: List[Dict[str, str]] = []
+        prompt_text: str = ""
+
+        for media_object in request.multimodal_prompt.media_objects:
+            if media_object.is_type("audio") and media_object.location:
+                query.append({"audio": media_object.location})
+            elif media_object.is_type(TEXT_TYPE):
+                if media_object.text is None:
+                    raise ValueError("MediaObject of text type has missing text field value")
+
+                query.append({"text": media_object.text})
+                prompt_text += media_object.text
+            else:
+                raise ValueError(f"Unrecognized MediaObject type {media_object.type}")
+
+        completions: List[GeneratedOutput] = []
+        request_time: float = 0
+        request_datetime: Optional[int] = None
+        all_cached: bool = True
+
+        with htrack_block(f"Generating for prompt: {prompt_text}"):
+            for completion_index in range(request.num_completions):
+                try:
+
+                    def do_it() -> Dict[str, Any]:
+                        completion, _ = model.chat(tokenizer, query=tokenizer.from_list_format(query), history=None)
+                        tokens: List[str] = tokenizer.tokenize(completion)
+                        return {"output": (completion, tokens)}
+
+                    # Include the prompt and model name in the cache key
+                    cache_key = CachingClient.make_cache_key(
+                        raw_request={
+                            "completion_index": completion_index,
+                            "model": request.model,
+                            "prompt": generate_uid_for_multimodal_prompt(request.multimodal_prompt),
+                            **generation_args,
+                        },
+                        request=request,
+                    )
+                    result, cached = self.cache.get(cache_key, wrap_request_time(do_it))
+                except RuntimeError as model_error:
+                    return RequestResult(
+                        success=False, cached=False, error=str(model_error), completions=[], embedding=[]
+                    )
+
+                text, tokens = result["output"]
+
+                # Tokenize truncated text to get the list of tokens
+                completions.append(
+                    GeneratedOutput(
+                        text=text, logprob=0, tokens=[Token(text=str(token), logprob=0) for token in tokens]
+                    )
+                )
+
+                request_time += result["request_time"]
+                # Use the datetime from the first completion because that's when the request was fired
+                request_datetime = request_datetime or result.get("request_datetime")
+                all_cached = all_cached and cached
+
+        return RequestResult(
+            success=True,
+            cached=all_cached,
+            request_time=request_time,
+            request_datetime=request_datetime,
+            completions=completions,
+            embedding=[],
+        )

helm/clients/auto_client.py

@@ -71,6 +71,7 @@ class AutoClient(Client):
                 model_deployment.client_spec,
                 constant_bindings={
                     "cache_config": cache_config,
+                    "model_name": model_deployment.model_name,
                     "tokenizer_name": model_deployment.tokenizer_name,
                 },
                 provider_bindings={
@@ -81,6 +82,7 @@ class AutoClient(Client):
                     "org_id": lambda: self.credentials.get(
                         host_organization + "OrgId", None
                     ),  # OpenAI, GooseAI, Microsoft
+                    "base_url": lambda: self.credentials.get(host_organization + "BaseUrl", None),
                     "moderation_api_client": lambda: self.get_moderation_api_client(),  # OpenAI DALL-E
                     "lock_file_path": lambda: os.path.join(
                         self.file_storage_path, f"{host_organization}.lock"
@@ -131,7 +133,7 @@ class AutoClient(Client):
             return replace(last_attempt.value, error=f"{retry_error}. Error: {last_attempt.value.error}")
 
     def get_gcs_client(self):
-        from .gcs_client import GCSClient
+        from helm.clients.gcs_client import GCSClient
 
         bucket_name: str = self.credentials["gcsBucketName"]
         cache_config: CacheConfig = self.cache_backend_config.get_cache_config("gcs")
@@ -144,7 +146,7 @@ class AutoClient(Client):
         return NudityCheckClient(cache_config)
 
     def get_clip_score_client(self):
-        from .clip_score_client import CLIPScoreClient
+        from helm.clients.clip_score_client import CLIPScoreClient
 
         cache_config: CacheConfig = self.cache_backend_config.get_cache_config("clip_score")
         return CLIPScoreClient(cache_config)

helm/clients/azure_openai_client.py

@@ -0,0 +1,55 @@
+import os
+from typing import Dict, Optional
+
+from helm.clients.openai_client import OpenAIClient
+from helm.common.cache import CacheConfig
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.proxy.retry import NonRetriableException
+from helm.tokenizers.tokenizer import Tokenizer
+
+try:
+    from openai import AzureOpenAI
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["openai"])
+
+
+class AzureOpenAIClient(OpenAIClient):
+    API_VERSION = "2024-07-01-preview"
+
+    def __init__(
+        self,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        cache_config: CacheConfig,
+        api_key: Optional[str] = None,
+        endpoint: Optional[str] = None,
+        api_version: Optional[str] = None,
+        default_headers: Optional[Dict[str, str]] = None,
+        base_url: Optional[str] = None,
+        azure_openai_deployment_name: Optional[str] = None,
+    ):
+        super().__init__(
+            tokenizer=tokenizer,
+            tokenizer_name=tokenizer_name,
+            cache_config=cache_config,
+            api_key="unused",
+            openai_model_name=azure_openai_deployment_name,
+        )
+        azure_endpoint = endpoint or os.getenv("AZURE_OPENAI_ENDPOINT")
+        assert (base_url is None) != (azure_endpoint is None), "Exactly one of endoint or base_url must be provided"
+        if base_url:
+            self.client = AzureOpenAI(
+                api_key=api_key,
+                api_version=api_version or AzureOpenAIClient.API_VERSION,
+                base_url=base_url,
+                default_headers=default_headers,
+            )
+        elif azure_endpoint:
+            self.client = AzureOpenAI(
+                api_key=api_key,
+                api_version=api_version or AzureOpenAIClient.API_VERSION,
+                azure_endpoint=azure_endpoint,
+                default_headers=default_headers,
+            )
+        else:
+            raise NonRetriableException("Must provide either endpoint or base_url")

helm/clients/bedrock_client.py

@@ -2,12 +2,13 @@ from abc import abstractmethod
 from copy import deepcopy
 import json
 import os
-from typing import Any, Dict, List, Mapping, Optional
+from typing import Any, Dict, List, Mapping, Optional, TypedDict
+from datetime import datetime
 
 from helm.common.cache import CacheConfig
 from helm.clients.client import CachingClient, truncate_and_tokenize_response_text
 from helm.common.request import Request, RequestResult, GeneratedOutput, wrap_request_time
-from helm.clients.bedrock_utils import get_bedrock_client
+from helm.clients.bedrock_utils import get_bedrock_client, get_bedrock_client_v1
 from helm.tokenizers.tokenizer import Tokenizer
 
 
@@ -23,27 +24,41 @@ class BedrockClient(CachingClient):
     def convert_raw_response_to_completions(self, response: Dict, request: Request) -> List[GeneratedOutput]:
         raise NotImplementedError()
 
+    """
+    Amazon Bedrock is a fully managed service that provides s selection of leading foundation models (FMs) from Amazon
+    and other partner model providers.
+    """
+
+    @property
+    @abstractmethod
+    def model_provider(self) -> str:
+        raise NotImplementedError()
+
     def __init__(
         self,
         cache_config: CacheConfig,
         tokenizer: Tokenizer,
         tokenizer_name: str,
-        bedrock_model_id: Optional[str] = None,
         assumed_role: Optional[str] = None,
         region: Optional[str] = None,
     ):
         super().__init__(cache_config=cache_config)
         self.tokenizer = tokenizer
         self.tokenizer_name = tokenizer_name
-        self.bedrock_model_id = bedrock_model_id
         self.bedrock_client = get_bedrock_client(
             assumed_role=assumed_role or os.environ.get("BEDROCK_ASSUME_ROLE", None),
-            region=region or os.environ.get("AWS_DEFAULT_REGION", None),
+            region=region,
         )
 
     def make_request(self, request: Request) -> RequestResult:
-        # model_id should be something like "amazon.titan-tg1-large"
-        model_id = self.bedrock_model_id if self.bedrock_model_id else request.model.replace("/", ".")
+        # model_id should be something like "amazon.titan-tg1-large", replace amazon- prefix with model creator name
+        model_name = request.model.split("/")[-1]
+        # check if model_name starts with "amazon-"
+        if self.model_provider == "amazon":
+            model_id = f"{self.model_provider}.{model_name}"
+        else:
+            model_id = model_name.replace("amazon-", f"{self.model_provider}.")
+
         raw_request = self.convert_request_to_raw_request(request)
 
         # modelId isn't part of raw_request, so it must be explicitly passed into the input to
@@ -58,6 +73,7 @@ class BedrockClient(CachingClient):
 
         try:
             response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
+
         except Exception as error:
             return RequestResult(
                 success=False,
@@ -79,12 +95,109 @@ class BedrockClient(CachingClient):
         )
 
 
+class _ContentBlock(TypedDict):
+    text: str
+
+
+class _Message(TypedDict):
+    role: str
+    content: List[_ContentBlock]
+
+
+class BedrockNovaClient(CachingClient):
+    """
+    Amazon Bedrock is a fully managed service that provides s selection of leading foundation models (FMs) from Amazon
+    and other partner model providers.
+    """
+
+    def __init__(
+        self,
+        cache_config: CacheConfig,
+        tokenizer: Tokenizer,
+        tokenizer_name: str,
+        assumed_role: Optional[str] = None,
+        region: Optional[str] = None,
+    ):
+        super().__init__(cache_config=cache_config)
+        self.tokenizer = tokenizer
+        self.tokenizer_name = tokenizer_name
+        self.bedrock_client = get_bedrock_client_v1(
+            assumed_role=assumed_role or os.environ.get("BEDROCK_ASSUME_ROLE", None),
+            region=region,
+        )
+
+    def _get_messages_from_request(self, request: Request) -> List[_Message]:
+        if request.prompt and request.messages:
+            raise ValueError(f"Only one of `prompt` and `messages` may be set in request: {request}")
+        if request.multimodal_prompt:
+            raise ValueError(f"`multimodal_prompt` is not supported in request: {request}")
+
+        if request.messages:
+            return [
+                {"role": message["role"], "content": [{"text": message["content"]}]} for message in request.messages
+            ]
+        else:
+            return [{"role": "user", "content": [{"text": request.prompt}]}]
+
+    def convert_request_to_raw_request(self, request: Request) -> Dict:
+        model_id = request.model.replace("/", ".")
+        messages = self._get_messages_from_request(request)
+
+        return {
+            "modelId": model_id,
+            "inferenceConfig": {
+                "temperature": request.temperature,
+                "maxTokens": request.max_tokens,
+                "topP": request.top_p,
+            },
+            "messages": messages,
+        }
+
+    def make_request(self, request: Request) -> RequestResult:
+        raw_request = self.convert_request_to_raw_request(request)
+        cache_key = CachingClient.make_cache_key(raw_request, request)
+
+        def do_it() -> Dict[Any, Any]:
+            return self.bedrock_client.converse(**raw_request)
+
+        response, cached = self.cache.get(cache_key, do_it)
+
+        completions = self.convert_raw_response_to_completions(response, request)
+        dt = datetime.strptime(response["ResponseMetadata"]["HTTPHeaders"]["date"], "%a, %d %b %Y %H:%M:%S GMT")
+        # Use API reported latency rather than client measured latency
+        request_time = response["metrics"]["latencyMs"] / 1000
+
+        return RequestResult(
+            success=True,
+            cached=cached,
+            request_time=request_time,
+            request_datetime=int(dt.timestamp()),
+            completions=completions,
+            embedding=[],
+        )
+
+    def convert_raw_response_to_completions(self, response: Dict, request: Request) -> List[GeneratedOutput]:
+        completions: List[GeneratedOutput] = []
+        raw_completion = response["output"]
+        output_text = raw_completion["message"]["content"][0]["text"]
+        finish_reason = response["stopReason"]
+        completion = truncate_and_tokenize_response_text(
+            output_text.lstrip(), request, self.tokenizer, self.tokenizer_name, finish_reason
+        )
+        completions.append(completion)
+        return completions
+
+
+# Amazon Bedrock Client for Titan Models
 class BedrockTitanClient(BedrockClient):
     _COMPLETION_REASON_TO_FINISH_REASON = {
         "LENGTH": "length",
         "FINISH": "endoftext",
     }
 
+    # creator org for titan
+    model_provider = "amazon"
+
     def convert_request_to_raw_request(self, request: Request) -> Dict:
         # TODO: Support the following:
         # - top_k_per_token
@@ -115,6 +228,7 @@ class BedrockTitanClient(BedrockClient):
         # - tokens
         # - logprob
         completions: List[GeneratedOutput] = []
+
         for raw_completion in response["results"]:
             output_text = raw_completion["outputText"]
             # Call lstrip() Titan has the tendency to emit "\n" as the first token in the generated text output.
@@ -126,3 +240,83 @@ class BedrockTitanClient(BedrockClient):
             )
             completions.append(completion)
         return completions
+
+
+# Amazon Bedrock Client for Mistral Models
+class BedrockMistralClient(BedrockClient):
+    _COMPLETION_REASON_TO_FINISH_REASON = {
+        "length": "length",
+        "stop": "endoftext",
+    }
+
+    model_provider = "mistral"
+
+    def convert_request_to_raw_request(self, request: Request) -> Dict:
+        # TODO: Support the following:
+        # - top_k_per_token
+        # - echo_prompt
+        # - num_completions
+        return {
+            "prompt": f"[INST]{request.prompt}[/INST]",
+            "temperature": request.temperature,
+            "top_p": request.top_p,
+            "max_tokens": request.max_tokens,
+        }
+
+    def convert_raw_response_to_completions(self, response: Dict, request: Request) -> List[GeneratedOutput]:
+        # - logprob
+        completions: List[GeneratedOutput] = []
+
+        for raw_completion in response["outputs"]:
+            output_text = raw_completion["text"]
+
+            finish_reason = BedrockMistralClient._COMPLETION_REASON_TO_FINISH_REASON.get(
+                raw_completion["stop_reason"], raw_completion["stop_reason"].lower()
+            )
+            # Work around generated outputs with leading whitespace due to issue #2467
+            # TODO(#2467): Remove workaround
+            completion = truncate_and_tokenize_response_text(
+                output_text.lstrip(), request, self.tokenizer, self.tokenizer_name, finish_reason
+            )
+            completions.append(completion)
+
+        return completions
+
+
+# Amazon Bedrock Client for LLAMA Models
+class BedrockLlamaClient(BedrockClient):
+    _COMPLETION_REASON_TO_FINISH_REASON = {
+        "length": "length",
+        "stop": "endoftext",
+    }
+
+    model_provider = "meta"
+
+    def convert_request_to_raw_request(self, request: Request) -> Dict:
+        # TODO: Support the following:
+        # - top_k_per_token
+        # - echo_prompt
+        # - num_completions
+        return {
+            "prompt": f"[INST]{request.prompt}[/INST]",
+            "temperature": request.temperature,
+            "top_p": request.top_p,
+            "max_gen_len": request.max_tokens,
+        }
+
+    def convert_raw_response_to_completions(self, response: Dict, request: Request) -> List[GeneratedOutput]:
+        # - logprob
+        completions: List[GeneratedOutput] = []
+        output_text = response["generation"]
+
+        finish_reason = BedrockLlamaClient._COMPLETION_REASON_TO_FINISH_REASON.get(
+            response["stop_reason"], response["stop_reason"].lower()
+        )
+        # Work around generated outputs with leading whitespace due to issue #2467
+        # TODO(#2467): Remove workaround
+        completion = truncate_and_tokenize_response_text(
+            output_text.lstrip(), request, self.tokenizer, self.tokenizer_name, finish_reason
+        )
+        completions.append(completion)
+
+        return completions

helm/clients/bedrock_utils.py

@@ -8,6 +8,7 @@ from helm.common.optional_dependencies import handle_module_not_found_error
 
 try:
     import boto3
+    from boto3 import Session
     from botocore.config import Config
 except ModuleNotFoundError as e:
     handle_module_not_found_error(e, ["aws"])
@@ -70,3 +71,35 @@ def get_bedrock_client(
 
     hlog(f"Amazon Bedrock client successfully created with endpoint {bedrock_client._endpoint}")
     return bedrock_client
+
+
+def get_bedrock_client_v1(
+    region: Optional[str] = None,
+    service_name: str = "bedrock-runtime",
+    assumed_role: Optional[str] = None,
+    read_timeout: int = 5000,
+    connect_timeout: int = 5000,
+    max_attempts: int = 10,
+):
+    boto_config = Config(
+        read_timeout=read_timeout, connect_timeout=connect_timeout, retries={"max_attempts": max_attempts}
+    )
+
+    if assumed_role:
+        session = boto3.Session(region_name=region)
+        # Assume role and get credentials
+        sts = session.client("sts")
+        creds = sts.assume_role(RoleArn=str(assumed_role), RoleSessionName="crfm-helm")["Credentials"]
+        session = Session(
+            aws_access_key_id=creds["AccessKeyId"],
+            aws_secret_access_key=creds["SecretAccessKey"],
+            aws_session_token=creds["SessionToken"],
+        )
+        return session.client(
+            service_name=service_name,
+            region_name=region,
+            config=boto_config,
+        )
+
+    # default to instance role to get the aws credentials or aws configured credentials
+    return boto3.client(service_name=service_name, region_name=region, config=boto_config)

helm/clients/clip_scorers/clip_scorer.py

@@ -6,7 +6,7 @@ import torch
 from helm.common.gpu_utils import get_torch_device
 from helm.common.images_utils import open_image
 from helm.common.optional_dependencies import handle_module_not_found_error
-from .base_clip_scorer import BaseCLIPScorer
+from helm.clients.clip_scorers.base_clip_scorer import BaseCLIPScorer
 
 
 _ = torch.manual_seed(42)

helm/clients/clip_scorers/multilingual_clip_scorer.py

@@ -4,7 +4,7 @@ import transformers
 from helm.common.gpu_utils import get_torch_device, get_torch_device_name
 from helm.common.images_utils import open_image
 from helm.common.optional_dependencies import handle_module_not_found_error
-from .base_clip_scorer import BaseCLIPScorer
+from helm.clients.clip_scorers.base_clip_scorer import BaseCLIPScorer
 
 _ = torch.manual_seed(42)
 

helm/clients/cohere_client.py

@@ -164,12 +164,12 @@ class CohereRawChatRequest(TypedDict):
     message: str
     model: Optional[str]
     preamble: Optional[str]
-    chat_history: Optional[Sequence[cohere.ChatMessage]]
+    chat_history: Optional[Sequence[cohere.ChatbotMessage]]
     temperature: Optional[float]
     max_tokens: Optional[int]
     k: Optional[int]
     p: Optional[float]
-    seed: Optional[float]
+    seed: Optional[int]
     stop_sequences: Optional[Sequence[str]]
     frequency_penalty: Optional[float]
     presence_penalty: Optional[float]
@@ -188,7 +188,7 @@ def convert_to_raw_chat_request(request: Request) -> CohereRawChatRequest:
         "k": request.top_k_per_token,
         "p": request.top_p,
         "stop_sequences": request.stop_sequences,
-        "seed": float(request.random) if request.random is not None else None,
+        "seed": int(request.random) if request.random is not None else None,
         "frequency_penalty": request.frequency_penalty,
         "presence_penalty": request.presence_penalty,
     }

helm/clients/google_client.py

@@ -2,7 +2,7 @@ from typing import List, Dict
 
 from helm.common.cache import CacheConfig
 from helm.common.request import Request, RequestResult, GeneratedOutput, Token
-from .client import CachingClient, truncate_sequence
+from helm.clients.client import CachingClient, truncate_sequence
 
 
 class GoogleClient(CachingClient):

helm/clients/http_model_client.py

@@ -11,7 +11,7 @@ from helm.common.request import (
     Token,
     EMBEDDING_UNAVAILABLE_REQUEST_RESULT,
 )
-from .client import CachingClient
+from helm.clients.client import CachingClient
 
 import requests
 

helm/clients/huggingface_client.py

@@ -19,7 +19,7 @@ from helm.common.request import (
     Token,
 )
 from helm.tokenizers.tokenizer import Tokenizer
-from .client import CachingClient, truncate_sequence
+from helm.clients.client import CachingClient, truncate_sequence
 from helm.tokenizers.huggingface_tokenizer import HuggingFaceTokenizer, WrappedPreTrainedTokenizer
 from threading import Lock
 
@@ -59,17 +59,23 @@ class HuggingFaceServer:
         self,
         pretrained_model_name_or_path: str,
         wrapped_tokenizer: WrappedPreTrainedTokenizer,
-        openvino: bool = False,
         **kwargs,
     ):
         self.device: Optional[str]
         if "device_map" in kwargs:
+            if "device" in kwargs:
+                raise ValueError("At most one of one of `device` and `device_map` may be specified.")
             try:
                 import accelerate  # noqa: F401
             except ModuleNotFoundError as e:
                 handle_module_not_found_error(e, ["accelerate"])
-            hlog(f'Hugging Face device_map set to "{kwargs["device_map"]}".')
+            hlog(f'Hugging Face device_map set to "{kwargs["device_map"]}" from kwargs.')
             self.device = None
+        elif "device" in kwargs:
+            if "device_map" in kwargs:
+                raise ValueError("At most one of one of `device` and `device_map` may be specified.")
+            hlog(f'Hugging Face device set to "{kwargs["device"]}" from kwargs.')
+            self.device = kwargs.pop("device")
         elif torch.cuda.is_available():
             hlog('Hugging Face device set to "cuda:0" because CUDA is available.')
             self.device = "cuda:0"
@@ -85,20 +91,7 @@ class HuggingFaceServer:
 
         with htrack_block(f"Loading Hugging Face model {pretrained_model_name_or_path}"):
             # WARNING this may fail if your GPU does not have enough memory
-            if openvino:
-                # Optimum Intel provides a simple interface to optimize Transformer models and convert them to \
-                # OpenVINO™ Intermediate Representation (IR) format to accelerate end-to-end pipelines on \
-                # Intel® architectures using OpenVINO™ runtime.
-                try:
-                    from optimum.intel.openvino import OVModelForCausalLM
-                except ModuleNotFoundError as e:
-                    handle_module_not_found_error(e, ["openvino"])
-
-                self.device = "cpu"
-                self.model = OVModelForCausalLM.from_pretrained(
-                    pretrained_model_name_or_path, export=True, **kwargs
-                ).to(self.device)
-            elif self.device is None:
+            if self.device is None:
                 # kwargs contains device_map=auto
                 # Do not call to() because accelerate will take care of model device placement.
                 self.model = AutoModelForCausalLM.from_pretrained(pretrained_model_name_or_path, **kwargs)
@@ -113,7 +106,6 @@ class HuggingFaceServer:
             encoded_input = tokenizer(raw_request["prompt"], return_tensors="pt", return_token_type_ids=False).to(
                 0 if self.device is None else self.device
             )
-
         stopping_criteria: Optional[StoppingCriteriaList] = None
         optional_args = {}
         if len(raw_request["stop_sequences"]) > 0: