crfm-helm 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

helm/benchmark/window_services/window_service_factory.py

@@ -1,41 +1,20 @@
-from helm.benchmark.model_deployment_registry import WindowServiceSpec, get_model_deployment
-from helm.proxy.models import (
-    get_model,
-    get_model_names_with_tag,
-    Model,
-    AI21_WIDER_CONTEXT_WINDOW_TAG,
-    AI21_JURASSIC_2_JUMBO_CONTEXT_WINDOW_TAG,
-    WIDER_CONTEXT_WINDOW_TAG,
-    GPT_TURBO_CONTEXT_WINDOW_TAG,
-    GPT_TURBO_16K_CONTEXT_WINDOW_TAG,
-    GPT4_CONTEXT_WINDOW_TAG,
-    GPT4_32K_CONTEXT_WINDOW_TAG,
-)
+from typing import Optional
 
-from helm.benchmark.window_services.huggingface_window_service import HuggingFaceWindowService
-from helm.benchmark.window_services.gpt2_window_service import GPT2WindowService
-from helm.benchmark.window_services.remote_window_service import get_remote_window_service
+from helm.benchmark.model_deployment_registry import ModelDeployment, WindowServiceSpec, get_model_deployment
+from helm.benchmark.tokenizer_config_registry import TokenizerConfig, get_tokenizer_config
 from helm.benchmark.window_services.window_service import WindowService
 from helm.benchmark.window_services.tokenizer_service import TokenizerService
-from helm.proxy.clients.remote_model_registry import get_remote_model
 from helm.common.object_spec import create_object, inject_object_spec_args
 
 
 class WindowServiceFactory:
     @staticmethod
-    def get_window_service(model_name: str, service: TokenizerService) -> WindowService:
+    def get_window_service(model_deployment_name: str, service: TokenizerService) -> WindowService:
         """
         Returns a `WindowService` given the name of the model.
         Make sure this function returns instantaneously on repeated calls.
         """
-        model: Model = get_model(model_name)
-        organization: str = model.organization
-        engine: str = model.engine
-
-        window_service: WindowService
-
-        # TODO: Migrate all window services to use use model deployments
-        model_deployment = get_model_deployment(model_name)
+        model_deployment: Optional[ModelDeployment] = get_model_deployment(model_deployment_name)
         if model_deployment:
             # If the model deployment specifies a WindowServiceSpec, instantiate it.
             window_service_spec: WindowServiceSpec
@@ -45,6 +24,16 @@ class WindowServiceFactory:
                 window_service_spec = WindowServiceSpec(
                     class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService", args={}
                 )
+
+            # If provided, look up special tokens from TokenizerConfig.
+            end_of_text_token: Optional[str] = None
+            prefix_token: Optional[str] = None
+            if model_deployment.tokenizer_name:
+                tokenizer_config: Optional[TokenizerConfig] = get_tokenizer_config(model_deployment.tokenizer_name)
+                if tokenizer_config:
+                    end_of_text_token = tokenizer_config.end_of_text_token
+                    prefix_token = tokenizer_config.prefix_token
+
             # Perform dependency injection to fill in remaining arguments.
             # Dependency injection is needed here for these reasons:
             #
@@ -54,253 +43,19 @@ class WindowServiceFactory:
             #    in the users configuration file. Instead, they have to be constructed dynamically at runtime.
             window_service_spec = inject_object_spec_args(
                 window_service_spec,
-                {
+                constant_bindings={
                     "service": service,
                     "tokenizer_name": model_deployment.tokenizer_name,
                     "max_sequence_length": model_deployment.max_sequence_length,
                     "max_request_length": model_deployment.max_request_length,
+                    "max_sequence_and_generated_tokens_length": model_deployment.max_sequence_and_generated_tokens_length,  # noqa
+                    "end_of_text_token": end_of_text_token,
+                    "prefix_token": prefix_token,
+                },
+                provider_bindings={
+                    "gpt2_window_service": lambda: WindowServiceFactory.get_window_service("huggingface/gpt2", service)
                 },
             )
-            window_service = create_object(window_service_spec)
-        elif get_remote_model(model_name):
-            window_service = get_remote_window_service(service, model_name)
-        elif organization == "neurips":
-            from helm.benchmark.window_services.http_model_window_service import HTTPModelWindowServce
-
-            window_service = HTTPModelWindowServce(service)
-        elif organization == "openai":
-            from helm.benchmark.window_services.openai_window_service import OpenAIWindowService
-            from helm.benchmark.window_services.wider_openai_window_service import (
-                WiderOpenAIWindowService,
-                GPTTurboWindowService,
-                GPTTurbo16KWindowService,
-                GPT4WindowService,
-                GPT432KWindowService,
-            )
-
-            if model_name in get_model_names_with_tag(GPT4_CONTEXT_WINDOW_TAG):
-                window_service = GPT4WindowService(service)
-            elif model_name in get_model_names_with_tag(GPT4_32K_CONTEXT_WINDOW_TAG):
-                window_service = GPT432KWindowService(service)
-            if model_name in get_model_names_with_tag(GPT_TURBO_CONTEXT_WINDOW_TAG):
-                window_service = GPTTurboWindowService(service)
-            elif model_name in get_model_names_with_tag(GPT_TURBO_16K_CONTEXT_WINDOW_TAG):
-                window_service = GPTTurbo16KWindowService(service)
-            elif model_name in get_model_names_with_tag(WIDER_CONTEXT_WINDOW_TAG):
-                window_service = WiderOpenAIWindowService(service)
-            else:
-                window_service = OpenAIWindowService(service)
-        # For the Google models, we approximate with the OpenAIWindowService
-        elif organization == "simple" or organization == "google":
-            from helm.benchmark.window_services.openai_window_service import OpenAIWindowService
-
-            window_service = OpenAIWindowService(service)
-        elif organization == "AlephAlpha":
-            from helm.benchmark.window_services.luminous_window_service import (
-                LuminousBaseWindowService,
-                LuminousExtendedWindowService,
-                LuminousSupremeWindowService,
-                LuminousWorldWindowService,
-            )
-
-            if engine == "luminous-base":
-                window_service = LuminousBaseWindowService(service)
-            elif engine == "luminous-extended":
-                window_service = LuminousExtendedWindowService(service)
-            elif engine == "luminous-supreme":
-                window_service = LuminousSupremeWindowService(service)
-            elif engine == "luminous-world":
-                window_service = LuminousWorldWindowService(service)
-            else:
-                raise ValueError(f"Unhandled Aleph Alpha model: {engine}")
-        elif organization == "microsoft":
-            from helm.benchmark.window_services.mt_nlg_window_service import MTNLGWindowService
-
-            window_service = MTNLGWindowService(service)
-        elif organization == "anthropic":
-            from helm.benchmark.window_services.anthropic_window_service import (
-                AnthropicWindowService,
-                LegacyAnthropicWindowService,
-            )
-
-            if engine == "stanford-online-all-v4-s3":
-                window_service = LegacyAnthropicWindowService(service)
-            else:
-                window_service = AnthropicWindowService(service)
-        elif organization == "writer":
-            from helm.benchmark.window_services.palmyra_window_service import (
-                PalmyraWindowService,
-                LongerPalmyraWindowService,
-            )
-
-            if engine in ["palmyra-base", "palmyra-large", "palmyra-instruct-30", "palmyra-e"]:
-                window_service = PalmyraWindowService(service)
-            elif engine in ["palmyra-x", "silk-road"]:
-                window_service = LongerPalmyraWindowService(service)
-            else:
-                raise ValueError(f"Unhandled Writer model: {engine}")
-        elif engine == "santacoder":
-            from helm.benchmark.window_services.santacoder_window_service import SantaCoderWindowService
-
-            window_service = SantaCoderWindowService(service)
-        elif engine == "starcoder":
-            from helm.benchmark.window_services.starcoder_window_service import StarCoderWindowService
-
-            window_service = StarCoderWindowService(service)
-        elif model_name == "huggingface/gpt2":
-            window_service = GPT2WindowService(service)
-        elif model_name == "together/bloom":
-            from helm.benchmark.window_services.bloom_window_service import BloomWindowService
-
-            window_service = BloomWindowService(service)
-        elif model_name == "together/glm":
-            # From https://github.com/THUDM/GLM-130B, "the tokenizer is implemented based on
-            # icetk---a unified multimodal tokenizer for images, Chinese, and English."
-            from helm.benchmark.window_services.ice_window_service import ICEWindowService
-
-            window_service = ICEWindowService(service)
-        elif model_name in ["huggingface/gpt-j-6b", "together/gpt-j-6b", "together/gpt-jt-6b-v1", "gooseai/gpt-j-6b"]:
-            from helm.benchmark.window_services.gptj_window_service import GPTJWindowService
-
-            window_service = GPTJWindowService(service)
-        elif model_name in [
-            "together/gpt-neox-20b",
-            "gooseai/gpt-neo-20b",
-            "together/gpt-neoxt-chat-base-20b",
-            "together/redpajama-incite-base-3b-v1",
-            "together/redpajama-incite-instruct-3b-v1",
-            "together/redpajama-incite-base-7b",
-            "together/redpajama-incite-instruct-7b",
-            # Pythia uses the same tokenizer as GPT-NeoX-20B.
-            # See: https://huggingface.co/EleutherAI/pythia-6.9b#training-procedure
-            "eleutherai/pythia-1b-v0",
-            "eleutherai/pythia-2.8b-v0",
-            "eleutherai/pythia-6.9b",
-            "eleutherai/pythia-12b-v0",
-            # MPT-7B model was trained with the EleutherAI/gpt-neox-20b tokenizer
-            # See: https://huggingface.co/mosaicml/mpt-7b
-            "mosaicml/mpt-7b",
-            "mosaicml/mpt-instruct-7b",
-            "mosaicml/mpt-30b",
-            "mosaicml/mpt-instruct-30b",
-            # Dolly models are based on Pythia.
-            # See: https://github.com/databrickslabs/dolly
-            "databricks/dolly-v2-3b",
-            "databricks/dolly-v2-7b",
-            "databricks/dolly-v2-12b",
-        ]:
-            from helm.benchmark.window_services.gptneox_window_service import GPTNeoXWindowService
-
-            window_service = GPTNeoXWindowService(service)
-        elif model_name in [
-            "tiiuae/falcon-7b",
-            "tiiuae/falcon-7b-instruct",
-            "tiiuae/falcon-40b",
-            "tiiuae/falcon-40b-instruct",
-        ]:
-            window_service = HuggingFaceWindowService(service=service, tokenizer_name="tiiuae/falcon-7b")
-        elif model_name in [
-            "stabilityai/stablelm-base-alpha-3b",
-            "stabilityai/stablelm-base-alpha-7b",
-        ]:
-            from helm.benchmark.window_services.gptneox_window_service import StableLMAlphaWindowService
-
-            window_service = StableLMAlphaWindowService(service)
-        elif model_name == "together/h3-2.7b":
-            window_service = GPT2WindowService(service)
-        elif model_name in [
-            "together/opt-1.3b",
-            "together/opt-6.7b",
-            "together/opt-66b",
-            "together/opt-175b",
-        ]:
-            from helm.benchmark.window_services.opt_window_service import OPTWindowService
-
-            window_service = OPTWindowService(service)
-        elif model_name == "together/t0pp":
-            from helm.benchmark.window_services.t0pp_window_service import T0ppWindowService
-
-            window_service = T0ppWindowService(service)
-        elif model_name == "together/t5-11b":
-            from helm.benchmark.window_services.t511b_window_service import T511bWindowService
-
-            window_service = T511bWindowService(service)
-        elif model_name == "together/flan-t5-xxl":
-            from helm.benchmark.window_services.flan_t5_window_service import FlanT5WindowService
-
-            window_service = FlanT5WindowService(service)
-        elif model_name == "together/ul2":
-            from helm.benchmark.window_services.ul2_window_service import UL2WindowService
-
-            window_service = UL2WindowService(service)
-        elif model_name == "together/yalm":
-            from helm.benchmark.window_services.yalm_window_service import YaLMWindowService
-
-            window_service = YaLMWindowService(service)
-        elif model_name == "nvidia/megatron-gpt2":
-            from helm.benchmark.window_services.megatron_window_service import MegatronWindowService
-
-            window_service = MegatronWindowService(service)
-        elif model_name in [
-            "lmsys/vicuna-7b-v1.3",
-            "lmsys/vicuna-13b-v1.3",
-            "meta/llama-7b",
-            "meta/llama-13b",
-            "meta/llama-30b",
-            "meta/llama-65b",
-            "stanford/alpaca-7b",
-        ]:
-            from helm.benchmark.window_services.llama_window_service import LlamaWindowService
-
-            window_service = LlamaWindowService(service)
-        elif model_name in [
-            "meta/llama-2-7b",
-            "meta/llama-2-13b",
-            "meta/llama-2-70b",
-        ]:
-            from helm.benchmark.window_services.llama_window_service import Llama2WindowService
-
-            window_service = Llama2WindowService(service)
-        elif organization == "cohere":
-            from helm.benchmark.window_services.cohere_window_service import (
-                CohereWindowService,
-                CohereCommandWindowService,
-            )
-
-            if "command" in engine:
-                window_service = CohereCommandWindowService(service)
-            else:
-                window_service = CohereWindowService(service)
-        elif organization == "ai21":
-            from helm.benchmark.window_services.wider_ai21_window_service import (
-                WiderAI21WindowService,
-                AI21Jurassic2JumboWindowService,
-            )
-            from helm.benchmark.window_services.ai21_window_service import AI21WindowService
-
-            if model_name in get_model_names_with_tag(AI21_WIDER_CONTEXT_WINDOW_TAG):
-                window_service = WiderAI21WindowService(service=service, gpt2_window_service=GPT2WindowService(service))
-            if model_name in get_model_names_with_tag(AI21_JURASSIC_2_JUMBO_CONTEXT_WINDOW_TAG):
-                window_service = AI21Jurassic2JumboWindowService(
-                    service=service, gpt2_window_service=GPT2WindowService(service)
-                )
-            else:
-                window_service = AI21WindowService(service=service, gpt2_window_service=GPT2WindowService(service))
-
-        elif organization == "lightningai":
-            from helm.benchmark.window_services.lit_gpt_window_service import LitGPTWindowServce
-
-            window_service = LitGPTWindowServce(service)
-        elif organization == "mistralai":
-            window_service = HuggingFaceWindowService(service, tokenizer_name="mistralai/Mistral-7B-v0.1")
-        elif model_name in [
-            "HuggingFaceM4/idefics-9b",
-            "HuggingFaceM4/idefics-9b-instruct",
-            "HuggingFaceM4/idefics-80b",
-            "HuggingFaceM4/idefics-80b-instruct",
-        ]:
-            window_service = HuggingFaceWindowService(service, model_name)
-        else:
-            raise ValueError(f"Unhandled model name: {model_name}")
+            return create_object(window_service_spec)
 
-        return window_service
+        raise ValueError(f"Unhandled model deployment name: {model_deployment_name}")

helm/benchmark/window_services/yalm_window_service.py

@@ -1,34 +1,7 @@
-from helm.proxy.tokenizers.yalm_tokenizer_data.yalm_tokenizer import YaLMTokenizer
 from .local_window_service import LocalWindowService
-from .tokenizer_service import TokenizerService
 
 
 class YaLMWindowService(LocalWindowService):
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def tokenizer_name(self) -> str:
-        return "Yandex/yalm"
-
-    @property
-    def max_sequence_length(self) -> int:
-        return YaLMTokenizer.MAX_SEQUENCE_LENGTH
-
-    @property
-    def max_request_length(self) -> int:
-        return self.max_sequence_length + 1
-
-    @property
-    def end_of_text_token(self) -> str:
-        """The end of text token."""
-        return YaLMTokenizer.EOS_TOKEN
-
-    @property
-    def prefix_token(self) -> str:
-        """The prefix token"""
-        return self.end_of_text_token
-
     def truncate_from_right(self, text: str, expected_completion_token_length: int = 0) -> str:
         """
         Truncates text from the right to fit within the context window given by `max_request_length`

helm/{proxy/clients → clients}/ai21_client.py

@@ -7,10 +7,9 @@ from helm.common.request import (
     EMBEDDING_UNAVAILABLE_REQUEST_RESULT,
     Request,
     RequestResult,
-    Sequence,
+    GeneratedOutput,
     Token,
 )
-from helm.proxy.tokenizers.tokenizer import Tokenizer
 from .client import CachingClient, truncate_sequence, cleanup_str
 from .ai21_utils import AI21RequestError, handle_failed_request
 
@@ -24,8 +23,8 @@ class AI21Client(CachingClient):
     COMPLETION_URL_TEMPLATE: str = "https://api.ai21.com/studio/v1/{model}/complete"
     EXPERIMENTAL_COMPLETION_URL_TEMPLATE: str = "https://api.ai21.com/studio/v1/experimental/{model}/complete"
 
-    def __init__(self, api_key: str, tokenizer: Tokenizer, cache_config: CacheConfig, url: Optional[str] = None):
-        super().__init__(cache_config=cache_config, tokenizer=tokenizer)
+    def __init__(self, api_key: str, cache_config: CacheConfig, url: Optional[str] = None):
+        super().__init__(cache_config=cache_config)
         self.api_key = api_key
         self.url = url
 
@@ -98,25 +97,19 @@ class AI21Client(CachingClient):
             # Compute the actual length of the token text
             # e.g. "▁burying"(0,8) -> 8 - 0 = 8; "▁burying"(0,7) -> 7 - 0 = 7
             text_length: int = raw["textRange"]["end"] - raw["textRange"]["start"]
-            # "topTokens" can be None when sending a request with topKReturn=0
-            # AI21 sends unscaled logprobs as `raw_logprob` so use this instead of `logprob`.
-            top_logprobs: Dict[str, float] = dict(
-                (fix_text(x["token"], first), x["raw_logprob"]) for x in raw["topTokens"] or []
-            )
 
             return Token(
                 # Text should not be longer than text_length. Since "▁" is always inserted
                 # in the beginning, we truncate the text from the right.
                 text=fix_text(raw["generatedToken"]["token"], first)[-text_length:] if text_length else "",
                 logprob=raw["generatedToken"]["raw_logprob"],
-                top_logprobs=top_logprobs,
             )
 
-        def parse_sequence(raw: Dict, first: bool, finish_reason: Optional[Dict] = None) -> Sequence:
+        def parse_sequence(raw: Dict, first: bool, finish_reason: Optional[Dict] = None) -> GeneratedOutput:
             text = raw["text"]
             tokens = [parse_token(token, first and i == 0) for i, token in enumerate(raw["tokens"])]
             logprob = sum(token.logprob for token in tokens)
-            return Sequence(text=text, logprob=logprob, tokens=tokens, finish_reason=finish_reason)
+            return GeneratedOutput(text=text, logprob=logprob, tokens=tokens, finish_reason=finish_reason)
 
         prompt = parse_sequence(response["prompt"], True)
         completions = []

helm/clients/aleph_alpha_client.py

@@ -0,0 +1,112 @@
+from typing import List
+
+from helm.common.cache import CacheConfig
+from helm.common.media_object import TEXT_TYPE
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.common.request import wrap_request_time, Request, RequestResult, GeneratedOutput, Token
+from .client import CachingClient, truncate_sequence, generate_uid_for_multimodal_prompt
+
+try:
+    from aleph_alpha_client import Client, CompletionRequest, CompletionResponse, Image, Prompt
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["aleph-alpha"])
+
+
+class AlephAlphaClient(CachingClient):
+    def __init__(self, api_key: str, cache_config: CacheConfig):
+        super().__init__(cache_config=cache_config)
+        self._api_key: str = api_key
+        self._aleph_alpha_client = Client(token=self._api_key) if self._api_key else None
+
+    def make_request(self, request: Request) -> RequestResult:
+        """Make a request following https://docs.aleph-alpha.com/api/complete."""
+        assert self._aleph_alpha_client is not None
+
+        model: str = request.model_engine
+        prompt: Prompt
+
+        # The prompt key is a unique identifier for the prompt
+        prompt_key: str = request.prompt
+
+        # Contents can either be text or a list of multimodal content made up of text, images or other content
+        if request.multimodal_prompt is not None:
+            from helm.common.images_utils import encode_base64
+
+            items = []
+            for media_object in request.multimodal_prompt.media_objects:
+                if media_object.is_type("image") and media_object.location:
+                    items.append(Image(base_64=encode_base64(media_object.location), cropping=None, controls=[]))
+                elif media_object.is_type(TEXT_TYPE):
+                    if media_object.text is None:
+                        raise ValueError("MediaObject of text type has missing text field value")
+                    items.append(media_object.text)
+                else:
+                    raise ValueError(f"Unrecognized MediaObject type {media_object.type}")
+
+            prompt = Prompt(items=items)
+            prompt_key = generate_uid_for_multimodal_prompt(request.multimodal_prompt)
+        else:
+            prompt = Prompt.from_text(request.prompt)
+
+        parameters = {
+            "maximum_tokens": request.max_tokens,
+            "temperature": request.temperature,
+            "top_k": request.top_k_per_token,
+            "top_p": request.top_p,
+            "presence_penalty": request.presence_penalty,
+            "frequency_penalty": request.frequency_penalty,
+            "n": request.num_completions,
+            "stop_sequences": request.stop_sequences,
+            "log_probs": request.top_k_per_token,
+            "echo": request.echo_prompt,
+            "tokens": True,  # Setting to True returns individual tokens of the completion
+        }
+
+        try:
+
+            def do_it():
+                assert self._aleph_alpha_client is not None
+                completion_response: CompletionResponse = self._aleph_alpha_client.complete(
+                    request=CompletionRequest(prompt=prompt, **parameters), model=model
+                )
+                result = dict(completion_response.to_json())
+                assert "completions" in result, f"Invalid response: {result}"
+                return result
+
+            cache_key = CachingClient.make_cache_key({"model": model, "prompt": prompt_key, **parameters}, request)
+            response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
+        except Exception as e:
+            error: str = f"AlephAlphaClient error: {e}"
+            return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
+
+        completions: List[GeneratedOutput] = []
+        for completion in response["completions"]:
+            sequence_logprob: float = 0
+            tokens: List[Token] = []
+
+            # `completion_tokens` is the list of selected tokens.
+            for i, token in enumerate(completion.get("completion_tokens", [])):
+                # Use the selected token value to get the logprob
+                logprob: float = completion["log_probs"][i][token]
+                sequence_logprob += logprob
+                tokens.append(
+                    Token(
+                        text=token,
+                        logprob=logprob,
+                    )
+                )
+
+            sequence: GeneratedOutput = GeneratedOutput(
+                text=completion["completion"], logprob=sequence_logprob, tokens=tokens
+            )
+            sequence = truncate_sequence(sequence, request)
+            completions.append(sequence)
+
+        return RequestResult(
+            success=True,
+            cached=cached,
+            request_time=response["request_time"],
+            request_datetime=response["request_datetime"],
+            completions=completions,
+            embedding=[],
+        )