crfm-helm 0.4.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

helm/benchmark/window_services/test_openai_window_service.py

@@ -1,6 +1,7 @@
 import shutil
 import tempfile
 
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from .test_utils import get_tokenizer_service, TEST_PROMPT, GPT2_TEST_TOKENS, GPT2_TEST_TOKEN_IDS
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
@@ -9,7 +10,7 @@ from .window_service_factory import WindowServiceFactory
 class TestOpenAIWindowService:
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
+        service: TokenizerService = get_tokenizer_service(self.path, BlackHoleCacheBackendConfig())
         self.window_service = WindowServiceFactory.get_window_service("openai/davinci", service)
 
     def teardown_method(self, method):

helm/benchmark/window_services/test_opt_window_service.py

@@ -1,6 +1,7 @@
 import shutil
 import tempfile
 
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from .test_utils import get_tokenizer_service, TEST_PROMPT
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
@@ -9,8 +10,8 @@ from .window_service_factory import WindowServiceFactory
 class TestOPTWindowService:
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
-        self.window_service = WindowServiceFactory.get_window_service("together/opt-175b", service)
+        service: TokenizerService = get_tokenizer_service(self.path, BlackHoleCacheBackendConfig())
+        self.window_service = WindowServiceFactory.get_window_service("huggingface/opt-175b", service)
 
     def teardown_method(self, method):
         shutil.rmtree(self.path)

helm/benchmark/window_services/test_palmyra_window_service.py

@@ -1,6 +1,7 @@
 from tempfile import TemporaryDirectory
 from typing import List
 
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
 from .test_utils import get_tokenizer_service, TEST_PROMPT
@@ -117,7 +118,7 @@ class TestPalmyraWindowService:
 
     def setup_method(self):
         self.temporary_directory = TemporaryDirectory()
-        service: TokenizerService = get_tokenizer_service(self.temporary_directory.name)
+        service: TokenizerService = get_tokenizer_service(self.temporary_directory.name, BlackHoleCacheBackendConfig())
         self.window_service = WindowServiceFactory.get_window_service("writer/palmyra-large", service)
 
     def teardown_method(self, method):

helm/benchmark/window_services/test_t0pp_window_service.py

@@ -2,6 +2,7 @@ import shutil
 import tempfile
 from typing import List
 
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
 from .test_utils import get_tokenizer_service, TEST_PROMPT
@@ -70,7 +71,7 @@ class TestT0ppWindowService:
 
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
+        service: TokenizerService = get_tokenizer_service(self.path, BlackHoleCacheBackendConfig())
         self.window_service = WindowServiceFactory.get_window_service("together/t0pp", service)
 
     def teardown_method(self, method):

helm/benchmark/window_services/test_t511b_window_service.py

@@ -2,6 +2,7 @@ import shutil
 import tempfile
 from typing import List
 
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
 from .test_utils import get_tokenizer_service, TEST_PROMPT
@@ -70,7 +71,7 @@ class TestT511bWindowService:
 
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
+        service: TokenizerService = get_tokenizer_service(self.path, BlackHoleCacheBackendConfig())
         self.window_service = WindowServiceFactory.get_window_service("together/t5-11b", service)
 
     def teardown_method(self, method):

helm/benchmark/window_services/test_ul2_window_service.py

@@ -2,6 +2,7 @@ import shutil
 import tempfile
 from typing import List
 
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
 from .test_utils import get_tokenizer_service, TEST_PROMPT
@@ -70,7 +71,7 @@ class TestUL2WindowService:
 
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
+        service: TokenizerService = get_tokenizer_service(self.path, BlackHoleCacheBackendConfig())
         self.window_service = WindowServiceFactory.get_window_service("together/ul2", service)
 
     def teardown_method(self, method):

helm/benchmark/window_services/test_utils.py

@@ -1,6 +1,7 @@
 from typing import List
 
 from helm.common.authentication import Authentication
+from helm.common.cache_backend_config import CacheBackendConfig
 from helm.proxy.services.server_service import ServerService
 from helm.benchmark.metrics.metric_service import MetricService
 from .tokenizer_service import TokenizerService
@@ -227,6 +228,6 @@ GPT4_TEST_TOKENS: List[str] = [
 ]
 
 
-def get_tokenizer_service(local_path: str) -> TokenizerService:
-    service = ServerService(base_path=local_path, root_mode=True)
+def get_tokenizer_service(local_path: str, cache_backend_config: CacheBackendConfig) -> TokenizerService:
+    service = ServerService(base_path=local_path, root_mode=True, cache_backend_config=cache_backend_config)
     return MetricService(service, Authentication("test"))

helm/benchmark/window_services/test_yalm_window_service.py

@@ -1,6 +1,7 @@
 import shutil
 import tempfile
 
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from .test_utils import get_tokenizer_service, TEST_PROMPT
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
@@ -9,7 +10,7 @@ from .window_service_factory import WindowServiceFactory
 class TestYaLMWindowService:
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
+        service: TokenizerService = get_tokenizer_service(self.path, BlackHoleCacheBackendConfig())
         self.window_service = WindowServiceFactory.get_window_service("together/yalm", service)
 
     def teardown_method(self, method):

helm/benchmark/window_services/window_service.py

@@ -110,3 +110,45 @@ class WindowService(ABC):
         minus the expected completion length (defaults to 0).
         """
         pass
+
+
+class ConfigurableWindowService(WindowService, ABC):
+    def __init__(
+        self,
+        tokenizer_name: str,
+        max_sequence_length: int,
+        max_request_length: Optional[int] = None,
+        max_sequence_and_generated_tokens_length: Optional[int] = None,
+        end_of_text_token: Optional[str] = None,
+        prefix_token: Optional[str] = None,
+    ):
+        self._tokenizer_name = tokenizer_name
+        self._max_sequence_length = max_sequence_length
+        self._max_request_length = max_request_length or max_sequence_length
+        self._max_sequence_and_generated_tokens_length = max_sequence_and_generated_tokens_length or INT_MAX
+        self._end_of_text_token = end_of_text_token or ""
+        self._prefix_token = prefix_token or ""
+
+    @property
+    def tokenizer_name(self) -> str:
+        return self._tokenizer_name
+
+    @property
+    def max_sequence_length(self) -> int:
+        return self._max_sequence_length
+
+    @property
+    def max_request_length(self) -> int:
+        return self._max_request_length
+
+    @property
+    def max_sequence_and_generated_tokens_length(self) -> int:
+        return self._max_sequence_and_generated_tokens_length
+
+    @property
+    def end_of_text_token(self) -> str:
+        return self._end_of_text_token
+
+    @property
+    def prefix_token(self) -> str:
+        return self._prefix_token

helm/benchmark/window_services/window_service_factory.py

@@ -43,7 +43,7 @@ class WindowServiceFactory:
             #    in the users configuration file. Instead, they have to be constructed dynamically at runtime.
             window_service_spec = inject_object_spec_args(
                 window_service_spec,
-                {
+                constant_bindings={
                     "service": service,
                     "tokenizer_name": model_deployment.tokenizer_name,
                     "max_sequence_length": model_deployment.max_sequence_length,
@@ -52,6 +52,9 @@ class WindowServiceFactory:
                     "end_of_text_token": end_of_text_token,
                     "prefix_token": prefix_token,
                 },
+                provider_bindings={
+                    "gpt2_window_service": lambda: WindowServiceFactory.get_window_service("huggingface/gpt2", service)
+                },
             )
             return create_object(window_service_spec)
 

helm/benchmark/window_services/yalm_window_service.py

@@ -1,34 +1,7 @@
-from helm.proxy.tokenizers.yalm_tokenizer_data.yalm_tokenizer import YaLMTokenizer
 from .local_window_service import LocalWindowService
-from .tokenizer_service import TokenizerService
 
 
 class YaLMWindowService(LocalWindowService):
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def tokenizer_name(self) -> str:
-        return "Yandex/yalm"
-
-    @property
-    def max_sequence_length(self) -> int:
-        return YaLMTokenizer.MAX_SEQUENCE_LENGTH
-
-    @property
-    def max_request_length(self) -> int:
-        return self.max_sequence_length + 1
-
-    @property
-    def end_of_text_token(self) -> str:
-        """The end of text token."""
-        return YaLMTokenizer.EOS_TOKEN
-
-    @property
-    def prefix_token(self) -> str:
-        """The prefix token"""
-        return self.end_of_text_token
-
     def truncate_from_right(self, text: str, expected_completion_token_length: int = 0) -> str:
         """
         Truncates text from the right to fit within the context window given by `max_request_length`

helm/{proxy/clients → clients}/ai21_client.py

@@ -7,7 +7,7 @@ from helm.common.request import (
     EMBEDDING_UNAVAILABLE_REQUEST_RESULT,
     Request,
     RequestResult,
-    Sequence,
+    GeneratedOutput,
     Token,
 )
 from .client import CachingClient, truncate_sequence, cleanup_str
@@ -97,25 +97,19 @@ class AI21Client(CachingClient):
             # Compute the actual length of the token text
             # e.g. "▁burying"(0,8) -> 8 - 0 = 8; "▁burying"(0,7) -> 7 - 0 = 7
             text_length: int = raw["textRange"]["end"] - raw["textRange"]["start"]
-            # "topTokens" can be None when sending a request with topKReturn=0
-            # AI21 sends unscaled logprobs as `raw_logprob` so use this instead of `logprob`.
-            top_logprobs: Dict[str, float] = dict(
-                (fix_text(x["token"], first), x["raw_logprob"]) for x in raw["topTokens"] or []
-            )
 
             return Token(
                 # Text should not be longer than text_length. Since "▁" is always inserted
                 # in the beginning, we truncate the text from the right.
                 text=fix_text(raw["generatedToken"]["token"], first)[-text_length:] if text_length else "",
                 logprob=raw["generatedToken"]["raw_logprob"],
-                top_logprobs=top_logprobs,
             )
 
-        def parse_sequence(raw: Dict, first: bool, finish_reason: Optional[Dict] = None) -> Sequence:
+        def parse_sequence(raw: Dict, first: bool, finish_reason: Optional[Dict] = None) -> GeneratedOutput:
             text = raw["text"]
             tokens = [parse_token(token, first and i == 0) for i, token in enumerate(raw["tokens"])]
             logprob = sum(token.logprob for token in tokens)
-            return Sequence(text=text, logprob=logprob, tokens=tokens, finish_reason=finish_reason)
+            return GeneratedOutput(text=text, logprob=logprob, tokens=tokens, finish_reason=finish_reason)
 
         prompt = parse_sequence(response["prompt"], True)
         completions = []

helm/clients/aleph_alpha_client.py

@@ -0,0 +1,112 @@
+from typing import List
+
+from helm.common.cache import CacheConfig
+from helm.common.media_object import TEXT_TYPE
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.common.request import wrap_request_time, Request, RequestResult, GeneratedOutput, Token
+from .client import CachingClient, truncate_sequence, generate_uid_for_multimodal_prompt
+
+try:
+    from aleph_alpha_client import Client, CompletionRequest, CompletionResponse, Image, Prompt
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["aleph-alpha"])
+
+
+class AlephAlphaClient(CachingClient):
+    def __init__(self, api_key: str, cache_config: CacheConfig):
+        super().__init__(cache_config=cache_config)
+        self._api_key: str = api_key
+        self._aleph_alpha_client = Client(token=self._api_key) if self._api_key else None
+
+    def make_request(self, request: Request) -> RequestResult:
+        """Make a request following https://docs.aleph-alpha.com/api/complete."""
+        assert self._aleph_alpha_client is not None
+
+        model: str = request.model_engine
+        prompt: Prompt
+
+        # The prompt key is a unique identifier for the prompt
+        prompt_key: str = request.prompt
+
+        # Contents can either be text or a list of multimodal content made up of text, images or other content
+        if request.multimodal_prompt is not None:
+            from helm.common.images_utils import encode_base64
+
+            items = []
+            for media_object in request.multimodal_prompt.media_objects:
+                if media_object.is_type("image") and media_object.location:
+                    items.append(Image(base_64=encode_base64(media_object.location), cropping=None, controls=[]))
+                elif media_object.is_type(TEXT_TYPE):
+                    if media_object.text is None:
+                        raise ValueError("MediaObject of text type has missing text field value")
+                    items.append(media_object.text)
+                else:
+                    raise ValueError(f"Unrecognized MediaObject type {media_object.type}")
+
+            prompt = Prompt(items=items)
+            prompt_key = generate_uid_for_multimodal_prompt(request.multimodal_prompt)
+        else:
+            prompt = Prompt.from_text(request.prompt)
+
+        parameters = {
+            "maximum_tokens": request.max_tokens,
+            "temperature": request.temperature,
+            "top_k": request.top_k_per_token,
+            "top_p": request.top_p,
+            "presence_penalty": request.presence_penalty,
+            "frequency_penalty": request.frequency_penalty,
+            "n": request.num_completions,
+            "stop_sequences": request.stop_sequences,
+            "log_probs": request.top_k_per_token,
+            "echo": request.echo_prompt,
+            "tokens": True,  # Setting to True returns individual tokens of the completion
+        }
+
+        try:
+
+            def do_it():
+                assert self._aleph_alpha_client is not None
+                completion_response: CompletionResponse = self._aleph_alpha_client.complete(
+                    request=CompletionRequest(prompt=prompt, **parameters), model=model
+                )
+                result = dict(completion_response.to_json())
+                assert "completions" in result, f"Invalid response: {result}"
+                return result
+
+            cache_key = CachingClient.make_cache_key({"model": model, "prompt": prompt_key, **parameters}, request)
+            response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
+        except Exception as e:
+            error: str = f"AlephAlphaClient error: {e}"
+            return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
+
+        completions: List[GeneratedOutput] = []
+        for completion in response["completions"]:
+            sequence_logprob: float = 0
+            tokens: List[Token] = []
+
+            # `completion_tokens` is the list of selected tokens.
+            for i, token in enumerate(completion.get("completion_tokens", [])):
+                # Use the selected token value to get the logprob
+                logprob: float = completion["log_probs"][i][token]
+                sequence_logprob += logprob
+                tokens.append(
+                    Token(
+                        text=token,
+                        logprob=logprob,
+                    )
+                )
+
+            sequence: GeneratedOutput = GeneratedOutput(
+                text=completion["completion"], logprob=sequence_logprob, tokens=tokens
+            )
+            sequence = truncate_sequence(sequence, request)
+            completions.append(sequence)
+
+        return RequestResult(
+            success=True,
+            cached=cached,
+            request_time=response["request_time"],
+            request_datetime=response["request_datetime"],
+            completions=completions,
+            embedding=[],
+        )