crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/proxy/clients/microsoft_client.py

@@ -1,180 +0,0 @@
-from typing import List, Optional, Dict
-
-from filelock import FileLock
-from openai.api_resources.abstract import engine_api_resource
-import openai as turing
-
-from helm.common.cache import CacheConfig
-from helm.common.request import (
-    wrap_request_time,
-    EMBEDDING_UNAVAILABLE_REQUEST_RESULT,
-    Request,
-    RequestResult,
-    Sequence,
-    Token,
-)
-from .client import CachingClient, truncate_sequence
-from .openai_client import ORIGINAL_COMPLETION_ATTRIBUTES
-
-
-class MicrosoftClient(CachingClient):
-    """
-    Client for the Microsoft's Megatron-Turing NLG models (https://arxiv.org/abs/2201.11990).
-
-    According to the internal documentation: https://github.com/microsoft/turing-academic-TNLG,
-    "the model will generate roughly 3 tokens per second. The response will be returned once
-    all tokens have been generated."
-    """
-
-    @staticmethod
-    def convert_to_raw_request(request: Request) -> Dict:
-        return {
-            "engine": request.model_engine,
-            "prompt": request.prompt,
-            "temperature": request.temperature,
-            "max_tokens": request.max_tokens,
-            "best_of": request.top_k_per_token,
-            "logprobs": request.top_k_per_token,
-            # Despite what was stated here: https://github.com/microsoft/turing-academic-TNLG#api-parameters,
-            # their API supports at most one stop sequence. Pass in the first one for now and handle the rest
-            # of the stop sequences during post processing (see `truncate_sequence` below).
-            "stop": None if len(request.stop_sequences) == 0 else request.stop_sequences[0],
-            "top_p": request.top_p,
-            "echo": request.echo_prompt,
-        }
-
-    def __init__(
-        self,
-        lock_file_path: str,
-        cache_config: CacheConfig,
-        api_key: Optional[str] = None,
-        org_id: Optional[str] = None,
-    ):
-        super().__init__(cache_config=cache_config)
-
-        # Adapted from their documentation: https://github.com/microsoft/turing-academic-TNLG
-        class EngineAPIResource(engine_api_resource.EngineAPIResource):
-            @classmethod
-            def class_url(
-                cls, engine: Optional[str] = None, api_type: Optional[str] = None, api_version: Optional[str] = None
-            ) -> str:
-                return f"/{engine}/inference"
-
-        self.org_id: Optional[str] = org_id
-        self.api_key: Optional[str] = api_key
-        self.api_base: str = "https://turingnlg-turingnlg-mstap-v2.turingase.p.azurewebsites.net"
-        self.completion_attributes = (EngineAPIResource,) + ORIGINAL_COMPLETION_ATTRIBUTES[1:]
-
-        # The Microsoft Turing server only allows a single request at a time, so acquire a
-        # process-safe lock before making a request.
-        # https://github.com/microsoft/turing-academic-TNLG#rate-limitations
-        #
-        # Since the model will generate roughly three tokens per second and the max context window
-        # is 2048 tokens, we expect the maximum time for a request to be fulfilled to be 700 seconds.
-        self._lock = FileLock(lock_file_path, timeout=700)
-
-    def make_request(self, request: Request) -> RequestResult:
-        """
-        Make a request for the Microsoft MT-NLG models.
-
-        They mimicked the OpenAI completions API, but not all the parameters are supported.
-
-        Supported parameters:
-            engine
-            prompt
-            temperature
-            max_tokens
-            best_of
-            logprobs
-            stop ("Only a single "stop" value (str) is currently supported.")
-            top_p
-            echo
-            n (Not originally supported, but we simulate n by making multiple requests)
-
-        Not supported parameters:
-            presence_penalty
-            frequency_penalty
-        """
-        # Embedding not supported for this model
-        if request.embedding:
-            return EMBEDDING_UNAVAILABLE_REQUEST_RESULT
-
-        raw_request = MicrosoftClient.convert_to_raw_request(request)
-        completions: List[Sequence] = []
-        request_time = 0
-        request_datetime: Optional[int] = None
-        all_cached = True
-
-        # API currently only supports 1 completion at a time, so we have to hit it multiple times.
-        for completion_index in range(request.num_completions):
-            try:
-
-                def do_it():
-                    with self._lock:
-                        # Following https://beta.openai.com/docs/api-reference/authentication
-                        # `organization` can be set to None.
-                        turing.organization = self.org_id
-                        turing.api_key = self.api_key
-                        turing.api_base = self.api_base
-                        turing.api_resources.completion.Completion.__bases__ = self.completion_attributes
-
-                        response: Dict = turing.Completion.create(**raw_request)
-                        # Validate the responses, so we don't cache malformed responses with null `logprobs` and `text`
-                        if (
-                            "choices" not in response
-                            or len(response["choices"]) == 0
-                            or response["choices"][0].get("text") is None
-                            or response["choices"][0].get("logprobs") is None
-                        ):
-                            raise turing.error.OpenAIError(
-                                f"For request: {raw_request}, invalid response from the MT-NLG server: {response}."
-                            )
-
-                        return response
-
-                def fail():
-                    raise RuntimeError(
-                        f"The result has not been uploaded to the cache for the following request: {cache_key}"
-                    )
-
-                # We want to make `request.num_completions` fresh requests,
-                # cache key should contain the completion_index.
-                cache_key = CachingClient.make_cache_key({"completion_index": completion_index, **raw_request}, request)
-                response, cached = self.cache.get(cache_key, wrap_request_time(do_it if self.api_key else fail))
-            except turing.error.OpenAIError as e:
-                error: str = f"OpenAI (Turing API) error: {e}"
-                return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
-
-            for raw_completion in response["choices"]:
-                sequence_logprob = 0
-                tokens: List[Token] = []
-
-                raw_data = raw_completion["logprobs"]
-                for text, logprob, top_logprobs in zip(
-                    raw_data["tokens"], raw_data["token_logprobs"], raw_data["top_logprobs"]
-                ):
-                    tokens.append(Token(text=text, logprob=logprob or 0, top_logprobs=dict(top_logprobs or {})))
-                    sequence_logprob += logprob or 0
-
-                completion = Sequence(
-                    text=raw_completion["text"],
-                    logprob=sequence_logprob,
-                    tokens=tokens,
-                    finish_reason={"reason": raw_completion["finish_reason"]},
-                )
-                completion = truncate_sequence(completion, request)
-                completions.append(completion)
-
-            request_time += response["request_time"]
-            # Use the datetime from the first completion because that's when the request was fired
-            request_datetime = request_datetime or response.get("request_datetime")
-            all_cached = all_cached and cached
-
-        return RequestResult(
-            success=True,
-            cached=all_cached,
-            request_time=request_time,
-            request_datetime=request_datetime,
-            completions=completions,
-            embedding=[],
-        )

helm/proxy/clients/openai_client.py

@@ -1,206 +0,0 @@
-# mypy: check_untyped_defs = False
-from dataclasses import replace
-from typing import Any, Dict, List, Optional, cast
-
-from helm.common.cache import CacheConfig
-from helm.common.request import wrap_request_time, Request, RequestResult, Sequence, Token
-from helm.common.hierarchical_logger import hlog
-from helm.common.optional_dependencies import handle_module_not_found_error
-from helm.common.tokenization_request import (
-    TokenizationRequest,
-    TokenizationRequestResult,
-)
-from helm.proxy.tokenizers.tokenizer import Tokenizer
-from .client import CachingClient, truncate_sequence
-
-try:
-    import openai
-except ModuleNotFoundError as e:
-    handle_module_not_found_error(e, ["openai"])
-
-
-ORIGINAL_COMPLETION_ATTRIBUTES = openai.api_resources.completion.Completion.__bases__
-
-
-class OpenAIClient(CachingClient):
-    END_OF_TEXT: str = "<|endoftext|>"
-
-    def __init__(
-        self,
-        tokenizer: Tokenizer,
-        tokenizer_name: str,
-        cache_config: CacheConfig,
-        api_key: Optional[str] = None,
-        org_id: Optional[str] = None,
-    ):
-        super().__init__(cache_config=cache_config)
-        self.tokenizer = tokenizer
-        self.tokenizer_name = tokenizer_name
-        self.org_id: Optional[str] = org_id
-        self.api_key: Optional[str] = api_key
-        self.api_base: str = "https://api.openai.com/v1"
-
-    def _is_chat_model_engine(self, model_engine: str):
-        return model_engine.startswith("gpt-3.5") or model_engine.startswith("gpt-4")
-
-    def make_request(self, request: Request) -> RequestResult:
-        if self.api_key is None:
-            raise ValueError("OpenAI API key is required")
-
-        raw_request: Dict[str, Any]
-        if request.embedding:
-            raw_request = {
-                "input": request.prompt,
-                "engine": request.model_engine,
-            }
-        elif self._is_chat_model_engine(request.model_engine):
-            messages: Optional[List[Dict[str, str]]] = request.messages
-            if request.messages and len(request.messages) > 1:
-                # Checks that all messages have a role and some content
-                for message in request.messages:
-                    if not message.get("role") or not message.get("content"):
-                        raise ValueError("All messages must have a role and content")
-                # Checks that the last role is "user"
-                if request.messages[-1]["role"] != "user":
-                    raise ValueError("Last message must have role 'user'")
-                if request.prompt != "":
-                    hlog("WARNING: Since message is set, prompt will be ignored")
-            else:
-                # Convert prompt into a single message
-                # For now, put the whole prompt in a single user message, and expect the response
-                # to be returned in a single assistant message.
-                # TODO: Support ChatML for creating multiple messages with different roles.
-                # See: https://github.com/openai/openai-python/blob/main/chatml.md
-                messages = [{"role": "user", "content": request.prompt}]
-            raw_request = {
-                "model": request.model_engine,
-                "messages": messages,
-                "temperature": request.temperature,
-                "top_p": request.top_p,
-                "n": request.num_completions,
-                "stop": request.stop_sequences or None,  # API doesn't like empty list
-                # Note: Chat models may require adding an extra token to max_tokens
-                # for the internal special role token.
-                "max_tokens": request.max_tokens,
-                "presence_penalty": request.presence_penalty,
-                "frequency_penalty": request.frequency_penalty,
-            }
-        else:
-            raw_request = {
-                "engine": request.model_engine,
-                "prompt": request.prompt,
-                "temperature": request.temperature,
-                "n": request.num_completions,
-                "max_tokens": request.max_tokens,
-                "best_of": request.top_k_per_token,
-                "logprobs": request.top_k_per_token,
-                "stop": request.stop_sequences or None,  # API doesn't like empty list
-                "top_p": request.top_p,
-                "presence_penalty": request.presence_penalty,
-                "frequency_penalty": request.frequency_penalty,
-                "echo": request.echo_prompt,
-            }
-
-            # OpenAI doesn't let you ask for more completions than the number of
-            # per-token candidates.
-            raw_request["best_of"] = max(raw_request["best_of"], raw_request["n"])
-            raw_request["logprobs"] = max(raw_request["logprobs"], raw_request["n"])
-
-        try:
-            if request.embedding:
-
-                def do_it():
-                    openai.organization = self.org_id
-                    openai.api_key = self.api_key
-                    openai.api_base = self.api_base
-                    return openai.Embedding.create(**raw_request)
-
-            elif self._is_chat_model_engine(request.model_engine):
-
-                def do_it():
-                    openai.organization = self.org_id
-                    openai.api_key = self.api_key
-                    openai.api_base = self.api_base
-                    return openai.ChatCompletion.create(**raw_request)
-
-            else:
-
-                def do_it():
-                    # Following https://beta.openai.com/docs/api-reference/authentication
-                    # `organization` can be set to None.
-                    openai.organization = self.org_id
-                    openai.api_key = self.api_key
-                    openai.api_base = self.api_base
-                    openai.api_resources.completion.Completion.__bases__ = ORIGINAL_COMPLETION_ATTRIBUTES
-                    return openai.Completion.create(**raw_request)
-
-            cache_key = CachingClient.make_cache_key(raw_request, request)
-            response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
-        except openai.error.OpenAIError as e:
-            error: str = f"OpenAI error: {e}"
-            return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
-
-        # If the user is requesting completions instead of an embedding, then `completions`
-        # needs to be populated, and `embedding` should be an empty list and vice-versa.
-        embedding: List[float] = []
-        completions: List[Sequence] = []
-        tokens: List[Token]
-        if request.embedding:
-            # If the user is requesting an embedding instead of completion
-            # then completions would be left as an empty list. The embedding needs to be set.
-            embedding = response["data"][0]["embedding"]
-        elif self._is_chat_model_engine(request.model_engine):
-            for raw_completion in response["choices"]:
-                # The OpenAI chat completion API doesn't support echo.
-                # If `echo_prompt` is true, combine the prompt and completion.
-                raw_completion_content = raw_completion["message"]["content"]
-                text: str = request.prompt + raw_completion_content if request.echo_prompt else raw_completion_content
-                # The OpenAI chat completion API doesn't return us tokens or logprobs, so we tokenize ourselves.
-                tokenization_result: TokenizationRequestResult = self.tokenizer.tokenize(
-                    TokenizationRequest(text, tokenizer=self.tokenizer_name)
-                )
-                # Log probs are not currently not supported by the OpenAI chat completion API, so set to 0 for now.
-                tokens = [
-                    Token(text=cast(str, raw_token), logprob=0, top_logprobs={})
-                    for raw_token in tokenization_result.raw_tokens
-                ]
-                completion = Sequence(
-                    text=text,
-                    logprob=0,  # OpenAI does not provide logprobs
-                    tokens=tokens,
-                    finish_reason={"reason": raw_completion["finish_reason"]},
-                )
-                completions.append(truncate_sequence(completion, request))  # Truncate the text by stop sequences
-        else:
-            for raw_completion in response["choices"]:
-                sequence_logprob = 0
-                tokens = []
-
-                raw_data = raw_completion["logprobs"]
-                for text, logprob, top_logprobs in zip(
-                    raw_data["tokens"], raw_data["token_logprobs"], raw_data["top_logprobs"]
-                ):
-                    tokens.append(Token(text=text, logprob=logprob or 0, top_logprobs=dict(top_logprobs or {})))
-                    sequence_logprob += logprob or 0
-                completion = Sequence(
-                    text=raw_completion["text"],
-                    logprob=sequence_logprob,
-                    tokens=tokens,
-                    finish_reason={"reason": raw_completion["finish_reason"]},
-                )
-                # OpenAI sends us back tokens past the end of text token,
-                # so we need to manually truncate the list of tokens.
-                # TODO: filed an issue with their support to check what the expected behavior here is.
-                completion = truncate_sequence(
-                    completion, replace(request, stop_sequences=request.stop_sequences + [OpenAIClient.END_OF_TEXT])
-                )
-                completions.append(completion)
-
-        return RequestResult(
-            success=True,
-            cached=cached,
-            request_time=response["request_time"],
-            request_datetime=response.get("request_datetime"),
-            completions=completions,
-            embedding=embedding,
-        )

helm/proxy/clients/simple_client.py

@@ -1,60 +0,0 @@
-from typing import List, Dict
-
-from helm.common.cache import CacheConfig
-from helm.common.request import wrap_request_time, Request, RequestResult, Sequence, Token
-from helm.proxy.tokenizers.simple_tokenizer import SimpleTokenizer
-from .client import CachingClient
-
-
-class SimpleClient(CachingClient):
-    """Implements some "models" that just generate silly things quickly just to debug the infrastructure."""
-
-    def __init__(self, cache_config: CacheConfig):
-        super().__init__(cache_config=cache_config)
-
-    def make_request(self, request: Request) -> RequestResult:
-        raw_request = {
-            "engine": request.model_engine,
-            "prompt": request.prompt,
-            "n": request.num_completions,
-        }
-
-        if request.model_engine == "model1":
-
-            def do_it():
-                return self.invoke_model1(raw_request)
-
-            cache_key = CachingClient.make_cache_key(raw_request, request)
-            response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
-            completions = [
-                Sequence(
-                    text=text,
-                    logprob=logprob,
-                    tokens=[Token(text=text, logprob=logprob, top_logprobs=response["completions"])],
-                )
-                for text, logprob in response["completions"].items()
-            ]
-        else:
-            raise ValueError(f"Invalid model: {request.model}")
-
-        return RequestResult(
-            success=True,
-            cached=False,
-            request_time=0,
-            request_datetime=response.get("request_datetime"),
-            completions=completions,
-            embedding=[],
-        )
-
-    def invoke_model1(self, raw_request: Dict) -> Dict:
-        """
-        Example: 7 2 4 6
-        Completions (num_completions = 3):
-        - 6
-        - 4
-        - 2
-        """
-        prompt_tokens: List[str] = SimpleTokenizer.tokenize_by_space(raw_request["prompt"])
-        choices = reversed(prompt_tokens[-raw_request["n"] :])
-        response = {"completions": dict((text, -i) for i, text in enumerate(choices))}
-        return response

helm/proxy/clients/test_client.py

@@ -1,49 +0,0 @@
-from .client import truncate_sequence
-from typing import List
-from helm.common.request import Request, Sequence, Token
-
-
-def truncate_sequence_helper(tokens: List[str], request: Request, expected_tokens: List[str]):
-    sequence = Sequence(
-        text="".join(tokens),
-        tokens=[Token(text=text, logprob=-1, top_logprobs={}) for text in tokens],
-        logprob=-len(tokens),
-    )
-
-    output_sequence = truncate_sequence(sequence, request)
-
-    assert expected_tokens == [token.text for token in output_sequence.tokens]
-    assert "".join(expected_tokens) == output_sequence.text
-    assert output_sequence.logprob == sum(token.logprob for token in output_sequence.tokens)
-
-
-def test_truncate_sequence():
-    # echo_prompt = True, nothing gets truncated
-    truncate_sequence_helper(
-        ["a", "b", "c"],
-        Request(
-            model="openai/text-davinci-002", model_deployment="openai/text-davinci-002", prompt="abc", echo_prompt=True
-        ),
-        ["a", "b", "c"],
-    )
-
-    # Nothing gets truncated
-    truncate_sequence_helper(
-        ["hello", " world"],
-        Request(model="openai/text-davinci-002", model_deployment="openai/text-davinci-002", stop_sequences=["#"]),
-        ["hello", " world"],
-    )
-
-    # Truncate using stop sequences
-    truncate_sequence_helper(
-        ["hello", " world", "\n", "what"],
-        Request(model="openai/text-davinci-002", model_deployment="openai/text-davinci-002", stop_sequences=["\n"]),
-        ["hello", " world"],
-    )
-
-    # Truncate using max tokens
-    truncate_sequence_helper(
-        ["a", "b", "c"],
-        Request(model="openai/text-davinci-002", model_deployment="openai/text-davinci-002", max_tokens=2),
-        ["a", "b"],
-    )

helm/proxy/clients/test_together_client.py

@@ -1,97 +0,0 @@
-import os
-import pytest
-import tempfile
-
-from helm.common.cache import SqliteCacheConfig
-from helm.common.request import Request
-
-from .together_client import TogetherClient, TogetherClientError
-
-
-class TestTogetherClient:
-    def setup_method(self, method):
-        cache_file = tempfile.NamedTemporaryFile(delete=False)
-        self.cache_path: str = cache_file.name
-        self.client = TogetherClient(cache_config=SqliteCacheConfig(self.cache_path))
-
-    def teardown_method(self, method):
-        os.remove(self.cache_path)
-
-    @pytest.mark.parametrize(
-        "test_input,expected",
-        [
-            (
-                Request(
-                    model="together/redpajama-incite-base-3b-v1",
-                    model_deployment="together/redpajama-incite-base-3b-v1",
-                ),
-                {
-                    "best_of": 1,
-                    "echo": False,
-                    "logprobs": 1,
-                    "max_tokens": 100,
-                    "model": "togethercomputer/RedPajama-INCITE-Base-3B-v1",
-                    "n": 1,
-                    "prompt": "",
-                    "request_type": "language-model-inference",
-                    "stop": None,
-                    "temperature": 1.0,
-                    "top_p": 1,
-                },
-            ),
-            (
-                Request(
-                    model="meta/llama-7b",
-                    model_deployment="together/llama-7b",
-                    prompt="I am a computer scientist.",
-                    temperature=0,
-                    num_completions=4,
-                    max_tokens=24,
-                    top_k_per_token=3,
-                    stop_sequences=["\n"],
-                    echo_prompt=True,
-                    top_p=0.3,
-                ),
-                {
-                    "best_of": 3,
-                    "echo": True,
-                    "logprobs": 3,
-                    "max_tokens": 24,
-                    "model": "huggyllama/llama-7b",
-                    "n": 4,
-                    "prompt": "I am a computer scientist.",
-                    "request_type": "language-model-inference",
-                    "stop": ["\n"],
-                    "temperature": 0,
-                    "top_p": 0.3,
-                },
-            ),
-            (
-                Request(
-                    model="stanford/alpaca-7b",
-                    model_deployment="together/alpaca-7b",
-                    stop_sequences=["\n"],
-                ),
-                {
-                    "best_of": 1,
-                    "echo": False,
-                    "logprobs": 1,
-                    "max_tokens": 100,
-                    "model": "togethercomputer/alpaca-7b",
-                    "n": 1,
-                    "prompt": "",
-                    "request_type": "language-model-inference",
-                    "stop": ["\n", "</s>"],
-                    "temperature": 1.0,
-                    "top_p": 1,
-                },
-            ),
-            # TODO(#1828): Add test for `SET_DETAILS_TO_TRUE` after Together supports it.
-        ],
-    )
-    def test_convert_to_raw_request(self, test_input, expected):
-        assert expected == TogetherClient.convert_to_raw_request(test_input)
-
-    def test_api_key_error(self):
-        with pytest.raises(TogetherClientError):
-            self.client.make_request(Request(model="bigscience/bloom", model_deployment="together/bloom"))