crfm-helm 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

helm/proxy/test_models.py

@@ -1,27 +0,0 @@
-from .models import get_model, get_model_group, get_models_by_organization, get_all_code_models, Model
-
-
-def test_get_model():
-    model: Model = get_model("ai21/j1-jumbo")
-    assert model.organization == "ai21"
-    assert model.engine == "j1-jumbo"
-
-
-def test_get_model_with_invalid_model_name():
-    try:
-        get_model("invalid/model")
-        assert False, "Expected to throw ValueError"
-    except ValueError:
-        pass
-
-
-def test_get_model_group():
-    assert get_model_group("openai/text-curie-001") == "gpt3"
-
-
-def test_get_models_by_organization():
-    assert get_models_by_organization("simple") == ["simple/model1"]
-
-
-def test_all_code_models():
-    assert "openai/code-davinci-002" in get_all_code_models()

helm/proxy/token_counters/ai21_token_counter.py

@@ -1,20 +0,0 @@
-from typing import List
-
-from helm.common.request import Request, Sequence
-from .token_counter import TokenCounter
-
-
-class AI21TokenCounter(TokenCounter):
-    def count_tokens(self, request: Request, completions: List[Sequence]) -> int:
-        """
-        Counts the number of generated tokens and NOT the number of tokens in the prompt
-        (https://studio.ai21.com/docs/calculating-usage).
-
-        The AI21 documentation (https://studio.ai21.com/docs/calculating-usage/) defines
-        generated tokens as:
-        "the total number of all completion tokens you generate. For example, assume you post
-        a complete request for J1-Jumbo with a prompt consisting of 10 tokens and requiring 3
-        completions, i.e. numResults = 3, and the model generates completions with 5, 15, and
-        20 tokens. In total this request will consume 5+15+20=40 generated tokens."
-        """
-        return sum(len(sequence.tokens) for sequence in completions)

helm/proxy/token_counters/cohere_token_counter.py

@@ -1,13 +0,0 @@
-from typing import List
-
-from helm.common.request import Request, Sequence
-from .token_counter import TokenCounter
-
-
-class CohereTokenCounter(TokenCounter):
-    def count_tokens(self, request: Request, completions: List[Sequence]) -> int:
-        """
-        Counts the number of generated tokens.
-        TODO: Cohere simply counts the number of generations, but we currently only support counting tokens.
-        """
-        return sum(len(sequence.tokens) for sequence in completions)

helm/proxy/token_counters/free_token_counter.py

@@ -1,12 +0,0 @@
-from typing import List
-
-from helm.common.request import Request, Sequence
-from .token_counter import TokenCounter
-
-
-class FreeTokenCounter(TokenCounter):
-    """For when we don't care about keeping track of the number of tokens."""
-
-    def count_tokens(self, request: Request, completions: List[Sequence]) -> int:
-        """No need to count tokens, since it's free. Return 0."""
-        return 0

helm/proxy/token_counters/gooseai_token_counter.py

@@ -1,24 +0,0 @@
-from typing import List
-
-from helm.common.request import Request, Sequence
-from .token_counter import TokenCounter
-
-
-class GooseAITokenCounter(TokenCounter):
-    # From https://goose.ai/pricing: "the base price includes your first 25 tokens
-    # generated, and you can scale beyond that on a per-token basis."
-    BASE_PRICE_TOKENS: int = 25
-
-    @staticmethod
-    def account_for_base_tokens(num_tokens: int):
-        """Subtracts the number of tokens included in the base price."""
-        return max(num_tokens - GooseAITokenCounter.BASE_PRICE_TOKENS, 0)
-
-    def count_tokens(self, request: Request, completions: List[Sequence]) -> int:
-        """
-        Counts the number of generated tokens and NOT the number of tokens in the prompt.
-        From https://goose.ai/pricing: "by charging only for output, you have control since
-        you can configure the maximum number of tokens generated per API call
-        (up to 2,048 tokens)."
-        """
-        return GooseAITokenCounter.account_for_base_tokens(sum(len(sequence.tokens) for sequence in completions))

helm/proxy/token_counters/openai_token_counter.py

@@ -1,22 +0,0 @@
-from typing import List
-
-from helm.common.request import Request, Sequence
-from helm.common.tokenization_request import TokenizationRequest, TokenizationRequestResult
-from helm.proxy.clients.huggingface_client import HuggingFaceClient
-from .token_counter import TokenCounter
-
-
-class OpenAITokenCounter(TokenCounter):
-    def __init__(self, huggingface_client: HuggingFaceClient):
-        self.huggingface_client: HuggingFaceClient = huggingface_client
-
-    def count_tokens(self, request: Request, completions: List[Sequence]) -> int:
-        """
-        Counts the total number of tokens using the suggestion here:
-        https://community.openai.com/t/how-do-i-calculate-the-pricing-for-generation-of-text/11662/5
-        """
-        tokenized_prompt: TokenizationRequestResult = self.huggingface_client.tokenize(
-            TokenizationRequest(request.prompt)
-        )
-        # Number of tokens in the prompt + number of tokens in all the completions
-        return len(tokenized_prompt.tokens) + sum([len(sequence.tokens) for sequence in completions])

helm/proxy/token_counters/test_ai21_token_counter.py

@@ -1,86 +0,0 @@
-from typing import List
-
-from helm.common.request import Request, Sequence, Token
-from .ai21_token_counter import AI21TokenCounter
-
-
-class TestAI21TokenCounter:
-    def setup_method(self, method):
-        self.token_counter = AI21TokenCounter()
-
-    def test_count_tokens(self):
-        request = Request(
-            prompt="The Center for Research on Foundation Models (CRFM) is "
-            "an interdisciplinary initiative born out of the Stanford "
-            "Institute for Human-Centered Artificial Intelligence (HAI) "
-            "that aims to make fundamental advances in the study, development, "
-            "and deployment of foundation models."
-        )
-        completions: List[Sequence] = [
-            Sequence(
-                text="\n\nFoundation models are an artificial intelligence paradigm that emphasizes: (1) reasoning "
-                "about symbolic, structured knowledge, (2) learning to perform tasks from observation, ",
-                logprob=-66.24831021729915,
-                tokens=[
-                    Token(text="\n", logprob=-1.034758448600769, top_logprobs={"\n": -1.034758448600769}),
-                    Token(text="\n", logprob=-2.325321674346924, top_logprobs={" Foundation": -1.2628217935562134}),
-                    Token(
-                        text=" Foundation",
-                        logprob=-1.2575088739395142,
-                        top_logprobs={" Foundation": -1.2575088739395142},
-                    ),
-                    Token(
-                        text=" models are",
-                        logprob=-0.9496442079544067,
-                        top_logprobs={" models are": -0.9496442079544067},
-                    ),
-                    Token(text=" an", logprob=-5.785625457763672, top_logprobs={" a class of": -2.762187957763672}),
-                    Token(
-                        text=" artificial intelligence",
-                        logprob=-2.073937177658081,
-                        top_logprobs={" increasingly popular": -1.714562177658081},
-                    ),
-                    Token(text=" paradigm", logprob=-2.855092763900757, top_logprobs={" ": -1.2613427639007568}),
-                    Token(text=" that", logprob=-1.380897879600525, top_logprobs={" that": -1.380897879600525}),
-                    Token(
-                        text=" emphasizes",
-                        logprob=-4.230795860290527,
-                        top_logprobs={" attempts to": -3.5276708602905273},
-                    ),
-                    Token(text=":", logprob=-5.380178451538086, top_logprobs={" reasoning": -2.192678689956665}),
-                    Token(text=" ", logprob=-1.1408348083496094, top_logprobs={"\n": -0.6095848083496094}),
-                    Token(text="(", logprob=-0.41460439562797546, top_logprobs={"(": -0.41460439562797546}),
-                    Token(text="1", logprob=-0.5666008591651917, top_logprobs={"1": -0.5666008591651917}),
-                    Token(text=")", logprob=-0.001801535952836275, top_logprobs={")": -0.001801535952836275}),
-                    Token(text=" reasoning", logprob=-3.4144058227539062, top_logprobs={" the": -2.3987808227539062}),
-                    Token(text=" about", logprob=-1.3604949712753296, top_logprobs={" about": -1.3604949712753296}),
-                    Token(text=" symbolic", logprob=-7.108627796173096, top_logprobs={" and": -2.5617527961730957}),
-                    Token(text=",", logprob=-2.8421378135681152, top_logprobs={" knowledge": -1.6233878135681152}),
-                    Token(
-                        text=" structured",
-                        logprob=-2.6082611083984375,
-                        top_logprobs={" structured": -2.6082611083984375},
-                    ),
-                    Token(text=" knowledge", logprob=-0.91008061170578, top_logprobs={" knowledge": -0.91008061170578}),
-                    Token(text=",", logprob=-1.0750247240066528, top_logprobs={",": -1.0750247240066528}),
-                    Token(text=" ", logprob=-0.5834965705871582, top_logprobs={" ": -0.5834965705871582}),
-                    Token(text="(", logprob=-0.0004963834653608501, top_logprobs={"(": -0.0004963834653608501}),
-                    Token(text="2", logprob=-0.0009141556802205741, top_logprobs={"2": -0.0009141556802205741}),
-                    Token(text=")", logprob=-5.686121585313231e-05, top_logprobs={")": -5.686121585313231e-05}),
-                    Token(text=" learning", logprob=-2.123058319091797, top_logprobs={" learning": -2.123058319091797}),
-                    Token(
-                        text=" to perform", logprob=-5.197870254516602, top_logprobs={" through": -1.7916204929351807}
-                    ),
-                    Token(text=" tasks", logprob=-1.5782833099365234, top_logprobs={" complex": -1.5470333099365234}),
-                    Token(text=" from", logprob=-1.1503676176071167, top_logprobs={" from": -1.1503676176071167}),
-                    Token(
-                        text=" observation", logprob=-4.8489789962768555, top_logprobs={" human": -2.5989787578582764}
-                    ),
-                    Token(text=",", logprob=-0.7239797711372375, top_logprobs={",": -0.7239797711372375}),
-                    Token(text=" ", logprob=-1.3241727352142334, top_logprobs={" and": -0.5897976756095886}),
-                ],
-            )
-        ]
-
-        # Verified against https://studio.ai21.com/playground.
-        assert self.token_counter.count_tokens(request, completions) == 32

helm/proxy/token_counters/test_openai_token_counter.py

@@ -1,79 +0,0 @@
-import os
-import tempfile
-from typing import List
-
-from helm.common.cache import SqliteCacheConfig
-from helm.common.request import Request, Sequence, Token
-from helm.proxy.clients.huggingface_client import HuggingFaceClient
-from helm.proxy.tokenizers.huggingface_tokenizer import HuggingFaceTokenizer
-from .openai_token_counter import OpenAITokenCounter
-
-
-class TestOpenAITokenCounter:
-    # The following prompt has 51 tokens according to the GPT-2 tokenizer
-    TEST_PROMPT: str = (
-        "The Center for Research on Foundation Models (CRFM) is "
-        "an interdisciplinary initiative born out of the Stanford "
-        "Institute for Human-Centered Artificial Intelligence (HAI) "
-        "that aims to make fundamental advances in the study, development, "
-        "and deployment of foundation models."
-    )
-
-    def setup_method(self, method):
-        self.cache_path: str = tempfile.NamedTemporaryFile(delete=False).name
-        self.token_counter = OpenAITokenCounter(
-            HuggingFaceClient(
-                tokenizer=HuggingFaceTokenizer(cache_config=SqliteCacheConfig(self.cache_path)),
-                cache_config=SqliteCacheConfig(self.cache_path),
-            )
-        )
-
-    def teardown_method(self, method):
-        os.remove(self.cache_path)
-
-    def test_count_tokens(self):
-        request = Request(prompt=TestOpenAITokenCounter.TEST_PROMPT)
-        completions: List[Sequence] = [
-            Sequence(
-                text=" The CRFM is dedicated to advancing our knowledge of the foundations of artificial intelligence "
-                "(AI) and related fields. It focuses on foundational questions in AI, which are",
-                logprob=-49.00783279519999,
-                tokens=[
-                    Token(text=" The", logprob=-1.8096403, top_logprobs={"\n": -1.6654028}),
-                    Token(text=" CR", logprob=-1.2861944, top_logprobs={" CR": -1.2861944}),
-                    Token(text="FM", logprob=-0.0032369632, top_logprobs={"FM": -0.0032369632}),
-                    Token(text=" is", logprob=-1.4355252, top_logprobs={" is": -1.4355252}),
-                    Token(text=" dedicated", logprob=-3.814422, top_logprobs={" a": -1.8003343}),
-                    Token(text=" to", logprob=-0.009623392, top_logprobs={" to": -0.009623392}),
-                    Token(text=" advancing", logprob=-2.6732886, top_logprobs={" the": -1.859751}),
-                    Token(text=" our", logprob=-3.123714, top_logprobs={" the": -1.0504603}),
-                    Token(text=" knowledge", logprob=-3.030337, top_logprobs={" understanding": -0.34646907}),
-                    Token(text=" of", logprob=-0.46280858, top_logprobs={" of": -0.46280858}),
-                    Token(text=" the", logprob=-1.4058315, top_logprobs={" the": -1.4058315}),
-                    Token(text=" foundations", logprob=-2.0638132, top_logprobs={" foundations": -2.0638132}),
-                    Token(text=" of", logprob=-0.2607486, top_logprobs={" of": -0.2607486}),
-                    Token(text=" artificial", logprob=-1.1653417, top_logprobs={" artificial": -1.1653417}),
-                    Token(text=" intelligence", logprob=-0.03756146, top_logprobs={" intelligence": -0.03756146}),
-                    Token(text=" (", logprob=-2.019812, top_logprobs={",": -1.3503861}),
-                    Token(text="AI", logprob=-0.03869382, top_logprobs={"AI": -0.03869382}),
-                    Token(text=")", logprob=-0.49895737, top_logprobs={")": -0.49895737}),
-                    Token(text=" and", logprob=-0.81909865, top_logprobs={" and": -0.81909865}),
-                    Token(text=" related", logprob=-2.611718, top_logprobs={" to": -2.3555496}),
-                    Token(text=" fields", logprob=-0.7640527, top_logprobs={" fields": -0.7640527}),
-                    Token(text=".", logprob=-1.8066244, top_logprobs={",": -1.2972366}),
-                    Token(text=" It", logprob=-2.2856107, top_logprobs={"\n": -1.7865871}),
-                    Token(text=" focuses", logprob=-3.3735154, top_logprobs={" is": -1.4982711}),
-                    Token(text=" on", logprob=-0.13244776, top_logprobs={" on": -0.13244776}),
-                    Token(text=" foundational", logprob=-1.2640914, top_logprobs={" foundational": -1.2640914}),
-                    Token(text=" questions", logprob=-2.010647, top_logprobs={" issues": -1.673752}),
-                    Token(text=" in", logprob=-1.980726, top_logprobs={" that": -1.8133409}),
-                    Token(text=" AI", logprob=-0.5709368, top_logprobs={" AI": -0.5709368}),
-                    Token(text=",", logprob=-1.036094, top_logprobs={",": -1.036094}),
-                    Token(text=" which", logprob=-3.826836, top_logprobs={" such": -2.0843854}),
-                    Token(text=" are", logprob=-1.3858839, top_logprobs={" are": -1.3858839}),
-                ],
-            )
-        ]
-
-        # Verified against https://beta.openai.com/tokenizer. Prompt + completions = 51 + 32.
-        assert self.token_counter.count_tokens(request, completions) == 51 + 32

helm/proxy/tokenizers/simple_tokenizer.py

@@ -1,32 +0,0 @@
-from typing import List
-
-from helm.common.tokenization_request import (
-    DecodeRequest,
-    DecodeRequestResult,
-    TokenizationRequest,
-    TokenizationRequestResult,
-    TokenizationToken,
-)
-from .tokenizer import Tokenizer
-
-
-class SimpleTokenizer(Tokenizer):
-    """Implements some "models" that just generate silly things quickly just to debug the infrastructure."""
-
-    @staticmethod
-    def tokenize_by_space(text: str) -> List[str]:
-        """Simply tokenizes by a single white space."""
-        return text.split(" ")
-
-    def tokenize(self, request: TokenizationRequest) -> TokenizationRequestResult:
-        # TODO: Does not support encoding
-        if request.tokenizer == "simple/model1":
-            raw_tokens: List[str] = SimpleTokenizer.tokenize_by_space(request.text)
-            return TokenizationRequestResult(
-                success=True, cached=False, tokens=[TokenizationToken(text) for text in raw_tokens], text=request.text
-            )
-        else:
-            raise ValueError("Unknown model")
-
-    def decode(self, request: DecodeRequest) -> DecodeRequestResult:
-        raise NotImplementedError

helm/proxy/tokenizers/test_huggingface_tokenizer.py

@@ -1,56 +0,0 @@
-from typing import Optional
-from helm.common.general import singleton
-from .huggingface_tokenizer import HuggingFaceTokenizer
-
-
-class TestHuggingFaceTokenizer:
-    # The following prompt has 51 tokens according to the GPT-2 tokenizer
-    TEST_PROMPT: str = (
-        "The Center for Research on Foundation Models (CRFM) is "
-        "an interdisciplinary initiative born out of the Stanford "
-        "Institute for Human-Centered Artificial Intelligence (HAI) "
-        "that aims to make fundamental advances in the study, development, "
-        "and deployment of foundation models."
-    )
-
-    @staticmethod
-    def verify_get_tokenizer(
-        tokenizer_name: str, expected_num_tokens: int, pretrained_model_name_or_path: Optional[str] = None
-    ):
-        tokenizer = HuggingFaceTokenizer.get_tokenizer(
-            helm_tokenizer_name=tokenizer_name,
-            pretrained_model_name_or_path=pretrained_model_name_or_path or tokenizer_name,
-        )
-        assert tokenizer_name in HuggingFaceTokenizer._tokenizers, "Tokenizer should be cached"
-        assert len(tokenizer.encode(TestHuggingFaceTokenizer.TEST_PROMPT)) == expected_num_tokens
-
-    def test_get_tokenizer_gpt2(self):
-        TestHuggingFaceTokenizer.verify_get_tokenizer("huggingface/gpt2", 51, pretrained_model_name_or_path="gpt2")
-
-    def test_get_tokenizer_gptj(self):
-        TestHuggingFaceTokenizer.verify_get_tokenizer("EleutherAI/gpt-j-6B", 51)
-
-    def test_get_tokenizer_gptneox(self):
-        TestHuggingFaceTokenizer.verify_get_tokenizer("EleutherAI/gpt-neox-20b", 52)
-
-    def test_get_tokenizer_bloom(self):
-        TestHuggingFaceTokenizer.verify_get_tokenizer("bigscience/bloom", 51)
-
-    def test_get_tokenizer_t0pp(self):
-        TestHuggingFaceTokenizer.verify_get_tokenizer("bigscience/T0pp", 58)
-
-    def test_get_tokenizer_t511b(self):
-        TestHuggingFaceTokenizer.verify_get_tokenizer("google/t5-11b", 58, pretrained_model_name_or_path="t5-11b")
-
-    def test_get_tokenizer_ul2(self):
-        TestHuggingFaceTokenizer.verify_get_tokenizer("google/ul2", 58)
-
-    def test_get_santacoder(self):
-        TestHuggingFaceTokenizer.verify_get_tokenizer("bigcode/santacoder", 62)
-
-    def test_gpt2_tokenize_eos(self):
-        eos_token: str = "<|endoftext|>"
-        tokenizer = HuggingFaceTokenizer.get_tokenizer("huggingface/gpt2", pretrained_model_name_or_path="gpt2")
-        token_ids = tokenizer.encode(eos_token)
-        assert singleton(token_ids) == 50256
-        assert tokenizer.decode(token_ids) == eos_token