crfm-helm 0.5.4__py3-none-any.whl → 0.5.6__py3-none-any.whl

helm/proxy/static/index.js

@@ -0,0 +1,456 @@
+/**
+ * This is a very quick and dirty frontend for just interacting with the models.
+ * Please refrain from adding additional functionality to this.
+ * TODO: Write this in React.
+ */
+$(function () {
+  const urlParams = decodeUrlParams(window.location.search);
+  let auth = null;
+
+  function censor(api_key) {
+    // Show only the first k letters
+    const k = 2;
+    if (api_key.length <= k) {
+      return api_key;
+    }
+    return api_key.substring(0, k) + "*".repeat(api_key.length - k);
+  }
+
+  // Logging in and out
+  function updateLogin() {
+    const $loginInfo = $("#loginInfo");
+    $loginInfo.empty();
+    let api_key = readCookie("api_key");
+    if (api_key) {
+      auth = { api_key };
+      $loginInfo.append(
+        $("<a>", { class: "nav-link", href: "#" })
+          .append("Logout of API key " + censor(api_key))
+          .click(() => {
+            eraseCookie("api_key");
+            updateLogin();
+          }),
+      );
+    } else {
+      auth = null;
+      $loginInfo.append(
+        $("<a>", { class: "nav-link", href: "#" })
+          .append("Login")
+          .click(() => {
+            api_key = prompt("Enter your API key:");
+            if (!api_key) {
+              return;
+            }
+
+            // Check the API key the user entered using endpoint /api/account
+            const args = { auth: JSON.stringify({ api_key }) };
+            $.getJSON("/api/account", args, (response) => {
+              console.log("/api/account", response);
+              if ("error" in response) {
+                alert("The API key you've entered is invalid. Try again.");
+              } else {
+                createCookie("api_key", api_key);
+                updateLogin();
+              }
+            });
+          }),
+      );
+    }
+  }
+
+  updateLogin();
+
+  ////////////////////////////////////////////////////////////
+  // Rendering functions
+
+  function renderExampleQueries(updateQuery) {
+    // Show links for each example query, so when you click on them, they populate the textboxes.
+    const $examplesBlock = $("<div>", { class: "examples-block" });
+    $examplesBlock.append($("<span>").append("Examples:"));
+    generalInfo.example_queries.forEach((query, i) => {
+      const href = "#";
+      const title =
+        "[Prompt]\n" +
+        query.prompt +
+        "\n[Settings]\n" +
+        query.settings +
+        "\n[Environments]\n" +
+        query.environments;
+      const $link = $("<a>", { href, title }).append(`[${i}]`);
+      $link.click(() => {
+        // Populate the query with the example
+        updateQuery(query);
+        urlParams.prompt = query.prompt;
+        urlParams.settings = query.settings;
+        urlParams.environments = query.environments;
+        updateBrowserLocation(urlParams);
+      });
+      $examplesBlock.append("&nbsp;");
+      $examplesBlock.append($link);
+    });
+    return $examplesBlock;
+  }
+
+  function renderQuery(handleQueryResult) {
+    // Render the textboxes for entering the query (which includes the prompt, settings, and environment)
+    const $queryBlock = $("<div>", { class: "block" });
+    const $prompt = $("<textarea>", {
+      cols: 90,
+      rows: 7,
+      placeholder: "Enter prompt",
+    }).val(urlParams.prompt);
+    const $settings = $("<textarea>", {
+      cols: 90,
+      rows: 5,
+      placeholder:
+        "Enter settings (e.g., model: openai/text-davinci-002 for Instruct GPT-3); click Help at the top to learn more",
+    }).val(urlParams.settings);
+    const $environments = $("<textarea>", {
+      cols: 90,
+      rows: 3,
+      placeholder:
+        "Enter environment variables (e.g., city: [Boston, New York]); click Help at the top to learn more",
+    }).val(urlParams.environments);
+
+    $queryBlock.data("prompt", $prompt);
+    $queryBlock.data("settings", $settings);
+    $queryBlock.data("environments", $environments);
+
+    function bindSubmit($text) {
+      $text.keypress((e) => {
+        if ((event.keyCode === 10 || event.keyCode === 13) && event.ctrlKey) {
+          submit();
+        }
+      });
+    }
+
+    bindSubmit($prompt);
+    bindSubmit($settings);
+    bindSubmit($environments);
+
+    function submit() {
+      if (!auth) {
+        alert("You must log in first.");
+        return;
+      }
+
+      const query = {
+        prompt: $prompt.val(),
+        settings: $settings.val(),
+        environments: $environments.val(),
+      };
+
+      urlParams.prompt = query.prompt;
+      urlParams.settings = query.settings;
+      urlParams.environments = query.environments;
+      updateBrowserLocation(urlParams);
+
+      $.getJSON("/api/query", query, handleQueryResult);
+    }
+
+    // Show examples of queries
+    const $exampleQueries = renderExampleQueries((query) => {
+      $queryBlock.data("prompt").val(query.prompt);
+      $queryBlock.data("settings").val(query.settings);
+      $queryBlock.data("environments").val(query.environments);
+      urlParams.prompt = query.prompt;
+      urlParams.settings = query.settings;
+      urlParams.environments = query.environments;
+      updateBrowserLocation();
+    });
+
+    const promptHelp =
+      "This is the text you feed into the language model to complete.\nExample:\n  Life is like";
+    const settingsHelp =
+      "Specifies what information we want from the language model (see [Help] for more details).\nExample:\n  temperature: ${temperature}\n  model: openai/davinci\n  max_tokens: 10\n  num_completions: 5";
+    const environmentsHelp =
+      "Specifies a list of values to try for each variable that appears in the prompt or settings.\nExample:\n  temperature: [0, 0.5, 1]";
+
+    const $promptLabel = $("<span>")
+      .append(helpIcon(promptHelp, "help.html#query"))
+      .append("Prompt");
+    const $settingsLabel = $("<span>")
+      .append(helpIcon(settingsHelp, "help.html#query"))
+      .append("Settings");
+    const $environmentsLabel = $("<span>")
+      .append(helpIcon(environmentsHelp, "help.html#query"))
+      .append("Environments");
+
+    $queryBlock.append($("<h4>").append("Query"));
+    $queryBlock.append($exampleQueries);
+    const $table = $("<table>", { class: "query-table" });
+    $table.append(
+      $("<tr>")
+        .append($("<td>").append($promptLabel))
+        .append($("<td>").append($prompt)),
+    );
+    $table.append(
+      $("<tr>")
+        .append($("<td>").append($settingsLabel))
+        .append($("<td>").append($settings)),
+    );
+    $table.append(
+      $("<tr>")
+        .append($("<td>").append($environmentsLabel))
+        .append($("<td>").append($environments)),
+    );
+    $queryBlock.append($table);
+    $queryBlock.append($("<button>").append("Submit").click(submit));
+
+    return $queryBlock;
+  }
+
+  function getChangingKeys(items) {
+    // Return the list of keys whose values vary across `items.`
+    if (items.length === 0) {
+      return [];
+    }
+    return Object.keys(items[0]).filter((key) => {
+      return !items.every(
+        (item) => JSON.stringify(item[key]) === JSON.stringify(items[0][key]),
+      );
+    });
+  }
+
+  function renderRequest(changingKeys, request) {
+    // Render the request metadata (e.g., temperature if it is changing)
+    const title = JSON.stringify(request);
+    // Always include model, never prompt (since that's shown right after).
+    const showKeys = ["model"].concat(
+      changingKeys.filter((key) => key !== "prompt" && key !== "model"),
+    );
+    const summary =
+      "[" + showKeys.map((key) => key + ":" + request[key]).join(", ") + "]";
+    return $("<div>", { title }).append(
+      summary + " " + multilineHtml(request.prompt),
+    );
+  }
+
+  function renderTime(time) {
+    return Math.round(time * 10) / 10 + "s";
+  }
+
+  function constructTokenGroups(tokens) {
+    // Note: sometimes multiple tokens correspond to one character, for example:
+    // ["bytes:\xe2\x80", "bytes:\x99"] => ’
+    // For these, we keep these in the buffer and collapse them, and concatenate the entries.
+    //
+    // get_num_bytes() and convert_tokens_to_text() in src/helm/benchmark/basic_metrics.py are adapted from this function.
+    const groups = [];
+    for (let i = 0; i < tokens.length;) {
+      // Aggregate consecutive tokens while they're "bytes:..."
+      const group = { tokens: [] };
+      if (tokens[i].text.startsWith("bytes:")) {
+        let bytestring = "";
+        while (i < tokens.length && tokens[i].text.startsWith("bytes:")) {
+          group.tokens.push(tokens[i]);
+          // Extract part after : (e.g., \xe2\x80)
+          bytestring += tokens[i].text.split(":")[1];
+          i++;
+        }
+        // Convert to encoded URI (e.g., %e2%80%99) and decode
+        group.text = decodeURIComponent(bytestring.replaceAll("\\x", "%"));
+      } else {
+        group.tokens.push(tokens[i]);
+        group.text = tokens[i].text;
+        i++;
+      }
+      groups.push(group);
+    }
+    return groups;
+  }
+
+  function renderTokens(tokens) {
+    // Render text as a sequence of tokens that you can interact with to see more information (e.g., logprobs)
+    const $result = $("<div>");
+    const groups = constructTokenGroups(tokens);
+    for (const group of groups) {
+      const $group = $("<span>", { class: "token" }).append(
+        multilineHtml(group.text),
+      );
+      $result.append($group);
+    }
+    return $result;
+  }
+
+  function renderRequestResult(requestResult) {
+    // Render the list of completions.
+    if (requestResult.error) {
+      return renderError(requestResult.error);
+    }
+    const $result = $("<div>");
+    requestResult.completions.forEach((completion) => {
+      const $contents = $("<span>", {
+        title: `logprob: ${completion.logprob}`,
+      }).append(renderTokens(completion.tokens));
+      const $metadata = $("<span>", { class: "metadata" });
+      $metadata.append(
+        $("<span>", { title: "Log probability" }).append(
+          round(completion.logprob, 2),
+        ),
+      );
+      if (completion.finish_reason) {
+        const title =
+          "Generation finished because of this reason: " +
+          JSON.stringify(completion.finish_reason);
+        $metadata
+          .append(" ")
+          .append(
+            $("<span>", { title }).append(completion.finish_reason.reason),
+          );
+      }
+      $result.append(
+        $("<div>", { class: "completion" }).append($metadata).append($contents),
+      );
+    });
+    $result.append($("<i>").append(renderTime(requestResult.request_time)));
+    return $result;
+  }
+
+  function renderAccount() {
+    // Render the account information (usage, quotas).
+    if (!auth) {
+      return null;
+    }
+
+    const $accountBlock = $("<div>", { class: "block" });
+    const args = { auth: JSON.stringify(auth) };
+    $.getJSON("/api/account", args, ([account]) => {
+      console.log("/api/account", account);
+      const items = [];
+      for (modelGroup in account.usages) {
+        for (granularity in account.usages[modelGroup]) {
+          const usage = account.usages[modelGroup][granularity];
+          // Only print out usage for model groups and granularities where there is a quota
+          if (usage.quota) {
+            const percent = Math.round((usage.used / usage.quota) * 100);
+            items.push(
+              `<b>${modelGroup}</b>: ${usage.period} (${usage.used} / ${usage.quota} = ${percent}%)`,
+            );
+          }
+        }
+      }
+      if (items.length === 0) {
+        items.push("no restrictions");
+      }
+      $accountBlock
+        .empty()
+        .append(
+          helpIcon(
+            "Specifies your usage/quota (321/10000) for each model group (e.g., gpt3) for the current period (e.g., 2022-1-2).",
+            "help.html#quotas",
+          ),
+        )
+        .append("Usage")
+        .append(": ")
+        .append(items.join(" | "));
+    });
+    return $accountBlock;
+  }
+
+  ////////////////////////////////////////////////////////////
+  // For index.html
+
+  function renderQueryInterface() {
+    // For index.html
+    const $accountBlock = $("<div>").append(renderAccount());
+
+    // Allow editing the query
+    const $queryBlock = renderQuery((queryResult) => {
+      // Create requests
+      console.log("/api/query", queryResult);
+      $requestsBlock.empty();
+
+      if (queryResult.error) {
+        $requestsBlock.append(renderError(queryResult.error));
+        return;
+      }
+
+      $requestsBlock.append(
+        $("<h4>").append(`Requests (${queryResult.requests.length})`),
+      );
+      if (queryResult.error) {
+        $requestsBlock.append(renderError(queryResult.error));
+        return;
+      }
+      const changingKeys = getChangingKeys(queryResult.requests);
+      queryResult.requests.forEach((request) => {
+        const $request = $("<div>", { class: "request" }).append(
+          renderRequest(changingKeys, request),
+        );
+        const $requestResult = $("<div>").append($("<i>").append("(waiting)"));
+        const args = {
+          auth: JSON.stringify(auth),
+          request: JSON.stringify(request),
+        };
+        $.getJSON("/api/request", args, (requestResult) => {
+          console.log("/api/request", request, requestResult);
+          $requestResult.empty().append(renderRequestResult(requestResult));
+          if (!requestResult.cached) {
+            $accountBlock.empty().append(renderAccount());
+          }
+        });
+        $request.append($requestResult);
+        $requestsBlock.append($request);
+      });
+    });
+
+    // Where the requests and responses come in
+    const $requestsBlock = $("<div>", { class: "block" });
+
+    const $group = $("<div>");
+    $group.append($accountBlock);
+    $group.append($queryBlock);
+    $group.append($requestsBlock);
+    return $group;
+  }
+
+  ////////////////////////////////////////////////////////////
+  // For help.html
+
+  function renderModelsTable() {
+    // Render the list of models
+    const $table = $("<table>", { class: "table" });
+    const $header = $("<tr>")
+      .append($("<td>").append("group"))
+      .append($("<td>").append("name"))
+      .append($("<td>").append("description"))
+      .append($("<td>").append("tags"));
+    $table.append($header);
+    generalInfo.all_models.forEach((model) => {
+      const $row = $("<tr>")
+        .append($("<td>").append($("<tt>").append(model.group)))
+        .append($("<td>").append($("<tt>").append(model.name)))
+        .append($("<td>").append(model.description))
+        .append($("<td>").append(model.tags.join(" ")));
+      $table.append($row);
+    });
+    return $table;
+  }
+
+  ////////////////////////////////////////////////////////////
+  // Main
+
+  let generalInfo;
+
+  $.getJSON("/api/general_info", (response) => {
+    generalInfo = response;
+    console.log("/api/general_info", generalInfo);
+    if (generalInfo.error) {
+      alert(generalInfo.error);
+      return;
+    }
+
+    // For index.html
+    const $main = $("#main");
+    if ($main.length > 0) {
+      $main.empty().append(renderQueryInterface());
+    }
+
+    // For help.html
+    const $helpModels = $("#help-models");
+    if ($helpModels.length > 0) {
+      $helpModels.empty().append(renderModelsTable());
+    }
+  });
+});

helm/proxy/test_retry.py

@@ -1,7 +1,7 @@
 from retrying import RetryError
 
 from helm.common.request import RequestResult
-from .retry import retry_request, get_retry_decorator, retry_if_request_failed
+from helm.proxy.retry import retry_request, get_retry_decorator, retry_if_request_failed
 
 
 def test_retry_for_successful_request():

helm/proxy/token_counters/auto_token_counter.py

@@ -4,7 +4,7 @@ from helm.benchmark.model_deployment_registry import ModelDeployment, get_model_
 from helm.common.request import Request, GeneratedOutput
 from helm.tokenizers.auto_tokenizer import AutoTokenizer
 from helm.common.tokenization_request import TokenizationRequest, TokenizationRequestResult
-from .token_counter import TokenCounter
+from helm.proxy.token_counters.token_counter import TokenCounter
 
 
 class AutoTokenCounter(TokenCounter):

helm/tokenizers/aleph_alpha_tokenizer.py

@@ -10,7 +10,7 @@ from helm.common.tokenization_request import (
     DecodeRequest,
     TokenizationToken,
 )
-from .caching_tokenizer import CachingTokenizer
+from helm.tokenizers.caching_tokenizer import CachingTokenizer
 
 try:
     from aleph_alpha_client import Client as AlephAlphaPythonClient

helm/tokenizers/caching_tokenizer.py

@@ -1,6 +1,6 @@
 from abc import abstractmethod
 from dataclasses import asdict
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List
 
 from helm.common.cache import Cache, CacheConfig
 from helm.common.request import wrap_request_time
@@ -11,7 +11,7 @@ from helm.common.tokenization_request import (
     DecodeRequestResult,
     TokenizationToken,
 )
-from .tokenizer import Tokenizer
+from helm.tokenizers.tokenizer import Tokenizer
 
 
 class CachingTokenizer(Tokenizer):
@@ -153,31 +153,3 @@ class CachingTokenizer(Tokenizer):
             )
         except Exception as error:
             raise ValueError(f"Failed to decode tokens with {self.__class__.__name__} tokenizer: {error}") from error
-
-
-def cleanup_str(token: str, tokenizer_name: Optional[str] = None) -> str:
-    """
-    Certain tokenizers introduce special characters to represent spaces, such as
-    "Ġ" or "▁". This function removes those characters.
-    """
-    if tokenizer_name in [
-        "TsinghuaKEG/ice",
-        "bigscience/T0pp",
-        "google/t5-11b",
-        "google/flan-t5-xxl",
-        "google/ul2",
-        "Yandex/yalm",
-        "ai21/j1",
-        "together",
-    ]:
-        return token.replace("▁", " ")
-    elif tokenizer_name is not None and tokenizer_name.startswith("huggingface"):
-        return token.replace("Ġ", " ")
-    return token
-
-
-def cleanup_tokens(tokens: List[str], tokenizer_name: Optional[str] = None) -> List[str]:
-    """
-    Applies `cleanup_str` to each token in `tokens`.
-    """
-    return [cleanup_str(token, tokenizer_name) for token in tokens]

helm/tokenizers/grok_tokenizer.py

@@ -0,0 +1,53 @@
+import dataclasses
+import os
+from typing import Any, Dict, List, Optional
+
+import requests
+
+from helm.common.cache import CacheConfig
+from helm.common.tokenization_request import (
+    TokenizationRequest,
+    TokenizationToken,
+)
+from helm.tokenizers.caching_tokenizer import CachingTokenizer
+
+
+class GrokAPITokenizer(CachingTokenizer):
+    """Tokenizer that uses the xAI Grok Tokenize Text API
+
+    Doc: https://docs.x.ai/docs/api-reference#tokenize-text"""
+
+    def __init__(self, cache_config: CacheConfig, api_key: Optional[str] = None) -> None:
+        super().__init__(cache_config)
+        self.api_key = api_key or os.environ.get("XAI_API_KEY")
+
+    def _tokenization_request_to_cache_key(self, request: TokenizationRequest) -> Dict[str, Any]:
+        cache_key = dataclasses.asdict(request)
+        # Delete encode because the Grok API simulateously gives string and integer tokens.
+        del cache_key["encode"]
+        return cache_key
+
+    def _tokenize_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:
+        if not self.api_key:
+            raise Exception(
+                "No Grok API key found. "
+                "Set grokApiKey in credentials.conf or set the GROK_API_KEY environment variable"
+            )
+        text = request["text"]
+        model = request["tokenizer"].split("/")[-1]
+        response = requests.post(
+            url="https://api.x.ai/v1/tokenize-text",
+            headers={"Authorization": f"Bearer {self.api_key}"},
+            json={"text": text, "model": model},
+        )
+        response.raise_for_status()
+        return response.json()
+
+    def _tokenization_raw_response_to_tokens(
+        self, response: Dict[str, Any], request: TokenizationRequest
+    ) -> List[TokenizationToken]:
+        raw_token_field_name = "token_id" if request.encode else "string_token"
+        return [TokenizationToken(raw_token[raw_token_field_name]) for raw_token in response["token_ids"]]
+
+    def _decode_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:
+        raise NotImplementedError("The xAI API does not support decoding.")

helm/tokenizers/http_model_tokenizer.py

@@ -11,7 +11,7 @@ from helm.common.tokenization_request import (
     TokenizationRequestResult,
     TokenizationToken,
 )
-from .tokenizer import Tokenizer
+from helm.tokenizers.tokenizer import Tokenizer
 
 import requests
 

helm/tokenizers/huggingface_tokenizer.py

@@ -7,8 +7,8 @@ from helm.common.concurrency import ThreadSafeWrapper
 from transformers import AutoTokenizer, PreTrainedTokenizerBase
 
 from helm.common.hierarchical_logger import htrack_block, hlog
-from .caching_tokenizer import CachingTokenizer
-from .tokenizer import cleanup_tokens
+from helm.tokenizers.caching_tokenizer import CachingTokenizer
+from helm.tokenizers.tokenizer import cleanup_tokens
 
 
 WrappedPreTrainedTokenizer = ThreadSafeWrapper[PreTrainedTokenizerBase]
@@ -119,7 +119,7 @@ class HuggingFaceTokenizer(CachingTokenizer):
                     tokens = tokenizer.encode(
                         request["text"],
                         truncation=request["truncation"],
-                        max_length=request["max_length"],
+                        max_length=max(request["max_length"], 0),
                         add_special_tokens=False,
                     )
             else:

helm/tokenizers/lit_gpt_tokenizer.py

@@ -5,7 +5,7 @@ import torch
 
 from helm.common.cache import CacheConfig
 from helm.common.optional_dependencies import OptionalDependencyNotInstalled
-from .caching_tokenizer import CachingTokenizer
+from helm.tokenizers.caching_tokenizer import CachingTokenizer
 
 try:
     from lit_gpt import Tokenizer as InternalTokenizer

helm/tokenizers/test_anthropic_tokenizer.py

@@ -10,7 +10,7 @@ from helm.common.tokenization_request import (
     TokenizationRequest,
     TokenizationRequestResult,
 )
-from helm.tokenizers.anthropic_tokenizer import AnthropicTokenizer
+from helm.tokenizers.huggingface_tokenizer import HuggingFaceTokenizer
 
 
 class TestAnthropicTokenizer:
@@ -21,7 +21,11 @@ class TestAnthropicTokenizer:
     def setup_method(self, method):
         cache_file = tempfile.NamedTemporaryFile(delete=False)
         self.cache_path: str = cache_file.name
-        self.tokenizer = AnthropicTokenizer(SqliteCacheConfig(self.cache_path))
+        self.tokenizer = HuggingFaceTokenizer(
+            SqliteCacheConfig(self.cache_path),
+            tokenizer_name="anthropic/claude",
+            pretrained_model_name_or_path="Xenova/claude-tokenizer",
+        )
 
     def teardown_method(self, method):
         os.remove(self.cache_path)

helm/tokenizers/test_grok_tokenizer.py

@@ -0,0 +1,33 @@
+import os
+import pytest
+
+from helm.common.cache import BlackHoleCacheConfig
+from helm.common.tokenization_request import (
+    TokenizationRequest,
+    TokenizationToken,
+)
+from helm.tokenizers.grok_tokenizer import GrokAPITokenizer
+
+
+@pytest.mark.models
+def test_tokenize():
+    if not os.environ.get("XAI_API_KEY"):
+        pytest.skip("No xAI API key found; skipping test")
+    tokenizer = GrokAPITokenizer(cache_config=BlackHoleCacheConfig())
+    request = TokenizationRequest(tokenizer="xai/grok-3-beta", text="otter 🦦")
+    result = tokenizer.tokenize(request)
+    assert result.success
+    assert not result.cached
+    assert result.tokens == [TokenizationToken(token) for token in ["otter", "", "", ""]]
+
+
+@pytest.mark.models
+def test_encode():
+    if not os.environ.get("XAI_API_KEY"):
+        pytest.skip("No xAI API key found; skipping test")
+    tokenizer = GrokAPITokenizer(cache_config=BlackHoleCacheConfig())
+    request = TokenizationRequest(tokenizer="xai/grok-3-beta", text="otter 🦦", encode=True)
+    result = tokenizer.tokenize(request)
+    assert result.success
+    assert not result.cached
+    assert result.tokens == [TokenizationToken(token) for token in [142507, 11637, 294, 294]]