crfm-helm 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

helm/proxy/server.py

@@ -1,5 +1,3 @@
-# mypy: check_untyped_defs = False
-
 """
 Starts a REST server for the frontend to interact with.
 Look at `index.js` to see how the functionality is invoked.
@@ -16,12 +14,21 @@ import time
 from dacite import from_dict
 import bottle
 
+from helm.benchmark.config_registry import (
+    register_configs_from_directory,
+    register_builtin_configs_from_helm_package,
+)
+from helm.benchmark.model_deployment_registry import get_default_model_deployment_for_model
 from helm.common.authentication import Authentication
+from helm.common.cache_backend_config import CacheBackendConfig, MongoCacheBackendConfig, SqliteCacheBackendConfig
+from helm.common.general import ensure_directory_exists
 from helm.common.hierarchical_logger import hlog
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.common.request import Request
 from helm.common.perspective_api_request import PerspectiveAPIRequest
+from helm.common.moderations_api_request import ModerationAPIRequest
 from helm.common.tokenization_request import TokenizationRequest, DecodeRequest
+from helm.proxy.services.service import CACHE_DIR
 from .accounts import Account
 from .services.server_service import ServerService
 from .query import Query
@@ -35,6 +42,7 @@ except ModuleNotFoundError as e:
 bottle.BaseRequest.MEMFILE_MAX = 1024 * 1024
 
 app = bottle.default_app()
+service: ServerService
 
 
 def safe_call(func, to_json=True):
@@ -83,9 +91,16 @@ def handle_static_filename(filename):
     return resp
 
 
+@app.get("/output/<filename:path>")
+def handle_output_filename(filename):
+    resp = bottle.static_file(filename, root=app.config["crfm.proxy.outputpath"])
+    return resp
+
+
 @app.get("/api/general_info")
 def handle_get_general_info():
     def perform(args):
+        global service
         return dataclasses.asdict(service.get_general_info())
 
     return safe_call(perform)
@@ -94,6 +109,7 @@ def handle_get_general_info():
 @app.get("/api/window_service_info")
 def handle_get_window_service_info():
     def perform(args):
+        global service
         return dataclasses.asdict(service.get_window_service_info(args["model_name"]))
 
     return safe_call(perform)
@@ -102,6 +118,7 @@ def handle_get_window_service_info():
 @app.post("/api/account")
 def handle_create_account():
     def perform(args):
+        global service
         auth = Authentication(**json.loads(args["auth"]))
         return dataclasses.asdict(service.create_account(auth))
 
@@ -111,6 +128,7 @@ def handle_create_account():
 @app.delete("/api/account")
 def handle_delete_account():
     def perform(args):
+        global service
         auth = Authentication(**json.loads(args["auth"]))
         api_key = args["api_key"]
         return dataclasses.asdict(service.delete_account(auth, api_key))
@@ -121,6 +139,7 @@ def handle_delete_account():
 @app.get("/api/account")
 def handle_get_account():
     def perform(args):
+        global service
         auth = Authentication(**json.loads(args["auth"]))
         if "all" in args and args["all"].lower() == "true":
             return [dataclasses.asdict(account) for account in service.get_accounts(auth)]
@@ -133,6 +152,7 @@ def handle_get_account():
 @app.put("/api/account")
 def handle_update_account():
     def perform(args):
+        global service
         auth = Authentication(**json.loads(args["auth"]))
         account = from_dict(Account, json.loads(args["account"]))
         return dataclasses.asdict(service.update_account(auth, account))
@@ -143,6 +163,7 @@ def handle_update_account():
 @app.put("/api/account/api_key")
 def handle_update_api_key():
     def perform(args):
+        global service
         auth = Authentication(**json.loads(args["auth"]))
         account = from_dict(Account, json.loads(args["account"]))
         return dataclasses.asdict(service.rotate_api_key(auth, account))
@@ -153,6 +174,7 @@ def handle_update_api_key():
 @app.get("/api/query")
 def handle_query():
     def perform(args):
+        global service
         query = Query(**args)
         return dataclasses.asdict(service.expand_query(query))
 
@@ -162,9 +184,28 @@ def handle_query():
 @app.get("/api/request")
 def handle_request():
     def perform(args):
+        global service
         auth = Authentication(**json.loads(args["auth"]))
         request = Request(**json.loads(args["request"]))
-        return dataclasses.asdict(service.make_request(auth, request))
+        # Hack to maintain reverse compatibility with clients with version <= 0.3.0.
+        # Clients with version <= 0.3.0 do not set model_deployment, but this is now
+        # required by Request.
+        if not request.model_deployment:
+            model_deployment = get_default_model_deployment_for_model(request.model)
+            if model_deployment is None:
+                raise ValueError(f"Unknown model '{request.model}'")
+            request = dataclasses.replace(request, model_deployment=model_deployment)
+
+        raw_response = dataclasses.asdict(service.make_request(auth, request))
+
+        # Hack to maintain reverse compatibility with clients with version <= 1.0.0.
+        # Clients with version <= 1.0.0 expect each token to contain a `top_logprobs`
+        # field of type dict.
+        for completion in raw_response["completions"]:
+            for token in completion["tokens"]:
+                token["top_logprobs"] = {}
+
+        return raw_response
 
     return safe_call(perform)
 
@@ -172,6 +213,7 @@ def handle_request():
 @app.get("/api/tokenize")
 def handle_tokenization():
     def perform(args):
+        global service
         auth = Authentication(**json.loads(args["auth"]))
         request = TokenizationRequest(**json.loads(args["request"]))
         return dataclasses.asdict(service.tokenize(auth, request))
@@ -182,6 +224,7 @@ def handle_tokenization():
 @app.get("/api/decode")
 def handle_decode():
     def perform(args):
+        global service
         auth = Authentication(**json.loads(args["auth"]))
         request = DecodeRequest(**json.loads(args["request"]))
         return dataclasses.asdict(service.decode(auth, request))
@@ -192,6 +235,7 @@ def handle_decode():
 @app.get("/api/toxicity")
 def handle_toxicity_request():
     def perform(args):
+        global service
         auth = Authentication(**json.loads(args["auth"]))
         request = PerspectiveAPIRequest(**json.loads(args["request"]))
         return dataclasses.asdict(service.get_toxicity_scores(auth, request))
@@ -199,9 +243,21 @@ def handle_toxicity_request():
     return safe_call(perform)
 
 
+@app.get("/api/moderation")
+def handle_moderation_request():
+    def perform(args):
+        global service
+        auth = Authentication(**json.loads(args["auth"]))
+        request = ModerationAPIRequest(**json.loads(args["request"]))
+        return dataclasses.asdict(service.get_moderation_results(auth, request))
+
+    return safe_call(perform)
+
+
 @app.get("/api/shutdown")
 def handle_shutdown():
     def perform(args):
+        global service
         auth = Authentication(**json.loads(args["auth"]))
         service.shutdown(auth)
 
@@ -214,6 +270,7 @@ def main():
     parser.add_argument("-p", "--port", type=int, help="What port to listen on", default=1959)
     parser.add_argument("--ssl-key-file", type=str, help="Path to SSL key file")
     parser.add_argument("--ssl-cert-file", type=str, help="Path to SSL cert file")
+    parser.add_argument("--ssl-ca-certs", type=str, help="Path to SSL CA certs")
     parser.add_argument("-b", "--base-path", help="What directory has credentials, etc.", default="prod_env")
     parser.add_argument("-w", "--workers", type=int, help="Number of worker processes to handle requests", default=8)
     parser.add_argument("-t", "--timeout", type=int, help="Request timeout in seconds", default=5 * 60)
@@ -225,17 +282,32 @@ def main():
     )
     args = parser.parse_args()
 
-    service = ServerService(base_path=args.base_path, mongo_uri=args.mongo_uri)
+    register_builtin_configs_from_helm_package()
+    register_configs_from_directory(args.base_path)
+
+    cache_backend_config: CacheBackendConfig
+    if args.mongo_uri:
+        cache_backend_config = MongoCacheBackendConfig(args.mongo_uri)
+    else:
+        sqlite_cache_path = os.path.join(args.base_path, CACHE_DIR)
+        ensure_directory_exists(sqlite_cache_path)
+        cache_backend_config = SqliteCacheBackendConfig(sqlite_cache_path)
+
+    service = ServerService(base_path=args.base_path, cache_backend_config=cache_backend_config)
 
     gunicorn_args = {
         "workers": args.workers,
         "timeout": args.timeout,
         "limit_request_line": 0,  # Controls the maximum size of HTTP request line in bytes. 0 = unlimited.
     }
-    if args.ssl_key_file and args.ssl_cert_file:
+    if args.ssl_key_file:
         gunicorn_args["keyfile"] = args.ssl_key_file
+    if args.ssl_cert_file:
         gunicorn_args["certfile"] = args.ssl_cert_file
+    if args.ssl_ca_certs:
+        gunicorn_args["ca_certs"] = args.ssl_ca_certs
 
     # Clear arguments before running gunicorn as it also uses argparse
     sys.argv = [sys.argv[0]]
+    app.config["crfm.proxy.outputpath"] = os.path.join(os.path.realpath(args.base_path), "cache", "output")
     app.run(host="0.0.0.0", port=args.port, server="gunicorn", **gunicorn_args)

helm/proxy/services/remote_service.py

@@ -5,9 +5,15 @@ import urllib.parse
 from dataclasses import asdict
 from typing import Any, List, Optional
 
+from helm.common.cache import CacheConfig
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from helm.common.authentication import Authentication
+from helm.common.moderations_api_request import ModerationAPIRequest, ModerationAPIRequestResult
 from helm.common.critique_request import CritiqueRequest, CritiqueRequestResult
+from helm.common.nudity_check_request import NudityCheckRequest, NudityCheckResult
+from helm.common.file_upload_request import FileUploadRequest, FileUploadResult
 from helm.common.perspective_api_request import PerspectiveAPIRequest, PerspectiveAPIRequestResult
+from helm.common.clip_score_request import CLIPScoreRequest, CLIPScoreResult
 from helm.common.tokenization_request import (
     WindowServiceInfo,
     TokenizationRequest,
@@ -27,6 +33,8 @@ class RemoteServiceError(Exception):
 
 
 class RemoteService(Service):
+    NOT_SUPPORTED_ERROR: str = "Not supported through the remote service."
+
     def __init__(self, base_url):
         self.base_url: str = base_url
 
@@ -84,6 +92,15 @@ class RemoteService(Service):
         RemoteService._check_response(response, request_json)
         return from_dict(DecodeRequestResult, response)
 
+    def upload(self, auth: Authentication, request: FileUploadRequest) -> FileUploadResult:
+        raise NotImplementedError(self.NOT_SUPPORTED_ERROR)
+
+    def check_nudity(self, auth: Authentication, request: NudityCheckRequest) -> NudityCheckResult:
+        raise NotImplementedError(self.NOT_SUPPORTED_ERROR)
+
+    def compute_clip_score(self, auth: Authentication, request: CLIPScoreRequest) -> CLIPScoreResult:
+        raise NotImplementedError(self.NOT_SUPPORTED_ERROR)
+
     def get_toxicity_scores(self, auth: Authentication, request: PerspectiveAPIRequest) -> PerspectiveAPIRequestResult:
         request_json: str = json.dumps(asdict(request))
         params = {
@@ -94,6 +111,16 @@ class RemoteService(Service):
         RemoteService._check_response(response, request_json)
         return from_dict(PerspectiveAPIRequestResult, response)
 
+    def get_moderation_results(self, auth: Authentication, request: ModerationAPIRequest) -> ModerationAPIRequestResult:
+        request_json: str = json.dumps(asdict(request))
+        params = {
+            "auth": json.dumps(asdict(auth)),
+            "request": request_json,
+        }
+        response = requests.get(f"{self.base_url}/api/moderation?{urllib.parse.urlencode(params)}").json()
+        RemoteService._check_response(response, request_json)
+        return from_dict(ModerationAPIRequestResult, response)
+
     def make_critique_request(self, auth: Authentication, request: CritiqueRequest) -> CritiqueRequestResult:
         raise NotImplementedError("make_critique_request is not supported by RemoteServer")
 
@@ -153,6 +180,10 @@ class RemoteService(Service):
             # A ConnectionError is expected when shutting down the server.
             pass
 
+    def get_cache_config(self, shard_name: str) -> CacheConfig:
+        """Returns a CacheConfig"""
+        return BlackHoleCacheBackendConfig().get_cache_config(shard_name)
+
 
 def add_service_args(parser: argparse.ArgumentParser):
     """Add command-line arguments to enable command-line utilities to specify how to connect to a remote server."""

helm/proxy/services/server_service.py

@@ -1,12 +1,16 @@
+import dataclasses
 import os
 import signal
 from typing import List, Optional
 
-from helm.benchmark.model_metadata_registry import maybe_register_model_metadata_from_base_path
-from helm.benchmark.model_deployment_registry import maybe_register_model_deployments_from_base_path
-from helm.benchmark.tokenizer_config_registry import maybe_register_tokenizer_configs_from_base_path
+from helm.common.cache import CacheConfig
+from helm.common.cache_backend_config import CacheBackendConfig, BlackHoleCacheBackendConfig
 from helm.common.critique_request import CritiqueRequest, CritiqueRequestResult
 from helm.common.authentication import Authentication
+from helm.common.moderations_api_request import ModerationAPIRequest, ModerationAPIRequestResult
+from helm.common.clip_score_request import CLIPScoreRequest, CLIPScoreResult
+from helm.common.nudity_check_request import NudityCheckRequest, NudityCheckResult
+from helm.common.file_upload_request import FileUploadRequest, FileUploadResult
 from helm.common.general import ensure_directory_exists, parse_hocon, get_credentials
 from helm.common.perspective_api_request import PerspectiveAPIRequest, PerspectiveAPIRequestResult
 from helm.common.tokenization_request import (
@@ -19,13 +23,20 @@ from helm.common.tokenization_request import (
 from helm.common.request import Request, RequestResult
 from helm.common.hierarchical_logger import hlog
 from helm.proxy.accounts import Accounts, Account
-from helm.proxy.clients.auto_client import AutoClient
-from helm.proxy.clients.toxicity_classifier_client import ToxicityClassifierClient
+from helm.clients.auto_client import AutoClient
+from helm.clients.moderation_api_client import ModerationAPIClient
+from helm.clients.perspective_api_client import PerspectiveAPIClient
+from helm.clients.image_generation.nudity_check_client import NudityCheckClient
+from helm.clients.gcs_client import GCSClient
+from helm.clients.clip_score_client import CLIPScoreClient
+from helm.clients.toxicity_classifier_client import ToxicityClassifierClient
 from helm.proxy.example_queries import example_queries
-from helm.proxy.models import ALL_MODELS, get_model_group
+from helm.benchmark.model_metadata_registry import ALL_MODELS_METADATA
+from helm.benchmark.model_deployment_registry import get_model_deployment_host_organization
 from helm.proxy.query import Query, QueryResult
 from helm.proxy.retry import retry_request
 from helm.proxy.token_counters.auto_token_counter import AutoTokenCounter
+from helm.tokenizers.auto_tokenizer import AutoTokenizer
 from .service import (
     Service,
     CACHE_DIR,
@@ -42,24 +53,38 @@ class ServerService(Service):
     Main class that supports various functionality for the server.
     """
 
-    def __init__(self, base_path: str = "prod_env", root_mode=False, mongo_uri: str = ""):
+    def __init__(
+        self,
+        base_path: str = "prod_env",
+        root_mode: bool = False,
+        cache_backend_config: CacheBackendConfig = BlackHoleCacheBackendConfig(),
+    ):
+        ensure_directory_exists(base_path)
+        client_file_storage_path = os.path.join(base_path, CACHE_DIR)
+        ensure_directory_exists(client_file_storage_path)
+
         credentials = get_credentials(base_path)
-        cache_path = os.path.join(base_path, CACHE_DIR)
-        ensure_directory_exists(cache_path)
         accounts_path = os.path.join(base_path, ACCOUNTS_FILE)
 
-        maybe_register_model_metadata_from_base_path(base_path)
-        maybe_register_model_deployments_from_base_path(base_path)
-        maybe_register_tokenizer_configs_from_base_path(base_path)
-
-        self.client = AutoClient(credentials, cache_path, mongo_uri)
-        self.token_counter = AutoTokenCounter(self.client.get_huggingface_client())
+        self.cache_backend_config = cache_backend_config
+        self.client = AutoClient(credentials, client_file_storage_path, cache_backend_config)
+        self.tokenizer = AutoTokenizer(credentials, cache_backend_config)
+        self.token_counter = AutoTokenCounter(self.tokenizer)
         self.accounts = Accounts(accounts_path, root_mode=root_mode)
-        # Lazily instantiated by get_toxicity_scores()
+
+        # Lazily instantiate the following clients
+        self.moderation_api_client: Optional[ModerationAPIClient] = None
         self.toxicity_classifier_client: Optional[ToxicityClassifierClient] = None
+        self.perspective_api_client: Optional[PerspectiveAPIClient] = None
+        self.nudity_check_client: Optional[NudityCheckClient] = None
+        self.clip_score_client: Optional[CLIPScoreClient] = None
+        self.gcs_client: Optional[GCSClient] = None
 
     def get_general_info(self) -> GeneralInfo:
-        return GeneralInfo(version=VERSION, example_queries=example_queries, all_models=ALL_MODELS)
+        # Can't send release_dates in ModelMetadata bacause dates cannot be round-tripped to and from JSON easily.
+        # TODO(#2158): Either fix this or delete get_general_info.
+        all_models = [dataclasses.replace(model_metadata, release_date=None) for model_metadata in ALL_MODELS_METADATA]
+        return GeneralInfo(version=VERSION, example_queries=example_queries, all_models=all_models)
 
     def get_window_service_info(self, model_name) -> WindowServiceInfo:
         # The import statement is placed here to avoid two problems, please refer to the link for details
@@ -88,6 +113,21 @@ class ServerService(Service):
             requests.append(request)
         return QueryResult(requests=requests)
 
+    def _get_model_group_for_model_deployment(self, model_deployment: str) -> str:
+        if model_deployment.startswith("openai/"):
+            if model_deployment.startswith("openai/code-"):
+                return "codex"
+            elif model_deployment.startswith("openai/dall-e-"):
+                return "dall_e"
+            elif model_deployment.startswith("openai/gpt-4-"):
+                return "gpt4"
+            else:
+                return "gpt3"
+        elif model_deployment.startswith("ai21/"):
+            return "jurassic"
+        else:
+            return get_model_deployment_host_organization(model_deployment)
+
     def make_request(self, auth: Authentication, request: Request) -> RequestResult:
         """Actually make a request to an API."""
         # TODO: try to invoke the API even if we're not authenticated, and if
@@ -95,7 +135,7 @@ class ServerService(Service):
         #       https://github.com/stanford-crfm/benchmarking/issues/56
 
         self.accounts.authenticate(auth)
-        model_group: str = get_model_group(request.model)
+        model_group: str = self._get_model_group_for_model_deployment(request.model_deployment)
         # Make sure we can use
         self.accounts.check_can_use(auth.api_key, model_group)
 
@@ -113,12 +153,42 @@ class ServerService(Service):
     def tokenize(self, auth: Authentication, request: TokenizationRequest) -> TokenizationRequestResult:
         """Tokenize via an API."""
         self.accounts.authenticate(auth)
-        return self.client.tokenize(request)
+        return self.tokenizer.tokenize(request)
 
     def decode(self, auth: Authentication, request: DecodeRequest) -> DecodeRequestResult:
         """Decodes to text."""
         self.accounts.authenticate(auth)
-        return self.client.decode(request)
+        return self.tokenizer.decode(request)
+
+    def upload(self, auth: Authentication, request: FileUploadRequest) -> FileUploadResult:
+        """Uploads a file to external storage."""
+        self.accounts.authenticate(auth)
+
+        if not self.gcs_client:
+            self.gcs_client = self.client.get_gcs_client()
+
+        assert self.gcs_client
+        return self.gcs_client.upload(request)
+
+    def check_nudity(self, auth: Authentication, request: NudityCheckRequest) -> NudityCheckResult:
+        """Check for nudity."""
+        self.accounts.authenticate(auth)
+
+        if not self.nudity_check_client:
+            self.nudity_check_client = self.client.get_nudity_check_client()
+
+        assert self.nudity_check_client
+        return self.nudity_check_client.check_nudity(request)
+
+    def compute_clip_score(self, auth: Authentication, request: CLIPScoreRequest) -> CLIPScoreResult:
+        """Computes CLIPScore for a given caption and image."""
+        self.accounts.authenticate(auth)
+
+        if not self.clip_score_client:
+            self.clip_score_client = self.client.get_clip_score_client()
+
+        assert self.clip_score_client
+        return self.clip_score_client.compute_score(request)
 
     def get_toxicity_scores(self, auth: Authentication, request: PerspectiveAPIRequest) -> PerspectiveAPIRequestResult:
         @retry_request
@@ -130,6 +200,16 @@ class ServerService(Service):
         self.accounts.authenticate(auth)
         return get_toxicity_scores_with_retry(request)
 
+    def get_moderation_results(self, auth: Authentication, request: ModerationAPIRequest) -> ModerationAPIRequestResult:
+        @retry_request
+        def get_moderation_results_with_retry(request: ModerationAPIRequest) -> ModerationAPIRequestResult:
+            if not self.moderation_api_client:
+                self.moderation_api_client = self.client.get_moderation_api_client()
+            return self.moderation_api_client.get_moderation_results(request)
+
+        self.accounts.authenticate(auth)
+        return get_moderation_results_with_retry(request)
+
     def make_critique_request(self, auth: Authentication, request: CritiqueRequest) -> CritiqueRequestResult:
         self.accounts.authenticate(auth)
         return self.client.get_critique_client().make_critique_request(request)
@@ -165,3 +245,6 @@ class ServerService(Service):
         hlog(f"Shutting down server by killing its own process {pid}...")
         os.kill(pid, signal.SIGTERM)
         hlog("Done.")
+
+    def get_cache_config(self, shard_name: str) -> CacheConfig:
+        return self.cache_backend_config.get_cache_config(shard_name)

helm/proxy/services/service.py

@@ -5,7 +5,11 @@ from typing import Dict, List, Tuple, Any
 
 from helm.common.general import parse_hocon
 from helm.common.critique_request import CritiqueRequest, CritiqueRequestResult
+from helm.common.clip_score_request import CLIPScoreRequest, CLIPScoreResult
+from helm.common.file_upload_request import FileUploadResult, FileUploadRequest
+from helm.common.nudity_check_request import NudityCheckRequest, NudityCheckResult
 from helm.common.perspective_api_request import PerspectiveAPIRequestResult, PerspectiveAPIRequest
+from helm.common.moderations_api_request import ModerationAPIRequest, ModerationAPIRequestResult
 from helm.common.tokenization_request import (
     WindowServiceInfo,
     TokenizationRequest,
@@ -14,9 +18,10 @@ from helm.common.tokenization_request import (
     DecodeRequestResult,
 )
 from helm.common.request import Request, RequestResult
-from helm.proxy.models import Model
+from helm.benchmark.model_metadata_registry import ModelMetadata
 from helm.proxy.query import Query, QueryResult
 from helm.proxy.accounts import Authentication, Account
+from helm.common.cache import CacheConfig
 
 VERSION = "1.0"
 ACCOUNTS_FILE = "accounts.sqlite"
@@ -29,7 +34,7 @@ MAX_EXPANSION = 1000
 class GeneralInfo:
     version: str
     example_queries: List[Query]
-    all_models: List[Model]
+    all_models: List[ModelMetadata]
 
 
 def expand_environments(environments: Dict[str, List[str]]):
@@ -69,6 +74,8 @@ def synthesize_request(prompt: str, settings: str, environment: Dict[str, str])
     request: Dict[str, Any] = {}
     request["prompt"] = substitute_text(prompt, environment)
     request.update(parse_hocon(substitute_text(settings, environment)))
+    if "model_deployment" not in request and "model" not in request:
+        request["model_deployment"] = "openai/text-davinci-002"
     return Request(**request)
 
 
@@ -103,11 +110,31 @@ class Service(ABC):
         """Decodes to text."""
         pass
 
+    @abstractmethod
+    def upload(self, auth: Authentication, request: FileUploadRequest) -> FileUploadResult:
+        """Uploads a file to external storage."""
+        pass
+
+    @abstractmethod
+    def check_nudity(self, auth: Authentication, request: NudityCheckRequest) -> NudityCheckResult:
+        """Check for nudity for a batch of images."""
+        pass
+
+    @abstractmethod
+    def compute_clip_score(self, auth: Authentication, request: CLIPScoreRequest) -> CLIPScoreResult:
+        """Computes CLIPScore for a given caption and image."""
+        pass
+
     @abstractmethod
     def get_toxicity_scores(self, auth: Authentication, request: PerspectiveAPIRequest) -> PerspectiveAPIRequestResult:
         """Get toxicity scores for a batch of text."""
         pass
 
+    @abstractmethod
+    def get_moderation_results(self, auth: Authentication, request: ModerationAPIRequest) -> ModerationAPIRequestResult:
+        """Get OpenAI's moderation results for some text."""
+        pass
+
     @abstractmethod
     def make_critique_request(self, auth: Authentication, request: CritiqueRequest) -> CritiqueRequestResult:
         """Get responses to a critique request."""
@@ -147,3 +174,8 @@ class Service(ABC):
     def shutdown(self, auth: Authentication):
         """Shutdown server."""
         pass
+
+    @abstractmethod
+    def get_cache_config(self, shard_name: str) -> CacheConfig:
+        """Returns a CacheConfig"""
+        pass

helm/proxy/services/test_remote_service.py

@@ -17,7 +17,7 @@ from sqlitedict import SqliteDict
 from helm.common.authentication import Authentication
 from helm.common.request import Request, RequestResult
 from helm.common.tokenization_request import TokenizationRequest, TokenizationRequestResult
-from helm.proxy.accounts import Account
+from helm.proxy.accounts import Account, set_default_quotas
 from .remote_service import RemoteService
 from .service import ACCOUNTS_FILE
 
@@ -55,6 +55,7 @@ class TestRemoteServerService:
 
             with SqliteDict(os.path.join(path, ACCOUNTS_FILE)) as cache:
                 account: Account = Account(TestRemoteServerService._ADMIN_API_KEY, is_admin=True)
+                set_default_quotas(account)
                 cache[TestRemoteServerService._ADMIN_API_KEY] = asdict(account)
                 cache.commit()
             return path
@@ -85,7 +86,7 @@ class TestRemoteServerService:
 
     @staticmethod
     def query(url: str, auth: Authentication, prompt: str):
-        request = Request(prompt=prompt, model="simple/model1")
+        request = Request(prompt=prompt, model="simple/model1", model_deployment="simple/model1")
         response: RequestResult = RemoteService(base_url=url).make_request(auth, request)
         response_text: str = response.completions[0].text
         # With the toy model (simple/model1), we should expect the same response as the prompt
@@ -121,18 +122,18 @@ class TestRemoteServerService:
         shutil.rmtree(cls.base_path)
 
     def test_make_request(self):
-        request = Request(prompt="1 2 3", model="simple/model1")
+        request = Request(prompt="1 2 3", model="simple/model1", model_deployment="simple/model1")
         response: RequestResult = self.service.make_request(self.auth, request)
         assert response.success
 
     def test_tokenize(self):
-        request = TokenizationRequest(text="1 2 3", tokenizer="simple/model1")
+        request = TokenizationRequest(text="1 2 3", tokenizer="simple/tokenizer1")
         response: TokenizationRequestResult = self.service.tokenize(self.auth, request)
-        assert [token.value for token in response.tokens] == ["1", "2", "3"]
+        assert [token.value for token in response.tokens] == ["1", " ", "2", " ", "3"]
 
     def test_make_request_plus_sign(self):
         # Ensure + in prompt doesn't get replaced by a blank space
-        request = Request(prompt="+", model="simple/model1")
+        request = Request(prompt="+", model="simple/model1", model_deployment="simple/model1")
         response: RequestResult = self.service.make_request(self.auth, request)
         assert response.completions[0].text == "+"
         assert response.success